Example usage for com.amazonaws ClientConfiguration setProxyDomain

List of usage examples for com.amazonaws ClientConfiguration setProxyDomain

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.*
  5. ClientConfiguration
  6. setProxyDomain

Introduction

In this page you can find the example usage for com.amazonaws ClientConfiguration setProxyDomain.

Prototype

public void setProxyDomain(String proxyDomain)

Source Link

Document

Sets the optional Windows domain name for configuration an NTLM proxy.

Usage

From source file:com.ibm.stocator.fs.cos.COSAPIClient.java

License:Apache License

@Override
public void initiate(String scheme) throws IOException, ConfigurationParseException {
    mCachedSparkOriginated = new HashMap<String, Boolean>();
    mCachedSparkJobsStatus = new HashMap<String, Boolean>();
    schemaProvided = scheme;//from  w w  w.  j  a  v  a  2 s  .com
    Properties props = ConfigurationHandler.initialize(filesystemURI, conf, scheme);
    // Set bucket name property
    int cacheSize = conf.getInt(CACHE_SIZE, GUAVA_CACHE_SIZE_DEFAULT);
    memoryCache = MemoryCache.getInstance(cacheSize);
    mBucket = props.getProperty(COS_BUCKET_PROPERTY);
    workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(filesystemURI,
            getWorkingDirectory());

    fModeAutomaticDelete = "true".equals(props.getProperty(FMODE_AUTOMATIC_DELETE_COS_PROPERTY, "false"));
    mIsV2Signer = "true".equals(props.getProperty(V2_SIGNER_TYPE_COS_PROPERTY, "false"));
    // Define COS client
    String accessKey = props.getProperty(ACCESS_KEY_COS_PROPERTY);
    String secretKey = props.getProperty(SECRET_KEY_COS_PROPERTY);

    if (accessKey == null) {
        throw new ConfigurationParseException("Access KEY is empty. Please provide valid access key");
    }
    if (secretKey == null) {
        throw new ConfigurationParseException("Secret KEY is empty. Please provide valid secret key");
    }

    BasicAWSCredentials creds = new BasicAWSCredentials(accessKey, secretKey);
    ClientConfiguration clientConf = new ClientConfiguration();

    int maxThreads = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, MAX_THREADS, DEFAULT_MAX_THREADS);
    if (maxThreads < 2) {
        LOG.warn(MAX_THREADS + " must be at least 2: forcing to 2.");
        maxThreads = 2;
    }
    int totalTasks = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, MAX_TOTAL_TASKS, DEFAULT_MAX_TOTAL_TASKS);
    long keepAliveTime = Utils.getLong(conf, FS_COS, FS_ALT_KEYS, KEEPALIVE_TIME, DEFAULT_KEEPALIVE_TIME);
    threadPoolExecutor = BlockingThreadPoolExecutorService.newInstance(maxThreads, maxThreads + totalTasks,
            keepAliveTime, TimeUnit.SECONDS, "s3a-transfer-shared");

    unboundedThreadPool = new ThreadPoolExecutor(maxThreads, Integer.MAX_VALUE, keepAliveTime, TimeUnit.SECONDS,
            new LinkedBlockingQueue<Runnable>(),
            BlockingThreadPoolExecutorService.newDaemonThreadFactory("s3a-transfer-unbounded"));

    boolean secureConnections = Utils.getBoolean(conf, FS_COS, FS_ALT_KEYS, SECURE_CONNECTIONS,
            DEFAULT_SECURE_CONNECTIONS);
    clientConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);

    String proxyHost = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_HOST, "");
    int proxyPort = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, PROXY_PORT, -1);
    if (!proxyHost.isEmpty()) {
        clientConf.setProxyHost(proxyHost);
        if (proxyPort >= 0) {
            clientConf.setProxyPort(proxyPort);
        } else {
            if (secureConnections) {
                LOG.warn("Proxy host set without port. Using HTTPS default 443");
                clientConf.setProxyPort(443);
            } else {
                LOG.warn("Proxy host set without port. Using HTTP default 80");
                clientConf.setProxyPort(80);
            }
        }
        String proxyUsername = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_USERNAME);
        String proxyPassword = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_PASSWORD);
        if ((proxyUsername == null) != (proxyPassword == null)) {
            String msg = "Proxy error: " + PROXY_USERNAME + " or " + PROXY_PASSWORD + " set without the other.";
            LOG.error(msg);
            throw new IllegalArgumentException(msg);
        }
        clientConf.setProxyUsername(proxyUsername);
        clientConf.setProxyPassword(proxyPassword);
        clientConf.setProxyDomain(Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_DOMAIN));
        clientConf.setProxyWorkstation(Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_WORKSTATION));
        if (LOG.isDebugEnabled()) {
            LOG.debug(
                    "Using proxy server {}:{} as user {} with password {} on " + "domain {} as workstation {}",
                    clientConf.getProxyHost(), clientConf.getProxyPort(),
                    String.valueOf(clientConf.getProxyUsername()), clientConf.getProxyPassword(),
                    clientConf.getProxyDomain(), clientConf.getProxyWorkstation());
        }
    } else if (proxyPort >= 0) {
        String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
        LOG.error(msg);
        throw new IllegalArgumentException(msg);
    }

    initConnectionSettings(conf, clientConf);
    if (mIsV2Signer) {
        clientConf.withSignerOverride("S3SignerType");
    }
    mClient = new AmazonS3Client(creds, clientConf);

    final String serviceUrl = props.getProperty(ENDPOINT_URL_COS_PROPERTY);
    if (serviceUrl != null && !serviceUrl.equals(amazonDefaultEndpoint)) {
        mClient.setEndpoint(serviceUrl);
    }
    mClient.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());

    // Set block size property
    String mBlockSizeString = props.getProperty(BLOCK_SIZE_COS_PROPERTY, "128");
    mBlockSize = Long.valueOf(mBlockSizeString).longValue() * 1024 * 1024L;

    boolean autoCreateBucket = "true"
            .equalsIgnoreCase((props.getProperty(AUTO_BUCKET_CREATE_COS_PROPERTY, "false")));

    partSize = Utils.getLong(conf, FS_COS, FS_ALT_KEYS, MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
    multiPartThreshold = Utils.getLong(conf, FS_COS, FS_ALT_KEYS, MIN_MULTIPART_THRESHOLD,
            DEFAULT_MIN_MULTIPART_THRESHOLD);
    readAhead = Utils.getLong(conf, FS_COS, FS_ALT_KEYS, READAHEAD_RANGE, DEFAULT_READAHEAD_RANGE);
    LOG.debug(READAHEAD_RANGE + ":" + readAhead);
    inputPolicy = COSInputPolicy
            .getPolicy(Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, INPUT_FADVISE, INPUT_FADV_NORMAL));

    initTransferManager();
    maxKeys = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
    flatListingFlag = Utils.getBoolean(conf, FS_COS, FS_ALT_KEYS, FLAT_LISTING, DEFAULT_FLAT_LISTING);

    if (autoCreateBucket) {
        try {
            boolean bucketExist = mClient.doesBucketExist(mBucket);
            if (bucketExist) {
                LOG.trace("Bucket {} exists", mBucket);
            } else {
                LOG.trace("Bucket {} doesn`t exists and autocreate", mBucket);
                String mRegion = props.getProperty(REGION_COS_PROPERTY);
                if (mRegion == null) {
                    mClient.createBucket(mBucket);
                } else {
                    LOG.trace("Creating bucket {} in region {}", mBucket, mRegion);
                    mClient.createBucket(mBucket, mRegion);
                }
            }
        } catch (AmazonServiceException ase) {
            /*
            *  we ignore the BucketAlreadyExists exception since multiple processes or threads
            *  might try to create the bucket in parrallel, therefore it is expected that
            *  some will fail to create the bucket
            */
            if (!ase.getErrorCode().equals("BucketAlreadyExists")) {
                LOG.error(ase.getMessage());
                throw (ase);
            }
        } catch (Exception e) {
            LOG.error(e.getMessage());
            throw (e);
        }
    }

    initMultipartUploads(conf);
    enableMultiObjectsDelete = Utils.getBoolean(conf, FS_COS, FS_ALT_KEYS, ENABLE_MULTI_DELETE, true);

    blockUploadEnabled = Utils.getBoolean(conf, FS_COS, FS_ALT_KEYS, FAST_UPLOAD, DEFAULT_FAST_UPLOAD);

    if (blockUploadEnabled) {
        blockOutputBuffer = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, FAST_UPLOAD_BUFFER,
                DEFAULT_FAST_UPLOAD_BUFFER);
        partSize = COSUtils.ensureOutputParameterInRange(MULTIPART_SIZE, partSize);
        blockFactory = COSDataBlocks.createFactory(this, blockOutputBuffer);
        blockOutputActiveBlocks = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, FAST_UPLOAD_ACTIVE_BLOCKS,
                DEFAULT_FAST_UPLOAD_ACTIVE_BLOCKS);
        LOG.debug("Using COSBlockOutputStream with buffer = {}; block={};" + " queue limit={}",
                blockOutputBuffer, partSize, blockOutputActiveBlocks);
    } else {
        LOG.debug("Using COSOutputStream");
    }
}

From source file:com.liferay.portal.store.s3.S3Store.java

License:Open Source License

protected void configureProxySettings(ClientConfiguration clientConfiguration) {

    String proxyHost = _s3StoreConfiguration.proxyHost();

    if (Validator.isNull(proxyHost)) {
        return;//from   ww w.  j  av  a2s. c om
    }

    clientConfiguration.setProxyHost(proxyHost);
    clientConfiguration.setProxyPort(_s3StoreConfiguration.proxyPort());

    String proxyAuthType = _s3StoreConfiguration.proxyAuthType();

    if (proxyAuthType.equals("ntlm") || proxyAuthType.equals("username-password")) {

        clientConfiguration.setProxyPassword(_s3StoreConfiguration.proxyPassword());
        clientConfiguration.setProxyUsername(_s3StoreConfiguration.proxyUsername());

        if (proxyAuthType.equals("ntlm")) {
            clientConfiguration.setProxyDomain(_s3StoreConfiguration.ntlmProxyDomain());
            clientConfiguration.setProxyWorkstation(_s3StoreConfiguration.ntlmProxyWorkstation());
        }
    }
}

From source file:com.netflix.spinnaker.clouddriver.aws.security.AWSProxy.java

License:Apache License

public void apply(ClientConfiguration clientConfiguration) {

    clientConfiguration.setProxyHost(proxyHost);
    clientConfiguration.setProxyPort(Integer.parseInt(proxyPort));
    clientConfiguration.setProxyUsername(proxyUsername);
    clientConfiguration.setProxyPassword(proxyPassword);

    Protocol awsProtocol = Protocol.HTTP;

    if ("HTTPS".equalsIgnoreCase(protocol)) {
        awsProtocol = Protocol.HTTPS;//from w w  w . j  av a2 s. co m
    }

    clientConfiguration.setProtocol(awsProtocol);

    if (isNTLMProxy()) {
        clientConfiguration.setProxyDomain(proxyDomain);
        clientConfiguration.setProxyWorkstation(proxyWorkstation);
    }
}

From source file:com.tcl.gateway.firehose.log4j.FirehoseAppender.java

License:Open Source License

/**
 * Set proxy configuration based on system properties. Some of the properties are standard
 * properties documented by Oracle (http.proxyHost, http.proxyPort, http.auth.ntlm.domain),
 * and others are from common convention (http.proxyUser, http.proxyPassword).
 *
 * Finally, for NTLM authentication the workstation name is taken from the environment as
 * COMPUTERNAME. We set this on the client configuration only if the NTLM domain was specified.
 *///from  w w w. j a  va  2 s .c  o m
private ClientConfiguration setProxySettingsFromSystemProperties(ClientConfiguration clientConfiguration) {

    final String proxyHost = System.getProperty("http.proxyHost");
    if (proxyHost != null) {
        clientConfiguration.setProxyHost(proxyHost);
    }

    final String proxyPort = System.getProperty("http.proxyPort");
    if (proxyPort != null) {
        clientConfiguration.setProxyPort(Integer.parseInt(proxyPort));
    }

    final String proxyUser = System.getProperty("http.proxyUser");
    if (proxyUser != null) {
        clientConfiguration.setProxyUsername(proxyUser);
    }

    final String proxyPassword = System.getProperty("http.proxyPassword");
    if (proxyPassword != null) {
        clientConfiguration.setProxyPassword(proxyPassword);
    }

    final String proxyDomain = System.getProperty("http.auth.ntlm.domain");
    if (proxyDomain != null) {
        clientConfiguration.setProxyDomain(proxyDomain);
    }

    final String workstation = System.getenv("COMPUTERNAME");
    if (proxyDomain != null && workstation != null) {
        clientConfiguration.setProxyWorkstation(workstation);
    }

    return clientConfiguration;
}

From source file:com.upplication.s3fs.S3FileSystemProvider.java

License:Open Source License

protected ClientConfiguration createClientConfig(Properties props) {
    ClientConfiguration config = new ClientConfiguration();

    if (props == null)
        return config;

    if (props.containsKey("connection_timeout")) {
        log.trace("AWS client config - connection_timeout: {}", props.getProperty("connection_timeout"));
        config.setConnectionTimeout(Integer.parseInt(props.getProperty("connection_timeout")));
    }//from ww  w .j a  v  a  2s . c  om

    if (props.containsKey("max_connections")) {
        log.trace("AWS client config - max_connections: {}", props.getProperty("max_connections"));
        config.setMaxConnections(Integer.parseInt(props.getProperty("max_connections")));
    }

    if (props.containsKey("max_error_retry")) {
        log.trace("AWS client config - max_error_retry: {}", props.getProperty("max_error_retry"));
        config.setMaxErrorRetry(Integer.parseInt(props.getProperty("max_error_retry")));
    }

    if (props.containsKey("protocol")) {
        log.trace("AWS client config - protocol: {}", props.getProperty("protocol"));
        config.setProtocol(Protocol.valueOf(props.getProperty("protocol").toUpperCase()));
    }

    if (props.containsKey("proxy_domain")) {
        log.trace("AWS client config - proxy_domain: {}", props.getProperty("proxy_domain"));
        config.setProxyDomain(props.getProperty("proxy_domain"));
    }

    if (props.containsKey("proxy_host")) {
        log.trace("AWS client config - proxy_host: {}", props.getProperty("proxy_host"));
        config.setProxyHost(props.getProperty("proxy_host"));
    }

    if (props.containsKey("proxy_port")) {
        log.trace("AWS client config - proxy_port: {}", props.getProperty("proxy_port"));
        config.setProxyPort(Integer.parseInt(props.getProperty("proxy_port")));
    }

    if (props.containsKey("proxy_username")) {
        log.trace("AWS client config - proxy_username: {}", props.getProperty("proxy_username"));
        config.setProxyUsername(props.getProperty("proxy_username"));
    }

    if (props.containsKey("proxy_password")) {
        log.trace("AWS client config - proxy_password: {}", props.getProperty("proxy_password"));
        config.setProxyPassword(props.getProperty("proxy_password"));
    }

    if (props.containsKey("proxy_workstation")) {
        log.trace("AWS client config - proxy_workstation: {}", props.getProperty("proxy_workstation"));
        config.setProxyWorkstation(props.getProperty("proxy_workstation"));
    }

    if (props.containsKey("signer_override")) {
        log.debug("AWS client config - signerOverride: {}", props.getProperty("signer_override"));
        config.setSignerOverride(props.getProperty("signer_override"));
    }

    if (props.containsKey("socket_send_buffer_size_hints")
            || props.containsKey("socket_recv_buffer_size_hints")) {
        log.trace("AWS client config - socket_send_buffer_size_hints: {}, socket_recv_buffer_size_hints: {}",
                props.getProperty("socket_send_buffer_size_hints", "0"),
                props.getProperty("socket_recv_buffer_size_hints", "0"));
        int send = Integer.parseInt(props.getProperty("socket_send_buffer_size_hints", "0"));
        int recv = Integer.parseInt(props.getProperty("socket_recv_buffer_size_hints", "0"));
        config.setSocketBufferSizeHints(send, recv);
    }

    if (props.containsKey("socket_timeout")) {
        log.trace("AWS client config - socket_timeout: {}", props.getProperty("socket_timeout"));
        config.setSocketTimeout(Integer.parseInt(props.getProperty("socket_timeout")));
    }

    if (props.containsKey("user_agent")) {
        log.trace("AWS client config - user_agent: {}", props.getProperty("user_agent"));
        config.setUserAgent(props.getProperty("user_agent"));
    }

    return config;
}

From source file:org.adroitlogic.build.aws.maven.S3Utils.java

License:Apache License

static ClientConfiguration getClientConfiguration(ProxyInfoProvider proxyInfoProvider) {
    ClientConfiguration clientConfiguration = new ClientConfiguration();

    if (proxyInfoProvider != null) {
        ProxyInfo proxyInfo = proxyInfoProvider.getProxyInfo("s3");

        if (proxyInfo != null) {
            //System.out.println("Proxy Info : NTLM host: " + proxyInfo.getNtlmHost() + " domain: " + proxyInfo.getNtlmDomain() + " username: " + proxyInfo.getUserName() + " password: " + proxyInfo.getPassword() + " host: " + proxyInfo.getHost() + " port: " + proxyInfo.getPort());
            clientConfiguration.withProxyHost(proxyInfo.getHost()).withProxyPort(proxyInfo.getPort());
            if (proxyInfo.getUserName() != null) {
                clientConfiguration.setProxyUsername(proxyInfo.getUserName());
                clientConfiguration.setProxyPassword(proxyInfo.getPassword());
            }/*from   w  w w .  ja  v  a  2s  .co m*/
            if (proxyInfo.getNtlmDomain() != null) {
                clientConfiguration.setProxyDomain(proxyInfo.getNtlmDomain());
            }
            if (proxyInfo.getNtlmHost() != null) {
                clientConfiguration.setProxyWorkstation(proxyInfo.getNtlmHost());
            }
        }
    }
    //clientConfiguration.setProtocol(com.amazonaws.Protocol.HTTP);

    return clientConfiguration;
}

From source file:org.adroitlogic.build.aws.maven.SimpleStorageServiceWagon.java

License:Apache License

@Override
protected void connectToRepository(Repository repository, AuthenticationInfo authenticationInfo,
        ProxyInfoProvider proxyInfoProvider) throws AuthenticationException {
    if (this.amazonS3 == null) {
        AuthenticationInfoAWSCredentialsProviderChain credentialsProvider = new AuthenticationInfoAWSCredentialsProviderChain(
                authenticationInfo);/*from  w  ww  .j a v  a2 s . c  o  m*/
        ClientConfiguration clientConfiguration = S3Utils.getClientConfiguration(proxyInfoProvider);

        this.bucketName = S3Utils.getBucketName(repository);
        this.baseDirectory = S3Utils.getBaseDirectory(repository);

        if (proxyInfoProvider != null) {
            ProxyInfo pi = proxyInfoProvider.getProxyInfo("s3");
            if (pi != null) {
                clientConfiguration.setProxyHost(pi.getHost());
                clientConfiguration.setProxyPort(pi.getPort());
                clientConfiguration.setProxyUsername(pi.getUserName());
                clientConfiguration.setProxyPassword(pi.getPassword());
                clientConfiguration.setProxyDomain(pi.getNtlmDomain());
                clientConfiguration.setProxyWorkstation(pi.getNtlmHost());
            }
        }

        this.amazonS3 = new AmazonS3Client(credentialsProvider, clientConfiguration);
        Region region = Region.fromLocationConstraint(this.amazonS3.getBucketLocation(this.bucketName));
        this.amazonS3.setEndpoint(region.getEndpoint());
    }
}

From source file:org.apache.hadoop.fs.s3r.S3RFileSystem.java

License:Apache License

/** Called after a new FileSystem instance is constructed.
 * @param name a uri whose authority section names the host, port, etc.
 *   for this FileSystem//  www .ja v  a  2  s .c o  m
 * @param conf the configuration
 */
public void initialize(URI name, Configuration conf) throws IOException {
    super.initialize(name, conf);

    uri = URI.create(name.getScheme() + "://" + name.getAuthority());
    workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(this.uri,
            this.getWorkingDirectory());

    // Try to get our credentials or just connect anonymously
    String accessKey = conf.get(ACCESS_KEY, null);
    String secretKey = conf.get(SECRET_KEY, null);

    String userInfo = name.getUserInfo();
    if (userInfo != null) {
        int index = userInfo.indexOf(':');
        if (index != -1) {
            accessKey = userInfo.substring(0, index);
            secretKey = userInfo.substring(index + 1);
        } else {
            accessKey = userInfo;
        }
    }

    AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain(
            new BasicAWSCredentialsProvider(accessKey, secretKey), new InstanceProfileCredentialsProvider(),
            new AnonymousAWSCredentialsProvider());

    bucket = name.getHost();

    ClientConfiguration awsConf = new ClientConfiguration();
    awsConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS, DEFAULT_MAXIMUM_CONNECTIONS));
    boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS);
    awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
    awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES));
    awsConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT));
    awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT));

    String proxyHost = conf.getTrimmed(PROXY_HOST, "");
    int proxyPort = conf.getInt(PROXY_PORT, -1);
    if (!proxyHost.isEmpty()) {
        awsConf.setProxyHost(proxyHost);
        if (proxyPort >= 0) {
            awsConf.setProxyPort(proxyPort);
        } else {
            if (secureConnections) {
                LOG.warn("Proxy host set without port. Using HTTPS default 443");
                awsConf.setProxyPort(443);
            } else {
                LOG.warn("Proxy host set without port. Using HTTP default 80");
                awsConf.setProxyPort(80);
            }
        }
        String proxyUsername = conf.getTrimmed(PROXY_USERNAME);
        String proxyPassword = conf.getTrimmed(PROXY_PASSWORD);
        if ((proxyUsername == null) != (proxyPassword == null)) {
            String msg = "Proxy error: " + PROXY_USERNAME + " or " + PROXY_PASSWORD + " set without the other.";
            LOG.error(msg);
            throw new IllegalArgumentException(msg);
        }
        awsConf.setProxyUsername(proxyUsername);
        awsConf.setProxyPassword(proxyPassword);
        awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
        awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
        if (LOG.isDebugEnabled()) {
            LOG.debug(
                    "Using proxy server {}:{} as user {} with password {} on " + "domain {} as workstation {}",
                    awsConf.getProxyHost(), awsConf.getProxyPort(), String.valueOf(awsConf.getProxyUsername()),
                    awsConf.getProxyPassword(), awsConf.getProxyDomain(), awsConf.getProxyWorkstation());
        }
    } else if (proxyPort >= 0) {
        String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
        LOG.error(msg);
        throw new IllegalArgumentException(msg);
    }

    s3 = new AmazonS3Client(credentials, awsConf);
    String endPoint = conf.getTrimmed(ENDPOINT, "");
    if (!endPoint.isEmpty()) {
        try {
            s3.setEndpoint(endPoint);
        } catch (IllegalArgumentException e) {
            String msg = "Incorrect endpoint: " + e.getMessage();
            LOG.error(msg);
            throw new IllegalArgumentException(msg, e);
        }
    }

    maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
    partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
    multiPartThreshold = conf.getInt(MIN_MULTIPART_THRESHOLD, DEFAULT_MIN_MULTIPART_THRESHOLD);

    if (partSize < 5 * 1024 * 1024) {
        LOG.error(MULTIPART_SIZE + " must be at least 5 MB");
        partSize = 5 * 1024 * 1024;
    }

    if (multiPartThreshold < 5 * 1024 * 1024) {
        LOG.error(MIN_MULTIPART_THRESHOLD + " must be at least 5 MB");
        multiPartThreshold = 5 * 1024 * 1024;
    }

    int maxThreads = conf.getInt(MAX_THREADS, DEFAULT_MAX_THREADS);
    int coreThreads = conf.getInt(CORE_THREADS, DEFAULT_CORE_THREADS);
    if (maxThreads == 0) {
        maxThreads = Runtime.getRuntime().availableProcessors() * 8;
    }
    if (coreThreads == 0) {
        coreThreads = Runtime.getRuntime().availableProcessors() * 8;
    }
    long keepAliveTime = conf.getLong(KEEPALIVE_TIME, DEFAULT_KEEPALIVE_TIME);
    LinkedBlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(
            maxThreads * conf.getInt(MAX_TOTAL_TASKS, DEFAULT_MAX_TOTAL_TASKS));
    threadPoolExecutor = new ThreadPoolExecutor(coreThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS,
            workQueue, newDaemonThreadFactory("s3a-transfer-shared-"));
    threadPoolExecutor.allowCoreThreadTimeOut(true);

    TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
    transferConfiguration.setMinimumUploadPartSize(partSize);
    transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);

    transfers = new TransferManager(s3, threadPoolExecutor);
    transfers.setConfiguration(transferConfiguration);

    String cannedACLName = conf.get(CANNED_ACL, DEFAULT_CANNED_ACL);
    if (!cannedACLName.isEmpty()) {
        cannedACL = CannedAccessControlList.valueOf(cannedACLName);
    } else {
        cannedACL = null;
    }

    if (!s3.doesBucketExist(bucket)) {
        throw new IOException("Bucket " + bucket + " does not exist");
    }

    boolean purgeExistingMultipart = conf.getBoolean(PURGE_EXISTING_MULTIPART,
            DEFAULT_PURGE_EXISTING_MULTIPART);
    long purgeExistingMultipartAge = conf.getLong(PURGE_EXISTING_MULTIPART_AGE,
            DEFAULT_PURGE_EXISTING_MULTIPART_AGE);

    if (purgeExistingMultipart) {
        Date purgeBefore = new Date(new Date().getTime() - purgeExistingMultipartAge * 1000);

        transfers.abortMultipartUploads(bucket, purgeBefore);
    }

    serverSideEncryptionAlgorithm = conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM);

    setConf(conf);
}

From source file:org.apache.tajo.storage.s3.S3TableSpace.java

License:Apache License

@Override
public void init(TajoConf tajoConf) throws IOException {
    super.init(tajoConf);

    try {/*from  www.j  a  va  2 s.c o  m*/
        // Try to get our credentials or just connect anonymously
        String accessKey = conf.get(ACCESS_KEY, null);
        String secretKey = conf.get(SECRET_KEY, null);

        String userInfo = uri.getUserInfo();
        if (userInfo != null) {
            int index = userInfo.indexOf(':');
            if (index != -1) {
                accessKey = userInfo.substring(0, index);
                secretKey = userInfo.substring(index + 1);
            } else {
                accessKey = userInfo;
            }
        }

        AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain(
                new BasicAWSCredentialsProvider(accessKey, secretKey), new InstanceProfileCredentialsProvider(),
                new AnonymousAWSCredentialsProvider());

        ClientConfiguration awsConf = new ClientConfiguration();
        awsConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS, DEFAULT_MAXIMUM_CONNECTIONS));
        boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS);
        awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
        awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES));
        awsConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT));
        awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT));

        String proxyHost = conf.getTrimmed(PROXY_HOST, "");
        int proxyPort = conf.getInt(PROXY_PORT, -1);
        if (!proxyHost.isEmpty()) {
            awsConf.setProxyHost(proxyHost);
            if (proxyPort >= 0) {
                awsConf.setProxyPort(proxyPort);
            } else {
                if (secureConnections) {
                    LOG.warn("Proxy host set without port. Using HTTPS default 443");
                    awsConf.setProxyPort(443);
                } else {
                    LOG.warn("Proxy host set without port. Using HTTP default 80");
                    awsConf.setProxyPort(80);
                }
            }
            String proxyUsername = conf.getTrimmed(PROXY_USERNAME);
            String proxyPassword = conf.getTrimmed(PROXY_PASSWORD);
            if ((proxyUsername == null) != (proxyPassword == null)) {
                String msg = "Proxy error: " + PROXY_USERNAME + " or " + PROXY_PASSWORD
                        + " set without the other.";
                LOG.error(msg);
            }
            awsConf.setProxyUsername(proxyUsername);
            awsConf.setProxyPassword(proxyPassword);
            awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
            awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
            if (LOG.isDebugEnabled()) {
                LOG.debug(String.format(
                        "Using proxy server %s:%d as user %s with password %s on domain %s as workstation "
                                + "%s",
                        awsConf.getProxyHost(), awsConf.getProxyPort(), awsConf.getProxyUsername(),
                        awsConf.getProxyPassword(), awsConf.getProxyDomain(), awsConf.getProxyWorkstation()));
            }
        } else if (proxyPort >= 0) {
            String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
            LOG.error(msg);
        }

        s3 = new AmazonS3Client(credentials, awsConf);
        String endPoint = conf.getTrimmed(ENDPOINT, "");
        if (!endPoint.isEmpty()) {
            try {
                s3.setEndpoint(endPoint);
            } catch (IllegalArgumentException e) {
                String msg = "Incorrect endpoint: " + e.getMessage();
                LOG.error(msg);
            }
        }

        maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
        s3Enabled = true;
    } catch (NoClassDefFoundError e) {
        // If the version of hadoop is less than 2.6.0, hadoop doesn't include aws dependencies because it doesn't provide
        // S3AFileSystem. In this case, tajo never uses aws s3 api directly.
        LOG.warn(e);
        s3Enabled = false;
    } catch (Exception e) {
        throw new TajoInternalError(e);
    }
}

From source file:org.geowebcache.s3.S3BlobStoreConfig.java

License:Open Source License

/**
 * @return {@link AmazonS3Client} constructed from this {@link S3BlobStoreConfig}.
 *//*from  ww w  . j  a  v  a2s. c o  m*/
public AmazonS3Client buildClient() {
    AWSCredentials awsCredentials = new BasicAWSCredentials(awsAccessKey, awsSecretKey);
    ClientConfiguration clientConfig = new ClientConfiguration();
    if (null != useHTTPS) {
        clientConfig.setProtocol(useHTTPS ? Protocol.HTTPS : Protocol.HTTP);
    }
    if (null != maxConnections && maxConnections > 0) {
        clientConfig.setMaxConnections(maxConnections);
    }
    clientConfig.setProxyDomain(proxyDomain);
    clientConfig.setProxyWorkstation(proxyWorkstation);
    clientConfig.setProxyHost(proxyHost);
    if (null != proxyPort) {
        clientConfig.setProxyPort(proxyPort);
    }
    clientConfig.setProxyUsername(proxyUsername);
    clientConfig.setProxyPassword(proxyPassword);
    if (null != useGzip) {
        clientConfig.setUseGzip(useGzip);
    }
    log.debug("Initializing AWS S3 connection");
    return new AmazonS3Client(awsCredentials, clientConfig);
}