List of usage examples for com.amazonaws ClientConfiguration setProxyUsername
public void setProxyUsername(String proxyUsername)
From source file:org.apache.hadoop.dynamodb.DynamoDBClient.java
License:Open Source License
@VisibleForTesting void applyProxyConfiguration(ClientConfiguration clientConfig, Configuration conf) { final String proxyHost = conf.get(DynamoDBConstants.PROXY_HOST); final int proxyPort = conf.getInt(DynamoDBConstants.PROXY_PORT, 0); final String proxyUsername = conf.get(DynamoDBConstants.PROXY_USERNAME); final String proxyPassword = conf.get(DynamoDBConstants.PROXY_PASSWORD); boolean proxyHostAndPortPresent = false; if (!Strings.isNullOrEmpty(proxyHost) && proxyPort > 0) { clientConfig.setProxyHost(proxyHost); clientConfig.setProxyPort(proxyPort); proxyHostAndPortPresent = true;/*from ww w.j ava2s . c o m*/ } else if (Strings.isNullOrEmpty(proxyHost) ^ proxyPort <= 0) { throw new RuntimeException("Only one of proxy host and port are set, when both are required"); } if (!Strings.isNullOrEmpty(proxyUsername) && !Strings.isNullOrEmpty(proxyPassword)) { if (!proxyHostAndPortPresent) { throw new RuntimeException( "Proxy host and port must be supplied if proxy username and " + "password are present"); } else { clientConfig.setProxyUsername(proxyUsername); clientConfig.setProxyPassword(proxyPassword); } } else if (Strings.isNullOrEmpty(proxyUsername) ^ Strings.isNullOrEmpty(proxyPassword)) { throw new RuntimeException( "Only one of proxy username and password are set, when both are " + "required"); } }
From source file:org.apache.hadoop.fs.s3r.S3RFileSystem.java
License:Apache License
/** Called after a new FileSystem instance is constructed. * @param name a uri whose authority section names the host, port, etc. * for this FileSystem// w ww . j av a 2s.c om * @param conf the configuration */ public void initialize(URI name, Configuration conf) throws IOException { super.initialize(name, conf); uri = URI.create(name.getScheme() + "://" + name.getAuthority()); workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(this.uri, this.getWorkingDirectory()); // Try to get our credentials or just connect anonymously String accessKey = conf.get(ACCESS_KEY, null); String secretKey = conf.get(SECRET_KEY, null); String userInfo = name.getUserInfo(); if (userInfo != null) { int index = userInfo.indexOf(':'); if (index != -1) { accessKey = userInfo.substring(0, index); secretKey = userInfo.substring(index + 1); } else { accessKey = userInfo; } } AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain( new BasicAWSCredentialsProvider(accessKey, secretKey), new InstanceProfileCredentialsProvider(), new AnonymousAWSCredentialsProvider()); bucket = name.getHost(); ClientConfiguration awsConf = new ClientConfiguration(); awsConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS, DEFAULT_MAXIMUM_CONNECTIONS)); boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS); awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES)); awsConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT)); awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT)); String proxyHost = conf.getTrimmed(PROXY_HOST, ""); int proxyPort = conf.getInt(PROXY_PORT, -1); if (!proxyHost.isEmpty()) { awsConf.setProxyHost(proxyHost); if (proxyPort >= 0) { awsConf.setProxyPort(proxyPort); } else { if (secureConnections) { LOG.warn("Proxy host set without port. Using HTTPS default 443"); awsConf.setProxyPort(443); } else { LOG.warn("Proxy host set without port. Using HTTP default 80"); awsConf.setProxyPort(80); } } String proxyUsername = conf.getTrimmed(PROXY_USERNAME); String proxyPassword = conf.getTrimmed(PROXY_PASSWORD); if ((proxyUsername == null) != (proxyPassword == null)) { String msg = "Proxy error: " + PROXY_USERNAME + " or " + PROXY_PASSWORD + " set without the other."; LOG.error(msg); throw new IllegalArgumentException(msg); } awsConf.setProxyUsername(proxyUsername); awsConf.setProxyPassword(proxyPassword); awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN)); awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION)); if (LOG.isDebugEnabled()) { LOG.debug( "Using proxy server {}:{} as user {} with password {} on " + "domain {} as workstation {}", awsConf.getProxyHost(), awsConf.getProxyPort(), String.valueOf(awsConf.getProxyUsername()), awsConf.getProxyPassword(), awsConf.getProxyDomain(), awsConf.getProxyWorkstation()); } } else if (proxyPort >= 0) { String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST; LOG.error(msg); throw new IllegalArgumentException(msg); } s3 = new AmazonS3Client(credentials, awsConf); String endPoint = conf.getTrimmed(ENDPOINT, ""); if (!endPoint.isEmpty()) { try { s3.setEndpoint(endPoint); } catch (IllegalArgumentException e) { String msg = "Incorrect endpoint: " + e.getMessage(); LOG.error(msg); throw new IllegalArgumentException(msg, e); } } maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS); partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE); multiPartThreshold = conf.getInt(MIN_MULTIPART_THRESHOLD, DEFAULT_MIN_MULTIPART_THRESHOLD); if (partSize < 5 * 1024 * 1024) { LOG.error(MULTIPART_SIZE + " must be at least 5 MB"); partSize = 5 * 1024 * 1024; } if (multiPartThreshold < 5 * 1024 * 1024) { LOG.error(MIN_MULTIPART_THRESHOLD + " must be at least 5 MB"); multiPartThreshold = 5 * 1024 * 1024; } int maxThreads = conf.getInt(MAX_THREADS, DEFAULT_MAX_THREADS); int coreThreads = conf.getInt(CORE_THREADS, DEFAULT_CORE_THREADS); if (maxThreads == 0) { maxThreads = Runtime.getRuntime().availableProcessors() * 8; } if (coreThreads == 0) { coreThreads = Runtime.getRuntime().availableProcessors() * 8; } long keepAliveTime = conf.getLong(KEEPALIVE_TIME, DEFAULT_KEEPALIVE_TIME); LinkedBlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>( maxThreads * conf.getInt(MAX_TOTAL_TASKS, DEFAULT_MAX_TOTAL_TASKS)); threadPoolExecutor = new ThreadPoolExecutor(coreThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue, newDaemonThreadFactory("s3a-transfer-shared-")); threadPoolExecutor.allowCoreThreadTimeOut(true); TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration(); transferConfiguration.setMinimumUploadPartSize(partSize); transferConfiguration.setMultipartUploadThreshold(multiPartThreshold); transfers = new TransferManager(s3, threadPoolExecutor); transfers.setConfiguration(transferConfiguration); String cannedACLName = conf.get(CANNED_ACL, DEFAULT_CANNED_ACL); if (!cannedACLName.isEmpty()) { cannedACL = CannedAccessControlList.valueOf(cannedACLName); } else { cannedACL = null; } if (!s3.doesBucketExist(bucket)) { throw new IOException("Bucket " + bucket + " does not exist"); } boolean purgeExistingMultipart = conf.getBoolean(PURGE_EXISTING_MULTIPART, DEFAULT_PURGE_EXISTING_MULTIPART); long purgeExistingMultipartAge = conf.getLong(PURGE_EXISTING_MULTIPART_AGE, DEFAULT_PURGE_EXISTING_MULTIPART_AGE); if (purgeExistingMultipart) { Date purgeBefore = new Date(new Date().getTime() - purgeExistingMultipartAge * 1000); transfers.abortMultipartUploads(bucket, purgeBefore); } serverSideEncryptionAlgorithm = conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM); setConf(conf); }
From source file:org.apache.heron.uploader.s3.S3Uploader.java
License:Apache License
@Override public void initialize(Config config) { bucket = S3Context.bucket(config); String accessKey = S3Context.accessKey(config); String accessSecret = S3Context.secretKey(config); String awsProfile = S3Context.awsProfile(config); String proxy = S3Context.proxyUri(config); String endpoint = S3Context.uri(config); String customRegion = S3Context.region(config); AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard(); if (Strings.isNullOrEmpty(bucket)) { throw new RuntimeException("Missing heron.uploader.s3.bucket config value"); }/*from w w w .j ava 2 s .co m*/ // If an accessKey is specified, use it. Otherwise check if an aws profile // is specified. If neither was set just use the DefaultAWSCredentialsProviderChain // by not specifying a CredentialsProvider. if (!Strings.isNullOrEmpty(accessKey) || !Strings.isNullOrEmpty(accessSecret)) { if (!Strings.isNullOrEmpty(awsProfile)) { throw new RuntimeException("Please provide access_key/secret_key " + "or aws_profile, not both."); } if (Strings.isNullOrEmpty(accessKey)) { throw new RuntimeException("Missing heron.uploader.s3.access_key config value"); } if (Strings.isNullOrEmpty(accessSecret)) { throw new RuntimeException("Missing heron.uploader.s3.secret_key config value"); } builder.setCredentials( new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, accessSecret))); } else if (!Strings.isNullOrEmpty(awsProfile)) { builder.setCredentials(new ProfileCredentialsProvider(awsProfile)); } if (!Strings.isNullOrEmpty(proxy)) { URI proxyUri; try { proxyUri = new URI(proxy); } catch (URISyntaxException e) { throw new RuntimeException("Invalid heron.uploader.s3.proxy_uri config value: " + proxy, e); } ClientConfiguration clientCfg = new ClientConfiguration(); clientCfg.withProtocol(Protocol.HTTPS).withProxyHost(proxyUri.getHost()) .withProxyPort(proxyUri.getPort()); if (!Strings.isNullOrEmpty(proxyUri.getUserInfo())) { String[] info = proxyUri.getUserInfo().split(":", 2); clientCfg.setProxyUsername(info[0]); if (info.length > 1) { clientCfg.setProxyPassword(info[1]); } } builder.setClientConfiguration(clientCfg); } s3Client = builder.withRegion(customRegion).withPathStyleAccessEnabled(true) .withChunkedEncodingDisabled(true).withPayloadSigningEnabled(true).build(); if (!Strings.isNullOrEmpty(endpoint)) { s3Client.setEndpoint(endpoint); } final String topologyName = Context.topologyName(config); final String topologyPackageLocation = Context.topologyPackageFile(config); pathPrefix = S3Context.pathPrefix(config); packageFileHandler = new File(topologyPackageLocation); // The path the packaged topology will be uploaded to remoteFilePath = generateS3Path(pathPrefix, topologyName, packageFileHandler.getName()); // Generate the location of the backup file incase we need to revert the deploy previousVersionFilePath = generateS3Path(pathPrefix, topologyName, "previous_" + packageFileHandler.getName()); }
From source file:org.apache.tajo.storage.s3.S3TableSpace.java
License:Apache License
@Override public void init(TajoConf tajoConf) throws IOException { super.init(tajoConf); try {//w w w.j ava2s . c o m // Try to get our credentials or just connect anonymously String accessKey = conf.get(ACCESS_KEY, null); String secretKey = conf.get(SECRET_KEY, null); String userInfo = uri.getUserInfo(); if (userInfo != null) { int index = userInfo.indexOf(':'); if (index != -1) { accessKey = userInfo.substring(0, index); secretKey = userInfo.substring(index + 1); } else { accessKey = userInfo; } } AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain( new BasicAWSCredentialsProvider(accessKey, secretKey), new InstanceProfileCredentialsProvider(), new AnonymousAWSCredentialsProvider()); ClientConfiguration awsConf = new ClientConfiguration(); awsConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS, DEFAULT_MAXIMUM_CONNECTIONS)); boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS); awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES)); awsConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT)); awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT)); String proxyHost = conf.getTrimmed(PROXY_HOST, ""); int proxyPort = conf.getInt(PROXY_PORT, -1); if (!proxyHost.isEmpty()) { awsConf.setProxyHost(proxyHost); if (proxyPort >= 0) { awsConf.setProxyPort(proxyPort); } else { if (secureConnections) { LOG.warn("Proxy host set without port. Using HTTPS default 443"); awsConf.setProxyPort(443); } else { LOG.warn("Proxy host set without port. Using HTTP default 80"); awsConf.setProxyPort(80); } } String proxyUsername = conf.getTrimmed(PROXY_USERNAME); String proxyPassword = conf.getTrimmed(PROXY_PASSWORD); if ((proxyUsername == null) != (proxyPassword == null)) { String msg = "Proxy error: " + PROXY_USERNAME + " or " + PROXY_PASSWORD + " set without the other."; LOG.error(msg); } awsConf.setProxyUsername(proxyUsername); awsConf.setProxyPassword(proxyPassword); awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN)); awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION)); if (LOG.isDebugEnabled()) { LOG.debug(String.format( "Using proxy server %s:%d as user %s with password %s on domain %s as workstation " + "%s", awsConf.getProxyHost(), awsConf.getProxyPort(), awsConf.getProxyUsername(), awsConf.getProxyPassword(), awsConf.getProxyDomain(), awsConf.getProxyWorkstation())); } } else if (proxyPort >= 0) { String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST; LOG.error(msg); } s3 = new AmazonS3Client(credentials, awsConf); String endPoint = conf.getTrimmed(ENDPOINT, ""); if (!endPoint.isEmpty()) { try { s3.setEndpoint(endPoint); } catch (IllegalArgumentException e) { String msg = "Incorrect endpoint: " + e.getMessage(); LOG.error(msg); } } maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS); s3Enabled = true; } catch (NoClassDefFoundError e) { // If the version of hadoop is less than 2.6.0, hadoop doesn't include aws dependencies because it doesn't provide // S3AFileSystem. In this case, tajo never uses aws s3 api directly. LOG.warn(e); s3Enabled = false; } catch (Exception e) { throw new TajoInternalError(e); } }
From source file:org.cloudfoundry.community.servicebroker.s3.config.AwsClientConfiguration.java
License:Apache License
public ClientConfiguration toClientConfiguration() { ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setProxyHost(proxyHost); if (proxyPort != null) { clientConfiguration.setProxyPort(Integer.parseInt(proxyPort)); }//w w w .ja v a2s . co m clientConfiguration.setProxyUsername(proxyUsername); clientConfiguration.setProxyPassword(proxyPassword); if (preemptiveBasicProxyAuth != null) { clientConfiguration.setPreemptiveBasicProxyAuth(preemptiveBasicProxyAuth); } return clientConfiguration; }
From source file:org.elasticsearch.repositories.s3.InternalAwsS3Service.java
License:Apache License
static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings, Settings repositorySettings) { ClientConfiguration clientConfiguration = new ClientConfiguration(); // the response metadata cache is only there for diagnostics purposes, // but can force objects from every response to the old generation. clientConfiguration.setResponseMetadataCacheSize(0); clientConfiguration.setProtocol(clientSettings.protocol); if (Strings.hasText(clientSettings.proxyHost)) { // TODO: remove this leniency, these settings should exist together and be validated clientConfiguration.setProxyHost(clientSettings.proxyHost); clientConfiguration.setProxyPort(clientSettings.proxyPort); clientConfiguration.setProxyUsername(clientSettings.proxyUsername); clientConfiguration.setProxyPassword(clientSettings.proxyPassword); }// ww w . j a v a 2 s . c om clientConfiguration.setMaxErrorRetry(clientSettings.maxRetries); clientConfiguration.setUseThrottleRetries(clientSettings.throttleRetries); clientConfiguration.setSocketTimeout(clientSettings.readTimeoutMillis); return clientConfiguration; }
From source file:org.elasticsearch.repositories.s3.S3Service.java
License:Apache License
static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) { final ClientConfiguration clientConfiguration = new ClientConfiguration(); // the response metadata cache is only there for diagnostics purposes, // but can force objects from every response to the old generation. clientConfiguration.setResponseMetadataCacheSize(0); clientConfiguration.setProtocol(clientSettings.protocol); if (Strings.hasText(clientSettings.proxyHost)) { // TODO: remove this leniency, these settings should exist together and be validated clientConfiguration.setProxyHost(clientSettings.proxyHost); clientConfiguration.setProxyPort(clientSettings.proxyPort); clientConfiguration.setProxyUsername(clientSettings.proxyUsername); clientConfiguration.setProxyPassword(clientSettings.proxyPassword); }/* w w w . j a v a 2s.c om*/ clientConfiguration.setMaxErrorRetry(clientSettings.maxRetries); clientConfiguration.setUseThrottleRetries(clientSettings.throttleRetries); clientConfiguration.setSocketTimeout(clientSettings.readTimeoutMillis); return clientConfiguration; }
From source file:org.geowebcache.s3.S3BlobStoreConfig.java
License:Open Source License
/** * @return {@link AmazonS3Client} constructed from this {@link S3BlobStoreConfig}. *//*from w w w . ja v a 2 s . c o m*/ public AmazonS3Client buildClient() { AWSCredentials awsCredentials = new BasicAWSCredentials(awsAccessKey, awsSecretKey); ClientConfiguration clientConfig = new ClientConfiguration(); if (null != useHTTPS) { clientConfig.setProtocol(useHTTPS ? Protocol.HTTPS : Protocol.HTTP); } if (null != maxConnections && maxConnections > 0) { clientConfig.setMaxConnections(maxConnections); } clientConfig.setProxyDomain(proxyDomain); clientConfig.setProxyWorkstation(proxyWorkstation); clientConfig.setProxyHost(proxyHost); if (null != proxyPort) { clientConfig.setProxyPort(proxyPort); } clientConfig.setProxyUsername(proxyUsername); clientConfig.setProxyPassword(proxyPassword); if (null != useGzip) { clientConfig.setUseGzip(useGzip); } log.debug("Initializing AWS S3 connection"); return new AmazonS3Client(awsCredentials, clientConfig); }
From source file:org.geowebcache.s3.S3BlobStoreInfo.java
License:Open Source License
/** @return {@link AmazonS3Client} constructed from this {@link S3BlobStoreInfo}. */ public AmazonS3Client buildClient() { ClientConfiguration clientConfig = new ClientConfiguration(); if (null != useHTTPS) { clientConfig.setProtocol(useHTTPS ? Protocol.HTTPS : Protocol.HTTP); }/*from www .j a v a 2s. c om*/ if (null != maxConnections && maxConnections > 0) { clientConfig.setMaxConnections(maxConnections); } clientConfig.setProxyDomain(proxyDomain); clientConfig.setProxyWorkstation(proxyWorkstation); clientConfig.setProxyHost(proxyHost); if (null != proxyPort) { clientConfig.setProxyPort(proxyPort); } clientConfig.setProxyUsername(proxyUsername); clientConfig.setProxyPassword(proxyPassword); if (null != useGzip) { clientConfig.setUseGzip(useGzip); } log.debug("Initializing AWS S3 connection"); AmazonS3Client client = new AmazonS3Client(getCredentialsProvider(), clientConfig); if (endpoint != null && !"".equals(endpoint)) { S3ClientOptions s3ClientOptions = new S3ClientOptions(); s3ClientOptions.setPathStyleAccess(true); client.setS3ClientOptions(s3ClientOptions); client.setEndpoint(endpoint); } if (!client.doesBucketExist(bucket)) { client.createBucket(bucket); } return client; }
From source file:org.gradle.internal.resource.transport.aws.s3.S3Client.java
License:Apache License
private ClientConfiguration createConnectionProperties() { ClientConfiguration clientConfiguration = new ClientConfiguration(); Optional<HttpProxySettings.HttpProxy> proxyOptional = s3ConnectionProperties.getProxy(); if (proxyOptional.isPresent()) { HttpProxySettings.HttpProxy proxy = s3ConnectionProperties.getProxy().get(); clientConfiguration.setProxyHost(proxy.host); clientConfiguration.setProxyPort(proxy.port); PasswordCredentials credentials = proxy.credentials; if (credentials != null) { clientConfiguration.setProxyUsername(credentials.getUsername()); clientConfiguration.setProxyPassword(credentials.getPassword()); }/*from w ww .j a v a 2 s. c om*/ } Optional<Integer> maxErrorRetryCount = s3ConnectionProperties.getMaxErrorRetryCount(); if (maxErrorRetryCount.isPresent()) { clientConfiguration.setMaxErrorRetry(maxErrorRetryCount.get()); } return clientConfiguration; }