List of usage examples for com.amazonaws ClientConfiguration setProtocol
public void setProtocol(Protocol protocol)
From source file:org.apache.streams.s3.S3PersistWriter.java
License:Apache License
public void prepare(Object configurationObject) { // Connect to S3 synchronized (this) { try {/*from w w w . j av a 2 s . c om*/ // if the user has chosen to not set the object mapper, then set a default object mapper for them. if (this.objectMapper == null) this.objectMapper = new StreamsJacksonMapper(); // Create the credentials Object if (this.amazonS3Client == null) { AWSCredentials credentials = new BasicAWSCredentials(s3WriterConfiguration.getKey(), s3WriterConfiguration.getSecretKey()); ClientConfiguration clientConfig = new ClientConfiguration(); clientConfig.setProtocol(Protocol.valueOf(s3WriterConfiguration.getProtocol().toString())); // We do not want path style access S3ClientOptions clientOptions = new S3ClientOptions(); clientOptions.setPathStyleAccess(false); this.amazonS3Client = new AmazonS3Client(credentials, clientConfig); if (!Strings.isNullOrEmpty(s3WriterConfiguration.getRegion())) this.amazonS3Client .setRegion(Region.getRegion(Regions.fromName(s3WriterConfiguration.getRegion()))); this.amazonS3Client.setS3ClientOptions(clientOptions); } } catch (Exception e) { LOGGER.error("Exception while preparing the S3 client: {}", e); } Preconditions.checkArgument(this.amazonS3Client != null); } }
From source file:org.apache.tajo.storage.s3.S3TableSpace.java
License:Apache License
@Override public void init(TajoConf tajoConf) throws IOException { super.init(tajoConf); try {//from w w w . j a v a 2 s .c o m // Try to get our credentials or just connect anonymously String accessKey = conf.get(ACCESS_KEY, null); String secretKey = conf.get(SECRET_KEY, null); String userInfo = uri.getUserInfo(); if (userInfo != null) { int index = userInfo.indexOf(':'); if (index != -1) { accessKey = userInfo.substring(0, index); secretKey = userInfo.substring(index + 1); } else { accessKey = userInfo; } } AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain( new BasicAWSCredentialsProvider(accessKey, secretKey), new InstanceProfileCredentialsProvider(), new AnonymousAWSCredentialsProvider()); ClientConfiguration awsConf = new ClientConfiguration(); awsConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS, DEFAULT_MAXIMUM_CONNECTIONS)); boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS); awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES)); awsConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT)); awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT)); String proxyHost = conf.getTrimmed(PROXY_HOST, ""); int proxyPort = conf.getInt(PROXY_PORT, -1); if (!proxyHost.isEmpty()) { awsConf.setProxyHost(proxyHost); if (proxyPort >= 0) { awsConf.setProxyPort(proxyPort); } else { if (secureConnections) { LOG.warn("Proxy host set without port. Using HTTPS default 443"); awsConf.setProxyPort(443); } else { LOG.warn("Proxy host set without port. Using HTTP default 80"); awsConf.setProxyPort(80); } } String proxyUsername = conf.getTrimmed(PROXY_USERNAME); String proxyPassword = conf.getTrimmed(PROXY_PASSWORD); if ((proxyUsername == null) != (proxyPassword == null)) { String msg = "Proxy error: " + PROXY_USERNAME + " or " + PROXY_PASSWORD + " set without the other."; LOG.error(msg); } awsConf.setProxyUsername(proxyUsername); awsConf.setProxyPassword(proxyPassword); awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN)); awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION)); if (LOG.isDebugEnabled()) { LOG.debug(String.format( "Using proxy server %s:%d as user %s with password %s on domain %s as workstation " + "%s", awsConf.getProxyHost(), awsConf.getProxyPort(), awsConf.getProxyUsername(), awsConf.getProxyPassword(), awsConf.getProxyDomain(), awsConf.getProxyWorkstation())); } } else if (proxyPort >= 0) { String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST; LOG.error(msg); } s3 = new AmazonS3Client(credentials, awsConf); String endPoint = conf.getTrimmed(ENDPOINT, ""); if (!endPoint.isEmpty()) { try { s3.setEndpoint(endPoint); } catch (IllegalArgumentException e) { String msg = "Incorrect endpoint: " + e.getMessage(); LOG.error(msg); } } maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS); s3Enabled = true; } catch (NoClassDefFoundError e) { // If the version of hadoop is less than 2.6.0, hadoop doesn't include aws dependencies because it doesn't provide // S3AFileSystem. In this case, tajo never uses aws s3 api directly. LOG.warn(e); s3Enabled = false; } catch (Exception e) { throw new TajoInternalError(e); } }
From source file:org.apache.usergrid.chop.api.store.amazon.AmazonUtils.java
License:Apache License
/** * @param accessKey/* w ww . ja v a 2 s. c om*/ * @param secretKey * @return */ public static AmazonEC2Client getEC2Client(String accessKey, String secretKey) { AWSCredentialsProvider provider; if (accessKey != null && secretKey != null) { AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); provider = new StaticCredentialsProvider(credentials); } else { provider = new DefaultAWSCredentialsProviderChain(); } AmazonEC2Client client = new AmazonEC2Client(provider); ClientConfiguration configuration = new ClientConfiguration(); configuration.setProtocol(Protocol.HTTPS); client.setConfiguration(configuration); return client; }
From source file:org.apache.usergrid.services.assets.data.AWSBinaryStore.java
License:Apache License
private AmazonS3 getS3Client() throws Exception { this.bucketName = properties.getProperty("usergrid.binary.bucketname"); if (bucketName == null) { logger.error("usergrid.binary.bucketname not properly set so amazon bucket is null"); throw new AwsPropertiesNotFoundException("usergrid.binary.bucketname"); }//from ww w . j a v a 2 s . c o m final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider(); AWSCredentials credentials = ugProvider.getCredentials(); ClientConfiguration clientConfig = new ClientConfiguration(); clientConfig.setProtocol(Protocol.HTTP); s3Client = new AmazonS3Client(credentials, clientConfig); if (regionName != null) s3Client.setRegion(Region.getRegion(Regions.fromName(regionName))); return s3Client; }
From source file:org.apache.usergrid.tools.WarehouseExport.java
License:Apache License
private void copyToS3(String fileName) { String bucketName = (String) properties.get(BUCKET_PROPNAME); String accessId = (String) properties.get(ACCESS_ID_PROPNAME); String secretKey = (String) properties.get(SECRET_KEY_PROPNAME); Properties overrides = new Properties(); overrides.setProperty("s3" + ".identity", accessId); overrides.setProperty("s3" + ".credential", secretKey); final Iterable<? extends Module> MODULES = ImmutableSet.of(new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(), new NettyPayloadModule()); AWSCredentials credentials = new BasicAWSCredentials(accessId, secretKey); ClientConfiguration clientConfig = new ClientConfiguration(); clientConfig.setProtocol(Protocol.HTTP); AmazonS3Client s3Client = new AmazonS3Client(credentials, clientConfig); s3Client.createBucket(bucketName);/*from ww w . jav a2 s .co m*/ File uploadFile = new File(fileName); PutObjectResult putObjectResult = s3Client.putObject(bucketName, uploadFile.getName(), uploadFile); logger.info("Uploaded file etag={}", putObjectResult.getETag()); }
From source file:org.elasticsearch.cloud.aws.AwsEc2Service.java
License:Apache License
public synchronized AmazonEC2 client() { if (client != null) { return client; }//from w ww . j a v a 2 s . com ClientConfiguration clientConfiguration = new ClientConfiguration(); String protocol = componentSettings.get("protocol", "http").toLowerCase(); if ("http".equals(protocol)) { clientConfiguration.setProtocol(Protocol.HTTP); } else if ("https".equals(protocol)) { clientConfiguration.setProtocol(Protocol.HTTPS); } else { throw new ElasticsearchIllegalArgumentException( "No protocol supported [" + protocol + "], can either be [http] or [https]"); } String account = componentSettings.get("access_key", settings.get("cloud.account")); String key = componentSettings.get("secret_key", settings.get("cloud.key")); String proxyHost = componentSettings.get("proxy_host"); if (proxyHost != null) { String portString = componentSettings.get("proxy_port", "80"); Integer proxyPort; try { proxyPort = Integer.parseInt(portString, 10); } catch (NumberFormatException ex) { throw new ElasticsearchIllegalArgumentException( "The configured proxy port value [" + portString + "] is invalid", ex); } clientConfiguration.withProxyHost(proxyHost).setProxyPort(proxyPort); } AWSCredentialsProvider credentials; if (account == null && key == null) { credentials = new AWSCredentialsProviderChain(new EnvironmentVariableCredentialsProvider(), new SystemPropertiesCredentialsProvider(), new InstanceProfileCredentialsProvider()); } else { credentials = new AWSCredentialsProviderChain( new StaticCredentialsProvider(new BasicAWSCredentials(account, key))); } this.client = new AmazonEC2Client(credentials, clientConfiguration); if (componentSettings.get("ec2.endpoint") != null) { String endpoint = componentSettings.get("ec2.endpoint"); logger.debug("using explicit ec2 endpoint [{}]", endpoint); client.setEndpoint(endpoint); } else if (componentSettings.get("region") != null) { String region = componentSettings.get("region").toLowerCase(); String endpoint; if (region.equals("us-east-1") || region.equals("us-east")) { endpoint = "ec2.us-east-1.amazonaws.com"; } else if (region.equals("us-west") || region.equals("us-west-1")) { endpoint = "ec2.us-west-1.amazonaws.com"; } else if (region.equals("us-west-2")) { endpoint = "ec2.us-west-2.amazonaws.com"; } else if (region.equals("ap-southeast") || region.equals("ap-southeast-1")) { endpoint = "ec2.ap-southeast-1.amazonaws.com"; } else if (region.equals("ap-southeast-2")) { endpoint = "ec2.ap-southeast-2.amazonaws.com"; } else if (region.equals("ap-northeast") || region.equals("ap-northeast-1")) { endpoint = "ec2.ap-northeast-1.amazonaws.com"; } else if (region.equals("eu-west") || region.equals("eu-west-1")) { endpoint = "ec2.eu-west-1.amazonaws.com"; } else if (region.equals("sa-east") || region.equals("sa-east-1")) { endpoint = "ec2.sa-east-1.amazonaws.com"; } else { throw new ElasticsearchIllegalArgumentException( "No automatic endpoint could be derived from region [" + region + "]"); } if (endpoint != null) { logger.debug("using ec2 region [{}], with endpoint [{}]", region, endpoint); client.setEndpoint(endpoint); } } return this.client; }
From source file:org.elasticsearch.cloud.aws.AwsEc2ServiceImpl.java
License:Apache License
protected static ClientConfiguration buildConfiguration(Logger logger, Settings settings) { ClientConfiguration clientConfiguration = new ClientConfiguration(); // the response metadata cache is only there for diagnostics purposes, // but can force objects from every response to the old generation. clientConfiguration.setResponseMetadataCacheSize(0); clientConfiguration.setProtocol(CLOUD_EC2.PROTOCOL_SETTING.get(settings)); if (PROXY_HOST_SETTING.exists(settings) || CLOUD_EC2.PROXY_HOST_SETTING.exists(settings)) { String proxyHost = CLOUD_EC2.PROXY_HOST_SETTING.get(settings); Integer proxyPort = CLOUD_EC2.PROXY_PORT_SETTING.get(settings); String proxyUsername = CLOUD_EC2.PROXY_USERNAME_SETTING.get(settings); String proxyPassword = CLOUD_EC2.PROXY_PASSWORD_SETTING.get(settings); clientConfiguration.withProxyHost(proxyHost).withProxyPort(proxyPort).withProxyUsername(proxyUsername) .withProxyPassword(proxyPassword); }//w ww.j a va2s.c om // #155: we might have 3rd party users using older EC2 API version String awsSigner = CLOUD_EC2.SIGNER_SETTING.get(settings); if (Strings.hasText(awsSigner)) { logger.debug("using AWS API signer [{}]", awsSigner); AwsSigner.configureSigner(awsSigner, clientConfiguration); } // Increase the number of retries in case of 5xx API responses final Random rand = Randomness.get(); RetryPolicy retryPolicy = new RetryPolicy(RetryPolicy.RetryCondition.NO_RETRY_CONDITION, new RetryPolicy.BackoffStrategy() { @Override public long delayBeforeNextRetry(AmazonWebServiceRequest originalRequest, AmazonClientException exception, int retriesAttempted) { // with 10 retries the max delay time is 320s/320000ms (10 * 2^5 * 1 * 1000) logger.warn("EC2 API request failed, retry again. Reason was:", exception); return 1000L * (long) (10d * Math.pow(2, retriesAttempted / 2.0d) * (1.0d + rand.nextDouble())); } }, 10, false); clientConfiguration.setRetryPolicy(retryPolicy); return clientConfiguration; }
From source file:org.elasticsearch.cloud.aws.AwsS3Service.java
License:Apache License
private synchronized AmazonS3 getClient(String endpoint, String account, String key) { Tuple<String, String> clientDescriptor = new Tuple<String, String>(endpoint, account); AmazonS3Client client = clients.get(clientDescriptor); if (client != null) { return client; }// www . ja v a 2 s . co m ClientConfiguration clientConfiguration = new ClientConfiguration(); String protocol = componentSettings.get("protocol", "http").toLowerCase(); if ("http".equals(protocol)) { clientConfiguration.setProtocol(Protocol.HTTP); } else if ("https".equals(protocol)) { clientConfiguration.setProtocol(Protocol.HTTPS); } else { throw new ElasticsearchIllegalArgumentException( "No protocol supported [" + protocol + "], can either be [http] or [https]"); } String proxyHost = componentSettings.get("proxy_host"); if (proxyHost != null) { String portString = componentSettings.get("proxy_port", "80"); Integer proxyPort; try { proxyPort = Integer.parseInt(portString, 10); } catch (NumberFormatException ex) { throw new ElasticsearchIllegalArgumentException( "The configured proxy port value [" + portString + "] is invalid", ex); } clientConfiguration.withProxyHost(proxyHost).setProxyPort(proxyPort); } AWSCredentialsProvider credentials; if (account == null && key == null) { credentials = new AWSCredentialsProviderChain(new EnvironmentVariableCredentialsProvider(), new SystemPropertiesCredentialsProvider(), new InstanceProfileCredentialsProvider()); } else { credentials = new AWSCredentialsProviderChain( new StaticCredentialsProvider(new BasicAWSCredentials(account, key))); } client = new AmazonS3Client(credentials, clientConfiguration); if (endpoint != null) { client.setEndpoint(endpoint); } clients.put(clientDescriptor, client); return client; }
From source file:org.elasticsearch.cloud.aws.InternalAwsS3Service.java
License:Apache License
private synchronized AmazonS3 getClient(String endpoint, String protocol, String account, String key, Integer maxRetries) {/* ww w. ja v a 2 s.com*/ Tuple<String, String> clientDescriptor = new Tuple<String, String>(endpoint, account); AmazonS3Client client = clients.get(clientDescriptor); if (client != null) { return client; } ClientConfiguration clientConfiguration = new ClientConfiguration(); // the response metadata cache is only there for diagnostics purposes, // but can force objects from every response to the old generation. clientConfiguration.setResponseMetadataCacheSize(0); if (protocol == null) { protocol = settings.get("cloud.aws.protocol", "https").toLowerCase(); protocol = settings.get("cloud.aws.s3.protocol", protocol).toLowerCase(); } if ("http".equals(protocol)) { clientConfiguration.setProtocol(Protocol.HTTP); } else if ("https".equals(protocol)) { clientConfiguration.setProtocol(Protocol.HTTPS); } else { throw new IllegalArgumentException( "No protocol supported [" + protocol + "], can either be [http] or [https]"); } String proxyHost = settings.get("cloud.aws.proxy_host"); proxyHost = settings.get("cloud.aws.s3.proxy_host", proxyHost); if (proxyHost != null) { String portString = settings.get("cloud.aws.proxy_port", "80"); portString = settings.get("cloud.aws.s3.proxy_port", portString); Integer proxyPort; try { proxyPort = Integer.parseInt(portString, 10); } catch (NumberFormatException ex) { throw new IllegalArgumentException( "The configured proxy port value [" + portString + "] is invalid", ex); } clientConfiguration.withProxyHost(proxyHost).setProxyPort(proxyPort); } if (maxRetries != null) { // If not explicitly set, default to 3 with exponential backoff policy clientConfiguration.setMaxErrorRetry(maxRetries); } // #155: we might have 3rd party users using older S3 API version String awsSigner = settings.get("cloud.aws.s3.signer", settings.get("cloud.aws.signer")); if (awsSigner != null) { logger.debug("using AWS API signer [{}]", awsSigner); try { AwsSigner.configureSigner(awsSigner, clientConfiguration); } catch (IllegalArgumentException e) { logger.warn("wrong signer set for [cloud.aws.s3.signer] or [cloud.aws.signer]: [{}]", awsSigner); } } AWSCredentialsProvider credentials; if (account == null && key == null) { credentials = new AWSCredentialsProviderChain(new EnvironmentVariableCredentialsProvider(), new SystemPropertiesCredentialsProvider(), new InstanceProfileCredentialsProvider()); } else { credentials = new AWSCredentialsProviderChain( new StaticCredentialsProvider(new BasicAWSCredentials(account, key))); } client = new AmazonS3Client(credentials, clientConfiguration); if (endpoint != null) { client.setEndpoint(endpoint); } clients.put(clientDescriptor, client); return client; }
From source file:org.elasticsearch.discovery.ec2.AwsEc2ServiceImpl.java
License:Apache License
protected static ClientConfiguration buildConfiguration(Logger logger, Settings settings) { ClientConfiguration clientConfiguration = new ClientConfiguration(); // the response metadata cache is only there for diagnostics purposes, // but can force objects from every response to the old generation. clientConfiguration.setResponseMetadataCacheSize(0); clientConfiguration.setProtocol(PROTOCOL_SETTING.get(settings)); if (PROXY_HOST_SETTING.exists(settings)) { String proxyHost = PROXY_HOST_SETTING.get(settings); Integer proxyPort = PROXY_PORT_SETTING.get(settings); try (SecureString proxyUsername = PROXY_USERNAME_SETTING.get(settings); SecureString proxyPassword = PROXY_PASSWORD_SETTING.get(settings)) { clientConfiguration.withProxyHost(proxyHost).withProxyPort(proxyPort) .withProxyUsername(proxyUsername.toString()).withProxyPassword(proxyPassword.toString()); }//w ww .jav a2 s. co m } // Increase the number of retries in case of 5xx API responses final Random rand = Randomness.get(); RetryPolicy retryPolicy = new RetryPolicy(RetryPolicy.RetryCondition.NO_RETRY_CONDITION, new RetryPolicy.BackoffStrategy() { @Override public long delayBeforeNextRetry(AmazonWebServiceRequest originalRequest, AmazonClientException exception, int retriesAttempted) { // with 10 retries the max delay time is 320s/320000ms (10 * 2^5 * 1 * 1000) logger.warn("EC2 API request failed, retry again. Reason was:", exception); return 1000L * (long) (10d * Math.pow(2, retriesAttempted / 2.0d) * (1.0d + rand.nextDouble())); } }, 10, false); clientConfiguration.setRetryPolicy(retryPolicy); clientConfiguration.setSocketTimeout((int) READ_TIMEOUT_SETTING.get(settings).millis()); return clientConfiguration; }