List of usage examples for com.amazonaws ClientConfiguration ClientConfiguration
public ClientConfiguration()
From source file:com.emc.vipr.services.s3.ViPRS3Config.java
License:Open Source License
public synchronized ClientConfiguration getClientConfiguration() { if (clientConfiguration == null) { clientConfiguration = new ClientConfiguration(); }//from w w w .j av a 2 s . co m return clientConfiguration; }
From source file:com.emc.vipr.sync.source.S3Source.java
License:Open Source License
@Override public void configure(SyncSource source, Iterator<SyncFilter> filters, SyncTarget target) { Assert.hasText(accessKey, "accessKey is required"); Assert.hasText(secretKey, "secretKey is required"); Assert.hasText(bucketName, "bucketName is required"); AWSCredentials creds = new BasicAWSCredentials(accessKey, secretKey); ClientConfiguration config = new ClientConfiguration(); if (protocol != null) config.setProtocol(Protocol.valueOf(protocol.toUpperCase())); s3 = new AmazonS3Client(creds, config); if (endpoint != null) s3.setEndpoint(endpoint);// w w w.ja v a 2 s .c o m if (disableVHosts) { l4j.info( "The use of virtual hosted buckets on the s3 source has been DISABLED. Path style buckets will be used."); S3ClientOptions opts = new S3ClientOptions(); opts.setPathStyleAccess(true); s3.setS3ClientOptions(opts); } if (!s3.doesBucketExist(bucketName)) { throw new ConfigurationException("The bucket " + bucketName + " does not exist."); } if (rootKey == null) rootKey = ""; // make sure rootKey isn't null if (rootKey.startsWith("/")) rootKey = rootKey.substring(1); // " " does not start with slash }
From source file:com.eucalyptus.objectstorage.client.GenericS3ClientFactory.java
License:Open Source License
protected static ClientConfiguration getDefaultConfiguration(boolean withHttps) { ClientConfiguration config = new ClientConfiguration(); config.setConnectionTimeout(CONNECTION_TIMEOUT_MS); config.setMaxConnections(DEFAULT_MAX_CONNECTIONS); config.setMaxErrorRetry(DEFAULT_MAX_ERROR_RETRY); config.setUseReaper(true);/* w w w . j a v a 2 s . c om*/ config.setSocketTimeout(DEFAULT_SOCKET_READ_TIMEOUT_MS); config.setProtocol(withHttps ? Protocol.HTTPS : Protocol.HTTP); return config; }
From source file:com.eucalyptus.objectstorage.client.OsgInternalS3Client.java
License:Open Source License
private synchronized void initializeNewClient(@Nonnull AWSCredentials credentials, @Nonnull String endpoint, @Nonnull Boolean https, @Nonnull Boolean useDns) { ClientConfiguration config = new ClientConfiguration(); config.setConnectionTimeout(CONNECTION_TIMEOUT_MS); //very short timeout config.setSocketTimeout(OSG_SOCKET_TIMEOUT_MS); config.setUseReaper(true);// w w w . j a va 2s .c o m config.setMaxConnections(OSG_MAX_CONNECTIONS); Protocol protocol = https ? Protocol.HTTPS : Protocol.HTTP; config.setProtocol(protocol); this.clientConfig = config; this.s3Client = new AmazonS3Client(credentials, config); this.ops = new S3ClientOptions().withPathStyleAccess(!useDns); this.s3Client.setS3ClientOptions(ops); this.instantiated = new Date(); this.currentCredentials = credentials; this.setS3Endpoint(endpoint); }
From source file:com.eucalyptus.portal.SimpleQueueClientManager.java
License:Open Source License
/** * Parse a JSON format string for AWS SDK for Java ClientConfiguration. * * @param text The configuration in JSON * @return The configuration object// w w w . ja v a2 s . c o m */ public static ClientConfiguration buildConfiguration(final String text) { try { return Strings.isNullOrEmpty(text) ? new ClientConfiguration() : mapper.readValue(source(text), ClientConfiguration.class); } catch (final IOException e) { throw new IllegalArgumentException("Invalid configuration: " + e.getMessage(), e); } }
From source file:com.eucalyptus.simpleworkflow.common.client.WorkflowClientStandalone.java
License:Open Source License
private ClientConfiguration buildClientConfig() { final ClientConfiguration configuration = new ClientConfiguration(); configuration.setConnectionTimeout(this.clientConnectionTimeout); configuration.setMaxConnections(this.clientMaxConnections); return configuration; }
From source file:com.facebook.presto.hive.metastore.glue.GlueHiveMetastore.java
License:Apache License
private static AWSGlueAsync createAsyncGlueClient(GlueHiveMetastoreConfig config) { ClientConfiguration clientConfig = new ClientConfiguration() .withMaxConnections(config.getMaxGlueConnections()); AWSGlueAsyncClientBuilder asyncGlueClientBuilder = AWSGlueAsyncClientBuilder.standard() .withClientConfiguration(clientConfig); if (config.getGlueRegion().isPresent()) { asyncGlueClientBuilder.setRegion(config.getGlueRegion().get()); } else if (config.getPinGlueClientToCurrentRegion()) { Region currentRegion = Regions.getCurrentRegion(); if (currentRegion != null) { asyncGlueClientBuilder.setRegion(currentRegion.getName()); }/*from ww w .j a v a 2 s . co m*/ } return asyncGlueClientBuilder.build(); }
From source file:com.facebook.presto.hive.PrestoS3FileSystem.java
License:Apache License
@Override public void initialize(URI uri, Configuration conf) throws IOException { checkNotNull(uri, "uri is null"); checkNotNull(conf, "conf is null"); this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); this.workingDirectory = new Path("/").makeQualified(this.uri, new Path("/")); HiveClientConfig defaults = new HiveClientConfig(); this.stagingDirectory = new File( conf.get(S3_STAGING_DIRECTORY, defaults.getS3StagingDirectory().toString())); this.maxClientRetries = conf.getInt(S3_MAX_CLIENT_RETRIES, defaults.getS3MaxClientRetries()); int maxErrorRetries = conf.getInt(S3_MAX_ERROR_RETRIES, defaults.getS3MaxErrorRetries()); boolean sslEnabled = conf.getBoolean(S3_SSL_ENABLED, defaults.isS3SslEnabled()); Duration connectTimeout = Duration .valueOf(conf.get(S3_CONNECT_TIMEOUT, defaults.getS3ConnectTimeout().toString())); ClientConfiguration configuration = new ClientConfiguration(); configuration.setMaxErrorRetry(maxErrorRetries); configuration.setProtocol(sslEnabled ? Protocol.HTTPS : Protocol.HTTP); configuration.setConnectionTimeout(Ints.checkedCast(connectTimeout.toMillis())); this.s3 = new AmazonS3Client(getAwsCredentials(uri, conf), configuration); }
From source file:com.facebook.presto.hive.s3.PrestoS3ClientFactory.java
License:Apache License
synchronized AmazonS3 getS3Client(Configuration config, HiveClientConfig clientConfig) { if (s3Client != null) { return s3Client; }//from ww w .j av a 2s. co m HiveS3Config defaults = new HiveS3Config(); String userAgentPrefix = config.get(S3_USER_AGENT_PREFIX, defaults.getS3UserAgentPrefix()); int maxErrorRetries = config.getInt(S3_MAX_ERROR_RETRIES, defaults.getS3MaxErrorRetries()); boolean sslEnabled = config.getBoolean(S3_SSL_ENABLED, defaults.isS3SslEnabled()); Duration connectTimeout = Duration .valueOf(config.get(S3_CONNECT_TIMEOUT, defaults.getS3ConnectTimeout().toString())); Duration socketTimeout = Duration .valueOf(config.get(S3_SOCKET_TIMEOUT, defaults.getS3SocketTimeout().toString())); int maxConnections = config.getInt(S3_SELECT_PUSHDOWN_MAX_CONNECTIONS, clientConfig.getS3SelectPushdownMaxConnections()); if (clientConfig.isS3SelectPushdownEnabled()) { s3UserAgentSuffix = "presto-select"; } ClientConfiguration clientConfiguration = new ClientConfiguration().withMaxErrorRetry(maxErrorRetries) .withProtocol(sslEnabled ? Protocol.HTTPS : Protocol.HTTP) .withConnectionTimeout(toIntExact(connectTimeout.toMillis())) .withSocketTimeout(toIntExact(socketTimeout.toMillis())).withMaxConnections(maxConnections) .withUserAgentPrefix(userAgentPrefix).withUserAgentSuffix(s3UserAgentSuffix); PrestoS3FileSystemStats stats = new PrestoS3FileSystemStats(); RequestMetricCollector metricCollector = new PrestoS3FileSystemMetricCollector(stats); AWSCredentialsProvider awsCredentialsProvider = getAwsCredentialsProvider(config, defaults); AmazonS3Builder<? extends AmazonS3Builder, ? extends AmazonS3> clientBuilder = AmazonS3Client.builder() .withCredentials(awsCredentialsProvider).withClientConfiguration(clientConfiguration) .withMetricsCollector(metricCollector).enablePathStyleAccess(); boolean regionOrEndpointSet = false; String endpoint = config.get(S3_ENDPOINT); boolean pinS3ClientToCurrentRegion = config.getBoolean(S3_PIN_CLIENT_TO_CURRENT_REGION, defaults.isPinS3ClientToCurrentRegion()); verify(!pinS3ClientToCurrentRegion || endpoint == null, "Invalid configuration: either endpoint can be set or S3 client can be pinned to the current region"); // use local region when running inside of EC2 if (pinS3ClientToCurrentRegion) { Region region = Regions.getCurrentRegion(); if (region != null) { clientBuilder.withRegion(region.getName()); regionOrEndpointSet = true; } } if (!isNullOrEmpty(endpoint)) { clientBuilder.withEndpointConfiguration(new EndpointConfiguration(endpoint, null)); regionOrEndpointSet = true; } if (!regionOrEndpointSet) { clientBuilder.withRegion(US_EAST_1); clientBuilder.setForceGlobalBucketAccessEnabled(true); } s3Client = clientBuilder.build(); return s3Client; }
From source file:com.facebook.presto.hive.s3.PrestoS3FileSystem.java
License:Apache License
@Override public void initialize(URI uri, Configuration conf) throws IOException { requireNonNull(uri, "uri is null"); requireNonNull(conf, "conf is null"); super.initialize(uri, conf); setConf(conf);/*from w w w .j a v a 2s. c o m*/ this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); this.workingDirectory = new Path(PATH_SEPARATOR).makeQualified(this.uri, new Path(PATH_SEPARATOR)); HiveS3Config defaults = new HiveS3Config(); this.stagingDirectory = new File( conf.get(S3_STAGING_DIRECTORY, defaults.getS3StagingDirectory().toString())); this.maxAttempts = conf.getInt(S3_MAX_CLIENT_RETRIES, defaults.getS3MaxClientRetries()) + 1; this.maxBackoffTime = Duration .valueOf(conf.get(S3_MAX_BACKOFF_TIME, defaults.getS3MaxBackoffTime().toString())); this.maxRetryTime = Duration.valueOf(conf.get(S3_MAX_RETRY_TIME, defaults.getS3MaxRetryTime().toString())); int maxErrorRetries = conf.getInt(S3_MAX_ERROR_RETRIES, defaults.getS3MaxErrorRetries()); boolean sslEnabled = conf.getBoolean(S3_SSL_ENABLED, defaults.isS3SslEnabled()); Duration connectTimeout = Duration .valueOf(conf.get(S3_CONNECT_TIMEOUT, defaults.getS3ConnectTimeout().toString())); Duration socketTimeout = Duration .valueOf(conf.get(S3_SOCKET_TIMEOUT, defaults.getS3SocketTimeout().toString())); int maxConnections = conf.getInt(S3_MAX_CONNECTIONS, defaults.getS3MaxConnections()); long minFileSize = conf.getLong(S3_MULTIPART_MIN_FILE_SIZE, defaults.getS3MultipartMinFileSize().toBytes()); long minPartSize = conf.getLong(S3_MULTIPART_MIN_PART_SIZE, defaults.getS3MultipartMinPartSize().toBytes()); this.isPathStyleAccess = conf.getBoolean(S3_PATH_STYLE_ACCESS, defaults.isS3PathStyleAccess()); this.useInstanceCredentials = conf.getBoolean(S3_USE_INSTANCE_CREDENTIALS, defaults.isS3UseInstanceCredentials()); this.pinS3ClientToCurrentRegion = conf.getBoolean(S3_PIN_CLIENT_TO_CURRENT_REGION, defaults.isPinS3ClientToCurrentRegion()); this.sseEnabled = conf.getBoolean(S3_SSE_ENABLED, defaults.isS3SseEnabled()); this.sseType = PrestoS3SseType.valueOf(conf.get(S3_SSE_TYPE, defaults.getS3SseType().name())); this.sseKmsKeyId = conf.get(S3_SSE_KMS_KEY_ID, defaults.getS3SseKmsKeyId()); String userAgentPrefix = conf.get(S3_USER_AGENT_PREFIX, defaults.getS3UserAgentPrefix()); ClientConfiguration configuration = new ClientConfiguration().withMaxErrorRetry(maxErrorRetries) .withProtocol(sslEnabled ? Protocol.HTTPS : Protocol.HTTP) .withConnectionTimeout(toIntExact(connectTimeout.toMillis())) .withSocketTimeout(toIntExact(socketTimeout.toMillis())).withMaxConnections(maxConnections) .withUserAgentPrefix(userAgentPrefix).withUserAgentSuffix(S3_USER_AGENT_SUFFIX); this.s3 = createAmazonS3Client(uri, conf, configuration); transferConfig.setMultipartUploadThreshold(minFileSize); transferConfig.setMinimumUploadPartSize(minPartSize); }