List of usage examples for com.amazonaws ClientConfiguration setConnectionTimeout
public void setConnectionTimeout(int connectionTimeout)
From source file:com.pearson.eidetic.driver.Common.java
public static ClientConfiguration setClientConfigurationSettings(ClientConfiguration clientConfiguration) { clientConfiguration.setConnectionTimeout(10000); clientConfiguration.setMaxConnections(10000); clientConfiguration.setProtocol(Protocol.HTTPS); clientConfiguration.setSocketTimeout(60000); return clientConfiguration; }
From source file:com.streamsets.pipeline.lib.aws.s3.S3Accessor.java
License:Apache License
ClientConfiguration createClientConfiguration() throws StageException { ClientConfiguration clientConfig = new ClientConfiguration(); clientConfig.setConnectionTimeout(connectionConfigs.getConnectionTimeoutMillis()); clientConfig.setSocketTimeout(connectionConfigs.getSocketTimeoutMillis()); clientConfig.withMaxErrorRetry(connectionConfigs.getMaxErrorRetry()); if (connectionConfigs.isProxyEnabled()) { clientConfig.setProxyHost(connectionConfigs.getProxyHost()); clientConfig.setProxyPort(connectionConfigs.getProxyPort()); if (connectionConfigs.isProxyAuthenticationEnabled()) { clientConfig.setProxyUsername(connectionConfigs.getProxyUser().get()); clientConfig.setProxyPassword(connectionConfigs.getProxyPassword().get()); }//from ww w . ja va 2 s .c o m } return clientConfig; }
From source file:com.upplication.s3fs.S3FileSystemProvider.java
License:Open Source License
protected ClientConfiguration createClientConfig(Properties props) { ClientConfiguration config = new ClientConfiguration(); if (props == null) return config; if (props.containsKey("connection_timeout")) { log.trace("AWS client config - connection_timeout: {}", props.getProperty("connection_timeout")); config.setConnectionTimeout(Integer.parseInt(props.getProperty("connection_timeout"))); }//from ww w . java 2s .c om if (props.containsKey("max_connections")) { log.trace("AWS client config - max_connections: {}", props.getProperty("max_connections")); config.setMaxConnections(Integer.parseInt(props.getProperty("max_connections"))); } if (props.containsKey("max_error_retry")) { log.trace("AWS client config - max_error_retry: {}", props.getProperty("max_error_retry")); config.setMaxErrorRetry(Integer.parseInt(props.getProperty("max_error_retry"))); } if (props.containsKey("protocol")) { log.trace("AWS client config - protocol: {}", props.getProperty("protocol")); config.setProtocol(Protocol.valueOf(props.getProperty("protocol").toUpperCase())); } if (props.containsKey("proxy_domain")) { log.trace("AWS client config - proxy_domain: {}", props.getProperty("proxy_domain")); config.setProxyDomain(props.getProperty("proxy_domain")); } if (props.containsKey("proxy_host")) { log.trace("AWS client config - proxy_host: {}", props.getProperty("proxy_host")); config.setProxyHost(props.getProperty("proxy_host")); } if (props.containsKey("proxy_port")) { log.trace("AWS client config - proxy_port: {}", props.getProperty("proxy_port")); config.setProxyPort(Integer.parseInt(props.getProperty("proxy_port"))); } if (props.containsKey("proxy_username")) { log.trace("AWS client config - proxy_username: {}", props.getProperty("proxy_username")); config.setProxyUsername(props.getProperty("proxy_username")); } if (props.containsKey("proxy_password")) { log.trace("AWS client config - proxy_password: {}", props.getProperty("proxy_password")); config.setProxyPassword(props.getProperty("proxy_password")); } if (props.containsKey("proxy_workstation")) { log.trace("AWS client config - proxy_workstation: {}", props.getProperty("proxy_workstation")); config.setProxyWorkstation(props.getProperty("proxy_workstation")); } if (props.containsKey("signer_override")) { log.debug("AWS client config - signerOverride: {}", props.getProperty("signer_override")); config.setSignerOverride(props.getProperty("signer_override")); } if (props.containsKey("socket_send_buffer_size_hints") || props.containsKey("socket_recv_buffer_size_hints")) { log.trace("AWS client config - socket_send_buffer_size_hints: {}, socket_recv_buffer_size_hints: {}", props.getProperty("socket_send_buffer_size_hints", "0"), props.getProperty("socket_recv_buffer_size_hints", "0")); int send = Integer.parseInt(props.getProperty("socket_send_buffer_size_hints", "0")); int recv = Integer.parseInt(props.getProperty("socket_recv_buffer_size_hints", "0")); config.setSocketBufferSizeHints(send, recv); } if (props.containsKey("socket_timeout")) { log.trace("AWS client config - socket_timeout: {}", props.getProperty("socket_timeout")); config.setSocketTimeout(Integer.parseInt(props.getProperty("socket_timeout"))); } if (props.containsKey("user_agent")) { log.trace("AWS client config - user_agent: {}", props.getProperty("user_agent")); config.setUserAgent(props.getProperty("user_agent")); } return config; }
From source file:com.zanox.vertx.mods.KinesisMessageProcessor.java
License:Apache License
private AmazonKinesisAsyncClient createClient() { // Building Kinesis configuration int connectionTimeout = getOptionalIntConfig(CONNECTION_TIMEOUT, ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT); int maxConnection = getOptionalIntConfig(MAX_CONNECTION, ClientConfiguration.DEFAULT_MAX_CONNECTIONS); // TODO: replace default retry policy RetryPolicy retryPolicy = ClientConfiguration.DEFAULT_RETRY_POLICY; int socketTimeout = getOptionalIntConfig(SOCKET_TIMEOUT, ClientConfiguration.DEFAULT_SOCKET_TIMEOUT); boolean useReaper = getOptionalBooleanConfig(USE_REAPER, ClientConfiguration.DEFAULT_USE_REAPER); String userAgent = getOptionalStringConfig(USER_AGENT, ClientConfiguration.DEFAULT_USER_AGENT); streamName = getMandatoryStringConfig(STREAM_NAME); partitionKey = getMandatoryStringConfig(PARTITION_KEY); region = getMandatoryStringConfig(REGION); logger.info(" --- Stream name: " + streamName); logger.info(" --- Partition key: " + partitionKey); logger.info(" --- Region: " + region); ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setConnectionTimeout(connectionTimeout); clientConfiguration.setMaxConnections(maxConnection); clientConfiguration.setRetryPolicy(retryPolicy); clientConfiguration.setSocketTimeout(socketTimeout); clientConfiguration.setUseReaper(useReaper); clientConfiguration.setUserAgent(userAgent); /*/*from ww w . j a va 2s. co m*/ AWS credentials provider chain that looks for credentials in this order: Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY Java System Properties - aws.accessKeyId and aws.secretKey Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI Instance profile credentials delivered through the Amazon EC2 metadata service */ AWSCredentialsProvider awsCredentialsProvider = new DefaultAWSCredentialsProviderChain(); // Configuring Kinesis-client with configuration AmazonKinesisAsyncClient kinesisAsyncClient = new AmazonKinesisAsyncClient(awsCredentialsProvider, clientConfiguration); Region awsRegion = RegionUtils.getRegion(region); kinesisAsyncClient.setRegion(awsRegion); return kinesisAsyncClient; }
From source file:hu.mta.sztaki.lpds.cloud.entice.imageoptimizer.iaashandler.amazontarget.Storage.java
License:Apache License
/** * @param endpoint S3 endpoint URL//from w w w . j a v a 2s. c om * @param accessKey Access key * @param secretKey Secret key * @param bucket Bucket name * @param path Key name of the object to download (path + file name) * @param file Local file to download to * @throws Exception On any error */ public static void download(String endpoint, String accessKey, String secretKey, String bucket, String path, File file) throws Exception { AmazonS3Client amazonS3Client = null; InputStream in = null; OutputStream out = null; try { AWSCredentials awsCredentials = new BasicAWSCredentials(accessKey, secretKey); ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setMaxConnections(MAX_CONNECTIONS); clientConfiguration.setMaxErrorRetry(PredefinedRetryPolicies.DEFAULT_MAX_ERROR_RETRY); clientConfiguration.setConnectionTimeout(ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT); amazonS3Client = new AmazonS3Client(awsCredentials, clientConfiguration); S3ClientOptions clientOptions = new S3ClientOptions().withPathStyleAccess(true); amazonS3Client.setS3ClientOptions(clientOptions); amazonS3Client.setEndpoint(endpoint); S3Object object = amazonS3Client.getObject(new GetObjectRequest(bucket, path)); in = object.getObjectContent(); byte[] buf = new byte[BUFFER_SIZE]; out = new FileOutputStream(file); int count; while ((count = in.read(buf)) != -1) out.write(buf, 0, count); out.close(); in.close(); } catch (AmazonServiceException x) { Shrinker.myLogger.info("download error: " + x.getMessage()); throw new Exception("download exception", x); } catch (AmazonClientException x) { Shrinker.myLogger.info("download error: " + x.getMessage()); throw new Exception("download exception", x); } catch (IOException x) { Shrinker.myLogger.info("download error: " + x.getMessage()); throw new Exception("download exception", x); } finally { if (in != null) { try { in.close(); } catch (Exception e) { } } if (out != null) { try { out.close(); } catch (Exception e) { } } if (amazonS3Client != null) { try { amazonS3Client.shutdown(); } catch (Exception e) { } } } }
From source file:hu.mta.sztaki.lpds.cloud.entice.imageoptimizer.iaashandler.amazontarget.Storage.java
License:Apache License
/** * @param file Local file to upload/*from www. j a v a2 s . co m*/ * @param endpoint S3 endpoint URL * @param accessKey Access key * @param secretKey Secret key * @param bucket Bucket name * @param path Key name (path + file name) * @throws Exception On any error */ public static void upload(File file, String endpoint, String accessKey, String secretKey, String bucket, String path) throws Exception { AmazonS3Client amazonS3Client = null; try { AWSCredentials awsCredentials = new BasicAWSCredentials(accessKey, secretKey); ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setMaxConnections(MAX_CONNECTIONS); clientConfiguration.setMaxErrorRetry(PredefinedRetryPolicies.DEFAULT_MAX_ERROR_RETRY); clientConfiguration.setConnectionTimeout(ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT); amazonS3Client = new AmazonS3Client(awsCredentials, clientConfiguration); S3ClientOptions clientOptions = new S3ClientOptions().withPathStyleAccess(true); amazonS3Client.setS3ClientOptions(clientOptions); amazonS3Client.setEndpoint(endpoint); // amazonS3Client.putObject(new PutObjectRequest(bucket, path, file)); // up to 5GB TransferManager tm = new TransferManager(amazonS3Client); // up to 5TB Upload upload = tm.upload(bucket, path, file); // while (!upload.isDone()) { upload.getProgress().getBytesTransferred(); Thread.sleep(1000); } // to get progress upload.waitForCompletion(); tm.shutdownNow(); } catch (AmazonServiceException x) { Shrinker.myLogger.info("upload error: " + x.getMessage()); throw new Exception("upload exception", x); } catch (AmazonClientException x) { Shrinker.myLogger.info("upload error: " + x.getMessage()); throw new Exception("upload exception", x); } finally { if (amazonS3Client != null) { try { amazonS3Client.shutdown(); } catch (Exception e) { } } } }
From source file:jp.aws.test.AmazonClientManager.java
License:Apache License
/** * ?//from w ww . j a v a2 s.c o m */ public void validateCredentials() { if (s3Client == null || ec2Client == null) { Log.i(LOG_TAG, "Creating New Clients."); // ??? SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this.context); String access_key = prefs.getString("prefs_account_access_key", ""); String secret_key = prefs.getString("prefs_account_secret_key", ""); // ? ClientConfiguration clientconfiguration = new ClientConfiguration(); clientconfiguration.setConnectionTimeout(this.connectionTimeout); // (ms) AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key); s3Client = new AmazonS3Client(credentials, clientconfiguration); ec2Client = new AmazonEC2Client(credentials, clientconfiguration); // this.changeRegion(); } }
From source file:org.apache.hadoop.fs.s3r.S3RFileSystem.java
License:Apache License
/** Called after a new FileSystem instance is constructed. * @param name a uri whose authority section names the host, port, etc. * for this FileSystem// ww w .ja v a 2s . c o m * @param conf the configuration */ public void initialize(URI name, Configuration conf) throws IOException { super.initialize(name, conf); uri = URI.create(name.getScheme() + "://" + name.getAuthority()); workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(this.uri, this.getWorkingDirectory()); // Try to get our credentials or just connect anonymously String accessKey = conf.get(ACCESS_KEY, null); String secretKey = conf.get(SECRET_KEY, null); String userInfo = name.getUserInfo(); if (userInfo != null) { int index = userInfo.indexOf(':'); if (index != -1) { accessKey = userInfo.substring(0, index); secretKey = userInfo.substring(index + 1); } else { accessKey = userInfo; } } AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain( new BasicAWSCredentialsProvider(accessKey, secretKey), new InstanceProfileCredentialsProvider(), new AnonymousAWSCredentialsProvider()); bucket = name.getHost(); ClientConfiguration awsConf = new ClientConfiguration(); awsConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS, DEFAULT_MAXIMUM_CONNECTIONS)); boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS); awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES, DEFAULT_MAX_ERROR_RETRIES)); awsConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT, DEFAULT_ESTABLISH_TIMEOUT)); awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT, DEFAULT_SOCKET_TIMEOUT)); String proxyHost = conf.getTrimmed(PROXY_HOST, ""); int proxyPort = conf.getInt(PROXY_PORT, -1); if (!proxyHost.isEmpty()) { awsConf.setProxyHost(proxyHost); if (proxyPort >= 0) { awsConf.setProxyPort(proxyPort); } else { if (secureConnections) { LOG.warn("Proxy host set without port. Using HTTPS default 443"); awsConf.setProxyPort(443); } else { LOG.warn("Proxy host set without port. Using HTTP default 80"); awsConf.setProxyPort(80); } } String proxyUsername = conf.getTrimmed(PROXY_USERNAME); String proxyPassword = conf.getTrimmed(PROXY_PASSWORD); if ((proxyUsername == null) != (proxyPassword == null)) { String msg = "Proxy error: " + PROXY_USERNAME + " or " + PROXY_PASSWORD + " set without the other."; LOG.error(msg); throw new IllegalArgumentException(msg); } awsConf.setProxyUsername(proxyUsername); awsConf.setProxyPassword(proxyPassword); awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN)); awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION)); if (LOG.isDebugEnabled()) { LOG.debug( "Using proxy server {}:{} as user {} with password {} on " + "domain {} as workstation {}", awsConf.getProxyHost(), awsConf.getProxyPort(), String.valueOf(awsConf.getProxyUsername()), awsConf.getProxyPassword(), awsConf.getProxyDomain(), awsConf.getProxyWorkstation()); } } else if (proxyPort >= 0) { String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST; LOG.error(msg); throw new IllegalArgumentException(msg); } s3 = new AmazonS3Client(credentials, awsConf); String endPoint = conf.getTrimmed(ENDPOINT, ""); if (!endPoint.isEmpty()) { try { s3.setEndpoint(endPoint); } catch (IllegalArgumentException e) { String msg = "Incorrect endpoint: " + e.getMessage(); LOG.error(msg); throw new IllegalArgumentException(msg, e); } } maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS); partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE); multiPartThreshold = conf.getInt(MIN_MULTIPART_THRESHOLD, DEFAULT_MIN_MULTIPART_THRESHOLD); if (partSize < 5 * 1024 * 1024) { LOG.error(MULTIPART_SIZE + " must be at least 5 MB"); partSize = 5 * 1024 * 1024; } if (multiPartThreshold < 5 * 1024 * 1024) { LOG.error(MIN_MULTIPART_THRESHOLD + " must be at least 5 MB"); multiPartThreshold = 5 * 1024 * 1024; } int maxThreads = conf.getInt(MAX_THREADS, DEFAULT_MAX_THREADS); int coreThreads = conf.getInt(CORE_THREADS, DEFAULT_CORE_THREADS); if (maxThreads == 0) { maxThreads = Runtime.getRuntime().availableProcessors() * 8; } if (coreThreads == 0) { coreThreads = Runtime.getRuntime().availableProcessors() * 8; } long keepAliveTime = conf.getLong(KEEPALIVE_TIME, DEFAULT_KEEPALIVE_TIME); LinkedBlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>( maxThreads * conf.getInt(MAX_TOTAL_TASKS, DEFAULT_MAX_TOTAL_TASKS)); threadPoolExecutor = new ThreadPoolExecutor(coreThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue, newDaemonThreadFactory("s3a-transfer-shared-")); threadPoolExecutor.allowCoreThreadTimeOut(true); TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration(); transferConfiguration.setMinimumUploadPartSize(partSize); transferConfiguration.setMultipartUploadThreshold(multiPartThreshold); transfers = new TransferManager(s3, threadPoolExecutor); transfers.setConfiguration(transferConfiguration); String cannedACLName = conf.get(CANNED_ACL, DEFAULT_CANNED_ACL); if (!cannedACLName.isEmpty()) { cannedACL = CannedAccessControlList.valueOf(cannedACLName); } else { cannedACL = null; } if (!s3.doesBucketExist(bucket)) { throw new IOException("Bucket " + bucket + " does not exist"); } boolean purgeExistingMultipart = conf.getBoolean(PURGE_EXISTING_MULTIPART, DEFAULT_PURGE_EXISTING_MULTIPART); long purgeExistingMultipartAge = conf.getLong(PURGE_EXISTING_MULTIPART_AGE, DEFAULT_PURGE_EXISTING_MULTIPART_AGE); if (purgeExistingMultipart) { Date purgeBefore = new Date(new Date().getTime() - purgeExistingMultipartAge * 1000); transfers.abortMultipartUploads(bucket, purgeBefore); } serverSideEncryptionAlgorithm = conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM); setConf(conf); }
From source file:org.apache.jackrabbit.aws.ext.Utils.java
License:Apache License
/** * Create AmazonS3Client from properties. * /* w w w .j a va 2s .c o m*/ * @param prop properties to configure @link {@link AmazonS3Client} * @return {@link AmazonS3Client} */ public static AmazonS3Client openService(final Properties prop) { AWSCredentials credentials = new BasicAWSCredentials(prop.getProperty(S3Constants.ACCESS_KEY), prop.getProperty(S3Constants.SECRET_KEY)); int connectionTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_CONN_TIMEOUT)); int socketTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_SOCK_TIMEOUT)); int maxConnections = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_CONNS)); int maxErrorRetry = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_ERR_RETRY)); ClientConfiguration cc = new ClientConfiguration(); cc.setConnectionTimeout(connectionTimeOut); cc.setSocketTimeout(socketTimeOut); cc.setMaxConnections(maxConnections); cc.setMaxErrorRetry(maxErrorRetry); return new AmazonS3Client(credentials, cc); }
From source file:org.apache.jackrabbit.oak.blob.cloud.aws.s3.Utils.java
License:Apache License
private static ClientConfiguration getClientConfiguration(Properties prop) { int connectionTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_CONN_TIMEOUT)); int socketTimeOut = Integer.parseInt(prop.getProperty(S3Constants.S3_SOCK_TIMEOUT)); int maxConnections = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_CONNS)); int maxErrorRetry = Integer.parseInt(prop.getProperty(S3Constants.S3_MAX_ERR_RETRY)); String protocol = prop.getProperty(S3Constants.S3_CONN_PROTOCOL); String proxyHost = prop.getProperty(S3Constants.PROXY_HOST); String proxyPort = prop.getProperty(S3Constants.PROXY_PORT); ClientConfiguration cc = new ClientConfiguration(); if (protocol != null && protocol.equalsIgnoreCase("http")) { cc.setProtocol(Protocol.HTTP);/*from w w w . j av a2 s . co m*/ } if (proxyHost != null && !proxyHost.isEmpty()) { cc.setProxyHost(proxyHost); } if (proxyPort != null && !proxyPort.isEmpty()) { cc.setProxyPort(Integer.parseInt(proxyPort)); } cc.setConnectionTimeout(connectionTimeOut); cc.setSocketTimeout(socketTimeOut); cc.setMaxConnections(maxConnections); cc.setMaxErrorRetry(maxErrorRetry); return cc; }