List of usage examples for com.amazonaws ClientConfiguration setMaxConnections
public void setMaxConnections(int maxConnections)
From source file:com.treasure_data.td_import.source.S3Source.java
License:Apache License
static AmazonS3Client createAmazonS3Client(SourceDesc desc) { String accessKey = desc.getUser(); if (accessKey == null || accessKey.isEmpty()) { throw new IllegalArgumentException("S3 AccessKey is null or empty."); }/*from w ww . j av a 2 s . c om*/ String secretAccessKey = desc.getPassword(); if (secretAccessKey == null || secretAccessKey.isEmpty()) { throw new IllegalArgumentException("S3 SecretAccessKey is null or empty."); } AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretAccessKey); ClientConfiguration conf = new ClientConfiguration(); conf.setProtocol(Configuration.BI_PREPARE_S3_PROTOCOL); conf.setMaxConnections(Configuration.BI_PREPARE_S3_MAX_CONNECTIONS); conf.setMaxErrorRetry(Configuration.BI_PREPARE_S3_MAX_ERRORRETRY); conf.setSocketTimeout(Configuration.BI_PREPARE_S3_SOCKET_TIMEOUT); return new AmazonS3Client(credentials, conf); }
From source file:com.upplication.s3fs.S3FileSystemProvider.java
License:Open Source License
protected ClientConfiguration createClientConfig(Properties props) { ClientConfiguration config = new ClientConfiguration(); if (props == null) return config; if (props.containsKey("connection_timeout")) { log.trace("AWS client config - connection_timeout: {}", props.getProperty("connection_timeout")); config.setConnectionTimeout(Integer.parseInt(props.getProperty("connection_timeout"))); }/* w w w. java 2 s. c o m*/ if (props.containsKey("max_connections")) { log.trace("AWS client config - max_connections: {}", props.getProperty("max_connections")); config.setMaxConnections(Integer.parseInt(props.getProperty("max_connections"))); } if (props.containsKey("max_error_retry")) { log.trace("AWS client config - max_error_retry: {}", props.getProperty("max_error_retry")); config.setMaxErrorRetry(Integer.parseInt(props.getProperty("max_error_retry"))); } if (props.containsKey("protocol")) { log.trace("AWS client config - protocol: {}", props.getProperty("protocol")); config.setProtocol(Protocol.valueOf(props.getProperty("protocol").toUpperCase())); } if (props.containsKey("proxy_domain")) { log.trace("AWS client config - proxy_domain: {}", props.getProperty("proxy_domain")); config.setProxyDomain(props.getProperty("proxy_domain")); } if (props.containsKey("proxy_host")) { log.trace("AWS client config - proxy_host: {}", props.getProperty("proxy_host")); config.setProxyHost(props.getProperty("proxy_host")); } if (props.containsKey("proxy_port")) { log.trace("AWS client config - proxy_port: {}", props.getProperty("proxy_port")); config.setProxyPort(Integer.parseInt(props.getProperty("proxy_port"))); } if (props.containsKey("proxy_username")) { log.trace("AWS client config - proxy_username: {}", props.getProperty("proxy_username")); config.setProxyUsername(props.getProperty("proxy_username")); } if (props.containsKey("proxy_password")) { log.trace("AWS client config - proxy_password: {}", props.getProperty("proxy_password")); config.setProxyPassword(props.getProperty("proxy_password")); } if (props.containsKey("proxy_workstation")) { log.trace("AWS client config - proxy_workstation: {}", props.getProperty("proxy_workstation")); config.setProxyWorkstation(props.getProperty("proxy_workstation")); } if (props.containsKey("signer_override")) { log.debug("AWS client config - signerOverride: {}", props.getProperty("signer_override")); config.setSignerOverride(props.getProperty("signer_override")); } if (props.containsKey("socket_send_buffer_size_hints") || props.containsKey("socket_recv_buffer_size_hints")) { log.trace("AWS client config - socket_send_buffer_size_hints: {}, socket_recv_buffer_size_hints: {}", props.getProperty("socket_send_buffer_size_hints", "0"), props.getProperty("socket_recv_buffer_size_hints", "0")); int send = Integer.parseInt(props.getProperty("socket_send_buffer_size_hints", "0")); int recv = Integer.parseInt(props.getProperty("socket_recv_buffer_size_hints", "0")); config.setSocketBufferSizeHints(send, recv); } if (props.containsKey("socket_timeout")) { log.trace("AWS client config - socket_timeout: {}", props.getProperty("socket_timeout")); config.setSocketTimeout(Integer.parseInt(props.getProperty("socket_timeout"))); } if (props.containsKey("user_agent")) { log.trace("AWS client config - user_agent: {}", props.getProperty("user_agent")); config.setUserAgent(props.getProperty("user_agent")); } return config; }
From source file:com.yahoo.ycsb.db.DynamoDBClient.java
License:Open Source License
/** * Initialize any state for this DB. Called once per DB instance; there is * one DB instance per client thread.//from w w w . j a v a 2 s .co m */ public void init() throws DBException { // initialize DynamoDb driver & table. String debug = getProperties().getProperty("dynamodb.debug", null); if (null != debug && "true".equalsIgnoreCase(debug)) { logger.setLevel(Level.DEBUG); } String endpoint = getProperties().getProperty("dynamodb.endpoint", null); String credentialsFile = getProperties().getProperty("dynamodb.awsCredentialsFile", null); String primaryKey = getProperties().getProperty("dynamodb.primaryKey", null); String consistentReads = getProperties().getProperty("dynamodb.consistentReads", null); String connectMax = getProperties().getProperty("dynamodb.connectMax", null); if (null != connectMax) { this.maxConnects = Integer.parseInt(connectMax); } if (null != consistentReads && "true".equalsIgnoreCase(consistentReads)) { this.consistentRead = true; } if (null != endpoint) { this.endpoint = endpoint; } if (null == primaryKey || primaryKey.length() < 1) { String errMsg = "Missing primary key attribute name, cannot continue"; logger.error(errMsg); } try { AWSCredentials credentials = new PropertiesCredentials(new File(credentialsFile)); ClientConfiguration cconfig = new ClientConfiguration(); cconfig.setMaxConnections(maxConnects); dynamoDB = new AmazonDynamoDBClient(credentials, cconfig); dynamoDB.setEndpoint(this.endpoint); primaryKeyName = primaryKey; logger.info("dynamodb connection created with " + this.endpoint); } catch (Exception e1) { String errMsg = "DynamoDBClient.init(): Could not initialize DynamoDB client: " + e1.getMessage(); logger.error(errMsg); } }
From source file:com.yahoo.ycsb.db.MailAppDynamoDBClient.java
License:Open Source License
/** * Initialize any state for this DB. Called once per DB instance; there is * one DB instance per client thread./*from www .ja v a2 s . co m*/ */ public void init() throws DBException { // initialize DynamoDb driver & table. String debug = getProperties().getProperty("dynamodb.debug", null); if (null != debug && "true".equalsIgnoreCase(debug)) { logger.setLevel(Level.DEBUG); } String endpoint = getProperties().getProperty("dynamodb.endpoint", null); String credentialsFile = getProperties().getProperty("dynamodb.awsCredentialsFile", null); String primaryKey = getProperties().getProperty("dynamodb.primaryKey", null); //range key support String rangeKey = getProperties().getProperty("dynamodb.rangeKey", "range"); this.rangeKeyName = rangeKey; String hashAndRangeKeyDelimiterProp = getProperties().getProperty("dynamodb.rangekeydelimiter", "_rangeKeyFollows_"); this.hashAndRangeKeyDelimiter = hashAndRangeKeyDelimiterProp; String consistentReads = getProperties().getProperty("dynamodb.consistentReads", null); String connectMax = getProperties().getProperty("dynamodb.connectMax", null); this.incrementTag = getProperties().getProperty("incrementtag", "#inc#"); this.decrementTag = getProperties().getProperty("decrementtag", "#dec#"); if (null != connectMax) { this.maxConnects = Integer.parseInt(connectMax); } if (null != consistentReads && "true".equalsIgnoreCase(consistentReads)) { this.consistentRead = true; } if (null != endpoint) { this.endpoint = endpoint; } if (null == primaryKey || primaryKey.length() < 1) { String errMsg = "Missing primary key attribute name, cannot continue"; logger.error(errMsg); } try { AWSCredentials credentials = new PropertiesCredentials(new File(credentialsFile)); ClientConfiguration cconfig = new ClientConfiguration(); cconfig.setMaxConnections(maxConnects); dynamoDB = new AmazonDynamoDBClient(credentials, cconfig); dynamoDB.setEndpoint(this.endpoint); primaryKeyName = primaryKey; logger.info("dynamodb connection created with " + this.endpoint); } catch (Exception e1) { String errMsg = "MailAppDynamoDBClient.init(): Could not initialize MailAppDynamoDB client: " + e1.getMessage(); logger.error(errMsg); } }
From source file:com.yahoo.ycsb.db.S3Client.java
License:Open Source License
/** * Initialize any state for the storage./* w ww. ja v a2 s . co m*/ * Called once per S3 instance; If the client is not null it is re-used. */ @Override public void init() throws DBException { final int count = INIT_COUNT.incrementAndGet(); synchronized (S3Client.class) { Properties propsCL = getProperties(); int recordcount = Integer.parseInt(propsCL.getProperty("recordcount")); int operationcount = Integer.parseInt(propsCL.getProperty("operationcount")); int numberOfOperations = 0; if (recordcount > 0) { if (recordcount > operationcount) { numberOfOperations = recordcount; } else { numberOfOperations = operationcount; } } else { numberOfOperations = operationcount; } if (count <= numberOfOperations) { String accessKeyId = null; String secretKey = null; String endPoint = null; String region = null; String maxErrorRetry = null; String maxConnections = null; String protocol = null; BasicAWSCredentials s3Credentials; ClientConfiguration clientConfig; if (s3Client != null) { System.out.println("Reusing the same client"); return; } try { InputStream propFile = S3Client.class.getClassLoader().getResourceAsStream("s3.properties"); Properties props = new Properties(System.getProperties()); props.load(propFile); accessKeyId = props.getProperty("s3.accessKeyId"); if (accessKeyId == null) { accessKeyId = propsCL.getProperty("s3.accessKeyId"); } System.out.println(accessKeyId); secretKey = props.getProperty("s3.secretKey"); if (secretKey == null) { secretKey = propsCL.getProperty("s3.secretKey"); } System.out.println(secretKey); endPoint = props.getProperty("s3.endPoint"); if (endPoint == null) { endPoint = propsCL.getProperty("s3.endPoint", "s3.amazonaws.com"); } System.out.println(endPoint); region = props.getProperty("s3.region"); if (region == null) { region = propsCL.getProperty("s3.region", "us-east-1"); } System.out.println(region); maxErrorRetry = props.getProperty("s3.maxErrorRetry"); if (maxErrorRetry == null) { maxErrorRetry = propsCL.getProperty("s3.maxErrorRetry", "15"); } maxConnections = props.getProperty("s3.maxConnections"); if (maxConnections == null) { maxConnections = propsCL.getProperty("s3.maxConnections"); } protocol = props.getProperty("s3.protocol"); if (protocol == null) { protocol = propsCL.getProperty("s3.protocol", "HTTPS"); } sse = props.getProperty("s3.sse"); if (sse == null) { sse = propsCL.getProperty("s3.sse", "false"); } String ssec = props.getProperty("s3.ssec"); if (ssec == null) { ssec = propsCL.getProperty("s3.ssec", null); } else { ssecKey = new SSECustomerKey(ssec); } } catch (Exception e) { System.err.println("The file properties doesn't exist " + e.toString()); e.printStackTrace(); } try { System.out.println("Inizializing the S3 connection"); s3Credentials = new BasicAWSCredentials(accessKeyId, secretKey); clientConfig = new ClientConfiguration(); clientConfig.setMaxErrorRetry(Integer.parseInt(maxErrorRetry)); if (protocol.equals("HTTP")) { clientConfig.setProtocol(Protocol.HTTP); } else { clientConfig.setProtocol(Protocol.HTTPS); } if (maxConnections != null) { clientConfig.setMaxConnections(Integer.parseInt(maxConnections)); } s3Client = new AmazonS3Client(s3Credentials, clientConfig); s3Client.setRegion(Region.getRegion(Regions.fromName(region))); s3Client.setEndpoint(endPoint); System.out.println("Connection successfully initialized"); } catch (Exception e) { System.err.println("Could not connect to S3 storage: " + e.toString()); e.printStackTrace(); throw new DBException(e); } } else { System.err.println("The number of threads must be less or equal than the operations"); throw new DBException(new Error("The number of threads must be less or equal than the operations")); } } }
From source file:com.yahoo.ycsb.utils.connection.S3Connection.java
License:Open Source License
public S3Connection(String bucket, String region, String endPoint) throws ClientException { super(bucket, region, endPoint); logger.debug("S3Client.establishConnection(" + region + "," + endPoint + ") bucket: " + bucket); org.apache.log4j.Logger.getLogger("com.amazonaws").setLevel(Level.OFF); /*if (S3Connection.init == true) { init();/*from www . ja va 2s. com*/ S3Connection.init = false; }*/ this.bucket = bucket; this.region = region; try { BasicAWSCredentials s3Credentials = new BasicAWSCredentials(accessKeyId, secretKey); ClientConfiguration clientConfig = new ClientConfiguration(); clientConfig.setMaxErrorRetry(Integer.parseInt(maxErrorRetry)); if (protocol.equals("HTTP")) { clientConfig.setProtocol(Protocol.HTTP); } else { clientConfig.setProtocol(Protocol.HTTPS); } if (maxConnections != null) { clientConfig.setMaxConnections(Integer.parseInt(maxConnections)); } logger.debug("Inizializing the S3 connection..."); awsClient = new AmazonS3Client(s3Credentials, clientConfig); awsClient.setRegion(Region.getRegion(Regions.fromName(region))); awsClient.setEndpoint(endPoint); logger.debug("Connection successfully initialized"); } catch (Exception e) { logger.error("Could not connect to S3 storage: " + e.toString()); e.printStackTrace(); throw new ClientException(e); } }
From source file:com.zanox.vertx.mods.KinesisMessageProcessor.java
License:Apache License
private AmazonKinesisAsyncClient createClient() { // Building Kinesis configuration int connectionTimeout = getOptionalIntConfig(CONNECTION_TIMEOUT, ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT); int maxConnection = getOptionalIntConfig(MAX_CONNECTION, ClientConfiguration.DEFAULT_MAX_CONNECTIONS); // TODO: replace default retry policy RetryPolicy retryPolicy = ClientConfiguration.DEFAULT_RETRY_POLICY; int socketTimeout = getOptionalIntConfig(SOCKET_TIMEOUT, ClientConfiguration.DEFAULT_SOCKET_TIMEOUT); boolean useReaper = getOptionalBooleanConfig(USE_REAPER, ClientConfiguration.DEFAULT_USE_REAPER); String userAgent = getOptionalStringConfig(USER_AGENT, ClientConfiguration.DEFAULT_USER_AGENT); streamName = getMandatoryStringConfig(STREAM_NAME); partitionKey = getMandatoryStringConfig(PARTITION_KEY); region = getMandatoryStringConfig(REGION); logger.info(" --- Stream name: " + streamName); logger.info(" --- Partition key: " + partitionKey); logger.info(" --- Region: " + region); ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setConnectionTimeout(connectionTimeout); clientConfiguration.setMaxConnections(maxConnection); clientConfiguration.setRetryPolicy(retryPolicy); clientConfiguration.setSocketTimeout(socketTimeout); clientConfiguration.setUseReaper(useReaper); clientConfiguration.setUserAgent(userAgent); /*/*www.jav a 2s . c o m*/ AWS credentials provider chain that looks for credentials in this order: Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY Java System Properties - aws.accessKeyId and aws.secretKey Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI Instance profile credentials delivered through the Amazon EC2 metadata service */ AWSCredentialsProvider awsCredentialsProvider = new DefaultAWSCredentialsProviderChain(); // Configuring Kinesis-client with configuration AmazonKinesisAsyncClient kinesisAsyncClient = new AmazonKinesisAsyncClient(awsCredentialsProvider, clientConfiguration); Region awsRegion = RegionUtils.getRegion(region); kinesisAsyncClient.setRegion(awsRegion); return kinesisAsyncClient; }
From source file:hu.mta.sztaki.lpds.cloud.entice.imageoptimizer.iaashandler.amazontarget.Storage.java
License:Apache License
/** * @param endpoint S3 endpoint URL/*w ww. ja v a2 s. co m*/ * @param accessKey Access key * @param secretKey Secret key * @param bucket Bucket name * @param path Key name of the object to download (path + file name) * @param file Local file to download to * @throws Exception On any error */ public static void download(String endpoint, String accessKey, String secretKey, String bucket, String path, File file) throws Exception { AmazonS3Client amazonS3Client = null; InputStream in = null; OutputStream out = null; try { AWSCredentials awsCredentials = new BasicAWSCredentials(accessKey, secretKey); ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setMaxConnections(MAX_CONNECTIONS); clientConfiguration.setMaxErrorRetry(PredefinedRetryPolicies.DEFAULT_MAX_ERROR_RETRY); clientConfiguration.setConnectionTimeout(ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT); amazonS3Client = new AmazonS3Client(awsCredentials, clientConfiguration); S3ClientOptions clientOptions = new S3ClientOptions().withPathStyleAccess(true); amazonS3Client.setS3ClientOptions(clientOptions); amazonS3Client.setEndpoint(endpoint); S3Object object = amazonS3Client.getObject(new GetObjectRequest(bucket, path)); in = object.getObjectContent(); byte[] buf = new byte[BUFFER_SIZE]; out = new FileOutputStream(file); int count; while ((count = in.read(buf)) != -1) out.write(buf, 0, count); out.close(); in.close(); } catch (AmazonServiceException x) { Shrinker.myLogger.info("download error: " + x.getMessage()); throw new Exception("download exception", x); } catch (AmazonClientException x) { Shrinker.myLogger.info("download error: " + x.getMessage()); throw new Exception("download exception", x); } catch (IOException x) { Shrinker.myLogger.info("download error: " + x.getMessage()); throw new Exception("download exception", x); } finally { if (in != null) { try { in.close(); } catch (Exception e) { } } if (out != null) { try { out.close(); } catch (Exception e) { } } if (amazonS3Client != null) { try { amazonS3Client.shutdown(); } catch (Exception e) { } } } }
From source file:hu.mta.sztaki.lpds.cloud.entice.imageoptimizer.iaashandler.amazontarget.Storage.java
License:Apache License
/** * @param file Local file to upload/*from w ww . java 2 s.c o m*/ * @param endpoint S3 endpoint URL * @param accessKey Access key * @param secretKey Secret key * @param bucket Bucket name * @param path Key name (path + file name) * @throws Exception On any error */ public static void upload(File file, String endpoint, String accessKey, String secretKey, String bucket, String path) throws Exception { AmazonS3Client amazonS3Client = null; try { AWSCredentials awsCredentials = new BasicAWSCredentials(accessKey, secretKey); ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setMaxConnections(MAX_CONNECTIONS); clientConfiguration.setMaxErrorRetry(PredefinedRetryPolicies.DEFAULT_MAX_ERROR_RETRY); clientConfiguration.setConnectionTimeout(ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT); amazonS3Client = new AmazonS3Client(awsCredentials, clientConfiguration); S3ClientOptions clientOptions = new S3ClientOptions().withPathStyleAccess(true); amazonS3Client.setS3ClientOptions(clientOptions); amazonS3Client.setEndpoint(endpoint); // amazonS3Client.putObject(new PutObjectRequest(bucket, path, file)); // up to 5GB TransferManager tm = new TransferManager(amazonS3Client); // up to 5TB Upload upload = tm.upload(bucket, path, file); // while (!upload.isDone()) { upload.getProgress().getBytesTransferred(); Thread.sleep(1000); } // to get progress upload.waitForCompletion(); tm.shutdownNow(); } catch (AmazonServiceException x) { Shrinker.myLogger.info("upload error: " + x.getMessage()); throw new Exception("upload exception", x); } catch (AmazonClientException x) { Shrinker.myLogger.info("upload error: " + x.getMessage()); throw new Exception("upload exception", x); } finally { if (amazonS3Client != null) { try { amazonS3Client.shutdown(); } catch (Exception e) { } } } }
From source file:io.konig.camel.component.aws.s3.client.impl.S3ClientIAMOptimizedImpl.java
License:Apache License
/** * Getting the s3 aws client that is used. * @return Amazon S3 Client./*from ww w . j a v a2s. c om*/ */ public AmazonS3 getS3Client() { AmazonS3 client = null; AmazonS3ClientBuilder clientBuilder = null; AmazonS3EncryptionClientBuilder encClientBuilder = null; ClientConfiguration clientConfiguration = null; if (configuration.hasProxyConfiguration()) { clientConfiguration = new ClientConfiguration(); clientConfiguration.setProxyHost(configuration.getProxyHost()); clientConfiguration.setProxyPort(configuration.getProxyPort()); clientConfiguration.setMaxConnections(maxConnections); } else { clientConfiguration = new ClientConfiguration(); clientConfiguration.setMaxConnections(maxConnections); } if (configuration.getAccessKey() != null || configuration.getSecretKey() != null) { LOG.trace("Do not pass in unnecessary static credentials when selecting the IAM credential option."); } if (!configuration.isUseEncryption()) { clientBuilder = AmazonS3ClientBuilder.standard() .withCredentials(new InstanceProfileCredentialsProvider(false)); } else if (configuration.isUseEncryption()) { StaticEncryptionMaterialsProvider encryptionMaterialsProvider = new StaticEncryptionMaterialsProvider( configuration.getEncryptionMaterials()); encClientBuilder = AmazonS3EncryptionClientBuilder.standard() .withClientConfiguration(clientConfiguration) .withEncryptionMaterials(encryptionMaterialsProvider) .withCredentials(new InstanceProfileCredentialsProvider(false)); } else { clientBuilder = AmazonS3ClientBuilder.standard().withClientConfiguration(clientConfiguration) .withCredentials(new InstanceProfileCredentialsProvider(false)); } if (!configuration.isUseEncryption()) { if (ObjectHelper.isNotEmpty(configuration.getRegion())) { clientBuilder = clientBuilder.withRegion(Regions.valueOf(configuration.getRegion())); } clientBuilder = clientBuilder.withPathStyleAccessEnabled(configuration.isPathStyleAccess()); client = clientBuilder.build(); } else { if (ObjectHelper.isNotEmpty(configuration.getRegion())) { encClientBuilder = encClientBuilder.withRegion(Regions.valueOf(configuration.getRegion())); } encClientBuilder = encClientBuilder.withPathStyleAccessEnabled(configuration.isPathStyleAccess()); client = encClientBuilder.build(); } return client; }