Example usage for com.amazonaws ClientConfiguration setMaxErrorRetry

List of usage examples for com.amazonaws ClientConfiguration setMaxErrorRetry

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.*
  5. ClientConfiguration
  6. setMaxErrorRetry

Introduction

In this page you can find the example usage for com.amazonaws ClientConfiguration setMaxErrorRetry.

Prototype

public void setMaxErrorRetry(int maxErrorRetry)

Source Link

Document

Sets the maximum number of retry attempts for failed retryable requests (ex: 5xx error responses from services).

Usage

From source file:com.sitewhere.aws.SqsOutboundEventProcessor.java

License:Open Source License

@Override
public void start() throws SiteWhereException {
    super.start();

    ClientConfiguration config = new ClientConfiguration();
    config.setMaxConnections(250);/*from   w w  w.j a  va2s.c  om*/
    config.setMaxErrorRetry(5);

    if (getAccessKey() == null) {
        throw new SiteWhereException("Amazon access key not provided.");
    }

    if (getSecretKey() == null) {
        throw new SiteWhereException("Amazon secret key not provided.");
    }

    sqs = new AmazonSQSClient(new BasicAWSCredentials(getAccessKey(), getSecretKey()), config);
    Region usEast1 = Region.getRegion(Regions.US_EAST_1);
    sqs.setRegion(usEast1);
}

From source file:com.sitewhere.connectors.aws.sqs.SqsOutboundEventProcessor.java

License:Open Source License

@Override
public void start(ILifecycleProgressMonitor monitor) throws SiteWhereException {
    super.start(monitor);

    ClientConfiguration config = new ClientConfiguration();
    config.setMaxConnections(250);/*from  w w w  .  j  av a 2  s  .c o m*/
    config.setMaxErrorRetry(5);

    if (getAccessKey() == null) {
        throw new SiteWhereException("Amazon access key not provided.");
    }

    if (getSecretKey() == null) {
        throw new SiteWhereException("Amazon secret key not provided.");
    }

    sqs = new AmazonSQSClient(new BasicAWSCredentials(getAccessKey(), getSecretKey()), config);
    Region usEast1 = Region.getRegion(Regions.US_EAST_1);
    sqs.setRegion(usEast1);
}

From source file:com.streamsets.pipeline.stage.common.s3.S3ConnectionBaseConfig.java

License:Apache License

private void createConnection(Stage.Context context, String configPrefix, ProxyConfig proxyConfig,
        List<Stage.ConfigIssue> issues, int maxErrorRetries) throws StageException {
    AWSCredentialsProvider credentials = AWSUtil.getCredentialsProvider(awsConfig);
    ClientConfiguration clientConfig = AWSUtil.getClientConfiguration(proxyConfig);

    if (maxErrorRetries >= 0) {
        clientConfig.setMaxErrorRetry(maxErrorRetries);
    }//from w ww  .  j a v a2  s  .co m

    AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withCredentials(credentials)
            .withClientConfiguration(clientConfig).withChunkedEncodingDisabled(awsConfig.disableChunkedEncoding)
            .withPathStyleAccessEnabled(true);

    if (region == AwsRegion.OTHER) {
        if (endpoint == null || endpoint.isEmpty()) {
            issues.add(context.createConfigIssue(Groups.S3.name(), configPrefix + "endpoint",
                    Errors.S3_SPOOLDIR_10));
            return;
        }
        builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null));
    } else {
        builder.withRegion(region.getId());
    }
    s3Client = builder.build();
}

From source file:com.streamsets.pipeline.stage.origin.s3.AmazonS3Runnable.java

License:Apache License

private AmazonS3 createConnection() throws StageException {
    AwsRegion region = s3ConfigBean.s3Config.region;
    AWSConfig awsConfig = s3ConfigBean.s3Config.awsConfig;
    int maxErrorRetries = s3ConfigBean.s3Config.getMaxErrorRetries();

    AWSCredentialsProvider credentials = AWSUtil.getCredentialsProvider(awsConfig);
    ClientConfiguration clientConfig = AWSUtil.getClientConfiguration(s3ConfigBean.proxyConfig);

    if (maxErrorRetries >= 0) {
        clientConfig.setMaxErrorRetry(maxErrorRetries);
    }//w w  w.ja  va  2s  . c o m

    AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withCredentials(credentials)
            .withClientConfiguration(clientConfig).withChunkedEncodingDisabled(awsConfig.disableChunkedEncoding)
            .withPathStyleAccessEnabled(true);

    builder.withRegion(region.getId());
    return builder.build();
}

From source file:com.tcl.gateway.firehose.log4j.FirehoseAppender.java

License:Open Source License

/**
 * Configures this appender instance and makes it ready for use by the
 * consumers. It validates mandatory parameters and confirms if the configured
 * stream is ready for publishing data yet.
 * //  www  .  j  a  va  2  s.  co m
 * Error details are made available through the fallback handler for this
 * appender
 * 
 * @throws IllegalStateException
 *           if we encounter issues configuring this appender instance
 */
@Override
public void activateOptions() {
    if (deliveryStreamName == null) {
        initializationFailed = true;
        error("Invalid configuration - streamName cannot be null for appender: " + name);
    }

    if (layout == null) {
        initializationFailed = true;
        error("Invalid configuration - No layout for appender: " + name);
    }

    ClientConfiguration clientConfiguration = new ClientConfiguration();
    clientConfiguration = setProxySettingsFromSystemProperties(clientConfiguration);

    clientConfiguration.setMaxErrorRetry(maxRetries);
    clientConfiguration.setRetryPolicy(new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION,
            PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, maxRetries, true));
    clientConfiguration.setUserAgent(AppenderConstants.USER_AGENT_STRING);

    final BlockingQueue<Runnable> taskBuffer = new LinkedBlockingDeque<Runnable>(bufferSize);
    ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(threadCount, threadCount,
            AppenderConstants.DEFAULT_THREAD_KEEP_ALIVE_SEC, TimeUnit.SECONDS, taskBuffer,
            new BlockFastProducerPolicy());
    threadPoolExecutor.prestartAllCoreThreads();
    firehoseClient = new AmazonKinesisFirehoseAsyncClient(new CustomCredentialsProviderChain(),
            clientConfiguration, threadPoolExecutor);

    boolean regionProvided = !Validator.isBlank(region);
    if (!regionProvided) {
        region = AppenderConstants.DEFAULT_REGION;
    }
    if (!Validator.isBlank(endpoint)) {
        if (regionProvided) {
            LOGGER.warn("Received configuration for both region as well as Amazon Kinesis endpoint. ("
                    + endpoint + ") will be used as endpoint instead of default endpoint for region (" + region
                    + ")");
        }
        firehoseClient.setEndpoint(endpoint);
    } else {
        firehoseClient.setRegion(Region.getRegion(Regions.fromName(region)));
    }

    DescribeDeliveryStreamResult describeResult = null;
    try {
        describeResult = firehoseClient.describeDeliveryStream(
                new DescribeDeliveryStreamRequest().withDeliveryStreamName(deliveryStreamName));
        String streamStatus = describeResult.getDeliveryStreamDescription().getDeliveryStreamStatus();
        if (!StreamStatus.ACTIVE.name().equals(streamStatus)
                && !StreamStatus.UPDATING.name().equals(streamStatus)) {
            initializationFailed = true;
            error("Delivery Stream " + deliveryStreamName
                    + " is not ready (in active/updating status) for appender: " + name);
        }
    } catch (ResourceNotFoundException rnfe) {
        initializationFailed = true;
        error("Delivery  Stream " + deliveryStreamName + " doesn't exist for appender: " + name, rnfe);
    }

    asyncCallHander = new AsyncPutCallStatsReporter(name);

    if (metric) {
        MetricRegistry registry = new MetricRegistry();
        registry.register("Gauge", new Gauge<Integer>() {

            @Override
            public Integer getValue() {
                return taskBuffer.size();
            }

        });

        ConsoleReporter.forRegistry(registry).build().start(3, TimeUnit.SECONDS);
    }
}

From source file:com.treasure_data.td_import.source.S3Source.java

License:Apache License

static AmazonS3Client createAmazonS3Client(SourceDesc desc) {
    String accessKey = desc.getUser();
    if (accessKey == null || accessKey.isEmpty()) {
        throw new IllegalArgumentException("S3 AccessKey is null or empty.");
    }/*from ww w . j  a v  a  2 s  . c om*/
    String secretAccessKey = desc.getPassword();
    if (secretAccessKey == null || secretAccessKey.isEmpty()) {
        throw new IllegalArgumentException("S3 SecretAccessKey is null or empty.");
    }
    AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretAccessKey);

    ClientConfiguration conf = new ClientConfiguration();
    conf.setProtocol(Configuration.BI_PREPARE_S3_PROTOCOL);
    conf.setMaxConnections(Configuration.BI_PREPARE_S3_MAX_CONNECTIONS);
    conf.setMaxErrorRetry(Configuration.BI_PREPARE_S3_MAX_ERRORRETRY);
    conf.setSocketTimeout(Configuration.BI_PREPARE_S3_SOCKET_TIMEOUT);

    return new AmazonS3Client(credentials, conf);
}

From source file:com.upplication.s3fs.S3FileSystemProvider.java

License:Open Source License

protected ClientConfiguration createClientConfig(Properties props) {
    ClientConfiguration config = new ClientConfiguration();

    if (props == null)
        return config;

    if (props.containsKey("connection_timeout")) {
        log.trace("AWS client config - connection_timeout: {}", props.getProperty("connection_timeout"));
        config.setConnectionTimeout(Integer.parseInt(props.getProperty("connection_timeout")));
    }//from  w ww  .j  a  v a  2 s  . co  m

    if (props.containsKey("max_connections")) {
        log.trace("AWS client config - max_connections: {}", props.getProperty("max_connections"));
        config.setMaxConnections(Integer.parseInt(props.getProperty("max_connections")));
    }

    if (props.containsKey("max_error_retry")) {
        log.trace("AWS client config - max_error_retry: {}", props.getProperty("max_error_retry"));
        config.setMaxErrorRetry(Integer.parseInt(props.getProperty("max_error_retry")));
    }

    if (props.containsKey("protocol")) {
        log.trace("AWS client config - protocol: {}", props.getProperty("protocol"));
        config.setProtocol(Protocol.valueOf(props.getProperty("protocol").toUpperCase()));
    }

    if (props.containsKey("proxy_domain")) {
        log.trace("AWS client config - proxy_domain: {}", props.getProperty("proxy_domain"));
        config.setProxyDomain(props.getProperty("proxy_domain"));
    }

    if (props.containsKey("proxy_host")) {
        log.trace("AWS client config - proxy_host: {}", props.getProperty("proxy_host"));
        config.setProxyHost(props.getProperty("proxy_host"));
    }

    if (props.containsKey("proxy_port")) {
        log.trace("AWS client config - proxy_port: {}", props.getProperty("proxy_port"));
        config.setProxyPort(Integer.parseInt(props.getProperty("proxy_port")));
    }

    if (props.containsKey("proxy_username")) {
        log.trace("AWS client config - proxy_username: {}", props.getProperty("proxy_username"));
        config.setProxyUsername(props.getProperty("proxy_username"));
    }

    if (props.containsKey("proxy_password")) {
        log.trace("AWS client config - proxy_password: {}", props.getProperty("proxy_password"));
        config.setProxyPassword(props.getProperty("proxy_password"));
    }

    if (props.containsKey("proxy_workstation")) {
        log.trace("AWS client config - proxy_workstation: {}", props.getProperty("proxy_workstation"));
        config.setProxyWorkstation(props.getProperty("proxy_workstation"));
    }

    if (props.containsKey("signer_override")) {
        log.debug("AWS client config - signerOverride: {}", props.getProperty("signer_override"));
        config.setSignerOverride(props.getProperty("signer_override"));
    }

    if (props.containsKey("socket_send_buffer_size_hints")
            || props.containsKey("socket_recv_buffer_size_hints")) {
        log.trace("AWS client config - socket_send_buffer_size_hints: {}, socket_recv_buffer_size_hints: {}",
                props.getProperty("socket_send_buffer_size_hints", "0"),
                props.getProperty("socket_recv_buffer_size_hints", "0"));
        int send = Integer.parseInt(props.getProperty("socket_send_buffer_size_hints", "0"));
        int recv = Integer.parseInt(props.getProperty("socket_recv_buffer_size_hints", "0"));
        config.setSocketBufferSizeHints(send, recv);
    }

    if (props.containsKey("socket_timeout")) {
        log.trace("AWS client config - socket_timeout: {}", props.getProperty("socket_timeout"));
        config.setSocketTimeout(Integer.parseInt(props.getProperty("socket_timeout")));
    }

    if (props.containsKey("user_agent")) {
        log.trace("AWS client config - user_agent: {}", props.getProperty("user_agent"));
        config.setUserAgent(props.getProperty("user_agent"));
    }

    return config;
}

From source file:com.yahoo.ycsb.db.S3Client.java

License:Open Source License

/**
* Initialize any state for the storage./*from  w  w w .ja v a  2s  . c  om*/
* Called once per S3 instance; If the client is not null it is re-used.
*/
@Override
public void init() throws DBException {
    final int count = INIT_COUNT.incrementAndGet();
    synchronized (S3Client.class) {
        Properties propsCL = getProperties();
        int recordcount = Integer.parseInt(propsCL.getProperty("recordcount"));
        int operationcount = Integer.parseInt(propsCL.getProperty("operationcount"));
        int numberOfOperations = 0;
        if (recordcount > 0) {
            if (recordcount > operationcount) {
                numberOfOperations = recordcount;
            } else {
                numberOfOperations = operationcount;
            }
        } else {
            numberOfOperations = operationcount;
        }
        if (count <= numberOfOperations) {
            String accessKeyId = null;
            String secretKey = null;
            String endPoint = null;
            String region = null;
            String maxErrorRetry = null;
            String maxConnections = null;
            String protocol = null;
            BasicAWSCredentials s3Credentials;
            ClientConfiguration clientConfig;
            if (s3Client != null) {
                System.out.println("Reusing the same client");
                return;
            }
            try {
                InputStream propFile = S3Client.class.getClassLoader().getResourceAsStream("s3.properties");
                Properties props = new Properties(System.getProperties());
                props.load(propFile);
                accessKeyId = props.getProperty("s3.accessKeyId");
                if (accessKeyId == null) {
                    accessKeyId = propsCL.getProperty("s3.accessKeyId");
                }
                System.out.println(accessKeyId);
                secretKey = props.getProperty("s3.secretKey");
                if (secretKey == null) {
                    secretKey = propsCL.getProperty("s3.secretKey");
                }
                System.out.println(secretKey);
                endPoint = props.getProperty("s3.endPoint");
                if (endPoint == null) {
                    endPoint = propsCL.getProperty("s3.endPoint", "s3.amazonaws.com");
                }
                System.out.println(endPoint);
                region = props.getProperty("s3.region");
                if (region == null) {
                    region = propsCL.getProperty("s3.region", "us-east-1");
                }
                System.out.println(region);
                maxErrorRetry = props.getProperty("s3.maxErrorRetry");
                if (maxErrorRetry == null) {
                    maxErrorRetry = propsCL.getProperty("s3.maxErrorRetry", "15");
                }
                maxConnections = props.getProperty("s3.maxConnections");
                if (maxConnections == null) {
                    maxConnections = propsCL.getProperty("s3.maxConnections");
                }
                protocol = props.getProperty("s3.protocol");
                if (protocol == null) {
                    protocol = propsCL.getProperty("s3.protocol", "HTTPS");
                }
                sse = props.getProperty("s3.sse");
                if (sse == null) {
                    sse = propsCL.getProperty("s3.sse", "false");
                }
                String ssec = props.getProperty("s3.ssec");
                if (ssec == null) {
                    ssec = propsCL.getProperty("s3.ssec", null);
                } else {
                    ssecKey = new SSECustomerKey(ssec);
                }
            } catch (Exception e) {
                System.err.println("The file properties doesn't exist " + e.toString());
                e.printStackTrace();
            }
            try {
                System.out.println("Inizializing the S3 connection");
                s3Credentials = new BasicAWSCredentials(accessKeyId, secretKey);
                clientConfig = new ClientConfiguration();
                clientConfig.setMaxErrorRetry(Integer.parseInt(maxErrorRetry));
                if (protocol.equals("HTTP")) {
                    clientConfig.setProtocol(Protocol.HTTP);
                } else {
                    clientConfig.setProtocol(Protocol.HTTPS);
                }
                if (maxConnections != null) {
                    clientConfig.setMaxConnections(Integer.parseInt(maxConnections));
                }
                s3Client = new AmazonS3Client(s3Credentials, clientConfig);
                s3Client.setRegion(Region.getRegion(Regions.fromName(region)));
                s3Client.setEndpoint(endPoint);
                System.out.println("Connection successfully initialized");
            } catch (Exception e) {
                System.err.println("Could not connect to S3 storage: " + e.toString());
                e.printStackTrace();
                throw new DBException(e);
            }
        } else {
            System.err.println("The number of threads must be less or equal than the operations");
            throw new DBException(new Error("The number of threads must be less or equal than the operations"));
        }
    }
}

From source file:com.yahoo.ycsb.utils.connection.S3Connection.java

License:Open Source License

public S3Connection(String bucket, String region, String endPoint) throws ClientException {
    super(bucket, region, endPoint);
    logger.debug("S3Client.establishConnection(" + region + "," + endPoint + ") bucket: " + bucket);
    org.apache.log4j.Logger.getLogger("com.amazonaws").setLevel(Level.OFF);

    /*if (S3Connection.init == true) {
    init();//from ww  w . j  a  v a 2s . c om
    S3Connection.init = false;
    }*/

    this.bucket = bucket;
    this.region = region;

    try {
        BasicAWSCredentials s3Credentials = new BasicAWSCredentials(accessKeyId, secretKey);
        ClientConfiguration clientConfig = new ClientConfiguration();
        clientConfig.setMaxErrorRetry(Integer.parseInt(maxErrorRetry));
        if (protocol.equals("HTTP")) {
            clientConfig.setProtocol(Protocol.HTTP);
        } else {
            clientConfig.setProtocol(Protocol.HTTPS);
        }
        if (maxConnections != null) {
            clientConfig.setMaxConnections(Integer.parseInt(maxConnections));
        }

        logger.debug("Inizializing the S3 connection...");
        awsClient = new AmazonS3Client(s3Credentials, clientConfig);
        awsClient.setRegion(Region.getRegion(Regions.fromName(region)));
        awsClient.setEndpoint(endPoint);
        logger.debug("Connection successfully initialized");
    } catch (Exception e) {
        logger.error("Could not connect to S3 storage: " + e.toString());
        e.printStackTrace();
        throw new ClientException(e);
    }
}

From source file:hu.mta.sztaki.lpds.cloud.entice.imageoptimizer.iaashandler.amazontarget.Storage.java

License:Apache License

/**
 * @param endpoint S3 endpoint URL//w  w  w .  j  a  v a 2  s. c  om
 * @param accessKey Access key
 * @param secretKey Secret key
 * @param bucket Bucket name 
 * @param path Key name of the object to download (path + file name)
 * @param file Local file to download to 
 * @throws Exception On any error
 */
public static void download(String endpoint, String accessKey, String secretKey, String bucket, String path,
        File file) throws Exception {
    AmazonS3Client amazonS3Client = null;
    InputStream in = null;
    OutputStream out = null;
    try {
        AWSCredentials awsCredentials = new BasicAWSCredentials(accessKey, secretKey);
        ClientConfiguration clientConfiguration = new ClientConfiguration();
        clientConfiguration.setMaxConnections(MAX_CONNECTIONS);
        clientConfiguration.setMaxErrorRetry(PredefinedRetryPolicies.DEFAULT_MAX_ERROR_RETRY);
        clientConfiguration.setConnectionTimeout(ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT);
        amazonS3Client = new AmazonS3Client(awsCredentials, clientConfiguration);
        S3ClientOptions clientOptions = new S3ClientOptions().withPathStyleAccess(true);
        amazonS3Client.setS3ClientOptions(clientOptions);
        amazonS3Client.setEndpoint(endpoint);
        S3Object object = amazonS3Client.getObject(new GetObjectRequest(bucket, path));
        in = object.getObjectContent();
        byte[] buf = new byte[BUFFER_SIZE];
        out = new FileOutputStream(file);
        int count;
        while ((count = in.read(buf)) != -1)
            out.write(buf, 0, count);
        out.close();
        in.close();
    } catch (AmazonServiceException x) {
        Shrinker.myLogger.info("download error: " + x.getMessage());
        throw new Exception("download exception", x);
    } catch (AmazonClientException x) {
        Shrinker.myLogger.info("download error: " + x.getMessage());
        throw new Exception("download exception", x);
    } catch (IOException x) {
        Shrinker.myLogger.info("download error: " + x.getMessage());
        throw new Exception("download exception", x);
    } finally {
        if (in != null) {
            try {
                in.close();
            } catch (Exception e) {
            }
        }
        if (out != null) {
            try {
                out.close();
            } catch (Exception e) {
            }
        }
        if (amazonS3Client != null) {
            try {
                amazonS3Client.shutdown();
            } catch (Exception e) {
            }
        }
    }
}