Example usage for com.amazonaws.services.s3 AmazonS3Client AmazonS3Client

List of usage examples for com.amazonaws.services.s3 AmazonS3Client AmazonS3Client

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.*
  5. AmazonS3Client
  6. AmazonS3Client

Introduction

In this page you can find the example usage for com.amazonaws.services.s3 AmazonS3Client AmazonS3Client.

Prototype

@Deprecated
public AmazonS3Client(AWSCredentialsProvider credentialsProvider, ClientConfiguration clientConfiguration)

Source Link

Document

Constructs a new Amazon S3 client using the specified AWS credentials and client configuration to access Amazon S3.

Usage

From source file:com.bigstep.S3Sampler.java

License:Apache License

@Override
public SampleResult runTest(JavaSamplerContext context) {
    // pull parameters
    String bucket = context.getParameter("bucket");
    String object = context.getParameter("object");
    String method = context.getParameter("method");
    String local_file_path = context.getParameter("local_file_path");
    String key_id = context.getParameter("key_id");
    String secret_key = context.getParameter("secret_key");
    String proxy_host = context.getParameter("proxy_host");
    String proxy_port = context.getParameter("proxy_port");
    String endpoint = context.getParameter("endpoint");

    log.debug("runTest:method=" + method + " local_file_path=" + local_file_path + " bucket=" + bucket
            + " object=" + object);

    SampleResult result = new SampleResult();
    result.sampleStart(); // start stopwatch

    try {//w  w  w . j  a  v  a2 s .  com
        ClientConfiguration config = new ClientConfiguration();
        if (proxy_host != null && !proxy_host.isEmpty()) {
            config.setProxyHost(proxy_host);
        }
        if (proxy_port != null && !proxy_port.isEmpty()) {
            config.setProxyPort(Integer.parseInt(proxy_port));
        }
        //config.setProtocol(Protocol.HTTP);

        AWSCredentials credentials = new BasicAWSCredentials(key_id, secret_key);

        AmazonS3 s3Client = new AmazonS3Client(credentials, config);
        if (endpoint != null && !endpoint.isEmpty()) {
            s3Client.setEndpoint(endpoint);
        }
        ObjectMetadata meta = null;

        if (method.equals("GET")) {
            File file = new File(local_file_path);
            //meta= s3Client.getObject(new GetObjectRequest(bucket, object), file);
            S3Object s3object = s3Client.getObject(bucket, object);
            S3ObjectInputStream stream = s3object.getObjectContent();
            //while(stream.skip(1024*1024)>0);
            stream.close();
        } else if (method.equals("PUT")) {
            File file = new File(local_file_path);
            s3Client.putObject(bucket, object, file);
        }

        result.sampleEnd(); // stop stopwatch
        result.setSuccessful(true);
        if (meta != null) {
            result.setResponseMessage(
                    "OK on url:" + bucket + "/" + object + ". Length=" + meta.getContentLength());
        } else {
            result.setResponseMessage("OK on url:" + bucket + "/" + object + ".No metadata");
        }
        result.setResponseCodeOK(); // 200 code

    } catch (Exception e) {
        result.sampleEnd(); // stop stopwatch
        result.setSuccessful(false);
        result.setResponseMessage("Exception: " + e);

        // get stack trace as a String to return as document data
        java.io.StringWriter stringWriter = new java.io.StringWriter();
        e.printStackTrace(new java.io.PrintWriter(stringWriter));
        result.setResponseData(stringWriter.toString());
        result.setDataType(org.apache.jmeter.samplers.SampleResult.TEXT);
        result.setResponseCode("500");
    }

    return result;
}

From source file:com.cloud.utils.S3Utils.java

License:Apache License

private static AmazonS3 acquireClient(final ClientOptions clientOptions) {

    final AWSCredentials credentials = new BasicAWSCredentials(clientOptions.getAccessKey(),
            clientOptions.getSecretKey());

    final ClientConfiguration configuration = new ClientConfiguration();

    if (clientOptions.isHttps() != null) {
        configuration.setProtocol(clientOptions.isHttps() == true ? HTTPS : HTTP);
    }/*from  w ww  .j av  a  2s.c o  m*/

    if (clientOptions.getConnectionTimeout() != null) {
        configuration.setConnectionTimeout(clientOptions.getConnectionTimeout());
    }

    if (clientOptions.getMaxErrorRetry() != null) {
        configuration.setMaxErrorRetry(clientOptions.getMaxErrorRetry());
    }

    if (clientOptions.getSocketTimeout() != null) {
        configuration.setSocketTimeout(clientOptions.getSocketTimeout());
    }

    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug(format(
                "Creating S3 client with configuration: [protocol: %1$s, connectionTimeOut: "
                        + "%2$s, maxErrorRetry: %3$s, socketTimeout: %4$s]",
                configuration.getProtocol(), configuration.getConnectionTimeout(),
                configuration.getMaxErrorRetry(), configuration.getSocketTimeout()));
    }

    final AmazonS3Client client = new AmazonS3Client(credentials, configuration);

    if (isNotBlank(clientOptions.getEndPoint())) {
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(format("Setting the end point for S3 client %1$s to %2$s.", client,
                    clientOptions.getEndPoint()));
        }
        client.setEndpoint(clientOptions.getEndPoint());
    }

    return client;

}

From source file:com.digitalpebble.stormcrawler.aws.s3.AbstractS3CacheBolt.java

License:Apache License

/** Returns an S3 client given the configuration **/
public static AmazonS3Client getS3Client(Map conf) {
    AWSCredentialsProvider provider = new DefaultAWSCredentialsProviderChain();
    AWSCredentials credentials = provider.getCredentials();
    ClientConfiguration config = new ClientConfiguration();

    AmazonS3Client client = new AmazonS3Client(credentials, config);

    String regionName = ConfUtils.getString(conf, REGION);
    if (StringUtils.isNotBlank(regionName)) {
        client.setRegion(RegionUtils.getRegion(regionName));
    }/* w w w  . jav  a  2  s  . c om*/

    String endpoint = ConfUtils.getString(conf, ENDPOINT);
    if (StringUtils.isNotBlank(endpoint)) {
        client.setEndpoint(endpoint);
    }
    return client;
}

From source file:com.emc.ecs.sync.source.S3Source.java

License:Open Source License

@Override
public void configure(SyncSource source, Iterator<SyncFilter> filters, SyncTarget target) {
    Assert.hasText(accessKey, "accessKey is required");
    Assert.hasText(secretKey, "secretKey is required");
    Assert.hasText(bucketName, "bucketName is required");
    Assert.isTrue(bucketName.matches("[A-Za-z0-9._-]+"), bucketName + " is not a valid bucket name");

    AWSCredentials creds = new BasicAWSCredentials(accessKey, secretKey);
    ClientConfiguration config = new ClientConfiguration();

    if (protocol != null)
        config.setProtocol(Protocol.valueOf(protocol.toUpperCase()));

    if (legacySignatures)
        config.setSignerOverride("S3SignerType");

    if (socketTimeoutMs >= 0)
        config.setSocketTimeout(socketTimeoutMs);

    s3 = new AmazonS3Client(creds, config);

    if (endpoint != null)
        s3.setEndpoint(endpoint);//from w  w w. j a v  a  2  s  . c o m

    // TODO: generalize uri translation
    AwsS3Util.S3Uri s3Uri = new AwsS3Util.S3Uri();
    s3Uri.protocol = protocol;
    s3Uri.endpoint = endpoint;
    s3Uri.accessKey = accessKey;
    s3Uri.secretKey = secretKey;
    s3Uri.rootKey = rootKey;
    if (sourceUri == null)
        sourceUri = s3Uri.toUri();

    if (disableVHosts) {
        log.info(
                "The use of virtual hosted buckets on the s3 source has been DISABLED.  Path style buckets will be used.");
        S3ClientOptions opts = new S3ClientOptions();
        opts.setPathStyleAccess(true);
        s3.setS3ClientOptions(opts);
    }

    if (!s3.doesBucketExist(bucketName)) {
        throw new ConfigurationException("The bucket " + bucketName + " does not exist.");
    }

    if (rootKey == null)
        rootKey = ""; // make sure rootKey isn't null

    // for version support. TODO: genericize version support
    if (target instanceof S3Target) {
        s3Target = (S3Target) target;
        if (s3Target.isIncludeVersions()) {
            BucketVersioningConfiguration versioningConfig = s3.getBucketVersioningConfiguration(bucketName);
            List<String> versionedStates = Arrays.asList(BucketVersioningConfiguration.ENABLED,
                    BucketVersioningConfiguration.SUSPENDED);
            versioningEnabled = versionedStates.contains(versioningConfig.getStatus());
        }
    }
}

From source file:com.emc.ecs.sync.target.S3Target.java

License:Open Source License

@Override
public void configure(SyncSource source, Iterator<SyncFilter> filters, SyncTarget target) {
    Assert.hasText(accessKey, "accessKey is required");
    Assert.hasText(secretKey, "secretKey is required");
    Assert.hasText(bucketName, "bucketName is required");
    Assert.isTrue(bucketName.matches("[A-Za-z0-9._-]+"), bucketName + " is not a valid bucket name");

    AWSCredentials creds = new BasicAWSCredentials(accessKey, secretKey);
    ClientConfiguration config = new ClientConfiguration();

    if (protocol != null)
        config.setProtocol(Protocol.valueOf(protocol.toUpperCase()));

    if (legacySignatures)
        config.setSignerOverride("S3SignerType");

    if (socketTimeoutMs >= 0)
        config.setSocketTimeout(socketTimeoutMs);

    s3 = new AmazonS3Client(creds, config);

    if (endpoint != null)
        s3.setEndpoint(endpoint);/*from w  ww .  ja  v a  2s  .  co m*/

    // TODO: generalize uri translation
    AwsS3Util.S3Uri s3Uri = new AwsS3Util.S3Uri();
    s3Uri.protocol = protocol;
    s3Uri.endpoint = endpoint;
    s3Uri.accessKey = accessKey;
    s3Uri.secretKey = secretKey;
    s3Uri.rootKey = rootKey;
    if (targetUri == null)
        targetUri = s3Uri.toUri();

    if (disableVHosts) {
        log.info(
                "The use of virtual hosted buckets on the s3 source has been DISABLED.  Path style buckets will be used.");
        S3ClientOptions opts = new S3ClientOptions();
        opts.setPathStyleAccess(true);
        s3.setS3ClientOptions(opts);
    }

    // for version support. TODO: genericize version support
    if (source instanceof S3Source) {
        s3Source = (S3Source) source;
        if (!s3Source.isVersioningEnabled())
            includeVersions = false; // don't include versions if source versioning is off
    } else if (includeVersions) {
        throw new ConfigurationException(
                "Object versions are currently only supported with the S3 source & target plugins.");
    }

    if (!s3.doesBucketExist(bucketName)) {
        if (createBucket) {
            s3.createBucket(bucketName);
            if (includeVersions)
                s3.setBucketVersioningConfiguration(new SetBucketVersioningConfigurationRequest(bucketName,
                        new BucketVersioningConfiguration(BucketVersioningConfiguration.ENABLED)));
        } else {
            throw new ConfigurationException("The bucket " + bucketName + " does not exist.");
        }
    }

    if (rootKey == null)
        rootKey = ""; // make sure rootKey isn't null

    if (includeVersions) {
        String status = s3.getBucketVersioningConfiguration(bucketName).getStatus();
        if (BucketVersioningConfiguration.OFF.equals(status))
            throw new ConfigurationException("The specified bucket does not have versioning enabled.");
    }

    if (mpuThresholdMB > AwsS3Util.MAX_PUT_SIZE_MB) {
        log.warn("{}MB is above the maximum PUT size of {}MB. the maximum will be used instead", mpuThresholdMB,
                AwsS3Util.MAX_PUT_SIZE_MB);
        mpuThresholdMB = AwsS3Util.MAX_PUT_SIZE_MB;
    }
    if (mpuPartSizeMB < AwsS3Util.MIN_PART_SIZE_MB) {
        log.warn("{}MB is below the minimum MPU part size of {}MB. the minimum will be used instead",
                mpuPartSizeMB, AwsS3Util.MIN_PART_SIZE_MB);
        mpuPartSizeMB = AwsS3Util.MIN_PART_SIZE_MB;
    }
}

From source file:com.emc.vipr.sync.source.S3Source.java

License:Open Source License

@Override
public void configure(SyncSource source, Iterator<SyncFilter> filters, SyncTarget target) {
    Assert.hasText(accessKey, "accessKey is required");
    Assert.hasText(secretKey, "secretKey is required");
    Assert.hasText(bucketName, "bucketName is required");

    AWSCredentials creds = new BasicAWSCredentials(accessKey, secretKey);
    ClientConfiguration config = new ClientConfiguration();

    if (protocol != null)
        config.setProtocol(Protocol.valueOf(protocol.toUpperCase()));

    s3 = new AmazonS3Client(creds, config);

    if (endpoint != null)
        s3.setEndpoint(endpoint);//from   w  w w  . j a va 2  s .c om

    if (disableVHosts) {
        l4j.info(
                "The use of virtual hosted buckets on the s3 source has been DISABLED.  Path style buckets will be used.");
        S3ClientOptions opts = new S3ClientOptions();
        opts.setPathStyleAccess(true);
        s3.setS3ClientOptions(opts);
    }

    if (!s3.doesBucketExist(bucketName)) {
        throw new ConfigurationException("The bucket " + bucketName + " does not exist.");
    }

    if (rootKey == null)
        rootKey = ""; // make sure rootKey isn't null
    if (rootKey.startsWith("/"))
        rootKey = rootKey.substring(1); // " " does not start with slash
}

From source file:com.eucalyptus.objectstorage.client.GenericS3ClientFactory.java

License:Open Source License

public static AmazonS3Client getS3Client(AWSCredentialsProvider provider, boolean https)
        throws NoSuchElementException {
    ClientConfiguration config = getDefaultConfiguration(https);
    AmazonS3Client s3Client = new AmazonS3Client(provider, config);
    s3Client.setS3ClientOptions(getDefaultClientOptions());
    s3Client.setEndpoint(getRandomOSGUri().toString());
    return s3Client;
}

From source file:com.eucalyptus.objectstorage.client.GenericS3ClientFactory.java

License:Open Source License

/**
 * Returns a configured S3 client for the specified set of credentials.
 * @param credentials//  w w w .j a  v  a 2s  . c  om
 * @param https
 * @return
 * @throws NoSuchElementException if no ENABLED OSG found
 */
public static AmazonS3Client getS3Client(AWSCredentials credentials, boolean https)
        throws NoSuchElementException {
    ClientConfiguration config = getDefaultConfiguration(https);
    AmazonS3Client s3Client = new AmazonS3Client(credentials, config);
    s3Client.setS3ClientOptions(getDefaultClientOptions());
    s3Client.setEndpoint(getRandomOSGUri().toString());
    return s3Client;
}

From source file:com.eucalyptus.objectstorage.client.OsgInternalS3Client.java

License:Open Source License

private synchronized void initializeNewClient(@Nonnull AWSCredentials credentials, @Nonnull String endpoint,
        @Nonnull Boolean https, @Nonnull Boolean useDns) {
    ClientConfiguration config = new ClientConfiguration();
    config.setConnectionTimeout(CONNECTION_TIMEOUT_MS); //very short timeout
    config.setSocketTimeout(OSG_SOCKET_TIMEOUT_MS);
    config.setUseReaper(true);//from   w  w  w  . j a  v  a  2s. c  o  m
    config.setMaxConnections(OSG_MAX_CONNECTIONS);
    Protocol protocol = https ? Protocol.HTTPS : Protocol.HTTP;
    config.setProtocol(protocol);
    this.clientConfig = config;
    this.s3Client = new AmazonS3Client(credentials, config);
    this.ops = new S3ClientOptions().withPathStyleAccess(!useDns);
    this.s3Client.setS3ClientOptions(ops);
    this.instantiated = new Date();
    this.currentCredentials = credentials;
    this.setS3Endpoint(endpoint);
}

From source file:com.facebook.presto.hive.PrestoS3FileSystem.java

License:Apache License

@Override
public void initialize(URI uri, Configuration conf) throws IOException {
    checkNotNull(uri, "uri is null");
    checkNotNull(conf, "conf is null");

    this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
    this.workingDirectory = new Path("/").makeQualified(this.uri, new Path("/"));

    HiveClientConfig defaults = new HiveClientConfig();
    this.stagingDirectory = new File(
            conf.get(S3_STAGING_DIRECTORY, defaults.getS3StagingDirectory().toString()));
    this.maxClientRetries = conf.getInt(S3_MAX_CLIENT_RETRIES, defaults.getS3MaxClientRetries());
    int maxErrorRetries = conf.getInt(S3_MAX_ERROR_RETRIES, defaults.getS3MaxErrorRetries());
    boolean sslEnabled = conf.getBoolean(S3_SSL_ENABLED, defaults.isS3SslEnabled());
    Duration connectTimeout = Duration
            .valueOf(conf.get(S3_CONNECT_TIMEOUT, defaults.getS3ConnectTimeout().toString()));

    ClientConfiguration configuration = new ClientConfiguration();
    configuration.setMaxErrorRetry(maxErrorRetries);
    configuration.setProtocol(sslEnabled ? Protocol.HTTPS : Protocol.HTTP);
    configuration.setConnectionTimeout(Ints.checkedCast(connectTimeout.toMillis()));

    this.s3 = new AmazonS3Client(getAwsCredentials(uri, conf), configuration);
}