Example usage for com.amazonaws.services.s3 AmazonS3Client AmazonS3Client

Introduction

In this page you can find the example usage for com.amazonaws.services.s3 AmazonS3Client AmazonS3Client.

Prototype

@SdkInternalApi
AmazonS3Client(AmazonS3ClientParams s3ClientParams) 

Source Link

Document

Constructs a new client to invoke service methods on S3 using the specified parameters.

Usage

From source file:org.apache.flink.cloudsort.io.aws.AwsCurlInput.java

License:Apache License

@Override
public void open(String objectName) throws IOException {
    Preconditions.checkNotNull(bucket);/*from   www.  ja  va2s . c  om*/

    AmazonS3 s3Client = new AmazonS3Client(new ProfileCredentialsProvider());

    java.util.Date expiration = new java.util.Date();
    long msec = expiration.getTime();
    msec += 1000 * 60 * 60; // Add 1 hour.
    expiration.setTime(msec);

    GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucket,
            objectName);
    generatePresignedUrlRequest.setMethod(HttpMethod.GET);
    generatePresignedUrlRequest.setContentType(CONTENT_TYPE);
    generatePresignedUrlRequest.setExpiration(expiration);

    String url = s3Client.generatePresignedUrl(generatePresignedUrlRequest).toExternalForm();

    downloader = new ProcessBuilder("curl", "-s", "-X", "GET", "-H", "Content-Type: " + CONTENT_TYPE, url)
            .redirectError(Redirect.INHERIT).start();
}

---

From source file:org.apache.flink.cloudsort.io.aws.AwsCurlOutput.java

License:Apache License

@Override
public Process open(String filename, String taskId) throws IOException {
    Preconditions.checkNotNull(bucket);/*from   w  ww. j  a va2 s  . c  om*/
    Preconditions.checkNotNull(prefix);
    Preconditions.checkNotNull(filename);
    Preconditions.checkNotNull(taskId);

    String objectName = prefix + taskId;

    AmazonS3 s3Client = new AmazonS3Client(new ProfileCredentialsProvider());

    java.util.Date expiration = new java.util.Date();
    long msec = expiration.getTime();
    msec += 1000 * 60 * 60; // Add 1 hour.
    expiration.setTime(msec);

    GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucket,
            objectName);
    generatePresignedUrlRequest.setMethod(HttpMethod.PUT);
    generatePresignedUrlRequest.setContentType(CONTENT_TYPE);
    generatePresignedUrlRequest.setExpiration(expiration);

    String url = s3Client.generatePresignedUrl(generatePresignedUrlRequest).toExternalForm();

    return new ProcessBuilder("curl", "-s", "--max-time", "60", "--retry", "1024", "-X", "PUT", "-H",
            "Content-Type: " + CONTENT_TYPE, "--data-binary", "@" + filename, url)
                    .redirectError(Redirect.INHERIT).redirectOutput(Redirect.INHERIT).start();
}

---

From source file:org.apache.flink.cloudsort.io.aws.AwsInput.java

License:Apache License

@Override
public List<InputSplit> list() {
    Preconditions.checkNotNull(bucket);/*  www . ja  v  a 2 s .  c o  m*/
    Preconditions.checkNotNull(prefix);

    List<InputSplit> objectNames = new ArrayList<>();

    // this will read credentials from user's home directory
    AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());

    final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucket).withPrefix(prefix);

    ListObjectsV2Result result;
    int index = 0;
    do {
        result = s3client.listObjectsV2(req);

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            String objectName = objectSummary.getKey();
            long objectSize = objectSummary.getSize();
            objectNames.add(new InputSplit(index++, objectName, objectSize));
        }
        req.setContinuationToken(result.getNextContinuationToken());
    } while (result.isTruncated());

    return objectNames;
}

---

From source file:org.apache.gobblin.aws.AWSSdkClient.java

License:Apache License

/***
 * Initialize the AWS SDK Client/*from   w w  w  .  j  a  v  a  2s .  c o  m*/
 *
 * @param awsClusterSecurityManager The {@link AWSClusterSecurityManager} to fetch AWS credentials
 * @param region The Amazon AWS {@link Region}
 */
public AWSSdkClient(final AWSClusterSecurityManager awsClusterSecurityManager, final Region region) {
    this.amazonEC2Supplier = Suppliers.memoize(new Supplier<AmazonEC2>() {
        @Override
        public AmazonEC2 get() {
            AmazonEC2Client amazonEC2 = new AmazonEC2Client(awsClusterSecurityManager.getCredentialsProvider());
            amazonEC2.setRegion(region);
            return amazonEC2;
        }
    });
    this.amazonS3Supplier = Suppliers.memoize(new Supplier<AmazonS3>() {
        @Override
        public AmazonS3 get() {
            AmazonS3Client amazonS3 = new AmazonS3Client(awsClusterSecurityManager.getCredentialsProvider());
            amazonS3.setRegion(region);
            return amazonS3;
        }
    });
    this.amazonAutoScalingSupplier = Suppliers.memoize(new Supplier<AmazonAutoScaling>() {
        @Override
        public AmazonAutoScaling get() {
            AmazonAutoScalingClient amazonAutoScaling = new AmazonAutoScalingClient(
                    awsClusterSecurityManager.getCredentialsProvider());
            amazonAutoScaling.setRegion(region);
            return amazonAutoScaling;
        }
    });
}

---

From source file:org.apache.jackrabbit.oak.blob.cloud.aws.s3.Utils.java

License:Apache License

/**
 * Create AmazonS3Client from properties.
 * /* w ww  .j a va 2 s .  c om*/
 * @param prop properties to configure @link {@link AmazonS3Client}
 * @return {@link AmazonS3Client}
 */
public static AmazonS3Client openService(final Properties prop) {
    String accessKey = prop.getProperty(S3Constants.ACCESS_KEY);
    String secretKey = prop.getProperty(S3Constants.SECRET_KEY);
    AmazonS3Client s3service = null;
    if (StringUtils.isNullOrEmpty(accessKey) || StringUtils.isNullOrEmpty(secretKey)) {
        LOG.info("Configuring Amazon Client from environment");
        s3service = new AmazonS3Client(getClientConfiguration(prop));
    } else {
        LOG.info("Configuring Amazon Client from property file.");
        AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
        s3service = new AmazonS3Client(credentials, getClientConfiguration(prop));
    }
    String region = prop.getProperty(S3Constants.S3_REGION);
    String endpoint = null;
    String propEndPoint = prop.getProperty(S3Constants.S3_END_POINT);
    if ((propEndPoint != null) & !"".equals(propEndPoint)) {
        endpoint = propEndPoint;
    } else {
        if (StringUtils.isNullOrEmpty(region)) {
            com.amazonaws.regions.Region s3Region = Regions.getCurrentRegion();
            if (s3Region != null) {
                region = s3Region.getName();
            } else {
                throw new AmazonClientException("parameter [" + S3Constants.S3_REGION
                        + "] not configured and cannot be derived from environment");
            }
        }
        if (DEFAULT_AWS_BUCKET_REGION.equals(region)) {
            endpoint = S3 + DOT + AWSDOTCOM;
        } else if (Region.EU_Ireland.toString().equals(region)) {
            endpoint = "s3-eu-west-1" + DOT + AWSDOTCOM;
        } else {
            endpoint = S3 + DASH + region + DOT + AWSDOTCOM;
        }
    }
    /*
     * setting endpoint to remove latency of redirection. If endpoint is
     * not set, invocation first goes us standard region, which
     * redirects it to correct location.
     */
    s3service.setEndpoint(endpoint);
    LOG.info("S3 service endpoint [{}] ", endpoint);
    return s3service;
}

---

From source file:org.apache.manifoldcf.authorities.authorities.amazons3.AmazonS3Authority.java

License:Apache License

/**
 * Get the Amazons3 client, relevant access keys should have been posted
 * already/* w w  w  . ja  va  2 s  . com*/
 * @return
 */
protected AmazonS3 getClient() {
    if (amazonS3 == null) {
        try {
            BasicAWSCredentials awsCreds = new BasicAWSCredentials(amazons3AwsAccessKey, amazons3AwsSecretKey);
            amazonS3 = new AmazonS3Client(awsCreds);
        } catch (Exception e) {
            Logging.authorityConnectors.error("Error while amazon s3 connectionr", e);

        }
    }
    lastSessionFetch = System.currentTimeMillis();
    return amazonS3;
}

---

From source file:org.apache.manifoldcf.crawler.connectors.amazons3.AmazonS3Connector.java

License:Apache License

/**
 * Get the Amazons3 client, relevant access keys should have been posted
 * already/*from  ww w.  ja  v  a 2  s.  co  m*/
 * @return
 */
protected AmazonS3 getClient() {
    if (amazonS3 == null) {
        try {
            BasicAWSCredentials awsCreds = new BasicAWSCredentials(amazons3AwsAccessKey, amazons3AwsSecretKey);
            amazonS3 = new AmazonS3Client(awsCreds);
        } catch (Exception e) {
            Logging.connectors.error("Error while amazon s3 connectionr", e);

        }
    }
    lastSessionFetch = System.currentTimeMillis();
    return amazonS3;
}

---

From source file:org.apache.nifi.processors.aws.s3.AbstractS3IT.java

License:Apache License

@BeforeClass
public static void oneTimeSetup() {
    // Creates a client and bucket for this test

    final FileInputStream fis;
    try {/*from w  w  w  .  ja va2  s. c o m*/
        fis = new FileInputStream(CREDENTIALS_FILE);
    } catch (FileNotFoundException e1) {
        fail("Could not open credentials file " + CREDENTIALS_FILE + ": " + e1.getLocalizedMessage());
        return;
    }
    try {
        final PropertiesCredentials credentials = new PropertiesCredentials(fis);
        client = new AmazonS3Client(credentials);

        if (client.doesBucketExist(BUCKET_NAME)) {
            fail("Bucket " + BUCKET_NAME + " exists. Choose a different bucket name to continue test");
        }

        CreateBucketRequest request = REGION.contains("east") ? new CreateBucketRequest(BUCKET_NAME) // See https://github.com/boto/boto3/issues/125
                : new CreateBucketRequest(BUCKET_NAME, REGION);
        client.createBucket(request);

    } catch (final AmazonS3Exception e) {
        fail("Can't create the key " + BUCKET_NAME + ": " + e.getLocalizedMessage());
    } catch (final IOException e) {
        fail("Caught IOException preparing tests: " + e.getLocalizedMessage());
    } finally {
        FileUtils.closeQuietly(fis);
    }

    if (!client.doesBucketExist(BUCKET_NAME)) {
        fail("Setup incomplete, tests will fail");
    }
}

---

From source file:org.apache.oodt.cas.filemgr.datatransfer.S3DataTransfererFactory.java

License:Apache License

@Override
public S3DataTransferer createDataTransfer() {
    String bucketName = System.getProperty(BUCKET_NAME_PROPERTY);
    String region = System.getProperty(REGION_PROPERTY);
    String accessKey = System.getProperty(ACCESS_KEY_PROPERTY);
    String secretKey = System.getProperty(SECRET_KEY_PROPERTY);
    boolean encrypt = Boolean.getBoolean(ENCRYPT_PROPERTY);

    AmazonS3Client s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey));
    s3.setRegion(Region.getRegion(Regions.valueOf(region)));

    return new S3DataTransferer(s3, bucketName, encrypt);
}

---

From source file:org.apache.usergrid.apm.service.AWSUtil.java

License:Apache License

public static String generatePresignedURLForCrashLog(String fullAppName, String fileName) {
    DeploymentConfig config = DeploymentConfig.geDeploymentConfig();
    AWSCredentials credentials = new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey());
    AmazonS3Client client = new AmazonS3Client(credentials);
    String s3FullFileName = AWSUtil.formS3CrashFileUrl(fullAppName, fileName);
    GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(config.getS3LogBucket(),
            s3FullFileName, HttpMethod.GET);
    request.setExpiration(new Date(System.currentTimeMillis() + (120 * 60 * 1000))); //expires in 2 hour
    return client.generatePresignedUrl(request).toString();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• »