Example usage for com.amazonaws.services.s3.model S3ObjectSummary getKey

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model S3ObjectSummary getKey.

Prototype

public String getKey() 

Source Link

Document

Gets the key under which this object is stored in Amazon S3.

Usage

From source file:org.symphonyoss.vb.services.JsonPersist.java

License:Apache License

public static Set<VoteSession> getVoteSessions() throws Exception {

    Set<VoteSession> voteSessions = new HashSet<>();
    Gson gson = new Gson();

    if (Boolean.parseBoolean(System.getProperty(BotConfig.S3_ENABLED))) {

        AwsS3Client awsS3Client = new AwsS3Client();
        List<S3ObjectSummary> allObjects = awsS3Client.getAllObjects(System.getProperty(BotConfig.S3_BUCKET),
                System.getProperty(BotConfig.VB_S3_PREFIX_JSON));

        for (S3ObjectSummary objectSummary : allObjects) {

            if (!objectSummary.getKey().contains(".json"))
                continue;

            logger.info("Loading vote session from aws cache [s3://{}/{}]", objectSummary.getBucketName(),
                    objectSummary.getKey());

            VoteProposal voteProposal = gson
                    .fromJson(new InputStreamReader(awsS3Client.getObject(objectSummary)), VoteProposal.class);
            voteSessions.add(voteProposal);

        }// ww w .j  a v a  2  s  . c o  m

    } else {
        File[] files = new File(System.getProperty("files.json")).listFiles();

        if (files == null) {
            logger.error("Failed to load locate directory [{}] for json pre-load..exiting",
                    System.getProperty("files.json"));
            System.exit(1);
        }

        for (File file : files) {

            if (!file.getName().contains(".json"))
                continue;

            logger.info("Loading vote session from cache [{}]", file.getName());
            try {
                VoteProposal voteProposal = gson.fromJson(new FileReader(file), VoteProposal.class);
                voteSessions.add(voteProposal);

            } catch (IOException e) {
                logger.error("Could not load json {} ", file.getName(), e);
                throw (e);
            }
        }

    }
    return voteSessions;

}

---

From source file:org.symphonyoss.vb.util.AwsS3Client.java

License:Apache License

public InputStream getObject(S3ObjectSummary objectSummary) {

    S3Object object = null;/*from  w  w  w . jav a 2 s. c o m*/

    try {
        logger.info("Retrieving object inputstream for s3://{}/{}", objectSummary.getBucketName(),
                objectSummary.getKey());
        object = s3Client
                .getObject(new GetObjectRequest(objectSummary.getBucketName(), objectSummary.getKey()));

    } catch (AmazonServiceException ase) {
        logger.error("Caught an AmazonServiceException, " + "which means your request made it "
                + "to Amazon S3, but was rejected with an error response " + "for some reason.");
        logger.error("Error Message:    " + ase.getMessage());
        logger.error("HTTP Status Code: " + ase.getStatusCode());
        logger.error("AWS Error Code:   " + ase.getErrorCode());
        logger.error("Error Type:       " + ase.getErrorType());
        logger.error("Request ID:       " + ase.getRequestId());

    } catch (AmazonClientException ace) {
        logger.error("Caught an AmazonClientException, " + "which means the client encountered "
                + "an internal error while trying to communicate" + " with S3, "
                + "such as not being able to access the network.");
        logger.error("Error Message: " + ace.getMessage());
    }
    return object == null ? null : object.getObjectContent();
}

---

From source file:org.symphonyoss.vb.util.AwsS3Client.java

License:Apache License

public void moveObject(S3ObjectSummary objectSummary, String destBucket, String destKey) {
    try {//from  w  w w  .  j av  a  2s .  com
        // Copying object
        CopyObjectRequest copyObjRequest = new CopyObjectRequest(objectSummary.getBucketName(),
                objectSummary.getKey(), destBucket, destKey);

        s3Client.copyObject(copyObjRequest);

        DeleteObjectRequest deleteObjectRequest = new DeleteObjectRequest(objectSummary.getBucketName(),
                objectSummary.getKey());

        s3Client.deleteObject(deleteObjectRequest);
    } catch (AmazonServiceException ase) {
        logger.error("Caught an AmazonServiceException, " + "which means your request made it "
                + "to Amazon S3, but was rejected with an error response " + "for some reason.");
        logger.error("Error Message:    " + ase.getMessage());
        logger.error("HTTP Status Code: " + ase.getStatusCode());
        logger.error("AWS Error Code:   " + ase.getErrorCode());
        logger.error("Error Type:       " + ase.getErrorType());
        logger.error("Request ID:       " + ase.getRequestId());

    } catch (AmazonClientException ace) {
        logger.error("Caught an AmazonClientException, " + "which means the client encountered "
                + "an internal error while trying to communicate" + " with S3, "
                + "such as not being able to access the network.");
        logger.error("Error Message: " + ace.getMessage());
    }
}

---

From source file:org.yardstickframework.spark.S3MasterUrlProvider.java

License:Apache License

/**
 * Resolves url to master node.//  w ww . java2 s .co  m
 *
 * @return Url to master node.
 */
public String getMasterUrl() {
    initAwsClient();

    String masterUrl = null;

    try {
        ObjectListing list = s3.listObjects(bucketName, SparkMaster.SPARK_URL_PREFIX);

        while (true) {
            for (S3ObjectSummary s3Entry : list.getObjectSummaries())
                if (s3Entry.getKey() != null && s3Entry.getKey().contains(SparkMaster.SPARK_URL_PREFIX)) {
                    masterUrl = s3KeyToUrl(s3Entry.getKey());

                    break;
                }

            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    } catch (AmazonClientException e) {
        throw new RuntimeException("Failed to list objects in the bucket: " + bucketName, e);
    }

    return masterUrl;
}

---

From source file:org.zalando.stups.fullstop.controller.S3Controller.java

License:Apache License

@RequestMapping(method = RequestMethod.GET, value = "/download")
public void downloadFiles(@RequestParam(value = "bucket") final String bucket,
        @RequestParam(value = "location") final String location, @RequestParam(value = "page") final int page) {

    try {/*from  w  w  w . j  a v  a 2  s . c  o m*/
        log.info("Creating fullstop directory here: {}", fullstopLoggingDir);

        boolean mkdirs = new File(fullstopLoggingDir).mkdirs();
    } catch (SecurityException e) {
        // do nothing
    }

    AmazonS3Client amazonS3Client = new AmazonS3Client();
    amazonS3Client.setRegion(Region.getRegion(Regions
            .fromName((String) cloudTrailProcessingLibraryProperties.getAsProperties().get(S3_REGION_KEY))));

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) //
            .withPrefix(location) //
            .withMaxKeys(page);

    ObjectListing objectListing = amazonS3Client.listObjects(listObjectsRequest);

    final List<S3ObjectSummary> s3ObjectSummaries = objectListing.getObjectSummaries();

    while (objectListing.isTruncated()) {

        objectListing = amazonS3Client.listNextBatchOfObjects(objectListing);
        s3ObjectSummaries.addAll(objectListing.getObjectSummaries());

    }

    for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) {
        String bucketName = s3ObjectSummary.getBucketName();
        String key = s3ObjectSummary.getKey();

        S3Object object = amazonS3Client.getObject(new GetObjectRequest(bucketName, key));
        InputStream inputStream = object.getObjectContent();

        File file = new File(fullstopLoggingDir,
                object.getBucketName() + object.getObjectMetadata().getETag() + JSON_GZ);

        copyInputStreamToFile(inputStream, file);
        log.info("File saved here: {}", file.getAbsolutePath());

    }
}

---

From source file:org.zalando.stups.fullstop.plugin.SaveSecurityGroupsPlugin.java

License:Apache License

private List<String> listS3Objects(String bucketName, String prefix) {
    final List<String> commonPrefixes = Lists.newArrayList();

    AmazonS3Client s3client = new AmazonS3Client();

    try {//ww  w. j a v  a 2s  .c o  m
        System.out.println("Listing objects");

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withDelimiter("/")
                .withBucketName(bucketName).withPrefix(prefix);

        ObjectListing objectListing;

        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            commonPrefixes.addAll(objectListing.getCommonPrefixes());
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                System.out.println(
                        " - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")");
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

    } catch (AmazonServiceException ase) {
        System.out.println("Caught an AmazonServiceException, " + "which means your request made it "
                + "to Amazon S3, but was rejected with an error response " + "for some reason.");
        System.out.println("Error Message:    " + ase.getMessage());
        System.out.println("HTTP Status Code: " + ase.getStatusCode());
        System.out.println("AWS Error Code:   " + ase.getErrorCode());
        System.out.println("Error Type:       " + ase.getErrorType());
        System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
        System.out.println("Caught an AmazonClientException, " + "which means the client encountered "
                + "an internal error while trying to communicate" + " with S3, "
                + "such as not being able to access the network.");
        System.out.println("Error Message: " + ace.getMessage());
    }

    return commonPrefixes;
}

---

From source file:oulib.aws.s3.S3TiffProcessor.java

/**
* 
* @param bookInfo : contains the information of the source bucket name, target bucket name, and the name of the book
* @param context : lambda function runtime context
* @return ://from   w w  w .  j  a va  2s.  c  om
* 
*/
@Override
public String handleRequest(S3BookInfo bookInfo, Context context) {

    AmazonS3 s3client = new AmazonS3Client();
    Region usEast = Region.getRegion(Regions.US_EAST_1);
    s3client.setRegion(usEast);

    try {
        String sourceBucketName = bookInfo.getBucketSourceName();
        String targetBucketName = bookInfo.getBucketTargetName();
        String bookName = bookInfo.getBookName();

        // Every book has a folder in the target bucket:
        Map targetBucketKeyMap = S3Util.getBucketObjectKeyMap(targetBucketName, bookName, s3client);
        if (!S3Util.folderExitsts(bookName, targetBucketKeyMap)) {
            S3Util.createFolder(targetBucketName, bookName, s3client);
        }

        final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(sourceBucketName)
                .withPrefix(bookName + "/data/");
        ListObjectsV2Result result;

        do {
            result = s3client.listObjectsV2(req);

            for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
                String key = objectSummary.getKey();
                if (key.endsWith(".tif") && !targetBucketKeyMap.containsKey(key + ".tif")) {
                    S3Object object = s3client.getObject(new GetObjectRequest(sourceBucketName, key));
                    System.out.println("Start to generate smaller tif image for the object " + key);
                    S3Util.generateSmallTiffWithTargetSize(s3client, object, targetBucketName,
                            bookInfo.getCompressionSize());
                    //                       S3Util.copyS3ObjectTiffMetadata(s3client, object, s3client.getObject(new GetObjectRequest(targetBucketName, key)), targetBucketName, key+".tif");
                    System.out.println("Finished to generate smaller tif image for the object " + key + ".tif");
                    //                       break;
                }
            }
            System.out.println("Next Continuation Token : " + result.getNextContinuationToken());
            req.setContinuationToken(result.getNextContinuationToken());
        } while (result.isTruncated() == true);

    } catch (AmazonServiceException ase) {
        System.out.println(
                "Caught an AmazonServiceException, which means your request made it to Amazon S3, but was rejected with an error response for some reason.");
        System.out.println("Error Message:    " + ase.getMessage());
        System.out.println("HTTP Status Code: " + ase.getStatusCode());
        System.out.println("AWS Error Code:   " + ase.getErrorCode());
        System.out.println("Error Type:       " + ase.getErrorType());
        System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
        System.out.println(
                "Caught an AmazonClientException, which means the client encountered an internal error while trying to communicate with S3, \nsuch as not being able to access the network.");
        System.out.println("Error Message: " + ace.getMessage());
    }
    return null;
}

---

From source file:oulib.aws.s3.S3Util.java

/**
 * //from  www.j  a v  a 2s  .c o  m
 * @param bucketName : bucket name
 * @param folderName : a unique folder name or partial path in the bucket
 * @param client : s3 client
 * @return : a map of keys with keyset of object keys
 */
public static Map<String, String> getBucketObjectKeyMap(String bucketName, String folderName, AmazonS3 client) {
    final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName);
    ListObjectsV2Result result;
    Map<String, String> keyMap = new HashMap<>();

    do {
        result = client.listObjectsV2(req);

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            String key = objectSummary.getKey();
            if (key.contains(folderName)) {
                keyMap.put(key, key);
            }
        }
        req.setContinuationToken(result.getNextContinuationToken());
    } while (result.isTruncated() == true);

    return keyMap;
}

---

From source file:oulib.aws.s3.S3Util.java

/**
 * //from www .ja  v a 2s . co  m
 * @param bucketName : bucket name
 * @param folderName : a unique folder name or partial path in the bucket
 * @param client : s3 client
 * @return : a list of keys
 */
public static List<String> getBucketObjectKeyList(String bucketName, String folderName, AmazonS3 client) {
    final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName);
    ListObjectsV2Result result;
    List<String> keyList = new ArrayList<>();

    do {
        result = client.listObjectsV2(req);

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            String key = objectSummary.getKey();
            if (key.contains(folderName)) {
                keyList.add(key);
            }
        }
        req.setContinuationToken(result.getNextContinuationToken());
    } while (result.isTruncated() == true);

    return keyList;
}

---

From source file:oulib.aws.s3.S3Util.java

public static void generateTifDerivativesByS3Bucket(AmazonS3 s3client, S3BookInfo bookInfo) {

    String sourceBucketName = bookInfo.getBucketSourceName();
    String targetBucketName = bookInfo.getBucketTargetName();
    String bookName = bookInfo.getBookName();

    try {//from   www. j  a v a2s .c o  m

        // Every book has a folder in the target bucket:
        Map targetBucketKeyMap = S3Util.getBucketObjectKeyMap(targetBucketName, bookName, s3client);
        if (!S3Util.folderExitsts(bookName, targetBucketKeyMap)) {
            S3Util.createFolder(targetBucketName, bookName, s3client);
        }

        final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(sourceBucketName)
                .withPrefix(bookName + "/data/");
        ListObjectsV2Result result;

        do {
            result = s3client.listObjectsV2(req);

            for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
                String key = objectSummary.getKey();
                if (key.contains(".tif") && (key.contains("047") || key.contains("049") || key.contains("054"))
                        && !targetBucketKeyMap.containsKey(key + ".tif")) {
                    S3Object object = s3client.getObject(new GetObjectRequest(sourceBucketName, key));
                    System.out.println("Start to generate smaller tif image for the object " + key + "\n");
                    S3Util.generateSmallTiffWithTargetSize(s3client, object, targetBucketName,
                            bookInfo.getCompressionSize());
                    //                   S3Util.copyS3ObjectTiffMetadata(s3client, object, s3client.getObject(new GetObjectRequest(targetBucketName, key)), targetBucketName, key+".tif");
                    System.out.println("Finished to generate smaller tif image for the object " + key + "\n");
                    //                   break;
                }
            }
            System.out.println("Next Continuation Token : " + result.getNextContinuationToken() + "\n");
            req.setContinuationToken(result.getNextContinuationToken());
        } while (result.isTruncated() == true);

    } catch (AmazonServiceException ase) {
        System.out.println(
                "Caught an AmazonServiceException, which means your request made it to Amazon S3, but was rejected with an error response for some reason.\n");
        System.out.println("Error Message:    " + ase.getMessage() + "\n");
        System.out.println("HTTP Status Code: " + ase.getStatusCode() + "\n");
        System.out.println("AWS Error Code:   " + ase.getErrorCode() + "\n");
        System.out.println("Error Type:       " + ase.getErrorType() + "\n");
        System.out.println("Request ID:       " + ase.getRequestId() + "\n");
    } catch (AmazonClientException ace) {
        System.out.println(
                "Caught an AmazonClientException, which means the client encountered an internal error while trying to communicate with S3, \nsuch as not being able to access the network.\n");
        System.out.println("Error Message: " + ace.getMessage() + "\n");
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• »