Example usage for com.amazonaws.services.s3.model ObjectListing getObjectSummaries

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ObjectListing getObjectSummaries.

Prototype

public List<S3ObjectSummary> getObjectSummaries() 

Source Link

Document

Gets the list of object summaries describing the objects stored in the S3 bucket.

Usage

From source file:org.pieShare.pieDrive.adapter.s3.S3Adapter.java

public boolean find(PieDriveFile file) {
    ObjectListing listing = s3Auth.getClient().listObjects(bucketName);
    boolean ret = false;

    for (S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
        ret = objectSummary.getKey().equals(file.getUuid());
        if (ret == true) {
            break;
        }/*from ww w.  j  av a  2  s.c  o m*/
    }

    return ret;
}

---

From source file:org.rdswitchboard.importers.browser.s3.App.java

License:Open Source License

public static void main(String[] args) {
    try {//from   w  w w .  jav  a 2s .c  o  m
        if (args.length == 0 || StringUtils.isNullOrEmpty(args[0]))
            throw new Exception("Please provide properties file");

        String propertiesFile = args[0];
        Properties properties = new Properties();
        try (InputStream in = new FileInputStream(propertiesFile)) {
            properties.load(in);
        }

        String source = properties.getProperty("data.source.id");

        if (StringUtils.isNullOrEmpty(source))
            throw new IllegalArgumentException("Source can not be empty");

        System.out.println("Source: " + source);

        String baseUrl = properties.getProperty("base.url");

        if (StringUtils.isNullOrEmpty(baseUrl))
            throw new IllegalArgumentException("Base URL can not be empty");

        System.out.println("Base URL: " + baseUrl);

        String sessionId = properties.getProperty("session.id");

        if (StringUtils.isNullOrEmpty(sessionId))
            throw new IllegalArgumentException("Session Id can not be empty");

        System.out.println("Session Id: " + sessionId);

        String accessKey = properties.getProperty("aws.access.key");
        String secretKey = properties.getProperty("aws.secret.key");

        String bucket = properties.getProperty("s3.bucket");

        if (StringUtils.isNullOrEmpty(bucket))
            throw new IllegalArgumentException("AWS S3 Bucket can not be empty");

        System.out.println("S3 Bucket: " + bucket);

        String prefix = properties.getProperty("s3.prefix");

        if (StringUtils.isNullOrEmpty(prefix))
            throw new IllegalArgumentException("AWS S3 Prefix can not be empty");

        System.out.println("S3 Prefix: " + prefix);

        String crosswalk = properties.getProperty("crosswalk");
        Templates template = null;

        if (!StringUtils.isNullOrEmpty(crosswalk)) {
            System.out.println("Crosswalk: " + crosswalk);

            template = TransformerFactory.newInstance()
                    .newTemplates(new StreamSource(new FileInputStream(crosswalk)));
        }

        ObjectMapper mapper = new ObjectMapper();

        Client client = Client.create();
        Cookie cookie = new Cookie("PHPSESSID", properties.getProperty("session"));

        AmazonS3 s3client;

        if (!StringUtils.isNullOrEmpty(accessKey) && !StringUtils.isNullOrEmpty(secretKey)) {
            System.out.println(
                    "Connecting to AWS via Access and Secret Keys. This is not safe practice, consider to use IAM Role instead.");

            AWSCredentials awsCredentials = new BasicAWSCredentials(accessKey, secretKey);
            s3client = new AmazonS3Client(awsCredentials);
        } else {
            System.out.println("Connecting to AWS via Instance Profile Credentials");

            s3client = new AmazonS3Client(new InstanceProfileCredentialsProvider());
        }

        //String file = "rda/rif/class:collection/54800.xml";

        ListObjectsRequest listObjectsRequest;
        ObjectListing objectListing;

        String file = prefix + "/latest.txt";
        S3Object object = s3client.getObject(new GetObjectRequest(bucket, file));

        String latest;
        try (InputStream txt = object.getObjectContent()) {
            latest = prefix + "/" + IOUtils.toString(txt, StandardCharsets.UTF_8).trim() + "/";
        }

        System.out.println("S3 Repository: " + latest);

        listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(latest);
        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {

                file = objectSummary.getKey();

                System.out.println("Processing file: " + file);

                object = s3client.getObject(new GetObjectRequest(bucket, file));
                String xml = null;

                if (null != template) {
                    Source reader = new StreamSource(object.getObjectContent());
                    StringWriter writer = new StringWriter();

                    Transformer transformer = template.newTransformer();
                    transformer.transform(reader, new StreamResult(writer));

                    xml = writer.toString();
                } else {
                    InputStream is = object.getObjectContent();

                    xml = IOUtils.toString(is, ENCODING);
                }

                URL url = new URL(baseUrl + "/registry/import/import_s3/");

                StringBuilder sb = new StringBuilder();
                addParam(sb, "id", source);
                addParam(sb, "xml", xml);

                //System.out.println(sb.toString());

                WebResource webResource = client.resource(url.toString());
                ClientResponse response = webResource
                        .header("User-Agent",
                                "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0")
                        .accept(MediaType.APPLICATION_JSON, "*/*").acceptLanguage("en-US", "en")
                        .type(MediaType.APPLICATION_FORM_URLENCODED).cookie(cookie)
                        .post(ClientResponse.class, sb.toString());

                if (response.getStatus() != 200) {
                    throw new RuntimeException("Failed : HTTP error code : " + response.getStatus());
                }

                String output = response.getEntity(String.class);

                Result result = mapper.readValue(output, Result.class);

                if (!result.getStatus().equals("OK")) {
                    System.err.println(result.getMessage());

                    break;
                } else
                    System.out.println(result.getMessage());
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

    } catch (Exception e) {
        e.printStackTrace();
    }
}

---

From source file:org.rdswitchboard.utils.s3.find.App.java

License:Open Source License

public static void main(String[] args) {
    try {/*from   w ww  . j  a v a 2s . c o  m*/
        if (args.length != 2)
            throw new IllegalArgumentException("Bucket name and search string can not be empty");

        String buckey = args[0];
        String search = args[1];
        String prefix = null;
        int pos = buckey.indexOf('/');
        if (pos > 0) {
            prefix = buckey.substring(pos + 1);
            buckey = buckey.substring(0, pos);
        }

        AmazonS3 s3client = new AmazonS3Client(new InstanceProfileCredentialsProvider());

        //         AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());        

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(buckey);

        if (!StringUtils.isNullOrEmpty(prefix))
            listObjectsRequest.setPrefix(prefix);

        ObjectListing objectListing;

        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String key = objectSummary.getKey();
                System.out.println(" - " + key);

                S3Object object = s3client.getObject(new GetObjectRequest(buckey, key));
                String str = IOUtils.toString(object.getObjectContent());
                if (str.contains(search)) {
                    System.out.println("Found!");

                    FileUtils.writeStringToFile(new File("s3/" + key), str);
                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (Exception e) {
        e.printStackTrace();
    }
}

---

From source file:org.reswitchboard.utils.s3.access.App.java

License:Open Source License

public static void main(String[] args) {
    try {//  ww w  .  j a  va2  s  .com
        if (args.length == 0 || StringUtils.isNullOrEmpty(args[0]))
            throw new IllegalArgumentException("Bucket name can not be empty");

        String bucketName = args[0];
        String prefix = null;
        if (args.length > 1)
            prefix = args[1];

        AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName);

        if (!StringUtils.isNullOrEmpty(prefix))
            listObjectsRequest.setPrefix(prefix);

        ObjectListing objectListing;

        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String key = objectSummary.getKey();
                System.out.println(" - " + key);

                for (int nAttempt = 1;; ++nAttempt) {
                    try {

                        AccessControlList acl = s3client.getObjectAcl(bucketName, key);
                        List<Grant> grants = acl.getGrantsAsList();
                        for (Grant grant : grants) {
                            //   System.out.println( "      Grant: " + grant.toString());

                            if (grant.getGrantee().equals(GroupGrantee.AllUsers)) {
                                System.out.println("      Revoking public access");

                                acl.revokeAllPermissions(GroupGrantee.AllUsers);
                                s3client.setObjectAcl(bucketName, key, acl);

                                break;
                            }
                        }

                        break;
                    } catch (Exception e) {
                        System.out.println("Error: " + e.toString());

                        if (nAttempt >= 10) {
                            throw new Exception("Maximum number of invalid attempts has been reeched");
                        }

                        // double back-off delay
                        Thread.sleep((long) (Math.pow(2, nAttempt) * 50));
                    }
                }

            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (Exception e) {
        e.printStackTrace();
    }
}

---

From source file:org.serginho.awss3conn.Connection.java

public List<S3ObjectSummary> getFileList() {
    AmazonS3 s3Client = getS3Client();/*from   w  ww . ja v a  2  s  .c  o m*/

    ObjectListing objects = s3Client.listObjects(this.amazonBucket, this.key);

    List<S3ObjectSummary> fileList = new ArrayList<S3ObjectSummary>();

    do {
        for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
            if (objectSummary.getSize() != 0) // We don't need to display the root "key" (folder)
                fileList.add(objectSummary);
        }
        objects = s3Client.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());

    return fileList;
}

---

From source file:org.springframework.aws.ivy.S3Repository.java

License:Apache License

public List<String> list(String parent) throws IOException {
    String bucket = S3Utils.getBucket(parent);
    String key = S3Utils.getKey(parent);
    String marker = null;//from  w  w  w .ja  v a  2  s.  c  om
    List<String> keys = new ArrayList<String>();
    do {
        ObjectListing objects = getService().listObjects(
                new ListObjectsRequest().withBucketName(bucket).withPrefix(key).withMarker(marker));
        for (S3ObjectSummary summary : objects.getObjectSummaries()) {
            String uri = "s3://" + bucket + "/" + summary.getKey();
            if (!uri.equals(parent)) {
                keys.add(uri);
                resourceCache.put(uri, new S3Resource(service, summary));
            }
        }
        marker = objects.getNextMarker();
    } while (marker != null);
    return keys;
}

---

From source file:org.springframework.aws.ivy.S3Resource.java

License:Apache License

public S3Resource(AmazonS3 service, String uri) {
    this.service = service;
    this.uri = uri;
    ObjectListing objects = service.listObjects(S3Utils.getBucket(uri), S3Utils.getKey(uri));
    if (objects.getObjectSummaries().size() == 0) {
        summary = null;// w ww .  java  2 s . c o  m
    } else {
        summary = objects.getObjectSummaries().get(0);
    }
}

---

From source file:org.springframework.cloud.aws.core.io.s3.PathMatchingSimpleStorageResourcePatternResolver.java

License:Apache License

private void findAllResourcesThatMatches(String bucketName, Set<Resource> resources, String prefix,
        String keyPattern) {// w w  w .  j a va2 s .  c o  m
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName)
            .withPrefix(prefix);
    ObjectListing objectListing = null;

    do {
        try {
            if (objectListing == null) {
                objectListing = this.amazonS3.listObjects(listObjectsRequest);
            } else {
                objectListing = this.amazonS3.listNextBatchOfObjects(objectListing);
            }
            Set<Resource> newResources = getResourcesFromObjectSummaries(bucketName, keyPattern,
                    objectListing.getObjectSummaries());
            if (!newResources.isEmpty()) {
                resources.addAll(newResources);
            }
        } catch (AmazonS3Exception e) {
            if (301 != e.getStatusCode()) {
                throw e;
            }
        }
    } while (objectListing != null && objectListing.isTruncated());
}

---

From source file:org.springframework.cloud.aws.core.io.s3.PathMatchingSimpleStorageResourcePatternResolver.java

License:Apache License

/**
 * Searches for matching keys progressively. This means that instead of retrieving all keys given a prefix, it goes
 * down one level at a time and filters out all non-matching results. This avoids a lot of unused requests results.
 * WARNING: This method does not truncate results. Therefore all matching resources will be returned regardless of
 * the truncation.//from   ww  w  .  j  ava2 s. c o  m
 */
private void findProgressivelyWithPartialMatch(String bucketName, Set<Resource> resources, String prefix,
        String keyPattern) {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName)
            .withDelimiter("/").withPrefix(prefix);
    ObjectListing objectListing = null;

    do {
        if (objectListing == null) {
            objectListing = this.amazonS3.listObjects(listObjectsRequest);
        } else {
            objectListing = this.amazonS3.listNextBatchOfObjects(objectListing);
        }

        Set<Resource> newResources = getResourcesFromObjectSummaries(bucketName, keyPattern,
                objectListing.getObjectSummaries());
        if (!newResources.isEmpty()) {
            resources.addAll(newResources);
        }

        for (String commonPrefix : objectListing.getCommonPrefixes()) {
            if (isKeyPathMatchesPartially(keyPattern, commonPrefix)) {
                findPathMatchingKeyInBucket(bucketName, resources, commonPrefix, keyPattern);
            }
        }
    } while (objectListing.isTruncated());
}

---

From source file:org.springframework.integration.aws.s3.core.AmazonS3OperationsImpl.java

License:Apache License

public PaginatedObjectsView listObjects(String bucketName, String folder, String nextMarker, int pageSize) {
    if (logger.isDebugEnabled()) {
        logger.debug("Listing objects from bucket " + bucketName + " and folder " + folder);
        logger.debug("Next marker is " + nextMarker + " and pageSize is " + pageSize);
    }/*from ww w  . j  av a  2 s  .c  om*/

    Assert.notNull(StringUtils.hasText(bucketName), "Bucket name should be non null and non empty");
    String prefix = null;
    if (folder != null && !"/".equals(folder)) {
        prefix = folder;
    }
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName)
            .withPrefix(prefix).withMarker(nextMarker);

    if (pageSize > 0) {
        listObjectsRequest.withMaxKeys(pageSize);
    }

    ObjectListing listing = client.listObjects(listObjectsRequest);
    PaginatedObjectsView view = null;
    List<com.amazonaws.services.s3.model.S3ObjectSummary> summaries = listing.getObjectSummaries();
    if (summaries != null && !summaries.isEmpty()) {
        List<S3ObjectSummary> objectSummaries = new ArrayList<S3ObjectSummary>();
        for (final com.amazonaws.services.s3.model.S3ObjectSummary summary : summaries) {
            S3ObjectSummary summ = new S3ObjectSummary() {

                public long getSize() {
                    return summary.getSize();
                }

                public Date getLastModified() {
                    return summary.getLastModified();
                }

                public String getKey() {
                    return summary.getKey();
                }

                public String getETag() {
                    return summary.getETag();
                }

                public String getBucketName() {
                    return summary.getBucketName();
                }
            };
            objectSummaries.add(summ);
        }
        view = new PagninatedObjectsViewImpl(objectSummaries, listing.getNextMarker());
    }
    return view;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• »