Example usage for com.amazonaws.services.s3.model ObjectListing getObjectSummaries

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ObjectListing getObjectSummaries.

Prototype

public List<S3ObjectSummary> getObjectSummaries() 

Source Link

Document

Gets the list of object summaries describing the objects stored in the S3 bucket.

Usage

From source file:org.icgc.dcc.storage.server.repository.s3.S3UploadStateStore.java

License:Open Source License

@Override
@SneakyThrows// w w  w  .ja v a 2  s .c  o m
public void markCompletedParts(String objectId, String uploadId, List<Part> parts) {
    if (parts == null || parts.size() == 0) {
        return;
    }

    try {
        String bucketName = bucketNamingService.getStateBucketName(objectId);
        sortPartsByNumber(parts);
        val partIterator = parts.iterator();

        val request = new ListObjectsRequest().withBucketName(bucketName).withMaxKeys(MAX_KEYS)
                .withPrefix(getUploadStateKey(objectId, uploadId, PART));

        ObjectListing objectListing = null;
        do {
            objectListing = s3Client.listObjects(request);
            Part part = null;
            for (val objectSummary : objectListing.getObjectSummaries()) {
                CompletedPart completedPart = readCompletedPart(objectId, uploadId, objectSummary);
                do {
                    if (partIterator.hasNext()) {
                        part = partIterator.next();
                    } else {
                        return;
                    }
                } while (completedPart.getPartNumber() != part.getPartNumber());
                part.setSourceMd5(completedPart.getMd5());
            }
            request.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonServiceException e) {
        log.error(
                "Failed to mark completed parts for object metadata for objectId: {}, uploadId: {}, parts: {}",
                objectId, uploadId, parts, e);
        throw new RetryableException(e);
    }
}

---

From source file:org.icgc.dcc.storage.server.repository.s3.S3UploadStateStore.java

License:Open Source License

@Override
public boolean isCompleted(String objectId, String uploadId) {
    val spec = read(objectId, uploadId);

    sortPartsByNumber(spec.getParts());//  w  w  w.  j  av a  2 s. c  o  m
    val partIterator = spec.getParts().iterator();

    val request = new ListObjectsRequest().withBucketName(bucketNamingService.getStateBucketName(objectId))
            .withMaxKeys(MAX_KEYS).withPrefix(getUploadStateKey(objectId, uploadId, PART));

    if (partIterator.hasNext()) {
        Part part = partIterator.next();

        ObjectListing objectListing;
        do {
            objectListing = s3Client.listObjects(request);
            for (val objectSummary : objectListing.getObjectSummaries()) {
                int partNumber = extractPartNumber(objectId, uploadId, objectSummary.getKey());
                if (part.getPartNumber() != partNumber) {
                    return false;
                }
                if (partIterator.hasNext()) {
                    part = partIterator.next();
                } else {
                    return true;
                }
            }
            request.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
        return false;
    }

    return true;
}

---

From source file:org.icgc.dcc.storage.server.repository.s3.S3UploadStateStore.java

License:Open Source License

private void eachObjectSummary(String objectId, String prefix, Consumer<S3ObjectSummary> callback) {
    val request = new ListObjectsRequest().withBucketName(bucketNamingService.getStateBucketName(objectId))
            .withMaxKeys(MAX_KEYS).withPrefix(prefix);

    try {/*from w  w  w .  ja v a2  s .c  o m*/
        ObjectListing objectListing;
        do {
            objectListing = s3Client.listObjects(request);
            for (val objectSummary : objectListing.getObjectSummaries()) {
                log.debug("processing {}", objectSummary.getKey());
                callback.accept(objectSummary);
            }

            request.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonServiceException e) {
        log.error("Failed to list objects with prefix: {}: {}", prefix, e);
        throw new RetryableException(e);
    }
}

---

From source file:org.kuali.maven.wagon.S3Wagon.java

License:Educational Community License

/**
 * List all of the objects in a given directory
 *///from  w w  w . ja va2 s .  c  o  m
@Override
protected List<String> listDirectory(final String directory) throws Exception {
    ObjectListing objectListing = client.listObjects(bucket.getName(), basedir + directory);
    List<String> fileNames = new ArrayList<String>();
    for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
        fileNames.add(summary.getKey());
    }
    return fileNames;
}

---

From source file:org.nickelproject.util.sources.S3MultiFileSource.java

License:Apache License

private static List<String> listKeysInDirectory(final String bucketName, final String prefix) {
    final String delimiter = "/";
    final String fixedPrefix = prefix.endsWith(delimiter) ? prefix : prefix + delimiter;

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName)
            .withPrefix(fixedPrefix).withDelimiter(delimiter);
    ObjectListing objects = s3Client.listObjects(listObjectsRequest);
    return Lists.transform(objects.getObjectSummaries(), new Function<S3ObjectSummary, String>() {
        @Override/*from  ww  w  .  j  a v  a 2 s . c  om*/
        public String apply(@Nonnull final S3ObjectSummary input) {
            return input.getKey();
        }
    });
}

---

From source file:org.nuxeo.liveconnect.importer.aws.S3Importer.java

License:Apache License

public void importBucket(DocumentModel rootFolder) {
    LiveconnectS3Blobprovider blobprovider = (LiveconnectS3Blobprovider) Framework.getService(BlobManager.class)
            .getBlobProvider(provider);//www.j  ava 2 s .  c o  m

    AmazonS3 s3 = blobprovider.getClient();
    String bucketName = blobprovider.getBucketName();

    final ListObjectsRequest req = new ListObjectsRequest().withBucketName(bucketName);
    ObjectListing result;
    int docsCount = 0;
    do {
        result = s3.listObjects(req);
        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            String name = S3LiveConnectFile.getFilename(objectSummary.getKey());
            if (name == null)
                continue;

            DocumentModel fileDoc = getOrCreateFileDocument(rootFolder, objectSummary.getKey(), name);

            //import object
            LiveConnectFileInfo info = new LiveConnectFileInfo(
                    rootFolder.getCoreSession().getPrincipal().getName(), objectSummary.getKey(),
                    objectSummary.getETag());

            LiveConnectBlobProvider blobProvider = (LiveConnectBlobProvider) Framework
                    .getService(BlobManager.class).getBlobProvider(provider);
            try {
                Blob blob = blobProvider.toBlob(info);
                fileDoc.setPropertyValue("file:content", (Serializable) blob);
                fileDoc.getCoreSession().saveDocument(fileDoc);
            } catch (IOException e) {
                log.warn("Couldn't get Blob with ID " + info.getFileId(), e);
            }
            docsCount++;
            if (docsCount % 10 == 0) {
                rootFolder.getCoreSession().save();
                if (TransactionHelper.isTransactionActive()) {
                    TransactionHelper.commitOrRollbackTransaction();
                    TransactionHelper.startTransaction();
                }
            }
        }
        req.setMarker(result.getNextMarker());
    } while (result.isTruncated());

    rootFolder.getCoreSession().save();
    if (TransactionHelper.isTransactionActive()) {
        TransactionHelper.commitOrRollbackTransaction();
        TransactionHelper.startTransaction();
    }

}

---

From source file:org.onebusaway.admin.service.impl.S3FileServiceImpl.java

License:Apache License

@Override
/**//from  w  ww.ja  v a  2  s . c om
 * check to see if the given bundle directory exists in the configured bucket.
 * Do not include leading slashes in the filename(key).
 */
public boolean bundleDirectoryExists(String filename) {
    ListObjectsRequest request = new ListObjectsRequest(_bucketName, filename, null, null, 1);
    ObjectListing listing = _s3.listObjects(request);
    return listing.getObjectSummaries().size() > 0;
}

---

From source file:org.onebusaway.admin.service.impl.S3FileServiceImpl.java

License:Apache License

@Override
/**/* w ww  .  j a  v  a 2  s .  co m*/
 * Return tabular data (filename, flag, modified date) about bundle directories.
 */
public List<String[]> listBundleDirectories(int maxResults) {
    List<String[]> rows = new ArrayList<String[]>();
    HashMap<String, String> map = new HashMap<String, String>();
    ListObjectsRequest request = new ListObjectsRequest(_bucketName, null, null, "/", maxResults);

    ObjectListing listing = null;
    do {
        if (listing == null) {
            listing = _s3.listObjects(request);
            if (listing.getCommonPrefixes() != null) {
                // short circuit if common prefixes works
                List<String> commonPrefixes = listing.getCommonPrefixes();
                for (String key : commonPrefixes) {
                    Date lastModified = getLastModifiedTimeForKey(key);
                    String lastModifiedStr = "n/a";
                    if (lastModified != null) {
                        lastModifiedStr = "" + lastModified.toString();
                    }
                    String[] columns = { parseKey(key), getStatus(key), lastModifiedStr };
                    rows.add(columns);
                }
                return rows;
            }
            _log.error("prefixes=" + listing.getCommonPrefixes());
        } else {
            listing = _s3.listNextBatchOfObjects(listing);
        }
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            String key = parseKey(summary.getKey());
            if (!map.containsKey(key)) {
                String[] columns = { key, " ", "" + summary.getLastModified().getTime() };
                rows.add(columns);
                map.put(key, key);
            }
        }

    } while (listing.isTruncated());
    return rows;
}

---

From source file:org.onebusaway.admin.service.impl.S3FileServiceImpl.java

License:Apache License

private Date getLastModifiedTimeForKey(String key) {
    ListObjectsRequest request = new ListObjectsRequest(_bucketName, key, null, "/", 1);
    ObjectListing listing = _s3.listObjects(request);
    if (!listing.getObjectSummaries().isEmpty())
        return listing.getObjectSummaries().get(0).getLastModified();
    return null;/*from  w  w  w .ja va  2 s . c om*/
}

---

From source file:org.onebusaway.admin.service.impl.S3FileServiceImpl.java

License:Apache License

@Override
/**/*from  w  ww . j  a v a  2  s. c  o m*/
 * list the files in the given directory.
 */
public List<String> list(String directory, int maxResults) {
    ListObjectsRequest request = new ListObjectsRequest(_bucketName, directory, null, null, maxResults);
    ObjectListing listing = _s3.listObjects(request);
    List<String> rows = new ArrayList<String>();
    for (S3ObjectSummary summary : listing.getObjectSummaries()) {
        // if its a directory at the root level
        if (!summary.getKey().endsWith("/")) {
            rows.add(summary.getKey());
        }
    }
    return rows;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• »