Example usage for com.amazonaws.services.s3.model ObjectListing getObjectSummaries

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ObjectListing getObjectSummaries.

Prototype

public List<S3ObjectSummary> getObjectSummaries() 

Source Link

Document

Gets the list of object summaries describing the objects stored in the S3 bucket.

Usage

From source file:org.elasticsearch.repositories.s3.S3BlobContainer.java

License:Apache License

@Override
public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String blobNamePrefix) throws IOException {
    return AccessController.doPrivileged((PrivilegedAction<Map<String, BlobMetaData>>) () -> {
        MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
        AmazonS3 client = blobStore.client();
        SocketAccess.doPrivilegedVoid(() -> {
            ObjectListing prevListing = null;
            while (true) {
                ObjectListing list;
                if (prevListing != null) {
                    list = client.listNextBatchOfObjects(prevListing);
                } else {
                    if (blobNamePrefix != null) {
                        list = client.listObjects(blobStore.bucket(), buildKey(blobNamePrefix));
                    } else {
                        list = client.listObjects(blobStore.bucket(), keyPath);
                    }//w  ww.j av a2 s .  co  m
                }
                for (S3ObjectSummary summary : list.getObjectSummaries()) {
                    String name = summary.getKey().substring(keyPath.length());
                    blobsBuilder.put(name, new PlainBlobMetaData(name, summary.getSize()));
                }
                if (list.isTruncated()) {
                    prevListing = list;
                } else {
                    break;
                }
            }
        });
        return blobsBuilder.immutableMap();
    });
}

---

From source file:org.elasticsearch.repositories.s3.S3BlobStore.java

License:Apache License

@Override
public void delete(BlobPath path) {
    AccessController.doPrivileged((PrivilegedAction<Object>) () -> {
        ObjectListing prevListing = null;
        //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
        //we can do at most 1K objects per delete
        //We don't know the bucket name until first object listing
        DeleteObjectsRequest multiObjectDeleteRequest = null;
        ArrayList<KeyVersion> keys = new ArrayList<>();
        while (true) {
            ObjectListing list;
            if (prevListing != null) {
                list = client.listNextBatchOfObjects(prevListing);
            } else {
                list = client.listObjects(bucket, path.buildAsString());
                multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
            }//  w  w w .j av  a 2s.  com
            for (S3ObjectSummary summary : list.getObjectSummaries()) {
                keys.add(new KeyVersion(summary.getKey()));
                //Every 500 objects batch the delete request
                if (keys.size() > 500) {
                    multiObjectDeleteRequest.setKeys(keys);
                    client.deleteObjects(multiObjectDeleteRequest);
                    multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
                    keys.clear();
                }
            }
            if (list.isTruncated()) {
                prevListing = list;
            } else {
                break;
            }
        }
        if (!keys.isEmpty()) {
            multiObjectDeleteRequest.setKeys(keys);
            client.deleteObjects(multiObjectDeleteRequest);
        }
        return null;
    });
}

---

From source file:org.entando.entando.plugins.jps3awsclient.aps.system.services.storage.AmazonS3StorageManager.java

License:Open Source License

private String[] list(String subPath, boolean isProtectedResource, boolean addFolders, boolean addFiles)
        throws ApsSystemException {
    if (!this.isActive()) {
        return null;
    }//from   ww w  .  java 2 s.c o  m
    String folder = this.getKey(subPath, isProtectedResource);
    ObjectListing objectListing = this.getS3Objects(folder);
    if (null == objectListing) {
        return null;
    }
    String[] objects = new String[] {};
    if (null != objectListing.getCommonPrefixes() && addFolders) {
        for (int i = 0; i < objectListing.getCommonPrefixes().size(); i++) {
            String object = objectListing.getCommonPrefixes().get(i);
            String name = object.substring(folder.length(), (object.length() - 1));
            objects = this.addChild(name, objects);
        }
    }
    if (null != objectListing.getObjectSummaries() && addFiles) {
        for (int i = 0; i < objectListing.getObjectSummaries().size(); i++) {
            S3ObjectSummary s3os = objectListing.getObjectSummaries().get(i);
            String key = s3os.getKey();
            String name = key.substring(folder.length());
            objects = this.addChild(name, objects);
        }
    }
    return objects;
}

---

From source file:org.entando.entando.plugins.jps3awsclient.aps.system.services.storage.AmazonS3StorageManager.java

License:Open Source License

private BasicFileAttributeView[] listAttributes(String subPath, boolean isProtectedResource, boolean addFolders,
        boolean addFiles) throws ApsSystemException {
    if (!this.isActive()) {
        return null;
    }/*from   w w w .j  a  va 2 s  .  c o  m*/
    String folder = this.getKey(subPath, isProtectedResource);
    ObjectListing objectListing = this.getS3Objects(folder);
    if (null == objectListing) {
        return null;
    }
    BasicFileAttributeView[] objects = new BasicFileAttributeView[] {};
    if (null != objectListing.getCommonPrefixes() && addFolders) {
        for (int i = 0; i < objectListing.getCommonPrefixes().size(); i++) {
            String object = objectListing.getCommonPrefixes().get(i);
            String name = object.substring(folder.length(), (object.length() - 1));
            BasicFileAttributeView bfav = new BasicFileAttributeView();
            bfav.setDirectory(true);
            bfav.setName(name);
            objects = this.addChildAttribute(bfav, objects);
        }
    }
    if (null != objectListing.getObjectSummaries() && addFiles) {
        for (int i = 0; i < objectListing.getObjectSummaries().size(); i++) {
            S3ObjectSummary s3os = objectListing.getObjectSummaries().get(i);
            String key = s3os.getKey();
            String name = key.substring(folder.length());
            BasicFileAttributeView bfav = new BasicFileAttributeView();
            bfav.setDirectory(false);
            bfav.setName(name);
            bfav.setLastModifiedTime(s3os.getLastModified());
            bfav.setSize(s3os.getSize());
            objects = this.addChildAttribute(bfav, objects);
        }
    }
    return objects;
}

---

From source file:org.entando.entando.plugins.jps3awsclient.aps.system.services.storage.AmazonS3StorageManager.java

License:Open Source License

public void delete(String bucketName, String key) throws ApsSystemException {
    if (!this.isActive()) {
        return;/*from ww  w .j av a  2s .  c o  m*/
    }
    try {
        if (key == null || key.trim().length() == 0) {
            _logger.warn("Empty storage path passed to delete method");
            return; // We don't want to delete everything in a path
        }
        AmazonS3Client client = this.getS3Client();
        // Go through the store structure and delete child objects
        ObjectListing listing = this.getS3Client().listObjects(bucketName, key);
        while (true) {
            List<S3ObjectSummary> objectList = listing.getObjectSummaries();
            for (int i = 0; i < objectList.size(); i++) {
                S3ObjectSummary summary = objectList.get(i);
                client.deleteObject(bucketName, summary.getKey());
            }
            if (listing.isTruncated()) {
                listing = client.listNextBatchOfObjects(listing);
            } else {
                break;
            }
        }
    } catch (Throwable t) {
        _logger.error("Error deleting objects : bucket {} - key {}", bucketName, key, t);
        throw new ApsSystemException("Error deleting objects : bucket " + bucketName + " - key " + key, t);
    }
}

---

From source file:org.finra.dm.dao.impl.MockS3OperationsImpl.java

License:Apache License

/**
 * Returns a list of objects. If the bucket does not exist, returns a listing with an empty list.
 * If a prefix is specified in listObjectsRequest, only keys starting with the prefix will be returned.
 *//* www.j  av  a 2 s  .  co m*/
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest, AmazonS3Client s3Client) {
    LOGGER.debug("listObjects(): listObjectsRequest.getBucketName() = " + listObjectsRequest.getBucketName());

    String bucketName = listObjectsRequest.getBucketName();

    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName)) {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(
                MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    }

    ObjectListing objectListing = new ObjectListing();
    objectListing.setBucketName(bucketName);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null) {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values()) {
            String s3ObjectKey = mockS3Object.getKey();
            if (listObjectsRequest.getPrefix() == null
                    || s3ObjectKey.startsWith(listObjectsRequest.getPrefix())) {
                S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                s3ObjectSummary.setBucketName(bucketName);
                s3ObjectSummary.setKey(s3ObjectKey);
                s3ObjectSummary.setSize(mockS3Object.getData().length);

                objectListing.getObjectSummaries().add(s3ObjectSummary);
            }
        }
    }

    return objectListing;
}

---

From source file:org.finra.dm.dao.impl.S3DaoImpl.java

License:Apache License

/**
 * Lists all S3 objects matching the S3 key prefix in the given bucket (S3 bucket name). The S3 bucket name and S3 key prefix that identify the S3 objects
 * to get listed are taken from the S3 file transfer request parameters DTO.
 *
 * @param params the S3 file transfer request parameters
 * @param ignoreZeroByteDirectoryMarkers specifies whether to ignore 0 byte objects that represent S3 directories
 *
 * @return the list of all S3 objects represented as storage files that match the prefix in the given bucket
 *//*w  w w  . j  av a2 s  . com*/
private List<StorageFile> listObjectsMatchingKeyPrefix(final S3FileTransferRequestParamsDto params,
        boolean ignoreZeroByteDirectoryMarkers) {
    AmazonS3Client s3Client = null;
    List<StorageFile> storageFiles = new ArrayList<>();

    try {
        s3Client = getAmazonS3(params);
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
                .withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        ObjectListing objectListing;

        do {
            objectListing = s3Operations.listObjects(listObjectsRequest, s3Client);

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                // Ignore 0 byte objects that represent S3 directories.
                if (!(ignoreZeroByteDirectoryMarkers && objectSummary.getKey().endsWith("/")
                        && objectSummary.getSize() == 0L)) {
                    storageFiles.add(new StorageFile(objectSummary.getKey(), objectSummary.getSize(), null));
                }
            }

            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonS3Exception amazonS3Exception) {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode())) {
            throw new IllegalArgumentException(
                    "The specified bucket '" + params.getS3BucketName() + "' does not exist.",
                    amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    } catch (AmazonClientException e) {
        throw new IllegalStateException(
                String.format("Failed to list keys/objects with prefix \"%s\" from bucket \"%s\". Reason: %s",
                        params.getS3KeyPrefix(), params.getS3BucketName(), e.getMessage()),
                e);
    } finally {
        // Shutdown the AmazonS3Client instance to release resources.
        if (s3Client != null) {
            s3Client.shutdown();
        }
    }

    return storageFiles;
}

---

From source file:org.finra.herd.dao.impl.MockS3OperationsImpl.java

License:Apache License

/**
 * {@inheritDoc}/*ww  w  .  j  a  va  2s.  c  o m*/
 * <p/>
 * If the bucket does not exist, returns a listing with an empty list. If a prefix is specified in listObjectsRequest, only keys starting with the prefix
 * will be returned.
 */
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest, AmazonS3 s3Client) {
    LOGGER.debug("listObjects(): listObjectsRequest.getBucketName() = " + listObjectsRequest.getBucketName());

    String bucketName = listObjectsRequest.getBucketName();

    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName)) {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(
                MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    }

    ObjectListing objectListing = new ObjectListing();
    objectListing.setBucketName(bucketName);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null) {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values()) {
            String s3ObjectKey = mockS3Object.getKey();
            if (listObjectsRequest.getPrefix() == null
                    || s3ObjectKey.startsWith(listObjectsRequest.getPrefix())) {
                S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                s3ObjectSummary.setBucketName(bucketName);
                s3ObjectSummary.setKey(s3ObjectKey);
                s3ObjectSummary.setSize(mockS3Object.getData().length);
                s3ObjectSummary.setStorageClass(mockS3Object.getObjectMetadata() != null
                        ? mockS3Object.getObjectMetadata().getStorageClass()
                        : null);

                objectListing.getObjectSummaries().add(s3ObjectSummary);
            }
        }
    }

    return objectListing;
}

---

From source file:org.finra.herd.dao.impl.S3DaoImpl.java

License:Apache License

@Override
public List<S3ObjectSummary> listDirectory(final S3FileTransferRequestParamsDto params,
        boolean ignoreZeroByteDirectoryMarkers) {
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()),
            "Listing of S3 objects from root directory is not allowed.");

    AmazonS3Client s3Client = getAmazonS3(params);
    List<S3ObjectSummary> s3ObjectSummaries = new ArrayList<>();

    try {//from  www .  j ava 2 s  . c o  m
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
                .withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        ObjectListing objectListing;

        do {
            objectListing = s3Operations.listObjects(listObjectsRequest, s3Client);

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                // Ignore 0 byte objects that represent S3 directories.
                if (!(ignoreZeroByteDirectoryMarkers && objectSummary.getKey().endsWith("/")
                        && objectSummary.getSize() == 0L)) {
                    s3ObjectSummaries.add(objectSummary);
                }
            }

            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonS3Exception amazonS3Exception) {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode())) {
            throw new IllegalArgumentException(
                    "The specified bucket '" + params.getS3BucketName() + "' does not exist.",
                    amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    } catch (AmazonClientException e) {
        throw new IllegalStateException(
                String.format("Failed to list keys with prefix \"%s\" from bucket \"%s\". Reason: %s",
                        params.getS3KeyPrefix(), params.getS3BucketName(), e.getMessage()),
                e);
    } finally {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }

    return s3ObjectSummaries;
}

---

From source file:org.geoserver.taskmanager.external.impl.S3FileServiceImpl.java

License:Open Source License

@Override
public List<String> listSubfolders() {
    Set<String> paths = new HashSet<>();
    ObjectListing listing = getS3Client().listObjects(rootFolder);
    for (S3ObjectSummary summary : listing.getObjectSummaries()) {
        String fullPath = summary.getKey();
        int countSeparators = fullPath.length() - fullPath.replace("/", "").length();
        int fromIndex = 0;
        for (int i = 0; i < countSeparators; i++) {
            int indexOfSeparator = fullPath.indexOf("/", fromIndex);
            fromIndex = indexOfSeparator + 1;
            paths.add(fullPath.substring(0, indexOfSeparator));
        }/*from   w ww  . j ava 2  s. c o m*/
    }
    return new ArrayList<>(paths);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• »