Example usage for com.amazonaws.services.s3.model S3ObjectSummary getKey

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model S3ObjectSummary getKey.

Prototype

public String getKey() 

Source Link

Document

Gets the key under which this object is stored in Amazon S3.

Usage

From source file:org.deeplearning4j.aws.s3.reader.S3Downloader.java

License:Apache License

/**
 * Return the keys for a bucket/* w  ww . j  ava 2 s.  c om*/
 * @param bucket the bucket to get the keys for
 * @return the bucket's keys
 */
public List<String> keysForBucket(String bucket) {
    AmazonS3 s3 = getClient();
    List<String> ret = new ArrayList<>();
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
    ObjectListing objectListing;

    do {
        objectListing = s3.listObjects(listObjectsRequest);
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            ret.add(objectSummary.getKey());
        }
        listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());

    return ret;
}

---

From source file:org.deeplearning4j.aws.s3.reader.S3Downloader.java

License:Apache License

/**
 * Paginates through a bucket's keys invoking the listener 
 * at each key/*from www . j a  va  2  s . c  o m*/
 * @param bucket the bucket to iterate
 * @param listener the listener
 */
public void paginate(String bucket, BucketKeyListener listener) {
    AmazonS3 s3 = getClient();
    ObjectListing list = s3.listObjects(bucket);
    for (S3ObjectSummary summary : list.getObjectSummaries()) {
        if (listener != null)
            listener.onKey(s3, bucket, summary.getKey());
    }

    while (list.isTruncated()) {
        list = s3.listNextBatchOfObjects(list);
        for (S3ObjectSummary summary : list.getObjectSummaries()) {
            if (listener != null)
                listener.onKey(s3, bucket, summary.getKey());
        }
    }

}

---

From source file:org.duracloud.s3storage.S3StorageProvider.java

License:Apache License

private List<String> getCompleteBucketContents(String bucketName, String prefix, long maxResults,
        String marker) {//from ww w  .j  av a2  s. c  o  m
    List<String> contentItems = new ArrayList<>();

    List<S3ObjectSummary> objects = listObjects(bucketName, prefix, maxResults, marker);
    for (S3ObjectSummary object : objects) {
        contentItems.add(object.getKey());
    }
    return contentItems;
}

---

From source file:org.elasticsearch.cloud.aws.blobstore.AbstarctS3BlobContainer.java

License:Apache License

@Override
public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(@Nullable String blobNamePrefix)
        throws IOException {
    ImmutableMap.Builder<String, BlobMetaData> blobsBuilder = ImmutableMap.builder();
    ObjectListing prevListing = null;//w  w  w  . java2 s  .  c  om
    while (true) {
        ObjectListing list;
        if (prevListing != null) {
            list = blobStore.client().listNextBatchOfObjects(prevListing);
        } else {
            if (blobNamePrefix != null) {
                list = blobStore.client().listObjects(blobStore.bucket(), buildKey(blobNamePrefix));
            } else {
                list = blobStore.client().listObjects(blobStore.bucket(), keyPath);
            }
        }
        for (S3ObjectSummary summary : list.getObjectSummaries()) {
            String name = summary.getKey().substring(keyPath.length());
            blobsBuilder.put(name, new PlainBlobMetaData(name, summary.getSize()));
        }
        if (list.isTruncated()) {
            prevListing = list;
        } else {
            break;
        }
    }
    return blobsBuilder.build();
}

---

From source file:org.elasticsearch.cloud.aws.blobstore.S3BlobStore.java

License:Apache License

@Override
public void delete(BlobPath path) {
    ObjectListing prevListing = null;//from   w w  w  .  j av a  2 s.  c  om
    //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
    //we can do at most 1K objects per delete
    //We don't know the bucket name until first object listing
    DeleteObjectsRequest multiObjectDeleteRequest = null;
    ArrayList<KeyVersion> keys = new ArrayList<KeyVersion>();
    while (true) {
        ObjectListing list;
        if (prevListing != null) {
            list = client.listNextBatchOfObjects(prevListing);
        } else {
            String keyPath = path.buildAsString("/");
            if (!keyPath.isEmpty()) {
                keyPath = keyPath + "/";
            }
            list = client.listObjects(bucket, keyPath);
            multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
        }
        for (S3ObjectSummary summary : list.getObjectSummaries()) {
            keys.add(new KeyVersion(summary.getKey()));
            //Every 500 objects batch the delete request
            if (keys.size() > 500) {
                multiObjectDeleteRequest.setKeys(keys);
                client.deleteObjects(multiObjectDeleteRequest);
                multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
                keys.clear();
            }
        }
        if (list.isTruncated()) {
            prevListing = list;
        } else {
            break;
        }
    }
    if (!keys.isEmpty()) {
        multiObjectDeleteRequest.setKeys(keys);
        client.deleteObjects(multiObjectDeleteRequest);
    }
}

---

From source file:org.elasticsearch.repositories.s3.S3BlobContainer.java

License:Apache License

@Override
public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String blobNamePrefix) throws IOException {
    return AccessController.doPrivileged((PrivilegedAction<Map<String, BlobMetaData>>) () -> {
        MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
        AmazonS3 client = blobStore.client();
        SocketAccess.doPrivilegedVoid(() -> {
            ObjectListing prevListing = null;
            while (true) {
                ObjectListing list;//from  ww  w  .  jav a2  s  .  co m
                if (prevListing != null) {
                    list = client.listNextBatchOfObjects(prevListing);
                } else {
                    if (blobNamePrefix != null) {
                        list = client.listObjects(blobStore.bucket(), buildKey(blobNamePrefix));
                    } else {
                        list = client.listObjects(blobStore.bucket(), keyPath);
                    }
                }
                for (S3ObjectSummary summary : list.getObjectSummaries()) {
                    String name = summary.getKey().substring(keyPath.length());
                    blobsBuilder.put(name, new PlainBlobMetaData(name, summary.getSize()));
                }
                if (list.isTruncated()) {
                    prevListing = list;
                } else {
                    break;
                }
            }
        });
        return blobsBuilder.immutableMap();
    });
}

---

From source file:org.elasticsearch.repositories.s3.S3BlobStore.java

License:Apache License

@Override
public void delete(BlobPath path) {
    AccessController.doPrivileged((PrivilegedAction<Object>) () -> {
        ObjectListing prevListing = null;
        //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
        //we can do at most 1K objects per delete
        //We don't know the bucket name until first object listing
        DeleteObjectsRequest multiObjectDeleteRequest = null;
        ArrayList<KeyVersion> keys = new ArrayList<>();
        while (true) {
            ObjectListing list;/*from w w w  .  java 2s. com*/
            if (prevListing != null) {
                list = client.listNextBatchOfObjects(prevListing);
            } else {
                list = client.listObjects(bucket, path.buildAsString());
                multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
            }
            for (S3ObjectSummary summary : list.getObjectSummaries()) {
                keys.add(new KeyVersion(summary.getKey()));
                //Every 500 objects batch the delete request
                if (keys.size() > 500) {
                    multiObjectDeleteRequest.setKeys(keys);
                    client.deleteObjects(multiObjectDeleteRequest);
                    multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
                    keys.clear();
                }
            }
            if (list.isTruncated()) {
                prevListing = list;
            } else {
                break;
            }
        }
        if (!keys.isEmpty()) {
            multiObjectDeleteRequest.setKeys(keys);
            client.deleteObjects(multiObjectDeleteRequest);
        }
        return null;
    });
}

---

From source file:org.entando.entando.plugins.jps3awsclient.aps.system.services.storage.AmazonS3StorageManager.java

License:Open Source License

private String[] list(String subPath, boolean isProtectedResource, boolean addFolders, boolean addFiles)
        throws ApsSystemException {
    if (!this.isActive()) {
        return null;
    }/*from ww w  .  j  a va 2  s  . c  om*/
    String folder = this.getKey(subPath, isProtectedResource);
    ObjectListing objectListing = this.getS3Objects(folder);
    if (null == objectListing) {
        return null;
    }
    String[] objects = new String[] {};
    if (null != objectListing.getCommonPrefixes() && addFolders) {
        for (int i = 0; i < objectListing.getCommonPrefixes().size(); i++) {
            String object = objectListing.getCommonPrefixes().get(i);
            String name = object.substring(folder.length(), (object.length() - 1));
            objects = this.addChild(name, objects);
        }
    }
    if (null != objectListing.getObjectSummaries() && addFiles) {
        for (int i = 0; i < objectListing.getObjectSummaries().size(); i++) {
            S3ObjectSummary s3os = objectListing.getObjectSummaries().get(i);
            String key = s3os.getKey();
            String name = key.substring(folder.length());
            objects = this.addChild(name, objects);
        }
    }
    return objects;
}

---

From source file:org.entando.entando.plugins.jps3awsclient.aps.system.services.storage.AmazonS3StorageManager.java

License:Open Source License

private BasicFileAttributeView[] listAttributes(String subPath, boolean isProtectedResource, boolean addFolders,
        boolean addFiles) throws ApsSystemException {
    if (!this.isActive()) {
        return null;
    }//from   w w  w . j  a  va  2 s. c o m
    String folder = this.getKey(subPath, isProtectedResource);
    ObjectListing objectListing = this.getS3Objects(folder);
    if (null == objectListing) {
        return null;
    }
    BasicFileAttributeView[] objects = new BasicFileAttributeView[] {};
    if (null != objectListing.getCommonPrefixes() && addFolders) {
        for (int i = 0; i < objectListing.getCommonPrefixes().size(); i++) {
            String object = objectListing.getCommonPrefixes().get(i);
            String name = object.substring(folder.length(), (object.length() - 1));
            BasicFileAttributeView bfav = new BasicFileAttributeView();
            bfav.setDirectory(true);
            bfav.setName(name);
            objects = this.addChildAttribute(bfav, objects);
        }
    }
    if (null != objectListing.getObjectSummaries() && addFiles) {
        for (int i = 0; i < objectListing.getObjectSummaries().size(); i++) {
            S3ObjectSummary s3os = objectListing.getObjectSummaries().get(i);
            String key = s3os.getKey();
            String name = key.substring(folder.length());
            BasicFileAttributeView bfav = new BasicFileAttributeView();
            bfav.setDirectory(false);
            bfav.setName(name);
            bfav.setLastModifiedTime(s3os.getLastModified());
            bfav.setSize(s3os.getSize());
            objects = this.addChildAttribute(bfav, objects);
        }
    }
    return objects;
}

---

From source file:org.entando.entando.plugins.jps3awsclient.aps.system.services.storage.AmazonS3StorageManager.java

License:Open Source License

public void delete(String bucketName, String key) throws ApsSystemException {
    if (!this.isActive()) {
        return;//from  w ww. j  ava  2 s.co  m
    }
    try {
        if (key == null || key.trim().length() == 0) {
            _logger.warn("Empty storage path passed to delete method");
            return; // We don't want to delete everything in a path
        }
        AmazonS3Client client = this.getS3Client();
        // Go through the store structure and delete child objects
        ObjectListing listing = this.getS3Client().listObjects(bucketName, key);
        while (true) {
            List<S3ObjectSummary> objectList = listing.getObjectSummaries();
            for (int i = 0; i < objectList.size(); i++) {
                S3ObjectSummary summary = objectList.get(i);
                client.deleteObject(bucketName, summary.getKey());
            }
            if (listing.isTruncated()) {
                listing = client.listNextBatchOfObjects(listing);
            } else {
                break;
            }
        }
    } catch (Throwable t) {
        _logger.error("Error deleting objects : bucket {} - key {}", bucketName, key, t);
        throw new ApsSystemException("Error deleting objects : bucket " + bucketName + " - key " + key, t);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• »