Example usage for com.amazonaws.services.s3 AmazonS3 listNextBatchOfObjects

Introduction

In this page you can find the example usage for com.amazonaws.services.s3 AmazonS3 listNextBatchOfObjects.

Prototype

public ObjectListing listNextBatchOfObjects(ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest)
        throws SdkClientException, AmazonServiceException;

Source Link

Document

Provides an easy way to continue a truncated object listing and retrieve the next page of results.

Usage

From source file:org.deeplearning4j.aws.s3.reader.S3Downloader.java

License:Apache License

/**
 * Paginates through a bucket's keys invoking the listener 
 * at each key//from   ww  w .  j av  a2  s. c  o  m
 * @param bucket the bucket to iterate
 * @param listener the listener
 */
public void paginate(String bucket, BucketKeyListener listener) {
    AmazonS3 s3 = getClient();
    ObjectListing list = s3.listObjects(bucket);
    for (S3ObjectSummary summary : list.getObjectSummaries()) {
        if (listener != null)
            listener.onKey(s3, bucket, summary.getKey());
    }

    while (list.isTruncated()) {
        list = s3.listNextBatchOfObjects(list);
        for (S3ObjectSummary summary : list.getObjectSummaries()) {
            if (listener != null)
                listener.onKey(s3, bucket, summary.getKey());
        }
    }

}

---

From source file:org.elasticsearch.repositories.s3.S3BlobContainer.java

License:Apache License

@Override
public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String blobNamePrefix) throws IOException {
    return AccessController.doPrivileged((PrivilegedAction<Map<String, BlobMetaData>>) () -> {
        MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
        AmazonS3 client = blobStore.client();
        SocketAccess.doPrivilegedVoid(() -> {
            ObjectListing prevListing = null;
            while (true) {
                ObjectListing list;// ww  w. j  a  va  2  s .c  o  m
                if (prevListing != null) {
                    list = client.listNextBatchOfObjects(prevListing);
                } else {
                    if (blobNamePrefix != null) {
                        list = client.listObjects(blobStore.bucket(), buildKey(blobNamePrefix));
                    } else {
                        list = client.listObjects(blobStore.bucket(), keyPath);
                    }
                }
                for (S3ObjectSummary summary : list.getObjectSummaries()) {
                    String name = summary.getKey().substring(keyPath.length());
                    blobsBuilder.put(name, new PlainBlobMetaData(name, summary.getSize()));
                }
                if (list.isTruncated()) {
                    prevListing = list;
                } else {
                    break;
                }
            }
        });
        return blobsBuilder.immutableMap();
    });
}

---

From source file:org.serginho.awss3conn.Connection.java

public List<S3ObjectSummary> getFileList() {
    AmazonS3 s3Client = getS3Client();

    ObjectListing objects = s3Client.listObjects(this.amazonBucket, this.key);

    List<S3ObjectSummary> fileList = new ArrayList<S3ObjectSummary>();

    do {/*from   www.  j a v a2  s.  co  m*/
        for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
            if (objectSummary.getSize() != 0) // We don't need to display the root "key" (folder)
                fileList.add(objectSummary);
        }
        objects = s3Client.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());

    return fileList;
}

---

From source file:org.xmlsh.aws.gradle.s3.DeleteBucketTask.java

License:BSD License

@TaskAction
public void deleteBucket() {
    // to enable conventionMappings feature
    String bucketName = getBucketName();
    boolean ifExists = isIfExists();

    if (bucketName == null)
        throw new GradleException("bucketName is not specified");

    AmazonS3PluginExtension ext = getProject().getExtensions().getByType(AmazonS3PluginExtension.class);
    AmazonS3 s3 = ext.getClient();

    if (ifExists == false || exists(s3)) {
        if (deleteObjects) {
            getLogger().info("Delete all S3 objects in bucket [{}]", bucketName);
            ObjectListing objectListing = s3.listObjects(bucketName);
            while (objectListing.getObjectSummaries().isEmpty() == false) {
                objectListing.getObjectSummaries().forEach(summary -> {
                    getLogger().info(" => delete s3://{}/{}", bucketName, summary.getKey());
                    s3.deleteObject(bucketName, summary.getKey());
                });//w w  w  .j a v a 2 s .c  o  m
                objectListing = s3.listNextBatchOfObjects(objectListing);
            }
        }
        s3.deleteBucket(bucketName);
        getLogger().info("S3 bucket {} is deleted", bucketName);
    } else {
        getLogger().debug("S3 bucket {} does not exist", bucketName);
    }
}

---

From source file:S3Controller.DownloadingImages.java

public static void main(String[] args) throws IOException {
    AWSCredentials credentials = null;/*  w  ww  . j  av a2  s  .  com*/
    String aws_access_key_id = "PUT_YOUR_aws_access_key_id_HERE";
    String aws_secret_access_key = "PUT_YOUR_aws_secret_access_key_HERE";
    try {
        credentials = new BasicAWSCredentials(aws_access_key_id, aws_secret_access_key);//.getCredentials();
    } catch (Exception e) {
        throw new AmazonClientException("Cannot load the credentials from the credential profiles file. "
                + "Please make sure that your credentials file is at the correct "
                + "location (~/.aws/credentials), and is in valid format.", e);
    }

    AmazonS3 s3 = new AmazonS3Client(credentials);
    Region AP_SOUTHEAST_1 = Region.getRegion(Regions.AP_SOUTHEAST_1);
    s3.setRegion(AP_SOUTHEAST_1);

    String bucketName = "PUT_YOUR_S3-BUCKET-NAME_HERE";
    String key = "PUT_YOUR_S3-BUCKET-KEY_HERE";

    try {

        ArrayList arr = new ArrayList();
        ArrayList EmailArray = new ArrayList();
        Bucket bucket = new Bucket(bucketName);
        ObjectListing objects = s3.listObjects(bucket.getName());
        do {
            for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                //                System.out.println(objectSummary.getKey() + "\t" +
                //                        objectSummary.getSize() + "\t" +
                //                        StringUtils.fromDate(objectSummary.getLastModified()));
                arr.add(objectSummary.getKey());
            }
            objects = s3.listNextBatchOfObjects(objects);
        } while (objects.isTruncated());

        KrakenIOExampleMain kraken = new KrakenIOExampleMain();
        for (int i = 0; i < arr.size(); i++) {
            System.out.println("Compressing: " + arr.get(i));
            String s = (String) arr.get(i);
            GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucket.getName(), s);
            System.out.println(s3.generatePresignedUrl(request));
            URL Glink = s3.generatePresignedUrl(request);
            String Dlink = Glink.toString();
            System.out.println("Download Link:" + Dlink);
            kraken.Compression(Dlink, bucketName);
            System.out.println("Compression completed: " + arr.get(i));
            EmailArray.add("Processed Image:" + arr.get(i));
        }
        System.out.println("Start Emailing list");
        EmailSender esender = new EmailSender();
        esender.EmailVerification(GetNotificationEmail, EmailArray);
        System.out.println("Kraken compression completed");
    } catch (AmazonServiceException ase) {
        System.out.println("Caught an AmazonServiceException, which means your request made it "
                + "to Amazon S3, but was rejected with an error response for some reason.");
        System.out.println("Error Message:    " + ase.getMessage());
        System.out.println("HTTP Status Code: " + ase.getStatusCode());
        System.out.println("AWS Error Code:   " + ase.getErrorCode());
        System.out.println("Error Type:       " + ase.getErrorType());
        System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
        System.out.println("Caught an AmazonClientException, which means the client encountered "
                + "a serious internal problem while trying to communicate with S3, "
                + "such as not being able to access the network.");
        System.out.println("Error Message: " + ace.getMessage());
    } catch (ExecutionException ex) {
        Logger.getLogger(DownloadingImages.class.getName()).log(Level.SEVERE, null, ex);
    } catch (InterruptedException ex) {
        Logger.getLogger(DownloadingImages.class.getName()).log(Level.SEVERE, null, ex);
    }
}

---

• «
• 1
• 2
• 3
• »