Example usage for com.amazonaws.services.s3.model ListObjectsRequest ListObjectsRequest

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ListObjectsRequest ListObjectsRequest.

Prototype

public ListObjectsRequest(String bucketName, String prefix, String marker, String delimiter, Integer maxKeys) 

Source Link

Document

Constructs a new ListObjectsRequest object and initializes all required and optional object fields.

Usage

From source file:com.proofpoint.event.collector.combiner.S3StorageSystem.java

License:Apache License

@Override
public List<URI> listDirectories(URI storageArea) {
    S3StorageHelper.checkValidS3Uri(storageArea);

    String bucket = getS3Bucket(storageArea);
    String key = getS3ObjectKey(storageArea);
    Iterator<String> iter = new S3PrefixListing(s3Service, new ListObjectsRequest(bucket, key, null, "/", null))
            .iterator();/*from  w w w  .  j ava2s. com*/

    ImmutableList.Builder<URI> builder = ImmutableList.builder();
    while (iter.hasNext()) {
        builder.add(buildS3Location("s3://", bucket, iter.next()));
    }
    return builder.build();
}

---

From source file:com.proofpoint.event.collector.combiner.S3StorageSystem.java

License:Apache License

@Override
public List<StoredObject> listObjects(URI storageArea) {
    S3StorageHelper.checkValidS3Uri(storageArea);

    String s3Path = getS3ObjectKey(storageArea);
    Iterator<S3ObjectSummary> iter = new S3ObjectListing(s3Service,
            new ListObjectsRequest(getS3Bucket(storageArea), s3Path, null, "/", null)).iterator();

    ImmutableList.Builder<StoredObject> builder = ImmutableList.builder();
    while (iter.hasNext()) {
        S3ObjectSummary summary = iter.next();
        builder.add(new StoredObject(buildS3Location(storageArea, summary.getKey().substring(s3Path.length())),
                summary.getETag(), summary.getSize(), summary.getLastModified().getTime()));
    }//from   w w w  .j  a v  a2s .c o  m
    return builder.build();
}

---

From source file:com.proofpoint.event.monitor.s3.S3StorageSystem.java

License:Apache License

public List<URI> listDirectoriesNewestFirst(URI storageArea) {
    S3StorageHelper.checkValidS3Uri(storageArea);

    String bucket = getS3Bucket(storageArea);
    String key = getS3ObjectKey(storageArea);
    Iterator<String> iter = new S3PrefixListing(s3Service, new ListObjectsRequest(bucket, key, null, "/", null))
            .iterator();/*from   w ww.  ja  v  a2  s  .c  om*/

    ImmutableList.Builder<URI> builder = ImmutableList.builder();
    while (iter.hasNext()) {
        builder.add(buildS3Location("s3://", bucket, iter.next()));
    }
    return builder.build().reverse();
}

---

From source file:com.proofpoint.event.monitor.s3.S3StorageSystem.java

License:Apache License

public List<URI> listObjects(URI storageArea) {
    S3StorageHelper.checkValidS3Uri(storageArea);

    String s3Path = getS3ObjectKey(storageArea);
    Iterable<S3ObjectSummary> objectListing = new S3ObjectListing(s3Service,
            new ListObjectsRequest(getS3Bucket(storageArea), s3Path, null, "/", null));

    ImmutableList.Builder<URI> builder = ImmutableList.builder();
    for (S3ObjectSummary summary : objectListing) {
        builder.add(buildS3Location(storageArea, summary.getKey().substring(s3Path.length())));
    }//  w  w w .j  a v  a2  s.  co m
    return builder.build();
}

---

From source file:com.streamsets.pipeline.stage.origin.s3.S3ConnectionSourceConfig.java

License:Apache License

private void validateConnection(Stage.Context context, String configPrefix, List<Stage.ConfigIssue> issues) {
    try {/*  w ww .java  2  s  . c o  m*/
        //check if the credentials are right by trying to list an object in the common prefix
        getS3Client().listObjects(
                new ListObjectsRequest(bucket, commonPrefix, null, delimiter, 1).withEncodingType("url"));
    } catch (AmazonS3Exception e) {
        LOG.debug(Errors.S3_SPOOLDIR_20.getMessage(), e.toString(), e);
        issues.add(context.createConfigIssue(Groups.S3.name(),
                configPrefix + S3ConnectionBaseConfig.AWS_CONFIG_PREFIX + "awsAccessKeyId",
                Errors.S3_SPOOLDIR_20, e.toString()));
    }
}

---

From source file:com.tango.BucketSyncer.KeyListers.S3KeyLister.java

License:Apache License

public S3KeyLister(Object client, String bucket, String prefix, MirrorContext context,
        Integer maxQueueCapacity) {
    super(bucket, prefix, context, maxQueueCapacity);
    this.s3Client = (AmazonS3Client) client;

    final MirrorOptions options = context.getOptions();
    int fetchSize = options.getMaxThreads();
    this.summaries = new ArrayList<S3ObjectSummary>(10 * fetchSize);

    final ListObjectsRequest request = new ListObjectsRequest(bucket, prefix, null, null, fetchSize);
    listing = s3getFirstBatch(s3Client, request);
    synchronized (summaries) {
        final List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
        summaries.addAll(objectSummaries);
        context.getStats().objectsRead.addAndGet(objectSummaries.size());
        if (options.isVerbose()) {
            log.info("added initial set of {} keys", objectSummaries.size());
        }//from  w  ww. j a  v  a 2 s  .c  o m
    }
}

---

From source file:com.treasure_data.td_import.source.S3Source.java

License:Apache License

static List<S3ObjectSummary> getSources(AmazonS3Client client, String bucket, String basePath) {
    String prefix;//from   w w  w. ja  v a  2  s. c o m
    int index = basePath.indexOf('*');
    if (index >= 0) {
        prefix = basePath.substring(0, index);
    } else {
        ObjectMetadata om = client.getObjectMetadata(bucket, basePath);
        S3ObjectSummary s3object = new S3ObjectSummary();
        s3object.setBucketName(bucket);
        s3object.setKey(basePath);
        s3object.setSize(om.getContentLength());

        return Arrays.asList(s3object);
    }

    LOG.info(String.format("list s3 files by client %s: bucket=%s, basePath=%s, prefix=%s", client, bucket,
            basePath, prefix));

    List<S3ObjectSummary> s3objects = new ArrayList<S3ObjectSummary>();
    String lastKey = prefix;
    do {
        ObjectListing listing = client.listObjects(new ListObjectsRequest(bucket, prefix, lastKey, null, 1024));
        for (S3ObjectSummary s3object : listing.getObjectSummaries()) {
            s3objects.add(s3object);
        }
        lastKey = listing.getNextMarker();
    } while (lastKey != null);

    return filterSources(s3objects, basePath);
}

---

From source file:edu.si.services.beans.cameratrap.AmazonS3ClientMock.java

License:Apache License

@Override
public ObjectListing listObjects(String bucketName) throws AmazonClientException, AmazonServiceException {
    //throw new UnsupportedOperationException();
    return listObjects(new ListObjectsRequest(bucketName, null, null, null, null));
}

---

From source file:internal.diff.aws.service.AmazonS3DirectoryMetadataServiceImpl.java

License:Apache License

public void populateMetadata(DirectoryMetadata directoryMetadata, String bucketName, String prefix) {

    String marker = null;//from  w ww  .j  a va2 s  .c  om

    do {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucketName, prefix, marker, "/", null);

        ObjectListing objectListing = this.amazonS3Client.listObjects(listObjectsRequest);

        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {

            if (objectSummary.getKey().equals(prefix)) {
                continue;
            }

            FileMetadata file = new FileMetadata();

            file.setName(objectSummary.getKey().substring(prefix.length()));
            file.setSizeInBytes(objectSummary.getSize());

            if (objectSummary.getETag().contains("-")) {
                file.addChecksum(ChecksumType.S3_MULTIPART_ETAG, objectSummary.getETag());
            } else {
                file.addChecksum(ChecksumType.MD5, objectSummary.getETag());
            }

            directoryMetadata.addFile(file);
        }

        for (String subdirectoryPath : objectListing.getCommonPrefixes()) {

            DirectoryMetadata subdirectory = new DirectoryMetadata();
            subdirectory.setName(subdirectoryPath.substring(prefix.length(), subdirectoryPath.length() - 1));

            directoryMetadata.addSubdirectory(subdirectory);

            populateMetadata(subdirectory, bucketName, subdirectoryPath);
        }

        marker = objectListing.getNextMarker();
    } while (marker != null);
}

---

From source file:io.jeffrey.web.assemble.S3PutObjectTarget.java

/**
 * @param bucket the bucket where we intend to upload the files
 * @param s3     the S3 client/*from   ww  w . j  a  v a  2s .com*/
 */
public S3PutObjectTarget(final String bucket, final AmazonS3 s3) {
    this.bucket = bucket;
    this.s3 = s3;
    this.etags = new HashMap<>();
    String marker = "";
    boolean again = true;
    while (again) {
        again = false;
        ListObjectsRequest request = new ListObjectsRequest(bucket, "", marker, null, 1000);
        for (S3ObjectSummary obj : s3.listObjects(request).getObjectSummaries()) {
            marker = obj.getKey();
            if (marker.startsWith("charts/")) {
                s3.deleteObject(bucket, marker);
            }
            System.out.println(marker + "->" + obj.getETag());
            etags.put(obj.getKey(), obj.getETag());
            again = true;
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• »