Example usage for com.amazonaws.services.s3.model ObjectListing getObjectSummaries

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ObjectListing getObjectSummaries.

Prototype

public List<S3ObjectSummary> getObjectSummaries() 

Source Link

Document

Gets the list of object summaries describing the objects stored in the S3 bucket.

Usage

From source file:com.netflix.ice.common.AwsUtils.java

License:Apache License

/**
 * List all object summary with given prefix in the s3 bucket.
 * @param bucket/*  w w w.j ava2 s  . c  o  m*/
 * @param prefix
 * @return
 */
public static List<S3ObjectSummary> listAllObjects(String bucket, String prefix, String accountId,
        String assumeRole, String externalId) {
    AmazonS3Client s3Client = AwsUtils.s3Client;

    try {
        ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix);
        List<S3ObjectSummary> result = Lists.newLinkedList();

        if (!StringUtils.isEmpty(accountId) && !StringUtils.isEmpty(assumeRole)) {
            Credentials assumedCredentials = getAssumedCredentials(accountId, assumeRole, externalId);
            s3Client = new AmazonS3Client(
                    new BasicSessionCredentials(assumedCredentials.getAccessKeyId(),
                            assumedCredentials.getSecretAccessKey(), assumedCredentials.getSessionToken()),
                    clientConfig);
        }

        ObjectListing page = null;
        do {
            if (page != null)
                request.setMarker(page.getNextMarker());
            page = s3Client.listObjects(request);
            result.addAll(page.getObjectSummaries());

        } while (page.isTruncated());

        return result;
    } finally {
        if (s3Client != AwsUtils.s3Client)
            s3Client.shutdown();
    }
}

---

From source file:com.netflix.spinnaker.front50.model.S3StorageService.java

License:Apache License

@Override
public <T extends Timestamped> Collection<T> loadObjectsWithPrefix(ObjectType objectType, String prefix,
        int maxResults) {
    ObjectListing bucketListing = amazonS3.listObjects(new ListObjectsRequest(bucket,
            (buildTypedFolder(rootFolder, objectType.group) + "/" + prefix).toLowerCase(), null, null,
            maxResults));/*from w w w.  j av  a 2 s  .  c o m*/
    List<S3ObjectSummary> summaries = bucketListing.getObjectSummaries();

    // TODO-AJ this is naive and inefficient
    return summaries.stream().map(
            s3ObjectSummary -> amazonS3.getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()))
            .map(s3Object -> {
                T item = null;
                try {
                    item = deserialize(s3Object, (Class<T>) objectType.clazz);
                    item.setLastModified(s3Object.getObjectMetadata().getLastModified().getTime());
                } catch (IOException e) {
                    // do nothing
                }
                return item;
            }).collect(Collectors.toSet());
}

---

From source file:com.netflix.spinnaker.front50.model.S3StorageService.java

License:Apache License

@Override
public Map<String, Long> listObjectKeys(ObjectType objectType) {
    ObjectListing bucketListing = amazonS3.listObjects(
            new ListObjectsRequest(bucket, buildTypedFolder(rootFolder, objectType.group), null, null, 10000));
    List<S3ObjectSummary> summaries = bucketListing.getObjectSummaries();

    while (bucketListing.isTruncated()) {
        bucketListing = amazonS3.listNextBatchOfObjects(bucketListing);
        summaries.addAll(bucketListing.getObjectSummaries());
    }// w  w w.jav  a2s . c om

    return summaries.stream().filter(s -> filterS3ObjectSummary(s, objectType.defaultMetadataFilename))
            .collect(Collectors.toMap((s -> buildObjectKey(objectType, s.getKey())),
                    (s -> s.getLastModified().getTime())));
}

---

From source file:com.netflix.spinnaker.front50.model.S3Support.java

License:Apache License

/**
 * Fetch any previously cached applications that have been updated since last retrieved.
 *
 * @param existingItems Previously cached applications
 * @return Refreshed applications/*  w  w w. j  a v a 2 s  .com*/
 */
protected Set<T> fetchAllItems(Set<T> existingItems) {
    if (existingItems == null) {
        existingItems = new HashSet<>();
    }

    Long refreshTime = System.currentTimeMillis();

    ObjectListing bucketListing = amazonS3
            .listObjects(new ListObjectsRequest(bucket, rootFolder, null, null, 10000));
    List<S3ObjectSummary> summaries = bucketListing.getObjectSummaries();

    while (bucketListing.isTruncated()) {
        bucketListing = amazonS3.listNextBatchOfObjects(bucketListing);
        summaries.addAll(bucketListing.getObjectSummaries());
    }

    Map<String, S3ObjectSummary> summariesByName = summaries.stream().filter(this::filterS3ObjectSummary)
            .collect(Collectors.toMap(S3ObjectSummary::getKey, Function.identity()));

    Map<String, T> existingItemsByName = existingItems.stream()
            .filter(a -> summariesByName.containsKey(buildS3Key(a)))
            .collect(Collectors.toMap(Timestamped::getId, Function.identity()));

    summaries = summariesByName.values().stream().filter(s3ObjectSummary -> {
        String itemName = extractItemName(s3ObjectSummary);
        T existingItem = existingItemsByName.get(itemName);

        return existingItem == null || existingItem.getLastModified() == null
                || s3ObjectSummary.getLastModified().after(new Date(existingItem.getLastModified()));
    }).collect(Collectors.toList());

    Observable.from(summaries).buffer(10).flatMap(ids -> Observable.from(ids).flatMap(s3ObjectSummary -> {
        try {
            return Observable
                    .just(amazonS3.getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()));
        } catch (AmazonS3Exception e) {
            if (e.getStatusCode() == 404) {
                // an item has been removed between the time that object summaries were fetched and now
                existingItemsByName.remove(extractItemName(s3ObjectSummary));
                return Observable.empty();
            }

            throw e;
        }
    }).subscribeOn(scheduler)).map(s3Object -> {
        try {
            T item = deserialize(s3Object);
            item.setLastModified(s3Object.getObjectMetadata().getLastModified().getTime());
            return item;
        } catch (IOException e) {
            throw new IllegalStateException(e);
        }
    }).subscribeOn(scheduler).toList().toBlocking().single().forEach(item -> {
        existingItemsByName.put(item.getId().toLowerCase(), item);
    });

    existingItems = existingItemsByName.values().stream().collect(Collectors.toSet());
    this.lastRefreshedTime = refreshTime;
    return existingItems;
}

---

From source file:com.nike.cerberus.operation.core.EnableConfigReplicationOperation.java

License:Apache License

private void touchCurrentFiles() {
    final String bucketName = environmentMetadata.getBucketName();
    final ObjectListing objectListing = s3Client.listObjects(bucketName);

    logger.info("Touching config files that already exist so they are replicated.");
    objectListing.getObjectSummaries().forEach(os -> {
        if (!StringUtils.startsWith(os.getKey(), "consul")) {
            logger.debug("Touching {}.", os.getKey());
            final S3Object object = s3Client.getObject(bucketName, os.getKey());
            s3Client.putObject(bucketName, object.getKey(), object.getObjectContent(),
                    object.getObjectMetadata());
        }/*www. ja v  a2  s .co  m*/
    });
}

---

From source file:com.nike.cerberus.service.S3StoreService.java

License:Apache License

public List<String> listKeysInPartialPath(String path) {
    ObjectListing objectListing = s3Client.listObjects(s3Bucket, getFullPath(path));

    List<String> s3PathKeys = objectListing.getObjectSummaries().stream()
            .map(objectSummary -> StringUtils.stripStart(objectSummary.getKey(), s3Prefix + "/"))
            .collect(Collectors.toList());
    return Collections.unmodifiableList(s3PathKeys);
}

---

From source file:com.nike.cerberus.service.S3StoreService.java

License:Apache License

public void deleteAllKeysOnPartialPath(String path) {
    ObjectListing objectListing = s3Client.listObjects(s3Bucket, getFullPath(path));

    if (objectListing.getObjectSummaries().isEmpty()) {
        return;/*from  w  w  w.j  a va  2s  .  com*/
    }

    List<DeleteObjectsRequest.KeyVersion> keys = objectListing.getObjectSummaries().stream()
            .map(objectSummary -> new DeleteObjectsRequest.KeyVersion(objectSummary.getKey()))
            .collect(Collectors.toList());

    DeleteObjectsRequest request = new DeleteObjectsRequest(s3Bucket);
    request.setKeys(keys);
    s3Client.deleteObjects(request);
}

---

From source file:com.openkm.util.backup.RepositoryS3Backup.java

License:Open Source License

private static boolean exists(AmazonS3 s3, String bucket, String key) {
    ObjectListing list = s3.listObjects(bucket, key);
    return list.getObjectSummaries().size() > 0;
}

---

From source file:com.pinterest.terrapin.TerrapinUtil.java

License:Apache License

static public List<Pair<Path, Long>> getS3FileList(AWSCredentials credentials, String s3Bucket,
        String s3KeyPrefix) {/*from www . j  av a2 s  . co  m*/
    List<Pair<Path, Long>> fileSizePairList = Lists.newArrayListWithCapacity(Constants.MAX_ALLOWED_SHARDS);
    AmazonS3Client s3Client = new AmazonS3Client(credentials);
    // List files and build the path using the s3n: prefix.
    // Note that keys > marker are retrieved where the > is by lexicographic order.
    String prefix = s3KeyPrefix;
    String marker = prefix;
    while (true) {
        boolean reachedEnd = false;
        ObjectListing listing = s3Client
                .listObjects(new ListObjectsRequest().withBucketName(s3Bucket).withMarker(marker));
        List<S3ObjectSummary> summaries = listing.getObjectSummaries();

        if (summaries.isEmpty()) {
            break;
        }

        for (S3ObjectSummary summary : summaries) {
            if (summary.getKey().startsWith(prefix)) {
                fileSizePairList.add(new ImmutablePair(new Path("s3n", s3Bucket, "/" + summary.getKey()),
                        summary.getSize()));
                if (fileSizePairList.size() > Constants.MAX_ALLOWED_SHARDS) {
                    throw new RuntimeException("Too many files " + fileSizePairList.size());
                }
            } else {
                // We found a key which does not match the prefix, stop.
                reachedEnd = true;
                break;
            }
        }
        if (reachedEnd) {
            break;
        }
        marker = summaries.get(summaries.size() - 1).getKey();
    }
    return fileSizePairList;
}

---

From source file:com.proofpoint.event.collector.combiner.S3ObjectListing.java

License:Apache License

@Override
public Iterator<S3ObjectSummary> iterator() {
    Iterator<ObjectListing> objectListings = new AbstractSequentialIterator<ObjectListing>(
            s3Client.listObjects(listObjectsRequest)) {
        @Override/* www  .  jav a2 s  . c o m*/
        protected ObjectListing computeNext(ObjectListing previous) {
            if (!previous.isTruncated()) {
                return null;
            }
            return s3Client.listNextBatchOfObjects(previous);
        }
    };

    return Iterators.concat(
            Iterators.transform(objectListings, new Function<ObjectListing, Iterator<S3ObjectSummary>>() {
                @Override
                public Iterator<S3ObjectSummary> apply(ObjectListing input) {
                    return input.getObjectSummaries().iterator();
                }
            }));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• »