List of usage examples for com.amazonaws.services.s3 AmazonS3Client listObjects
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
throws SdkClientException, AmazonServiceException
From source file:awslabs.lab41.SolutionCode.java
License:Open Source License
@Override public void removeLabBuckets(AmazonS3Client s3Client, List<String> bucketNames) { for (String bucketName : bucketNames) { try {//www. j av a 2 s . co m ObjectListing objectListing = s3Client .listObjects(new ListObjectsRequest().withBucketName(bucketName)); for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) { DeleteObjectRequest deleteObjectRequest = new DeleteObjectRequest( s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()); s3Client.deleteObject(deleteObjectRequest); } s3Client.deleteBucket(new DeleteBucketRequest(bucketName)); } catch (AmazonS3Exception s3E) { if (!s3E.getErrorCode().equals("NoSuchBucket")) { // This error wasn't expected, so rethrow. throw s3E; } } } }
From source file:backup.store.s3.S3BackupStoreUtil.java
License:Apache License
public static void removeAllObjects(String bucketName) throws Exception { AmazonS3Client client = new AmazonS3Client(new DefaultAWSCredentialsProviderChain()); ObjectListing listObjects = client.listObjects(bucketName); List<S3ObjectSummary> objectSummaries = listObjects.getObjectSummaries(); for (S3ObjectSummary objectSummary : objectSummaries) { String key = objectSummary.getKey(); client.deleteObject(bucketName, key); }/*from w w w . j a va2 s. c o m*/ }
From source file:backup.store.s3.S3BackupStoreUtil.java
License:Apache License
public static void removeAllObjects(String bucketName, String prefix) throws Exception { AmazonS3Client client = new AmazonS3Client(new DefaultAWSCredentialsProviderChain()); ObjectListing listObjects = client.listObjects(bucketName); List<S3ObjectSummary> objectSummaries = listObjects.getObjectSummaries(); for (S3ObjectSummary objectSummary : objectSummaries) { String key = objectSummary.getKey(); if (key.startsWith(prefix)) { client.deleteObject(bucketName, key); }/* w w w . j a v a 2 s .co m*/ } }
From source file:com.appdynamics.monitors.s3.AWSS3Monitor.java
License:Apache License
/** * This method calls Amazon WS to get required S3 statistics, set values * based on configured unit, and returns the result back * /*w w w. j a v a 2s . co m*/ * @param buckets * @param amazonS3Client * @return Map<String, String> * @throws TaskExecutionException */ private Map<String, String> getS3Result(List<Bucket> buckets, AmazonS3Client amazonS3Client) throws TaskExecutionException { // Declaring result variables with default values long size = 0; long count = 0; Date lastModified = new Date(0); try { // Fetching all bucket names if passed buckets is null if (buckets == null) { logger.debug("Calling Webservice to list all buckets"); buckets = amazonS3Client.listBuckets(); } // Looping over all buckets for (Bucket bucket : buckets) { logger.debug("Getting data for bucket: " + bucket.getName()); ObjectListing objectListing = null; do { // Getting objectListing while calling it for the first time if (objectListing == null) { logger.debug("Calling Webservice to get objectlisting for first time"); objectListing = amazonS3Client.listObjects(bucket.getName()); } else { // Calling listNextBatchOfObjects if previous response // is truncated logger.debug("Calling Webservice to get objectlisting subsequent time"); objectListing = amazonS3Client.listNextBatchOfObjects(objectListing); } // Incrementing the count count += objectListing.getObjectSummaries().size(); // Looping over all objects for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) { // Incrementing size size += s3ObjectSummary.getSize(); // Setting last modified if lastModifiedDate is latest if (lastModified.before(s3ObjectSummary.getLastModified())) { lastModified = s3ObjectSummary.getLastModified(); } } } // Continuing till objectListing is complete while (objectListing.isTruncated()); } } catch (AmazonS3Exception exception) { logger.error("AmazonS3Exception occurred", exception); throw new TaskExecutionException("Sending S3 metric failed due to AmazonS3Exception"); } return getResultWithRequiredUnit(size, count, lastModified); }
From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java
License:Open Source License
@Override public ListBucketResponseType listBucket(ListBucketType request) throws S3Exception { ListBucketResponseType reply = request.getReply(); User requestUser = getRequestUser(request); OsgInternalS3Client internalS3Client = null; try {//from www .ja v a 2s. c o m internalS3Client = getS3Client(requestUser); AmazonS3Client s3Client = internalS3Client.getS3Client(); ListObjectsRequest listRequest = new ListObjectsRequest(); listRequest.setBucketName(request.getBucket()); listRequest.setDelimiter(Strings.isNullOrEmpty(request.getDelimiter()) ? null : request.getDelimiter()); listRequest.setMarker(Strings.isNullOrEmpty(request.getMarker()) ? null : request.getMarker()); listRequest.setMaxKeys((request.getMaxKeys() == null ? null : Integer.parseInt(request.getMaxKeys()))); listRequest.setPrefix(Strings.isNullOrEmpty(request.getPrefix()) ? null : request.getPrefix()); ObjectListing response = s3Client.listObjects(listRequest); /* Non-optional, must have non-null values */ reply.setName(request.getBucket()); reply.setMaxKeys(response.getMaxKeys()); reply.setMarker(response.getMarker() == null ? "" : response.getMarker()); reply.setPrefix(response.getPrefix() == null ? "" : response.getPrefix()); reply.setIsTruncated(response.isTruncated()); /* Optional */ reply.setNextMarker(response.getNextMarker()); reply.setDelimiter(response.getDelimiter()); if (reply.getContents() == null) { reply.setContents(new ArrayList<ListEntry>()); } if (reply.getCommonPrefixesList() == null) { reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); } for (S3ObjectSummary obj : response.getObjectSummaries()) { //Add entry, note that the canonical user is set based on requesting user, not returned user reply.getContents() .add(new ListEntry(obj.getKey(), DateFormatter.dateToHeaderFormattedString(obj.getLastModified()), obj.getETag(), obj.getSize(), getCanonicalUser(requestUser), obj.getStorageClass())); } if (response.getCommonPrefixes() != null && response.getCommonPrefixes().size() > 0) { reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); for (String s : response.getCommonPrefixes()) { reply.getCommonPrefixesList().add(new CommonPrefixesEntry(s)); } } return reply; } catch (AmazonServiceException e) { LOG.debug("Error from backend", e); throw S3ExceptionMapper.fromAWSJavaSDK(e); } }
From source file:com.facebook.presto.kinesis.s3config.S3TableConfigClient.java
License:Apache License
/** * Call S3 to get the most recent object list. * * This is an object list request to AWS in the given "directory". * * @return//from w w w. ja v a2s . c o m */ protected List<S3ObjectSummary> getObjectSummaries() { AmazonS3Client s3client = this.clientManager.getS3Client(); AmazonS3URI directoryURI = new AmazonS3URI(this.bucketUrl); ArrayList<S3ObjectSummary> returnList = new ArrayList<S3ObjectSummary>(); try { log.info("Getting the listing of objects in the S3 table config directory: bucket %s prefix %s :", directoryURI.getBucket(), directoryURI.getKey()); ListObjectsRequest req = new ListObjectsRequest().withBucketName(directoryURI.getBucket()) .withPrefix(directoryURI.getKey() + "/").withDelimiter("/").withMaxKeys(25); ObjectListing result; do { result = s3client.listObjects(req); returnList.addAll(result.getObjectSummaries()); req.setMarker(result.getNextMarker()); } while (result.isTruncated()); log.info("Completed getting S3 object listing."); } catch (AmazonServiceException ase) { StringBuilder sb = new StringBuilder(); sb.append("Caught an AmazonServiceException, which means your request made it "); sb.append("to Amazon S3, but was rejected with an error response for some reason.\n"); sb.append("Error Message: " + ase.getMessage()); sb.append("HTTP Status Code: " + ase.getStatusCode()); sb.append("AWS Error Code: " + ase.getErrorCode()); sb.append("Error Type: " + ase.getErrorType()); sb.append("Request ID: " + ase.getRequestId()); log.error(sb.toString(), ase); } catch (AmazonClientException ace) { StringBuilder sb = new StringBuilder(); sb.append("Caught an AmazonClientException, " + "which means the client encountered " + "an internal error while trying to communicate" + " with S3, " + "such as not being able to access the network."); sb.append("Error Message: " + ace.getMessage()); log.error(sb.toString(), ace); } return returnList; }
From source file:com.netflix.exhibitor.core.s3.S3ClientFactoryImpl.java
License:Apache License
@Override public S3Client makeNewClient(final S3Credential credentials) throws Exception { return new S3Client() { private final AtomicReference<RefCountedClient> client = new AtomicReference<RefCountedClient>(null); {//from w w w . ja v a 2 s .c om changeCredentials(credentials); } @Override public void changeCredentials(S3Credential credential) throws Exception { RefCountedClient newRefCountedClient = (credential != null) ? new RefCountedClient( new AmazonS3Client(new BasicAWSCredentials(credentials.getAccessKeyId(), credentials.getSecretAccessKey()))) : null; RefCountedClient oldRefCountedClient = client.getAndSet(newRefCountedClient); if (oldRefCountedClient != null) { oldRefCountedClient.markForDelete(); } } @Override public void close() throws IOException { try { changeCredentials(null); } catch (Exception e) { throw new IOException(e); } } @Override public InitiateMultipartUploadResult initiateMultipartUpload(InitiateMultipartUploadRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.initiateMultipartUpload(request); } finally { holder.release(); } } @Override public PutObjectResult putObject(PutObjectRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.putObject(request); } finally { holder.release(); } } @Override public S3Object getObject(String bucket, String key) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.getObject(bucket, key); } finally { holder.release(); } } @Override public ObjectListing listObjects(ListObjectsRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.listObjects(request); } finally { holder.release(); } } @Override public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.listNextBatchOfObjects(previousObjectListing); } finally { holder.release(); } } @Override public void deleteObject(String bucket, String key) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { amazonS3Client.deleteObject(bucket, key); } finally { holder.release(); } } @Override public UploadPartResult uploadPart(UploadPartRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.uploadPart(request); } finally { holder.release(); } } @Override public void completeMultipartUpload(CompleteMultipartUploadRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { amazonS3Client.completeMultipartUpload(request); } finally { holder.release(); } } @Override public void abortMultipartUpload(AbortMultipartUploadRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { amazonS3Client.abortMultipartUpload(request); } finally { holder.release(); } } }; }
From source file:com.netflix.exhibitor.core.s3.S3ClientImpl.java
License:Apache License
@Override public ObjectListing listObjects(ListObjectsRequest request) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try {//from w ww . jav a2s .com return amazonS3Client.listObjects(request); } finally { holder.release(); } }
From source file:com.netflix.ice.common.AwsUtils.java
License:Apache License
/** * List all object summary with given prefix in the s3 bucket. * @param bucket/*from w w w . jav a2s .c o m*/ * @param prefix * @return */ public static List<S3ObjectSummary> listAllObjects(String bucket, String prefix, String accountId, String assumeRole, String externalId) { AmazonS3Client s3Client = AwsUtils.s3Client; try { ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix); List<S3ObjectSummary> result = Lists.newLinkedList(); if (!StringUtils.isEmpty(accountId) && !StringUtils.isEmpty(assumeRole)) { Credentials assumedCredentials = getAssumedCredentials(accountId, assumeRole, externalId); s3Client = new AmazonS3Client( new BasicSessionCredentials(assumedCredentials.getAccessKeyId(), assumedCredentials.getSecretAccessKey(), assumedCredentials.getSessionToken()), clientConfig); } ObjectListing page = null; do { if (page != null) request.setMarker(page.getNextMarker()); page = s3Client.listObjects(request); result.addAll(page.getObjectSummaries()); } while (page.isTruncated()); return result; } finally { if (s3Client != AwsUtils.s3Client) s3Client.shutdown(); } }
From source file:com.pinterest.terrapin.TerrapinUtil.java
License:Apache License
static public List<Pair<Path, Long>> getS3FileList(AWSCredentials credentials, String s3Bucket, String s3KeyPrefix) {/* ww w . java2 s . co m*/ List<Pair<Path, Long>> fileSizePairList = Lists.newArrayListWithCapacity(Constants.MAX_ALLOWED_SHARDS); AmazonS3Client s3Client = new AmazonS3Client(credentials); // List files and build the path using the s3n: prefix. // Note that keys > marker are retrieved where the > is by lexicographic order. String prefix = s3KeyPrefix; String marker = prefix; while (true) { boolean reachedEnd = false; ObjectListing listing = s3Client .listObjects(new ListObjectsRequest().withBucketName(s3Bucket).withMarker(marker)); List<S3ObjectSummary> summaries = listing.getObjectSummaries(); if (summaries.isEmpty()) { break; } for (S3ObjectSummary summary : summaries) { if (summary.getKey().startsWith(prefix)) { fileSizePairList.add(new ImmutablePair(new Path("s3n", s3Bucket, "/" + summary.getKey()), summary.getSize())); if (fileSizePairList.size() > Constants.MAX_ALLOWED_SHARDS) { throw new RuntimeException("Too many files " + fileSizePairList.size()); } } else { // We found a key which does not match the prefix, stop. reachedEnd = true; break; } } if (reachedEnd) { break; } marker = summaries.get(summaries.size() - 1).getKey(); } return fileSizePairList; }