List of usage examples for com.amazonaws.services.s3.model ObjectListing getObjectSummaries
public List<S3ObjectSummary> getObjectSummaries()
From source file:com.easarrive.aws.plugins.common.service.impl.S3Service.java
License:Open Source License
/** * {@inheritDoc}//from ww w . j av a 2s .co m */ @Override public DeleteObjectsResult deleteObjectsWithPrefix(AmazonS3 client, String bucketName, String prefix) { if (client == null) { return null; } else if (StringUtil.isEmpty(bucketName)) { return null; } else if (StringUtil.isEmpty(prefix)) { return null; } int pre_len = prefix.length(); ObjectListing objectListing = this.getObjectList(client, bucketName, prefix); List<KeyVersion> keyList = new ArrayList<KeyVersion>(); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String key = objectSummary.getKey(); int len = key.length(); if (len < pre_len) continue; int i; for (i = 0; i < pre_len; i++) if (key.charAt(i) != prefix.charAt(i)) break; if (i < pre_len) continue; KeyVersion keyVersion = new KeyVersion(key); keyList.add(keyVersion); } return this.deleteObjects2(client, bucketName, keyList); }
From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java
License:Open Source License
@Override public ListBucketResponseType listBucket(ListBucketType request) throws S3Exception { ListBucketResponseType reply = request.getReply(); User requestUser = getRequestUser(request); OsgInternalS3Client internalS3Client = null; try {//from w w w .j av a 2 s .c o m internalS3Client = getS3Client(requestUser); AmazonS3Client s3Client = internalS3Client.getS3Client(); ListObjectsRequest listRequest = new ListObjectsRequest(); listRequest.setBucketName(request.getBucket()); listRequest.setDelimiter(Strings.isNullOrEmpty(request.getDelimiter()) ? null : request.getDelimiter()); listRequest.setMarker(Strings.isNullOrEmpty(request.getMarker()) ? null : request.getMarker()); listRequest.setMaxKeys((request.getMaxKeys() == null ? null : Integer.parseInt(request.getMaxKeys()))); listRequest.setPrefix(Strings.isNullOrEmpty(request.getPrefix()) ? null : request.getPrefix()); ObjectListing response = s3Client.listObjects(listRequest); /* Non-optional, must have non-null values */ reply.setName(request.getBucket()); reply.setMaxKeys(response.getMaxKeys()); reply.setMarker(response.getMarker() == null ? "" : response.getMarker()); reply.setPrefix(response.getPrefix() == null ? "" : response.getPrefix()); reply.setIsTruncated(response.isTruncated()); /* Optional */ reply.setNextMarker(response.getNextMarker()); reply.setDelimiter(response.getDelimiter()); if (reply.getContents() == null) { reply.setContents(new ArrayList<ListEntry>()); } if (reply.getCommonPrefixesList() == null) { reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); } for (S3ObjectSummary obj : response.getObjectSummaries()) { //Add entry, note that the canonical user is set based on requesting user, not returned user reply.getContents() .add(new ListEntry(obj.getKey(), DateFormatter.dateToHeaderFormattedString(obj.getLastModified()), obj.getETag(), obj.getSize(), getCanonicalUser(requestUser), obj.getStorageClass())); } if (response.getCommonPrefixes() != null && response.getCommonPrefixes().size() > 0) { reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); for (String s : response.getCommonPrefixes()) { reply.getCommonPrefixesList().add(new CommonPrefixesEntry(s)); } } return reply; } catch (AmazonServiceException e) { LOG.debug("Error from backend", e); throw S3ExceptionMapper.fromAWSJavaSDK(e); } }
From source file:com.eucalyptus.vm.Bundles.java
License:Open Source License
static void deleteBucket(final User user, String bucketName, boolean deleteObject) throws ComputeException { final EucaS3Client s3c = EucaS3ClientFactory.getEucaS3Client(user); try {//from ww w.j av a 2 s. c o m final List<Bucket> buckets = s3c.listBuckets(); boolean bucketFound = false; for (final Bucket bucket : buckets) { if (bucketName.equals(bucket.getName())) { bucketFound = true; break; } } if (!bucketFound) { return; } final ObjectListing objects = s3c.listObjects(bucketName); final List<S3ObjectSummary> objectSummaries = objects.getObjectSummaries(); if (!deleteObject && objectSummaries.size() > 0) { throw new ClientComputeException("InvalidParameter", "Bucket is not empty"); } if (deleteObject) { for (final S3ObjectSummary object : objectSummaries) { s3c.deleteObject(bucketName, object.getKey()); } } s3c.deleteBucket(bucketName); } catch (final EucalyptusCloudException ex) { throw ex; } catch (final Exception ex) { LOG.debug("Unable to delete the bucket", ex); throw new ComputeException("InternalError", "Unable to delete the bucket"); } }
From source file:com.example.S3Sample02.java
License:Open Source License
public static void main(String[] args) throws IOException { /*/*from ww w . j a v a 2s . c o m*/ * The ProfileCredentialsProvider will return your [default] * credential profile by reading from the credentials file located at * (~/.aws/credentials). */ AWSCredentials credentials = null; try { credentials = new ProfileCredentialsProvider().getCredentials(); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (~/.aws/credentials), and is in valid format.", e); } AmazonS3 s3 = new AmazonS3Client(credentials); // AP_SOUTHEAST_2 // Region usWest2 = Region.getRegion(Regions.AP_SOUTHEAST_2 ); // s3.setRegion(usWest2); // String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String bucketName = "imos-test-data-1"; String key = "MyObjectKey" + UUID.randomUUID(); System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ // System.out.println("Creating bucket " + bucketName + "\n"); // s3.createBucket(bucketName); /* * List the buckets in your account */ /* System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); */ /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); System.out.println("done\n"); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); System.out.println("done\n"); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); System.out.println("done\n"); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ // System.out.println("Deleting an object\n"); // s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ /* System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); */ } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.exedosoft.plat.storage.s3.S3Sample.java
License:Open Source License
public static void main(String[] args) throws IOException { /*/*from ww w . j a v a2 s . c o m*/ * This credentials provider implementation loads your AWS credentials * from a properties file at the root of your classpath. * * Important: Be sure to fill in your AWS access credentials in the * AwsCredentials.properties file before you try to run this * sample. * http://aws.amazon.com/security-credentials */ AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ // System.out.println("Deleting an object\n"); // s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ // System.out.println("Deleting bucket " + bucketName + "\n"); // s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.facebook.presto.hive.PrestoS3FileSystem.java
License:Apache License
private Function<ObjectListing, Iterator<LocatedFileStatus>> statusFromListing() { return new Function<ObjectListing, Iterator<LocatedFileStatus>>() { @Override/*from w ww. j a v a2 s . c o m*/ public Iterator<LocatedFileStatus> apply(ObjectListing listing) { return Iterators.concat(statusFromPrefixes(listing.getCommonPrefixes()), statusFromObjects(listing.getObjectSummaries())); } }; }
From source file:com.facebook.presto.hive.s3.PrestoS3FileSystem.java
License:Apache License
private Iterator<LocatedFileStatus> statusFromListing(ObjectListing listing) { return Iterators.concat(statusFromPrefixes(listing.getCommonPrefixes()), statusFromObjects(listing.getObjectSummaries())); }
From source file:com.facebook.presto.kinesis.s3config.S3TableConfigClient.java
License:Apache License
/** * Call S3 to get the most recent object list. * * This is an object list request to AWS in the given "directory". * * @return// ww w . j a va 2s.co m */ protected List<S3ObjectSummary> getObjectSummaries() { AmazonS3Client s3client = this.clientManager.getS3Client(); AmazonS3URI directoryURI = new AmazonS3URI(this.bucketUrl); ArrayList<S3ObjectSummary> returnList = new ArrayList<S3ObjectSummary>(); try { log.info("Getting the listing of objects in the S3 table config directory: bucket %s prefix %s :", directoryURI.getBucket(), directoryURI.getKey()); ListObjectsRequest req = new ListObjectsRequest().withBucketName(directoryURI.getBucket()) .withPrefix(directoryURI.getKey() + "/").withDelimiter("/").withMaxKeys(25); ObjectListing result; do { result = s3client.listObjects(req); returnList.addAll(result.getObjectSummaries()); req.setMarker(result.getNextMarker()); } while (result.isTruncated()); log.info("Completed getting S3 object listing."); } catch (AmazonServiceException ase) { StringBuilder sb = new StringBuilder(); sb.append("Caught an AmazonServiceException, which means your request made it "); sb.append("to Amazon S3, but was rejected with an error response for some reason.\n"); sb.append("Error Message: " + ase.getMessage()); sb.append("HTTP Status Code: " + ase.getStatusCode()); sb.append("AWS Error Code: " + ase.getErrorCode()); sb.append("Error Type: " + ase.getErrorType()); sb.append("Request ID: " + ase.getRequestId()); log.error(sb.toString(), ase); } catch (AmazonClientException ace) { StringBuilder sb = new StringBuilder(); sb.append("Caught an AmazonClientException, " + "which means the client encountered " + "an internal error while trying to communicate" + " with S3, " + "such as not being able to access the network."); sb.append("Error Message: " + ace.getMessage()); log.error(sb.toString(), ace); } return returnList; }
From source file:com.ge.predix.sample.blobstore.repository.BlobstoreService.java
License:Apache License
/** * Gets the list of available Blobs for the binded bucket from the BlobStore. * * @return List<BlobFile> List of Blobs *///from w w w . ja v a 2s .c o m public List<S3Object> get() { List<S3Object> objs = new ArrayList<>(); try { // Get the List from BlobStore ObjectListing objectList = s3Client.listObjects(bucket); for (S3ObjectSummary objectSummary : objectList.getObjectSummaries()) { objs.add(s3Client.getObject(new GetObjectRequest(bucket, objectSummary.getKey()))); } } catch (Exception e) { log.error("Exception occurred in get(): " + e.getMessage()); throw e; } return objs; }
From source file:com.ge.predix.solsvc.blobstore.bootstrap.BlobstoreClientImpl.java
License:Apache License
/** * Gets the list of available Blobs for the binded bucket from the BlobStore. * * @return List of String Blobs/*from w w w . java2 s .co m*/ */ @Override public List<String> getAvailableBlobs() { List<String> objs = new ArrayList<>(); try { // Get the List from BlobStore ObjectListing objectList = this.s3Client.listObjects(this.blobstoreConfig.getBucketName()); for (S3ObjectSummary objectSummary : objectList.getObjectSummaries()) { objs.add(objectSummary.getKey()); } } catch (Exception e) { this.log.error("Exception occurred in get(): " + e.getMessage()); //$NON-NLS-1$ throw e; } return objs; }