List of usage examples for com.amazonaws.services.s3.model ObjectListing getObjectSummaries
public List<S3ObjectSummary> getObjectSummaries()
From source file:com.supprema.utils.S3Sample.java
License:Open Source License
public static void main(String[] args) throws IOException { /*/*from ww w . java 2 s .c o m*/ * The ProfileCredentialsProvider will return your [fabiano-user-s3] * credential profile by reading from the credentials file located at * (/Users/fabianorodriguesmatias/.aws/credentials). */ AWSCredentials credentials = null; try { credentials = new ProfileCredentialsProvider("fabiano-user-s3").getCredentials(); } catch (Exception e) { throw new AmazonClientException( "Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (/Users/fabianorodriguesmatias/.aws/credentials), and is in valid format.", e); } AmazonS3 s3 = new AmazonS3Client(credentials); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.tfnsnproject.util.S3StorageManager.java
License:Open Source License
/** * Deletes the specified S3 object from the S3 storage service. If a * storage path is passed in that has child S3 objects, it will recursively * delete the underlying objects.//from w w w .j a va2 s . co m * * @param s3Store the s3 object to be deleted */ public void delete(S3StorageObject s3Store) { if (s3Store.getStoragePath() == null || s3Store.getStoragePath().equals("")) { logger.log(Level.WARNING, "Empty storage path passed to delete method"); return; // We don't want to delete everything in a path } // Go through the store structure and delete child objects ObjectListing listing = s3client.listObjects(s3Store.getBucketName(), s3Store.getStoragePath()); while (true) { List<S3ObjectSummary> objectList = listing.getObjectSummaries(); for (S3ObjectSummary summary : objectList) { s3client.deleteObject(s3Store.getBucketName(), summary.getKey()); } if (listing.isTruncated()) { listing = s3client.listNextBatchOfObjects(listing); } else { break; } } }
From source file:com.topera.epoch.service.S3Util.java
License:Open Source License
public static void main(String[] args) throws IOException { /*// ww w . j a v a 2 s. c o m * Create your credentials file at ~/.aws/credentials (C:\Users\USER_NAME\.aws\credentials for Windows users) * and save the following lines after replacing the underlined values with your own. * * [default] * aws_access_key_id = YOUR_ACCESS_KEY_ID * aws_secret_access_key = YOUR_SECRET_ACCESS_KEY */ AWSCredentials creds = new AWSCredentials() { public String getAWSSecretKey() { // TODO Auto-generated method stub return "5VVtmI7vcecuVbw8JsG4uo2O1/9RwwLHrTT01Itz"; } public String getAWSAccessKeyId() { // TODO Auto-generated method stub return "AKIAJCMYALI46A2DIPRQ"; } }; AmazonS3 s3 = new AmazonS3Client(creds); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.treasure_data.td_import.source.S3Source.java
License:Apache License
static List<S3ObjectSummary> getSources(AmazonS3Client client, String bucket, String basePath) { String prefix;//w ww . ja v a 2 s . co m int index = basePath.indexOf('*'); if (index >= 0) { prefix = basePath.substring(0, index); } else { ObjectMetadata om = client.getObjectMetadata(bucket, basePath); S3ObjectSummary s3object = new S3ObjectSummary(); s3object.setBucketName(bucket); s3object.setKey(basePath); s3object.setSize(om.getContentLength()); return Arrays.asList(s3object); } LOG.info(String.format("list s3 files by client %s: bucket=%s, basePath=%s, prefix=%s", client, bucket, basePath, prefix)); List<S3ObjectSummary> s3objects = new ArrayList<S3ObjectSummary>(); String lastKey = prefix; do { ObjectListing listing = client.listObjects(new ListObjectsRequest(bucket, prefix, lastKey, null, 1024)); for (S3ObjectSummary s3object : listing.getObjectSummaries()) { s3objects.add(s3object); } lastKey = listing.getNextMarker(); } while (lastKey != null); return filterSources(s3objects, basePath); }
From source file:com.ub.ml.S3Sample.java
License:Open Source License
public static void main(String[] args) throws IOException { /*//from w ww. j a va 2s .c om * Create your credentials file at ~/.aws/credentials (C:\Users\USER_NAME\.aws\credentials for Windows users) * and save the following lines after replacing the underlined values with your own. * * [default] * aws_access_key_id = YOUR_ACCESS_KEY_ID * aws_secret_access_key = YOUR_SECRET_ACCESS_KEY */ AmazonS3 s3 = new AmazonS3Client(); Region usWest2 = Region.getRegion(Regions.US_EAST_1); s3.setRegion(usWest2); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ //System.out.println("Deleting an object\n"); //s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ //System.out.println("Deleting bucket " + bucketName + "\n"); //s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.uiintl.backup.agent.samples.S3Sample.java
License:Open Source License
public static void main2(String[] args) throws IOException { /*/*w ww .ja v a2 s.c o m*/ * This credentials provider implementation loads your AWS credentials * from a properties file at the root of your classpath. * * Important: Be sure to fill in your AWS access credentials in the * AwsCredentials.properties file before you try to run this * sample. * http://aws.amazon.com/security-credentials */ AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); Region usWest2 = Region.getRegion(Regions.AP_SOUTHEAST_2); s3.setRegion(usWest2); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.universal.storage.UniversalS3Storage.java
License:Open Source License
/** * This method wipes the root folder of a storage, basically, will remove all files and folder in it. * Be careful with this method because in too many cases this action won't provide a rollback action. * //from w w w.ja v a2 s. co m * This method loops over the versions for deletion, the bucket will be empty and without any version of its objects. */ public void wipe() throws UniversalIOException { ObjectListing object_listing = this.s3client.listObjects(this.settings.getRoot()); while (true) { for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext();) { S3ObjectSummary summary = (S3ObjectSummary) iterator.next(); this.s3client.deleteObject(this.settings.getRoot(), summary.getKey()); } if (object_listing.isTruncated()) { object_listing = this.s3client.listNextBatchOfObjects(object_listing); } else { break; } } ; VersionListing version_listing = this.s3client .listVersions(new ListVersionsRequest().withBucketName(this.settings.getRoot())); while (true) { for (Iterator<?> iterator = version_listing.getVersionSummaries().iterator(); iterator.hasNext();) { S3VersionSummary vs = (S3VersionSummary) iterator.next(); this.s3client.deleteVersion( this.settings.getRoot(), vs.getKey(), vs.getVersionId()); } if (version_listing.isTruncated()) { version_listing = this.s3client.listNextBatchOfVersions(version_listing); } else { break; } } }
From source file:com.upplication.s3fs.S3Iterator.java
License:Open Source License
/** * add to the listPath the elements at the same level that s3Path * @param listPath List not null list to add * @param current ObjectListing to walk//from w ww.ja va 2s. c o m */ private void parseObjectListing(List<S3Path> listPath, ObjectListing current) { // add all the objects i.e. the files for (final S3ObjectSummary objectSummary : current.getObjectSummaries()) { final String key = objectSummary.getKey(); final S3Path path = new S3Path(s3FileSystem, "/" + bucket, key.split("/")); path.setObjectSummary(objectSummary); listPath.add(path); } // add all the common prefixes i.e. the directories for (final String dir : current.getCommonPrefixes()) { if (dir.equals("/")) continue; listPath.add(new S3Path(s3FileSystem, "/" + bucket, dir)); } }
From source file:com.upplication.s3fs.util.AmazonS3ClientMock.java
License:Open Source License
/** * list all objects without and return ObjectListing with all elements * and with truncated to false//from w ww .j a v a2 s. com */ @Override public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException { ObjectListing objectListing = new ObjectListing(); objectListing.setBucketName(listObjectsRequest.getBucketName()); objectListing.setPrefix(listObjectsRequest.getPrefix()); objectListing.setMarker(listObjectsRequest.getMarker()); objectListing.setDelimiter(listObjectsRequest.getDelimiter()); Bucket bucket = find(listObjectsRequest.getBucketName()); Iterator<S3Element> iterator = objects.get(bucket).iterator(); int i = 0; while (iterator.hasNext()) { S3Element elem = iterator.next(); // TODO. add delimiter and marker support if (listObjectsRequest.getPrefix() != null && elem.getS3Object().getKey().startsWith(listObjectsRequest.getPrefix())) { S3ObjectSummary s3ObjectSummary = parseToS3ObjectSummary(elem); objectListing.getObjectSummaries().add(s3ObjectSummary); if (i + 1 == LIMIT_AWS_MAX_ELEMENTS && iterator.hasNext()) { objectListing.setTruncated(true); objectListing.setNextMarker(iterator.next().getS3Object().getKey()); return objectListing; } else { objectListing.setTruncated(false); } i++; } } return objectListing; }
From source file:com.upplication.s3fs.util.AmazonS3ClientMock.java
License:Open Source License
@Override public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) { ObjectListing objectListing = new ObjectListing(); objectListing.setBucketName(previousObjectListing.getBucketName()); objectListing.setPrefix(previousObjectListing.getPrefix()); objectListing.setMarker(previousObjectListing.getMarker()); objectListing.setDelimiter(previousObjectListing.getDelimiter()); if (!previousObjectListing.isTruncated() || previousObjectListing.getNextMarker() == null) { return objectListing; }// w ww . jav a2s .c o m Bucket bucket = find(previousObjectListing.getBucketName()); Iterator<S3Element> iterator = objects.get(bucket).iterator(); int i = 0; boolean continueElement = false; while (iterator.hasNext()) { S3Element elem = iterator.next(); if (!continueElement && elem.getS3Object().getKey().equals(previousObjectListing.getNextMarker())) { continueElement = true; } if (continueElement) { // TODO. add delimiter and marker support if (previousObjectListing.getPrefix() != null && elem.getS3Object().getKey().startsWith(previousObjectListing.getPrefix())) { S3ObjectSummary s3ObjectSummary = parseToS3ObjectSummary(elem); objectListing.getObjectSummaries().add(s3ObjectSummary); // max 1000 elements at same time. if (i + 1 == LIMIT_AWS_MAX_ELEMENTS && iterator.hasNext()) { objectListing.setTruncated(true); objectListing.setNextMarker(iterator.next().getS3Object().getKey()); return objectListing; } else { objectListing.setTruncated(false); } i++; } } } return objectListing; }