List of usage examples for com.amazonaws.services.s3 AmazonS3 listObjects
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws SdkClientException, AmazonServiceException;
Returns a list of summary information about the objects in the specified bucket.
From source file:br.com.unb.aws.client.S3ClientV1.java
License:Open Source License
public static void main(String[] args) throws IOException { /*/*from www . j a v a 2 s.c o m*/ * Create your credentials file at ~/.aws/credentials (C:\Users\USER_NAME\.aws\credentials for Windows users) * and save the following lines after replacing the underlined values with your own. * * [default] * aws_access_key_id = YOUR_ACCESS_KEY_ID * aws_secret_access_key = YOUR_SECRET_ACCESS_KEY */ AmazonS3 s3 = new AmazonS3Client(); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:c3.ops.priam.aws.S3FileIterator.java
License:Apache License
public S3FileIterator(Provider<AbstractBackupPath> pathProvider, AmazonS3 s3Client, String path, Date start, Date till) {/* w ww . j a va 2 s . co m*/ this.start = start; this.till = till; this.pathProvider = pathProvider; ListObjectsRequest listReq = new ListObjectsRequest(); String[] paths = path.split(String.valueOf(S3BackupPath.PATH_SEP)); listReq.setBucketName(paths[0]); listReq.setPrefix(pathProvider.get().remotePrefix(start, till, path)); this.s3Client = s3Client; objectListing = s3Client.listObjects(listReq); iterator = createIterator(); }
From source file:cloudExplorer.BucketClass.java
License:Open Source License
String listBucketContents(String access_key, String secret_key, String bucket, String endpoint) {
objectlist = null;/* w w w . j a va2 s . c om*/
AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
AmazonS3 s3Client = new AmazonS3Client(credentials,
new ClientConfiguration().withSignerOverride("S3SignerType"));
s3Client.setEndpoint(endpoint);
try {
ObjectListing current = s3Client.listObjects((bucket));
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
ObjectListing objectListing;
do {
objectListing = s3Client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
objectlist = objectlist + "@@" + objectSummary.getKey();
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (Exception listBucket) {
mainFrame.jTextArea1.append("\n" + listBucket.getMessage());
}
String parse = null;
if (objectlist != null) {
parse = objectlist;
} else {
parse = "No objects_found.";
}
return parse;
}
From source file:cloudExplorer.BucketClass.java
License:Open Source License
String getObjectInfo(String key, String access_key, String secret_key, String bucket, String endpoint,
String process) {//from ww w . j a v a 2 s. c om
AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
AmazonS3 s3Client = new AmazonS3Client(credentials,
new ClientConfiguration().withSignerOverride("S3SignerType"));
s3Client.setEndpoint(endpoint);
objectlist = null;
try {
ObjectListing current = s3Client.listObjects((bucket));
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
ObjectListing objectListing;
do {
objectListing = s3Client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
if (process.contains("objectsize")) {
if (objectSummary.getKey().contains(key)) {
objectlist = String.valueOf(objectSummary.getSize());
break;
}
}
if (process.contains("objectdate")) {
if (objectSummary.getKey().contains(key)) {
objectlist = String.valueOf(objectSummary.getLastModified());
break;
}
}
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (Exception listBucket) {
mainFrame.jTextArea1.append("\n" + listBucket.getMessage());
}
return objectlist;
}
From source file:cloudtrailviewer.components.S3FileChooserDialog.java
License:Open Source License
private void reloadContents() { List<String> tmp = new ArrayList<String>(); this.files.setAll(tmp); String bucketName = PropertiesSingleton.getInstance().getProperty("Bucket"); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(bucketName); listObjectsRequest.setPrefix(prefix); listObjectsRequest.setDelimiter("/"); AWSCredentials credentials = new BasicAWSCredentials(PropertiesSingleton.getInstance().getProperty("Key"), PropertiesSingleton.getInstance().getProperty("Secret")); AmazonS3 s3Client = new AmazonS3Client(credentials); ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); // these are directories List<String> directories = objectListing.getCommonPrefixes(); for (String directory : directories) { tmp.add(stripPrefix(directory)); }// w w w . j a va 2 s. c o m // these are files List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries(); for (final S3ObjectSummary objectSummary : objectSummaries) { tmp.add(stripPrefix(objectSummary.getKey())); } this.files.setAll(tmp); }
From source file:com.arc.cloud.aws.s3.S3Sample.java
License:Open Source License
public static void main(String[] args) throws IOException { /*// w w w. j a v a 2 s . c o m * The ProfileCredentialsProvider will return your [default] * credential profile by reading from the credentials file located at * (~/.aws/credentials). */ AWSCredentials credentials = null; try { credentials = new ProfileCredentialsProvider().getCredentials(); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (~/.aws/credentials), and is in valid format.", e); } AmazonS3 s3 = new AmazonS3Client(credentials); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.atlassian.localstack.sample.S3Sample.java
License:Open Source License
public static void runTest(AWSCredentials credentials) throws IOException { AmazonS3 s3 = new AmazonS3Client(credentials); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2);/*w ww. j a v a 2s . c o m*/ s3.setEndpoint(LocalstackTestRunner.getEndpointS3()); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println(" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); }
From source file:com.blacklocus.qs.worker.aws.AmazonS3TaskService.java
License:Apache License
public AmazonS3TaskService(String taskHandlerIdentifier, String bucket, String prefix, String delimiter, AmazonS3 s3) { this.taskHandlerIdentifier = taskHandlerIdentifier; this.bucket = bucket; this.prefix = prefix; this.delimiter = delimiter; this.s3 = s3; this.objectListing = s3.listObjects(new ListObjectsRequest(bucket, prefix, null, delimiter, 1000)); this.listingBatchId = IdSupplier.newId(); this.iterator = objectListing.getObjectSummaries().iterator(); }
From source file:com.climate.oada.dao.impl.S3ResourceDAO.java
License:Open Source License
@Override public List<FileResource> getFileUrls(Long userId, String type) { List<FileResource> retval = new ArrayList<FileResource>(); long validfor = new Long(validHours).longValue() * HOURS_TO_MILLISECONDS; try {/*from w ww . j a v a2s. c o m*/ AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); String prefix = userId.toString() + S3_SEPARATOR + type; LOG.debug("Listing objects from bucket " + bucketName + " with prefix " + prefix); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName) .withPrefix(prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { LOG.debug(" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); Date expiration = new Date(); long milliSeconds = expiration.getTime(); milliSeconds += validfor; expiration.setTime(milliSeconds); GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest( bucketName, objectSummary.getKey()); generatePresignedUrlRequest.setMethod(HttpMethod.GET); generatePresignedUrlRequest.setExpiration(expiration); FileResource res = new FileResource(); res.setFileURL(s3client.generatePresignedUrl(generatePresignedUrlRequest)); retval.add(res); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (AmazonServiceException ase) { logAWSServiceException(ase); } catch (AmazonClientException ace) { logAWSClientException(ace); } catch (Exception e) { LOG.error("Unable to retrieve S3 file URLs " + e.getMessage()); } return retval; }
From source file:com.conductor.s3.S3InputFormatUtils.java
License:Apache License
/** * Efficiently gets the Hadoop {@link org.apache.hadoop.fs.FileStatus} for all S3 files under the provided * {@code dirs}/* w w w. j av a2 s. co m*/ * * @param s3Client * s3 client * @param blockSize * the block size * @param dirs * the dirs to search through * @return the {@link org.apache.hadoop.fs.FileStatus} version of all S3 files under {@code dirs} */ static List<FileStatus> getFileStatuses(final AmazonS3 s3Client, final long blockSize, final Path... dirs) { final List<FileStatus> result = Lists.newArrayList(); for (final Path dir : dirs) { // get bucket and prefix from path final String bucket = S3HadoopUtils.getBucketFromPath(dir.toString()); final String prefix = S3HadoopUtils.getKeyFromPath(dir.toString()); // list request final ListObjectsRequest req = new ListObjectsRequest().withMaxKeys(Integer.MAX_VALUE) .withBucketName(bucket).withPrefix(prefix); // recursively page through all objects under the path for (ObjectListing listing = s3Client.listObjects(req); listing.getObjectSummaries() .size() > 0; listing = s3Client.listNextBatchOfObjects(listing)) { for (final S3ObjectSummary summary : listing.getObjectSummaries()) { final Path path = new Path( String.format("s3n://%s/%s", summary.getBucketName(), summary.getKey())); if (S3_PATH_FILTER.accept(path)) { result.add(new FileStatus(summary.getSize(), false, 1, blockSize, summary.getLastModified().getTime(), path)); } } // don't need to check the next listing if this one is not truncated if (!listing.isTruncated()) { break; } } } return result; }