List of usage examples for com.amazonaws.services.s3.model ObjectListing getObjectSummaries
public List<S3ObjectSummary> getObjectSummaries()
From source file:com.images3.data.impl.ImageContentAccessImplS3.java
License:Apache License
private void deleteAllImageContent(AmazonS3 client, AmazonS3Bucket bucket, ObjectListing objList) { boolean isFinished = false; while (!isFinished) { DeleteObjectsRequest request = new DeleteObjectsRequest(bucket.getName()); List<KeyVersion> keys = new ArrayList<KeyVersion>(objList.getMaxKeys()); for (S3ObjectSummary sum : objList.getObjectSummaries()) { keys.add(new KeyVersion(sum.getKey())); }/*from ww w . ja va 2 s.co m*/ request.setKeys(keys); client.deleteObjects(request); if (objList.isTruncated()) { objList = client.listNextBatchOfObjects(objList); } else { isFinished = true; } } }
From source file:com.imos.sample.S3SampleCheck.java
License:Open Source License
public static void main(String[] args) throws IOException { /*/*from www. ja v a2 s. c o m*/ * The ProfileCredentialsProvider will return your [default] * credential profile by reading from the credentials file located at * (/home/alok/.aws/credentials). */ AWSCredentials credentials = null; try { credentials = new ProfileCredentialsProvider("default").getCredentials(); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (/home/alok/.aws/credentials), and is in valid format.", e); } AmazonS3 s3 = new AmazonS3Client(credentials); // Region usWest2 = Region.getRegion(Regions.US_WEST_2); Region usWest2 = Region.getRegion(Regions.AP_SOUTHEAST_1); s3.setRegion(usWest2); String bucketName = "alok-test"; String key = "sample.json"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ // System.out.println("Creating bucket " + bucketName + "\n"); // s3.createBucket(bucketName); /* * List the buckets in your account */ // System.out.println("Listing buckets"); // for (Bucket bucket : s3.listBuckets()) { // System.out.println(" - " + bucket.getName()); // } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); //s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); // S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); S3Object object = s3.getObject(new GetObjectRequest("alok-test", key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest() // .withBucketName(bucketName) .withBucketName("alok-test")); // .withPrefix("My")); objectListing.getObjectSummaries().forEach((objectSummary) -> { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); }); System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ // System.out.println("Deleting an object\n"); // s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ // System.out.println("Deleting bucket " + bucketName + "\n"); // s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.indeed.imhotep.iql.cache.S3QueryCache.java
License:Apache License
@Override public boolean isFileCached(String fileName) { if (!enabled) { return false; }//w w w .j a va 2 s . c om try { ObjectListing objs; objs = client.listObjects(bucket, fileName); return !objs.getObjectSummaries().isEmpty(); } catch (Exception e) { return false; } }
From source file:com.intuit.s3encrypt.S3Encrypt.java
License:Open Source License
public static void main(String[] args) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { // create Options object Options options = new Options(); options.addOption(create_bucket);/*from w ww . j a v a 2 s . com*/ options.addOption(create_key); options.addOption(delete_bucket); options.addOption(get); options.addOption(help); options.addOption(inspect); options.addOption(keyfile); options.addOption(list_buckets); options.addOption(list_objects); options.addOption(put); options.addOption(remove); options.addOption(rotate); options.addOption(rotateall); options.addOption(rotateKey); // CommandLineParser parser = new GnuParser(); // Changed from above GnuParser to below PosixParser because I found code which allows for multiple arguments PosixParser parser = new PosixParser(); CommandLine cmd; try { cmd = parser.parse(options, args); Logger.getRootLogger().setLevel(Level.OFF); if (cmd.hasOption("help")) { HelpFormatter help = new HelpFormatter(); System.out.println(); help.printHelp("S3Encrypt", options); System.out.println(); System.exit(1); } else if (cmd.hasOption("create_key")) { keyname = cmd.getOptionValue("keyfile"); createKeyFile(keyname); key = new File(keyname); } else { if (cmd.hasOption("keyfile")) { keyname = cmd.getOptionValue("keyfile"); } key = new File(keyname); } if (!(key.exists())) { System.out.println("Key does not exist or not provided"); System.exit(1); } // AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); ClasspathPropertiesFileCredentialsProvider credentials = new ClasspathPropertiesFileCredentialsProvider( ".s3encrypt"); EncryptionMaterials encryptionMaterials = new EncryptionMaterials(getKeyFile(keyname)); AmazonS3EncryptionClient s3 = new AmazonS3EncryptionClient(credentials.getCredentials(), encryptionMaterials); // Region usWest2 = Region.getRegion(Regions.US_WEST_2); // s3.setRegion(usWest2); if (cmd.hasOption("create_bucket")) { String bucket = cmd.getOptionValue("create_bucket"); System.out.println("Creating bucket " + bucket + "\n"); s3.createBucket(bucket); } else if (cmd.hasOption("delete_bucket")) { String bucket = cmd.getOptionValue("delete_bucket"); System.out.println("Deleting bucket " + bucket + "\n"); s3.deleteBucket(bucket); } else if (cmd.hasOption("get")) { String[] searchArgs = cmd.getOptionValues("get"); String bucket = searchArgs[0]; String filename = searchArgs[1]; getS3Object(cmd, s3, bucket, filename); } else if (cmd.hasOption("inspect")) { String[] searchArgs = cmd.getOptionValues("inspect"); String bucket = searchArgs[0]; String filename = searchArgs[1]; String keyname = "encryption_key"; String metadata = inspectS3Object(cmd, s3, bucket, filename, keyname); System.out.println(metadata); } else if (cmd.hasOption("list_buckets")) { System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(bucket.getName()); } System.out.println(); } else if (cmd.hasOption("list_objects")) { String bucket = cmd.getOptionValue("list_objects"); System.out.println("Listing objects"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucket)); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println(objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); } else if (cmd.hasOption("put")) { String[] searchArgs = cmd.getOptionValues("put"); String bucket = searchArgs[0]; String filename = searchArgs[1]; String metadataKeyname = "encryption_key"; String key = keyname; putS3Object(cmd, s3, bucket, filename, metadataKeyname, key); } else if (cmd.hasOption("remove")) { String[] searchArgs = cmd.getOptionValues("remove"); String bucket = searchArgs[0]; String filename = searchArgs[1]; System.out.println("Removing object in S3 from BUCKET = " + bucket + " FILENAME = " + filename); s3.deleteObject(new DeleteObjectRequest(bucket, filename)); System.out.println(); } else if (cmd.hasOption("rotate")) { String[] searchArgs = cmd.getOptionValues("rotate"); String bucket = searchArgs[0]; String filename = searchArgs[1]; String key1 = cmd.getOptionValue("keyfile"); String key2 = cmd.getOptionValue("rotateKey"); String metadataKeyname = "encryption_key"; System.out.println("Supposed to get object from here OPTION VALUE = " + bucket + " FILENAME = " + filename + " KEY1 = " + key1 + " KEY2 = " + key2); EncryptionMaterials rotateEncryptionMaterials = new EncryptionMaterials(getKeyFile(key2)); AmazonS3EncryptionClient rotateS3 = new AmazonS3EncryptionClient(credentials.getCredentials(), rotateEncryptionMaterials); getS3Object(cmd, s3, bucket, filename); putS3Object(cmd, rotateS3, bucket, filename, metadataKeyname, key2); } else if (cmd.hasOption("rotateall")) { String[] searchArgs = cmd.getOptionValues("rotateall"); String bucket = searchArgs[0]; String key1 = searchArgs[1]; String key2 = searchArgs[2]; System.out.println("Supposed to rotateall here for BUCKET NAME = " + bucket + " KEY1 = " + key1 + " KEY2 = " + key2); } else { System.out.println("Something went wrong... "); System.exit(1); } } catch (ParseException e) { e.printStackTrace(); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which " + "means your request made it " + "to Amazon S3, but was rejected with an error response" + " for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which " + "means the client encountered " + "an internal error while trying to " + "communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.ipcglobal.fredimportaws.TsvsToRedshift.java
License:Apache License
/** * Empty bucket contents.// w ww.j a va 2 s.com * * @throws Exception the exception */ private void emptyBucketContents() throws Exception { try { while (true) { ObjectListing objectListing = s3Client.listObjects(new ListObjectsRequest() .withBucketName(awsBucketName).withPrefix(awsBucketTsvPrefix + "/").withDelimiter("/")); if (objectListing.getObjectSummaries().size() == 0) break; List<KeyVersion> keyVersions = new ArrayList<KeyVersion>(); for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) keyVersions.add(new KeyVersion(s3ObjectSummary.getKey())); if (keyVersions.size() > 0) { DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(awsBucketName); deleteObjectsRequest.setKeys(keyVersions); s3Client.deleteObjects(deleteObjectsRequest); } } } catch (AmazonServiceException ase) { log.error(ase); throw ase; } catch (AmazonClientException ace) { log.error(ace); throw ace; } }
From source file:com.jeet.s3.AmazonS3ClientWrapper.java
License:Open Source License
public List<String> getListOfFiles(String basepath) { try {/*ww w.j ava2 s .co m*/ List<String> keys = new ArrayList(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(Constants.BUCKET_NAME); // listObjectsRequest.setDelimiter(basepath); listObjectsRequest.setPrefix(basepath); ObjectListing objectListing; do { objectListing = s3Client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // System.out.println(" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); keys.add(objectSummary.getKey()); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); return keys; } catch (Exception e) { e.printStackTrace(); } return new ArrayList<>(); }
From source file:com.jfixby.scarabei.red.aws.test.S3Sample.java
License:Open Source License
public static void main(final String[] args) throws IOException { /*//from w ww . jav a 2s. com * The ProfileCredentialsProvider will return your [default] credential profile by reading from the credentials file located * at (C:\\Users\\JCode\\.aws\\credentials). */ AWSCredentials credentials = null; try { credentials = new ProfileCredentialsProvider("default").getCredentials(); } catch (final Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (C:\\Users\\%USERNAME%\\.aws\\credentials), and is in valid format.", e); } final AmazonS3 s3 = new AmazonS3Client(credentials); final Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); final String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); final String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, so once a bucket name has been taken by any user, * you can't create another bucket with that same name. * * You can optionally specify a location for your bucket if you want to keep your data closer to your applications or * users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (final Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to S3, or upload directly an InputStream if you know * the length of the data in the stream. You can also specify your own metadata when uploading to S3, which allows you * set a variety of options like content-type and content-encoding, plus additional metadata specific to your * applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of the object's metadata and a stream from which to read * the contents. It's important to read the contents of the stream as quickly as possibly since the data is streamed * directly from Amazon S3 and your network connection will remain open until you read all the data or close the input * stream. * * GetObjectRequest also supports several other options, including conditional downloading of objects based on * modification times, ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); final S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for listing the objects in your bucket. Keep in mind * that buckets with many objects might truncate their results when listing their objects, so be sure to check if the * returned object listing is truncated, and use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); final ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (final S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, there is no way to undelete an object, so use * caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be deleted, so remember to delete any objects from * your buckets before you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); } catch (final AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (final AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.jktsoftware.amazondownloader.download.S3TypeBucket.java
License:Open Source License
public List<IObject> getObjectsInRepo() { String repoid = getRepoId();/* ww w. j a v a 2 s . c om*/ AWSCredentials awscredentials = new BasicAWSCredentials(this.credentials.getAccessKey(), this.credentials.getSecretAccessKey()); AmazonS3 s3 = new AmazonS3Client(awscredentials); s3.setEndpoint(endpoint); System.out.println("Getting objects"); ObjectListing objectListing = s3.listObjects(repoid); List<IObject> objects = new ArrayList<IObject>(); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { S3TypeObject obj = new S3TypeObject(objectSummary.getKey(), objectSummary.getSize(), objectSummary.getBucketName(), objectSummary.getStorageClass(), s3); objects.add(obj); } return objects; }
From source file:com.liferay.amazontools.S3Cleaner.java
License:Open Source License
protected void deleteBucket(String bucketName) { ObjectListing objectListing = amazonS3Client.listObjects(bucketName); List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries(); if (!objectSummaries.isEmpty()) { DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName); List<DeleteObjectsRequest.KeyVersion> keyVersions = new ArrayList<>(); for (S3ObjectSummary objectSummary : objectSummaries) { keyVersions.add(new DeleteObjectsRequest.KeyVersion(objectSummary.getKey())); }/* w w w . jav a 2 s. c o m*/ deleteObjectsRequest.setKeys(keyVersions); amazonS3Client.deleteObjects(deleteObjectsRequest); } amazonS3Client.deleteBucket(bucketName); }
From source file:com.liferay.portal.store.s3.S3Store.java
License:Open Source License
protected List<S3ObjectSummary> getS3ObjectSummaries(String prefix) { try {/* w w w .ja va 2 s . co m*/ ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.withBucketName(_bucketName); listObjectsRequest.withPrefix(prefix); ObjectListing objectListing = _amazonS3.listObjects(listObjectsRequest); List<S3ObjectSummary> s3ObjectSummaries = new ArrayList<>(objectListing.getMaxKeys()); while (true) { s3ObjectSummaries.addAll(objectListing.getObjectSummaries()); if (objectListing.isTruncated()) { objectListing = _amazonS3.listNextBatchOfObjects(objectListing); } else { break; } } return s3ObjectSummaries; } catch (AmazonClientException ace) { throw transform(ace); } }