List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary getKey
public String getKey()
From source file:org.opendedup.sdfs.filestore.cloud.BatchAwsS3ChunkStore.java
License:Open Source License
public static void deleteBucket(String bucketName, String awsAccessKey, String awsSecretKey) { try {/*w w w . ja v a 2 s . c om*/ System.out.println(""); System.out.print("Deleting Bucket [" + bucketName + "]"); AWSCredentials bawsCredentials = new BasicAWSCredentials(awsAccessKey, awsSecretKey); AmazonS3Client bs3Service = new AmazonS3Client(bawsCredentials); ObjectListing ls = bs3Service.listObjects(bucketName); for (S3ObjectSummary objectSummary : ls.getObjectSummaries()) { bs3Service.deleteObject(bucketName, objectSummary.getKey()); System.out.print("."); } bs3Service.deleteBucket(bucketName); SDFSLogger.getLog().info("Bucket [" + bucketName + "] deleted"); System.out.println("Bucket [" + bucketName + "] deleted"); } catch (Exception e) { SDFSLogger.getLog().warn("Unable to delete bucket " + bucketName, e); } }
From source file:org.opendedup.sdfs.filestore.cloud.BatchAwsS3ChunkStore.java
License:Open Source License
public Iterator<String> getNextObjectList(String prefix) { this.s3clientLock.readLock().lock(); try {/*w w w.j a v a2 s. c om*/ List<String> keys = new ArrayList<String>(); if (ck == null) { ck = s3Service.listObjects(this.getName(), prefix); } else if (ck.isTruncated()) { ck = s3Service.listNextBatchOfObjects(ck); } else { return keys.iterator(); } List<S3ObjectSummary> objs = ck.getObjectSummaries(); for (S3ObjectSummary obj : objs) { if (obj.getKey().length() > prefix.length()) keys.add(obj.getKey()); } return keys.iterator(); } finally { this.s3clientLock.readLock().unlock(); } }
From source file:org.openflamingo.fs.s3.S3ObjectProvider.java
License:Apache License
public List<FileInfo> getFiles(String path) { String bucket = null;/* w w w.j ava 2 s . c o m*/ if (!"/".equals(path)) { bucket = S3Utils.getBucket(path + "/"); } String relativePath = S3Utils.getObjectKey(path); List<FileInfo> filesList = new ArrayList<FileInfo>(); if ("".equals(relativePath)) { return filesList; } try { ObjectListing objectListing = awsClient.listObjects( new ListObjectsRequest().withBucketName(bucket).withPrefix(relativePath).withDelimiter("/")); while (true) { List<S3ObjectSummary> summaries = objectListing.getObjectSummaries(); for (S3ObjectSummary objectSummary : summaries) { if (!objectSummary.getKey().endsWith("/")) { long size = objectSummary.getSize(); String filename = FileUtils.getFilename(objectSummary.getKey()); String bucketName = objectSummary.getBucketName(); long modified = objectSummary.getLastModified().getTime(); S3ObjectInfo info = new S3ObjectInfo(bucketName, objectSummary.getKey(), filename, modified, size); filesList.add(info); } } if (!objectListing.isTruncated()) { break; } objectListing = awsClient.listNextBatchOfObjects(objectListing); } return filesList; } catch (Exception ase) { // throw new FileSystemException("? ? ? ? ? ?.", ase); throw new FileSystemException("An error has occurred.", ase); } }
From source file:org.openflamingo.fs.s3.S3ObjectProvider.java
License:Apache License
@Override public boolean delete(String path) { Assert.hasLength(path, "Please enter the file path."); if (S3Utils.isDirectory(path)) { ObjectListing objectListing = awsClient.listObjects(new ListObjectsRequest() .withBucketName(S3Utils.getBucket(path)).withPrefix(S3Utils.getObjectKey(path))); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { awsClient.deleteObject(objectSummary.getBucketName(), objectSummary.getKey()); }/*from w ww . j a va 2 s . c o m*/ } else { String bucket = S3Utils.getBucket(path); String relativePath = StringUtils.remove(path, "/" + bucket + "/"); awsClient.deleteObject(bucket, relativePath); } // auditService.delete(FileSystemType.S3, username, path); return true; }
From source file:org.openinfinity.cloud.domain.repository.deployer.BucketRepositoryAWSImpl.java
License:Apache License
public void deleteBucketAndObjects(String bucketName) { try {// w ww. jav a 2 s .c o m // list and delete objects in a bucket LOGGER.debug("deleteBucketAndObjects called for bucket: <" + bucketName + ">."); ObjectListing objects = simpleStorageService.listObjects(bucketName); do { for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { LOGGER.debug("Trying to delete object <" + objectSummary.getKey() + "> from bucket <: " + objectSummary.getBucketName() + ">."); simpleStorageService.deleteObject(bucketName, objectSummary.getKey()); } objects = simpleStorageService.listNextBatchOfObjects(objects); } while (objects.isTruncated()); // Now that the bucket is empty, you can delete it. If you try to delete your bucket before it is empty, it will fail. LOGGER.debug("Trying to delete bucket <: " + bucketName + ">."); simpleStorageService.deleteBucket(bucketName); //System.out.println("Deleted bucket " + testBucket.getName()); } catch (Exception e) { // TODO: handle exception LOGGER.warn("Error in deleting objects and bucket: " + e + " -- " + e.getStackTrace().toString()); e.printStackTrace(); ExceptionUtil.throwSystemException(e.getMessage(), ExceptionLevel.ERROR, BucketRepository.EXCEPTION_MESSAGE_CONNECTION_FAILURE); } }
From source file:org.p365.S3Sample.java
License:Open Source License
public static void main(String[] args) throws IOException { /*//from w w w . ja v a2s . c o m * This credentials provider implementation loads your AWS credentials * from a properties file at the root of your classpath. * * Important: Be sure to fill in your AWS access credentials in the * AwsCredentials.properties file before you try to run this * sample. * http://aws.amazon.com/security-credentials */ AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); String bucketName = "mynewbuket"; String key = "Myobj/sd.jpg"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); if (!s3.doesBucketExist(bucketName)) { s3.createBucket(bucketName); } /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); String pathname = "D:\\Program Files\\apache-tomcat-7.0.42\\webapps\\WorkerForP365\\src\\AAA_1465.jpg"; File file = new File(pathname); s3.putObject( new PutObjectRequest(bucketName, key, file).withCannedAcl(CannedAccessControlList.PublicRead)); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ //System.out.println("Deleting an object\n"); //s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ //System.out.println("Deleting bucket " + bucketName + "\n"); //s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:org.pieShare.pieDrive.adapter.s3.S3Adapter.java
public boolean find(PieDriveFile file) { ObjectListing listing = s3Auth.getClient().listObjects(bucketName); boolean ret = false; for (S3ObjectSummary objectSummary : listing.getObjectSummaries()) { ret = objectSummary.getKey().equals(file.getUuid()); if (ret == true) { break; }// w w w . ja v a 2 s . co m } return ret; }
From source file:org.rdswitchboard.importers.browser.s3.App.java
License:Open Source License
public static void main(String[] args) { try {//from w w w. j a v a 2s. com if (args.length == 0 || StringUtils.isNullOrEmpty(args[0])) throw new Exception("Please provide properties file"); String propertiesFile = args[0]; Properties properties = new Properties(); try (InputStream in = new FileInputStream(propertiesFile)) { properties.load(in); } String source = properties.getProperty("data.source.id"); if (StringUtils.isNullOrEmpty(source)) throw new IllegalArgumentException("Source can not be empty"); System.out.println("Source: " + source); String baseUrl = properties.getProperty("base.url"); if (StringUtils.isNullOrEmpty(baseUrl)) throw new IllegalArgumentException("Base URL can not be empty"); System.out.println("Base URL: " + baseUrl); String sessionId = properties.getProperty("session.id"); if (StringUtils.isNullOrEmpty(sessionId)) throw new IllegalArgumentException("Session Id can not be empty"); System.out.println("Session Id: " + sessionId); String accessKey = properties.getProperty("aws.access.key"); String secretKey = properties.getProperty("aws.secret.key"); String bucket = properties.getProperty("s3.bucket"); if (StringUtils.isNullOrEmpty(bucket)) throw new IllegalArgumentException("AWS S3 Bucket can not be empty"); System.out.println("S3 Bucket: " + bucket); String prefix = properties.getProperty("s3.prefix"); if (StringUtils.isNullOrEmpty(prefix)) throw new IllegalArgumentException("AWS S3 Prefix can not be empty"); System.out.println("S3 Prefix: " + prefix); String crosswalk = properties.getProperty("crosswalk"); Templates template = null; if (!StringUtils.isNullOrEmpty(crosswalk)) { System.out.println("Crosswalk: " + crosswalk); template = TransformerFactory.newInstance() .newTemplates(new StreamSource(new FileInputStream(crosswalk))); } ObjectMapper mapper = new ObjectMapper(); Client client = Client.create(); Cookie cookie = new Cookie("PHPSESSID", properties.getProperty("session")); AmazonS3 s3client; if (!StringUtils.isNullOrEmpty(accessKey) && !StringUtils.isNullOrEmpty(secretKey)) { System.out.println( "Connecting to AWS via Access and Secret Keys. This is not safe practice, consider to use IAM Role instead."); AWSCredentials awsCredentials = new BasicAWSCredentials(accessKey, secretKey); s3client = new AmazonS3Client(awsCredentials); } else { System.out.println("Connecting to AWS via Instance Profile Credentials"); s3client = new AmazonS3Client(new InstanceProfileCredentialsProvider()); } //String file = "rda/rif/class:collection/54800.xml"; ListObjectsRequest listObjectsRequest; ObjectListing objectListing; String file = prefix + "/latest.txt"; S3Object object = s3client.getObject(new GetObjectRequest(bucket, file)); String latest; try (InputStream txt = object.getObjectContent()) { latest = prefix + "/" + IOUtils.toString(txt, StandardCharsets.UTF_8).trim() + "/"; } System.out.println("S3 Repository: " + latest); listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(latest); do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { file = objectSummary.getKey(); System.out.println("Processing file: " + file); object = s3client.getObject(new GetObjectRequest(bucket, file)); String xml = null; if (null != template) { Source reader = new StreamSource(object.getObjectContent()); StringWriter writer = new StringWriter(); Transformer transformer = template.newTransformer(); transformer.transform(reader, new StreamResult(writer)); xml = writer.toString(); } else { InputStream is = object.getObjectContent(); xml = IOUtils.toString(is, ENCODING); } URL url = new URL(baseUrl + "/registry/import/import_s3/"); StringBuilder sb = new StringBuilder(); addParam(sb, "id", source); addParam(sb, "xml", xml); //System.out.println(sb.toString()); WebResource webResource = client.resource(url.toString()); ClientResponse response = webResource .header("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0") .accept(MediaType.APPLICATION_JSON, "*/*").acceptLanguage("en-US", "en") .type(MediaType.APPLICATION_FORM_URLENCODED).cookie(cookie) .post(ClientResponse.class, sb.toString()); if (response.getStatus() != 200) { throw new RuntimeException("Failed : HTTP error code : " + response.getStatus()); } String output = response.getEntity(String.class); Result result = mapper.readValue(output, Result.class); if (!result.getStatus().equals("OK")) { System.err.println(result.getMessage()); break; } else System.out.println(result.getMessage()); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (Exception e) { e.printStackTrace(); } }
From source file:org.rdswitchboard.utils.s3.find.App.java
License:Open Source License
public static void main(String[] args) { try {/*from w ww .j a va 2s. c o m*/ if (args.length != 2) throw new IllegalArgumentException("Bucket name and search string can not be empty"); String buckey = args[0]; String search = args[1]; String prefix = null; int pos = buckey.indexOf('/'); if (pos > 0) { prefix = buckey.substring(pos + 1); buckey = buckey.substring(0, pos); } AmazonS3 s3client = new AmazonS3Client(new InstanceProfileCredentialsProvider()); // AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(buckey); if (!StringUtils.isNullOrEmpty(prefix)) listObjectsRequest.setPrefix(prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String key = objectSummary.getKey(); System.out.println(" - " + key); S3Object object = s3client.getObject(new GetObjectRequest(buckey, key)); String str = IOUtils.toString(object.getObjectContent()); if (str.contains(search)) { System.out.println("Found!"); FileUtils.writeStringToFile(new File("s3/" + key), str); } } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (Exception e) { e.printStackTrace(); } }
From source file:org.reswitchboard.utils.s3.access.App.java
License:Open Source License
public static void main(String[] args) { try {/* w ww. ja va 2 s. co m*/ if (args.length == 0 || StringUtils.isNullOrEmpty(args[0])) throw new IllegalArgumentException("Bucket name can not be empty"); String bucketName = args[0]; String prefix = null; if (args.length > 1) prefix = args[1]; AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); if (!StringUtils.isNullOrEmpty(prefix)) listObjectsRequest.setPrefix(prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String key = objectSummary.getKey(); System.out.println(" - " + key); for (int nAttempt = 1;; ++nAttempt) { try { AccessControlList acl = s3client.getObjectAcl(bucketName, key); List<Grant> grants = acl.getGrantsAsList(); for (Grant grant : grants) { // System.out.println( " Grant: " + grant.toString()); if (grant.getGrantee().equals(GroupGrantee.AllUsers)) { System.out.println(" Revoking public access"); acl.revokeAllPermissions(GroupGrantee.AllUsers); s3client.setObjectAcl(bucketName, key, acl); break; } } break; } catch (Exception e) { System.out.println("Error: " + e.toString()); if (nAttempt >= 10) { throw new Exception("Maximum number of invalid attempts has been reeched"); } // double back-off delay Thread.sleep((long) (Math.pow(2, nAttempt) * 50)); } } } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (Exception e) { e.printStackTrace(); } }