List of usage examples for com.amazonaws.services.s3.model S3VersionSummary getKey
public String getKey()
From source file:aws.example.s3.DeleteBucket.java
License:Open Source License
public static void main(String[] args) { final String USAGE = "\n" + "To run this example, supply the name of an S3 bucket\n" + "\n" + "Ex: DeleteBucket <bucketname>\n"; if (args.length < 1) { System.out.println(USAGE); System.exit(1);//from w w w. j a v a 2 s . c om } String bucket_name = args[0]; System.out.println("Deleting S3 bucket: " + bucket_name); final AmazonS3 s3 = new AmazonS3Client(); try { System.out.println(" - removing objects from bucket"); ObjectListing object_listing = s3.listObjects(bucket_name); while (true) { for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext();) { S3ObjectSummary summary = (S3ObjectSummary) iterator.next(); s3.deleteObject(bucket_name, summary.getKey()); } // more object_listing to retrieve? if (object_listing.isTruncated()) { object_listing = s3.listNextBatchOfObjects(object_listing); } else { break; } } ; System.out.println(" - removing versions from bucket"); VersionListing version_listing = s3.listVersions(new ListVersionsRequest().withBucketName(bucket_name)); while (true) { for (Iterator<?> iterator = version_listing.getVersionSummaries().iterator(); iterator.hasNext();) { S3VersionSummary vs = (S3VersionSummary) iterator.next(); s3.deleteVersion(bucket_name, vs.getKey(), vs.getVersionId()); } if (version_listing.isTruncated()) { version_listing = s3.listNextBatchOfVersions(version_listing); } else { break; } } System.out.println(" OK, bucket ready to delete!"); s3.deleteBucket(bucket_name); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } System.out.println("Done!"); }
From source file:cloudExplorer.Versioning.java
License:Open Source License
void getVersions(String key, String access_key, String secret_key, String bucket, String endpoint) { String results = null;/*from www .jav a 2 s. c o m*/ Boolean check_finished = false; try { mainFrame.jTextArea1.append("\nPlease wait, loading versions."); mainFrame.calibrateTextArea(); AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key); AmazonS3 s3Client = new AmazonS3Client(credentials, new ClientConfiguration().withSignerOverride("S3SignerType")); s3Client.setEndpoint(endpoint); VersionListing vListing; if (key == null) { vListing = s3Client.listVersions(bucket, null); } else { vListing = s3Client.listVersions(bucket, key); } List<S3VersionSummary> summary = vListing.getVersionSummaries(); if (summary.size() == 0) { check_finished = true; } else { check_finished = false; } for (S3VersionSummary object : summary) { if (!Versioning.delete) { mainFrame.versioning_date.add(object.getLastModified().toString()); } mainFrame.versioning_id.add(object.getVersionId()); mainFrame.versioning_name.add(object.getKey()); System.gc(); } if (Versioning.delete) { int i = 0; for (String delVer : mainFrame.versioning_name) { del = new Delete(delVer, mainFrame.cred.getAccess_key(), mainFrame.cred.getSecret_key(), mainFrame.cred.getBucket(), mainFrame.cred.getEndpoint(), mainFrame.versioning_id.get(i)); del.startc(delVer, mainFrame.cred.getAccess_key(), mainFrame.cred.getSecret_key(), mainFrame.cred.getBucket(), mainFrame.cred.getEndpoint(), mainFrame.versioning_id.get(i)); i++; } if (!check_finished) { getVersions(key, access_key, secret_key, bucket, endpoint); } } if (Versioning.delete) { mainFrame.jTextArea1.append( "\nCompleted deleting every object. Please observe this window for any tasks that are still running."); } else { mainFrame.jTextArea1 .append("\nDone gathering Versions. If files are found, the first 1000 will be displayed."); } mainFrame.calibrateTextArea(); } catch (Exception getVersions) { } if (Versioning.delete) { Versioning.delete = false; mainFrame.reloadObjects(false); } }
From source file:com.emc.ecs.sync.util.AwsS3Util.java
License:Open Source License
public static ListIterator<S3ObjectVersion> listVersions(SyncPlugin parentPlugin, AmazonS3 s3, String bucket, String key, String relativePath) { List<S3ObjectVersion> versions = new ArrayList<>(); VersionListing listing = null;/*from w w w . j ava2 s.com*/ do { if (listing == null) listing = s3.listVersions(bucket, key, null, null, "/", null); else listing = s3.listNextBatchOfVersions(listing); for (S3VersionSummary summary : listing.getVersionSummaries()) { if (summary.getKey().equals(key)) { versions.add(new S3ObjectVersion(parentPlugin, s3, bucket, key, summary.getVersionId(), summary.isLatest(), summary.isDeleteMarker(), summary.getLastModified(), summary.getETag(), relativePath, summary.getSize())); } } } while (listing.isTruncated()); // sort chronologically Collections.sort(versions, new VersionComparator()); return versions.listIterator(); }
From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java
License:Open Source License
@Override public ListVersionsResponseType listVersions(ListVersionsType request) throws S3Exception { ListVersionsResponseType reply = request.getReply(); User requestUser = getRequestUser(request); OsgInternalS3Client internalS3Client = null; try {/*ww w. j ava2 s. c om*/ internalS3Client = getS3Client(requestUser); AmazonS3Client s3Client = internalS3Client.getS3Client(); ListVersionsRequest listVersionsRequest = new ListVersionsRequest(request.getBucket(), request.getPrefix(), request.getKeyMarker(), request.getVersionIdMarker(), request.getDelimiter(), Integer.parseInt(request.getMaxKeys())); VersionListing result = s3Client.listVersions(listVersionsRequest); CanonicalUser owner; try { owner = AclUtils.buildCanonicalUser(requestUser.getAccount()); } catch (AuthException e) { LOG.error("Error getting request user's account during bucket version listing", e); owner = null; throw new AccountProblemException("Account for user " + requestUser.getUserId()); } //Populate result to euca reply.setBucket(request.getBucket()); reply.setMaxKeys(result.getMaxKeys()); reply.setDelimiter(result.getDelimiter()); reply.setNextKeyMarker(result.getNextKeyMarker()); reply.setNextVersionIdMarker(result.getNextVersionIdMarker()); reply.setIsTruncated(result.isTruncated()); reply.setVersionIdMarker(result.getVersionIdMarker()); reply.setKeyMarker(result.getKeyMarker()); if (result.getCommonPrefixes() != null && result.getCommonPrefixes().size() > 0) { reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); for (String s : result.getCommonPrefixes()) { reply.getCommonPrefixesList().add(new CommonPrefixesEntry(s)); } } ArrayList<KeyEntry> versions = new ArrayList<>(); VersionEntry v; DeleteMarkerEntry d; for (S3VersionSummary summary : result.getVersionSummaries()) { if (!summary.isDeleteMarker()) { v = new VersionEntry(); v.setKey(summary.getKey()); v.setVersionId(summary.getVersionId()); v.setLastModified(DateFormatter.dateToHeaderFormattedString(summary.getLastModified())); v.setEtag(summary.getETag()); v.setIsLatest(summary.isLatest()); v.setOwner(owner); v.setSize(summary.getSize()); versions.add(v); } else { d = new DeleteMarkerEntry(); d.setIsLatest(summary.isLatest()); d.setKey(summary.getKey()); d.setLastModified(DateFormatter.dateToHeaderFormattedString(summary.getLastModified())); d.setOwner(owner); d.setVersionId(summary.getVersionId()); versions.add(d); } } //Again, this is wrong, should be a single listing reply.setKeyEntries(versions); } catch (AmazonServiceException e) { LOG.debug("Error from backend", e); throw S3ExceptionMapper.fromAWSJavaSDK(e); } return reply; }
From source file:com.universal.storage.UniversalS3Storage.java
License:Open Source License
/** * This method wipes the root folder of a storage, basically, will remove all files and folder in it. * Be careful with this method because in too many cases this action won't provide a rollback action. * /* w w w.j av a 2 s . c om*/ * This method loops over the versions for deletion, the bucket will be empty and without any version of its objects. */ public void wipe() throws UniversalIOException { ObjectListing object_listing = this.s3client.listObjects(this.settings.getRoot()); while (true) { for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext();) { S3ObjectSummary summary = (S3ObjectSummary) iterator.next(); this.s3client.deleteObject(this.settings.getRoot(), summary.getKey()); } if (object_listing.isTruncated()) { object_listing = this.s3client.listNextBatchOfObjects(object_listing); } else { break; } } ; VersionListing version_listing = this.s3client .listVersions(new ListVersionsRequest().withBucketName(this.settings.getRoot())); while (true) { for (Iterator<?> iterator = version_listing.getVersionSummaries().iterator(); iterator.hasNext();) { S3VersionSummary vs = (S3VersionSummary) iterator.next(); this.s3client.deleteVersion( this.settings.getRoot(), vs.getKey(), vs.getVersionId()); } if (version_listing.isTruncated()) { version_listing = this.s3client.listNextBatchOfVersions(version_listing); } else { break; } } }
From source file:org.apache.nifi.processors.aws.s3.ListS3.java
License:Apache License
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) { try {/*from w ww . j a v a 2 s.c o m*/ restoreState(context); } catch (IOException ioe) { getLogger().error("Failed to restore processor state; yielding", ioe); context.yield(); return; } final long startNanos = System.nanoTime(); final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions().getValue(); final AmazonS3 client = getClient(); int listCount = 0; long maxTimestamp = 0L; String delimiter = context.getProperty(DELIMITER).getValue(); String prefix = context.getProperty(PREFIX).evaluateAttributeExpressions().getValue(); boolean useVersions = context.getProperty(USE_VERSIONS).asBoolean(); int listType = context.getProperty(LIST_TYPE).asInteger(); S3BucketLister bucketLister = useVersions ? new S3VersionBucketLister(client) : listType == 2 ? new S3ObjectBucketListerVersion2(client) : new S3ObjectBucketLister(client); bucketLister.setBucketName(bucket); if (delimiter != null && !delimiter.isEmpty()) { bucketLister.setDelimiter(delimiter); } if (prefix != null && !prefix.isEmpty()) { bucketLister.setPrefix(prefix); } VersionListing versionListing; do { versionListing = bucketLister.listVersions(); for (S3VersionSummary versionSummary : versionListing.getVersionSummaries()) { long lastModified = versionSummary.getLastModified().getTime(); if (lastModified < currentTimestamp || lastModified == currentTimestamp && currentKeys.contains(versionSummary.getKey())) { continue; } // Create the attributes final Map<String, String> attributes = new HashMap<>(); attributes.put(CoreAttributes.FILENAME.key(), versionSummary.getKey()); attributes.put("s3.bucket", versionSummary.getBucketName()); if (versionSummary.getOwner() != null) { // We may not have permission to read the owner attributes.put("s3.owner", versionSummary.getOwner().getId()); } attributes.put("s3.etag", versionSummary.getETag()); attributes.put("s3.lastModified", String.valueOf(lastModified)); attributes.put("s3.length", String.valueOf(versionSummary.getSize())); attributes.put("s3.storeClass", versionSummary.getStorageClass()); attributes.put("s3.isLatest", String.valueOf(versionSummary.isLatest())); if (versionSummary.getVersionId() != null) { attributes.put("s3.version", versionSummary.getVersionId()); } // Create the flowfile FlowFile flowFile = session.create(); flowFile = session.putAllAttributes(flowFile, attributes); session.transfer(flowFile, REL_SUCCESS); // Update state if (lastModified > maxTimestamp) { maxTimestamp = lastModified; currentKeys.clear(); } if (lastModified == maxTimestamp) { currentKeys.add(versionSummary.getKey()); } listCount++; } bucketLister.setNextMarker(); commit(context, session, listCount); listCount = 0; } while (bucketLister.isTruncated()); currentTimestamp = maxTimestamp; final long listMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); getLogger().info("Successfully listed S3 bucket {} in {} millis", new Object[] { bucket, listMillis }); if (!commit(context, session, listCount)) { if (currentTimestamp > 0) { persistState(context); } getLogger().debug("No new objects in S3 bucket {} to list. Yielding.", new Object[] { bucket }); context.yield(); } }
From source file:org.finra.herd.dao.impl.S3DaoImpl.java
License:Apache License
@Override public void deleteDirectory(final S3FileTransferRequestParamsDto params) { LOGGER.info("Deleting keys/key versions from S3... s3KeyPrefix=\"{}\" s3BucketName=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName()); Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Deleting from root directory is not allowed."); try {/* ww w . j a va 2 s . c o m*/ // List S3 versions. List<S3VersionSummary> s3VersionSummaries = listVersions(params); LOGGER.info( "Found keys/key versions in S3 for deletion. s3KeyCount={} s3KeyPrefix=\"{}\" s3BucketName=\"{}\"", s3VersionSummaries.size(), params.getS3KeyPrefix(), params.getS3BucketName()); // In order to avoid a MalformedXML AWS exception, we send delete request only when we have any key versions to delete. if (CollectionUtils.isNotEmpty(s3VersionSummaries)) { // Create an S3 client. AmazonS3Client s3Client = getAmazonS3(params); // Build a list of objects to be deleted. List<DeleteObjectsRequest.KeyVersion> keyVersions = new ArrayList<>(); for (S3VersionSummary s3VersionSummary : s3VersionSummaries) { keyVersions.add(new DeleteObjectsRequest.KeyVersion(s3VersionSummary.getKey(), s3VersionSummary.getVersionId())); } try { // Delete the key versions. deleteKeyVersions(s3Client, params.getS3BucketName(), keyVersions); } finally { s3Client.shutdown(); } } } catch (AmazonClientException e) { throw new IllegalStateException(String.format( "Failed to delete keys/key versions with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(), e.getMessage()), e); } }
From source file:org.finra.herd.dao.impl.S3DaoImpl.java
License:Apache License
@Override public void tagVersions(final S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto, final S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto, final List<S3VersionSummary> s3VersionSummaries, final Tag tag) { // Eliminate delete markers from the list of version summaries to be tagged. List<S3VersionSummary> s3VersionSummariesWithoutDeleteMarkers = null; if (CollectionUtils.isNotEmpty(s3VersionSummaries)) { s3VersionSummariesWithoutDeleteMarkers = s3VersionSummaries.stream() .filter(s3VersionSummary -> !s3VersionSummary.isDeleteMarker()).collect(Collectors.toList()); }//w w w. j a v a 2 s . co m LOGGER.info( "Tagging versions in S3... s3BucketName=\"{}\" s3KeyPrefix=\"{}\" s3VersionCount={} s3ObjectTagKey=\"{}\" s3ObjectTagValue=\"{}\" " + "Excluding from tagging S3 delete markers... s3DeleteMarkerCount={}", s3FileTransferRequestParamsDto.getS3BucketName(), s3FileTransferRequestParamsDto.getS3KeyPrefix(), CollectionUtils.size(s3VersionSummariesWithoutDeleteMarkers), tag.getKey(), tag.getValue(), CollectionUtils.size(s3VersionSummaries) - CollectionUtils.size(s3VersionSummariesWithoutDeleteMarkers)); if (CollectionUtils.isNotEmpty(s3VersionSummariesWithoutDeleteMarkers)) { // Tag S3 versions. tagVersionsHelper(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, s3VersionSummariesWithoutDeleteMarkers, tag); // Log a list of S3 versions that got tagged. if (LOGGER.isInfoEnabled()) { LOGGER.info("Successfully tagged versions in S3 bucket. " + "s3BucketName=\"{}\" s3KeyPrefix=\"{}\" s3VersionCount={} s3ObjectTagKey=\"{}\" s3ObjectTagValue=\"{}\"", s3FileTransferRequestParamsDto.getS3BucketName(), s3FileTransferRequestParamsDto.getS3KeyPrefix(), s3VersionSummariesWithoutDeleteMarkers.size(), tag.getKey(), tag.getValue()); for (S3VersionSummary s3VersionSummary : s3VersionSummariesWithoutDeleteMarkers) { LOGGER.info("s3Key=\"{}\" s3VersionId=\"{}\"", s3VersionSummary.getKey(), s3VersionSummary.getVersionId()); } } } }
From source file:org.finra.herd.dao.impl.S3DaoImpl.java
License:Apache License
private void tagVersionsHelper(final S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto, final S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto, final List<S3VersionSummary> s3VersionSummaries, final Tag tag) { // Initialize an S3 version for the error message in the catch block. S3VersionSummary currentS3VersionSummary = s3VersionSummaries.get(0); // Amazon S3 client to access S3 objects. AmazonS3Client s3Client = null;/* w w w . j a va 2 s . c om*/ // Amazon S3 client for S3 object tagging. AmazonS3Client s3ObjectTaggerClient = null; try { // Create an S3 client to access S3 objects. s3Client = getAmazonS3(s3FileTransferRequestParamsDto); // Create an S3 client for S3 object tagging. s3ObjectTaggerClient = getAmazonS3(s3ObjectTaggerParamsDto); // Create a get object tagging request. GetObjectTaggingRequest getObjectTaggingRequest = new GetObjectTaggingRequest( s3FileTransferRequestParamsDto.getS3BucketName(), null, null); // Create a set object tagging request. SetObjectTaggingRequest setObjectTaggingRequest = new SetObjectTaggingRequest( s3FileTransferRequestParamsDto.getS3BucketName(), null, null, null); for (S3VersionSummary s3VersionSummary : s3VersionSummaries) { // Set the current S3 version summary. currentS3VersionSummary = s3VersionSummary; // Retrieve the current tagging information for the S3 version. getObjectTaggingRequest.setKey(s3VersionSummary.getKey()); getObjectTaggingRequest.setVersionId(s3VersionSummary.getVersionId()); GetObjectTaggingResult getObjectTaggingResult = s3Operations .getObjectTagging(getObjectTaggingRequest, s3Client); // Update the list of tags to include the specified S3 object tag. List<Tag> updatedTags = new ArrayList<>(); updatedTags.add(tag); if (CollectionUtils.isNotEmpty(getObjectTaggingResult.getTagSet())) { for (Tag currentTag : getObjectTaggingResult.getTagSet()) { if (!StringUtils.equals(tag.getKey(), currentTag.getKey())) { updatedTags.add(currentTag); } } } // Update tagging information for the S3 version. setObjectTaggingRequest.setKey(s3VersionSummary.getKey()); setObjectTaggingRequest.setVersionId(s3VersionSummary.getVersionId()); setObjectTaggingRequest.setTagging(new ObjectTagging(updatedTags)); s3Operations.setObjectTagging(setObjectTaggingRequest, s3ObjectTaggerClient); } } catch (Exception e) { throw new IllegalStateException(String.format( "Failed to tag S3 object with \"%s\" key and \"%s\" version id in \"%s\" bucket. Reason: %s", currentS3VersionSummary.getKey(), currentS3VersionSummary.getVersionId(), s3FileTransferRequestParamsDto.getS3BucketName(), e.getMessage()), e); } finally { if (s3Client != null) { s3Client.shutdown(); } if (s3ObjectTaggerClient != null) { s3ObjectTaggerClient.shutdown(); } } }
From source file:org.mule.module.s3.simpleapi.SimpleAmazonS3AmazonDevKitImpl.java
License:Open Source License
public void deleteBucketAndObjects(@NotNull String bucketName) { Validate.notNull(bucketName);/*from w ww . j a v a 2 s.c o m*/ if (!s3.getBucketVersioningConfiguration(bucketName).getStatus() .equals(BucketVersioningConfiguration.OFF)) { ListVersionsRequest request = new ListVersionsRequest().withBucketName(bucketName); for (S3VersionSummary summary : listObjectVersions(request)) { s3.deleteVersion(bucketName, summary.getKey(), summary.getVersionId()); } } else { ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucketName); for (S3ObjectSummary summary : listObjects(request)) { s3.deleteObject(bucketName, summary.getKey()); } } deleteBucket(bucketName); }