List of usage examples for com.amazonaws.services.s3 AmazonS3 listObjectsV2
public ListObjectsV2Result listObjectsV2(ListObjectsV2Request listObjectsV2Request) throws SdkClientException, AmazonServiceException;
From source file:com.dustindoloff.s3websitedeploy.Main.java
License:Apache License
private static boolean emptyBucket(final AmazonS3 s3Client, final String bucket) { final ListObjectsV2Request request = new ListObjectsV2Request(); request.setBucketName(bucket);//from w w w. j av a2 s . co m String continuationToken = null; ListObjectsV2Result result; do { request.setContinuationToken(continuationToken); result = s3Client.listObjectsV2(bucket); for (final S3ObjectSummary summary : result.getObjectSummaries()) { s3Client.deleteObject(bucket, summary.getKey()); } continuationToken = result.getNextContinuationToken(); } while (result.isTruncated()); return true; }
From source file:com.epam.dlab.module.aws.S3FileList.java
License:Apache License
private List<S3ObjectSummary> reportFilesInBillingBucket(AmazonS3 s3Client) throws AdapterException { ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucket); ListObjectsV2Result result;//from w w w .j a v a 2s .c o m List<S3ObjectSummary> objectSummaries = new ArrayList<>(); try { do { result = s3Client.listObjectsV2(request); objectSummaries.addAll(notProcessedFiles(result)); } while (result.isTruncated()); } catch (Exception e) { throw new AdapterException( "Cannot get the file listing of bucket \"" + bucket + "*\". " + e.getLocalizedMessage(), e); } return objectSummaries; }
From source file:com.handywedge.binarystore.store.aws.BinaryStoreManagerImpl.java
License:MIT License
@Override public List<BinaryInfo> list(StorageInfo storage, BinaryInfo binary) throws StoreException { logger.debug("={}", storage); logger.debug("?={}", binary); List<BinaryInfo> objInfoList = new ArrayList<BinaryInfo>(); AmazonS3 s3client = getS3Client(binary.getBucketName()); try {/*from w w w . j a v a 2 s . c om*/ logger.debug("Listing binaries"); final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(binary.getBucketName()) .withMaxKeys(2); ListObjectsV2Result result; do { result = s3client.listObjectsV2(req); for (S3ObjectSummary binarySummary : result.getObjectSummaries()) { logger.debug(" - {}(size={})", binarySummary.getKey(), binarySummary.getSize()); if (binarySummary.getSize() != 0) { BinaryInfo objInfo = new BinaryInfo(binary.getBucketName()); objInfo.setFileName(binarySummary.getKey()); objInfo.setSize(binarySummary.getSize()); S3Object s3Object = s3client .getObject(new GetObjectRequest(binary.getBucketName(), binarySummary.getKey())); objInfo.setContentType(s3Object.getObjectMetadata().getContentType()); objInfo.setUrl(s3client.getUrl(binary.getBucketName(), binarySummary.getKey()).toString()); logger.debug("Generating pre-signed URL."); URL PresignedUrl = getPresignedUrl(s3client, binary.getBucketName(), binarySummary.getKey()); objInfo.setPresignedUrl(PresignedUrl.toString()); logger.debug("Pre-Signed URL = " + PresignedUrl.toString()); objInfoList.add(objInfo); } } logger.debug("Next Continuation Token : " + result.getNextContinuationToken()); req.setContinuationToken(result.getNextContinuationToken()); } while (result.isTruncated() == true); } catch (AmazonServiceException ase) { throw new StoreException(HttpStatus.SC_BAD_REQUEST, ErrorClassification.LIST_FAIL, ase, binary.getFileName()); } catch (AmazonClientException ace) { throw new StoreException(HttpStatus.SC_BAD_REQUEST, ErrorClassification.LIST_FAIL, ace, binary.getFileName()); } logger.info(" ={}", objInfoList.size()); return objInfoList; }
From source file:io.druid.storage.s3.S3Utils.java
License:Apache License
public static Iterator<S3ObjectSummary> objectSummaryIterator(final AmazonS3 s3Client, final String bucket, final String prefix, final int numMaxKeys) { final ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucket).withPrefix(prefix) .withMaxKeys(numMaxKeys);/* www . ja v a2 s . co m*/ return new Iterator<S3ObjectSummary>() { private ListObjectsV2Result result; private Iterator<S3ObjectSummary> objectSummaryIterator; { fetchNextBatch(); } private void fetchNextBatch() { result = s3Client.listObjectsV2(request); objectSummaryIterator = result.getObjectSummaries().iterator(); request.setContinuationToken(result.getContinuationToken()); } @Override public boolean hasNext() { return objectSummaryIterator.hasNext() || result.isTruncated(); } @Override public S3ObjectSummary next() { if (!hasNext()) { throw new NoSuchElementException(); } if (objectSummaryIterator.hasNext()) { return objectSummaryIterator.next(); } if (result.isTruncated()) { fetchNextBatch(); } if (!objectSummaryIterator.hasNext()) { throw new ISE( "Failed to further iterate on bucket[%s] and prefix[%s]. The last continuationToken was [%s]", bucket, prefix, result.getContinuationToken()); } return objectSummaryIterator.next(); } }; }
From source file:io.druid.storage.s3.S3Utils.java
License:Apache License
/** * Gets a single {@link S3ObjectSummary} from s3. Since this method might return a wrong object if there are multiple * objects that match the given key, this method should be used only when it's guaranteed that the given key is unique * in the given bucket./*from www .j av a2 s. c o m*/ * * @param s3Client s3 client * @param bucket s3 bucket * @param key unique key for the object to be retrieved */ public static S3ObjectSummary getSingleObjectSummary(AmazonS3 s3Client, String bucket, String key) { final ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucket).withPrefix(key) .withMaxKeys(1); final ListObjectsV2Result result = s3Client.listObjectsV2(request); if (result.getKeyCount() == 0) { throw new ISE("Cannot find object for bucket[%s] and key[%s]", bucket, key); } final S3ObjectSummary objectSummary = result.getObjectSummaries().get(0); if (!objectSummary.getBucketName().equals(bucket) || !objectSummary.getKey().equals(key)) { throw new ISE("Wrong object[%s] for bucket[%s] and key[%s]", objectSummary, bucket, key); } return objectSummary; }
From source file:net.solarnetwork.node.backup.s3.SdkS3Client.java
License:Open Source License
@Override public Set<S3ObjectReference> listObjects(String prefix) throws IOException { AmazonS3 client = getClient(); Set<S3ObjectReference> result = new LinkedHashSet<>(100); try {/*from www . ja va 2 s . c o m*/ final ListObjectsV2Request req = new ListObjectsV2Request(); req.setBucketName(bucketName); req.setMaxKeys(maximumKeysPerRequest); req.setPrefix(prefix); ListObjectsV2Result listResult; do { listResult = client.listObjectsV2(req); for (S3ObjectSummary objectSummary : listResult.getObjectSummaries()) { result.add(new S3ObjectReference(objectSummary.getKey(), objectSummary.getSize(), objectSummary.getLastModified())); } req.setContinuationToken(listResult.getNextContinuationToken()); } while (listResult.isTruncated() == true); } catch (AmazonServiceException e) { log.warn("AWS error: {}; HTTP code {}; AWS code {}; type {}; request ID {}", e.getMessage(), e.getStatusCode(), e.getErrorCode(), e.getErrorType(), e.getRequestId()); throw new RemoteServiceException("Error listing S3 objects at " + prefix, e); } catch (AmazonClientException e) { log.debug("Error communicating with AWS: {}", e.getMessage()); throw new IOException("Error communicating with AWS", e); } return result; }
From source file:org.apache.flink.cloudsort.io.aws.AwsInput.java
License:Apache License
@Override public List<InputSplit> list() { Preconditions.checkNotNull(bucket);/* w ww . j a v a 2s . com*/ Preconditions.checkNotNull(prefix); List<InputSplit> objectNames = new ArrayList<>(); // this will read credentials from user's home directory AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucket).withPrefix(prefix); ListObjectsV2Result result; int index = 0; do { result = s3client.listObjectsV2(req); for (S3ObjectSummary objectSummary : result.getObjectSummaries()) { String objectName = objectSummary.getKey(); long objectSize = objectSummary.getSize(); objectNames.add(new InputSplit(index++, objectName, objectSize)); } req.setContinuationToken(result.getNextContinuationToken()); } while (result.isTruncated()); return objectNames; }
From source file:oulib.aws.s3.S3TiffProcessor.java
/** * * @param bookInfo : contains the information of the source bucket name, target bucket name, and the name of the book * @param context : lambda function runtime context * @return ://from w w w .j a v a 2 s.co m * */ @Override public String handleRequest(S3BookInfo bookInfo, Context context) { AmazonS3 s3client = new AmazonS3Client(); Region usEast = Region.getRegion(Regions.US_EAST_1); s3client.setRegion(usEast); try { String sourceBucketName = bookInfo.getBucketSourceName(); String targetBucketName = bookInfo.getBucketTargetName(); String bookName = bookInfo.getBookName(); // Every book has a folder in the target bucket: Map targetBucketKeyMap = S3Util.getBucketObjectKeyMap(targetBucketName, bookName, s3client); if (!S3Util.folderExitsts(bookName, targetBucketKeyMap)) { S3Util.createFolder(targetBucketName, bookName, s3client); } final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(sourceBucketName) .withPrefix(bookName + "/data/"); ListObjectsV2Result result; do { result = s3client.listObjectsV2(req); for (S3ObjectSummary objectSummary : result.getObjectSummaries()) { String key = objectSummary.getKey(); if (key.endsWith(".tif") && !targetBucketKeyMap.containsKey(key + ".tif")) { S3Object object = s3client.getObject(new GetObjectRequest(sourceBucketName, key)); System.out.println("Start to generate smaller tif image for the object " + key); S3Util.generateSmallTiffWithTargetSize(s3client, object, targetBucketName, bookInfo.getCompressionSize()); // S3Util.copyS3ObjectTiffMetadata(s3client, object, s3client.getObject(new GetObjectRequest(targetBucketName, key)), targetBucketName, key+".tif"); System.out.println("Finished to generate smaller tif image for the object " + key + ".tif"); // break; } } System.out.println("Next Continuation Token : " + result.getNextContinuationToken()); req.setContinuationToken(result.getNextContinuationToken()); } while (result.isTruncated() == true); } catch (AmazonServiceException ase) { System.out.println( "Caught an AmazonServiceException, which means your request made it to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println( "Caught an AmazonClientException, which means the client encountered an internal error while trying to communicate with S3, \nsuch as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } return null; }
From source file:oulib.aws.s3.S3Util.java
/** * // ww w. jav a 2 s . c o m * @param bucketName : bucket name * @param folderName : a unique folder name or partial path in the bucket * @param client : s3 client * @return : a map of keys with keyset of object keys */ public static Map<String, String> getBucketObjectKeyMap(String bucketName, String folderName, AmazonS3 client) { final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName); ListObjectsV2Result result; Map<String, String> keyMap = new HashMap<>(); do { result = client.listObjectsV2(req); for (S3ObjectSummary objectSummary : result.getObjectSummaries()) { String key = objectSummary.getKey(); if (key.contains(folderName)) { keyMap.put(key, key); } } req.setContinuationToken(result.getNextContinuationToken()); } while (result.isTruncated() == true); return keyMap; }
From source file:oulib.aws.s3.S3Util.java
/** * /*from w w w . ja v a 2 s . c om*/ * @param bucketName : bucket name * @param folderName : a unique folder name or partial path in the bucket * @param client : s3 client * @return : a list of keys */ public static List<String> getBucketObjectKeyList(String bucketName, String folderName, AmazonS3 client) { final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName); ListObjectsV2Result result; List<String> keyList = new ArrayList<>(); do { result = client.listObjectsV2(req); for (S3ObjectSummary objectSummary : result.getObjectSummaries()) { String key = objectSummary.getKey(); if (key.contains(folderName)) { keyList.add(key); } } req.setContinuationToken(result.getNextContinuationToken()); } while (result.isTruncated() == true); return keyList; }