List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary getBucketName
public String getBucketName()
From source file:com.scoyo.tools.s3cacheenhancer.S3HeaderEnhancer.java
License:Apache License
private void setHeaders(ObjectListing listing, final String maxAgeHeader, ExecutorService executorService) { for (final S3ObjectSummary summary : listing.getObjectSummaries()) { executorService.submit(new Runnable() { @Override//w w w. j a v a2 s.com public void run() { String bucket = summary.getBucketName(); String key = summary.getKey(); ObjectMetadata metadata = null; try { metadata = s3.getObjectMetadata(bucket, key); } catch (AmazonS3Exception exception) { System.out.println("Could not update " + key + " [" + exception.getMessage() + "]"); return; } if ("application/x-directory".equals(metadata.getContentType())) { System.out.println("Skipping because content-type " + key); return; } if (!maxAgeHeader.equals(metadata.getCacheControl())) { metadata.setCacheControl(maxAgeHeader); } else { System.out.println("Skipping because header is already correct " + key); return; } AccessControlList acl = s3.getObjectAcl(summary.getBucketName(), summary.getKey()); CopyObjectRequest copyReq = new CopyObjectRequest(bucket, key, bucket, key) .withAccessControlList(acl).withNewObjectMetadata(metadata); CopyObjectResult result = s3.copyObject(copyReq); if (result != null) { System.out.println("Updated " + key); } else { System.out.println("Could not update " + key); } } }); } }
From source file:com.streamsets.pipeline.stage.origin.s3.AmazonS3Runnable.java
License:Apache License
private void handleWholeFileDataFormat(S3ObjectSummary s3ObjectSummary, String recordId) throws StageException { S3Object partialS3ObjectForMetadata; //partialObject with fetchSize 1 byte. //This is mostly used for extracting metadata and such. partialS3ObjectForMetadata = AmazonS3Util.getObjectRange(s3Client, s3ConfigBean.s3Config.bucket, s3ObjectSummary.getKey(), 1, s3ConfigBean.sseConfig.useCustomerSSEKey, s3ConfigBean.sseConfig.customerKey, s3ConfigBean.sseConfig.customerKeyMd5); S3FileRef.Builder s3FileRefBuilder = new S3FileRef.Builder().s3Client(s3Client) .s3ObjectSummary(s3ObjectSummary).useSSE(s3ConfigBean.sseConfig.useCustomerSSEKey) .customerKey(s3ConfigBean.sseConfig.customerKey) .customerKeyMd5(s3ConfigBean.sseConfig.customerKeyMd5) .bufferSize((int) dataParser.suggestedWholeFileBufferSize()).createMetrics(true) .totalSizeInBytes(s3ObjectSummary.getSize()).rateLimit(dataParser.wholeFileRateLimit()); if (dataParser.isWholeFileChecksumRequired()) { s3FileRefBuilder.verifyChecksum(true).checksumAlgorithm(HashingUtil.HashType.MD5) //128 bit hex encoded md5 checksum. .checksum(partialS3ObjectForMetadata.getObjectMetadata().getETag()); }/* w w w . ja v a 2 s .co m*/ Map<String, Object> metadata = AmazonS3Util.getMetaData(partialS3ObjectForMetadata); metadata.put(S3Constants.BUCKET, s3ObjectSummary.getBucketName()); metadata.put(S3Constants.OBJECT_KEY, s3ObjectSummary.getKey()); metadata.put(S3Constants.OWNER, s3ObjectSummary.getOwner()); metadata.put(S3Constants.SIZE, s3ObjectSummary.getSize()); metadata.put(HeaderAttributeConstants.FILE_NAME, s3ObjectSummary.getKey()); metadata.remove(S3Constants.CONTENT_LENGTH); parser = dataParser.getParser(recordId, metadata, s3FileRefBuilder.build()); //Object is assigned so that setHeaders() function can use this to get metadata //information about the object object = partialS3ObjectForMetadata; }
From source file:com.treasure_data.td_import.source.S3Source.java
License:Apache License
S3Source(AmazonS3Client client, String rawPath, S3ObjectSummary s3object) { super("s3://" + s3object.getBucketName() + "/" + s3object.getKey()); this.client = client; this.bucket = s3object.getBucketName(); this.key = s3object.getKey(); this.size = s3object.getSize(); this.rawPath = rawPath; }
From source file:com.upplication.s3fs.S3FileSystemProvider.java
License:Open Source License
/** * Get the Control List, if the path not exists * (because the path is a directory and this key isnt created at amazon s3) * then return the ACL of the first child. *//from w w w .j ava 2 s .co m * @param path {@link S3Path} * @return AccessControlList * @throws NoSuchFileException if not found the path and any child */ private AccessControlList getAccessControl(S3Path path) throws NoSuchFileException { S3ObjectSummary obj = s3ObjectSummaryLookup.lookup(path); // check first for file: return path.getFileSystem().getClient().getObjectAcl(obj.getBucketName(), obj.getKey()); }
From source file:gobblin.aws.AWSSdkClient.java
License:Apache License
/*** * Download a S3 object to local directory * * @param s3ObjectSummary S3 object summary for the object to download * @param targetDirectory Local target directory to download the object to * @throws IOException If any errors were encountered in downloading the object *//*from w w w . ja v a 2 s . co m*/ public void downloadS3Object(S3ObjectSummary s3ObjectSummary, String targetDirectory) throws IOException { final AmazonS3 amazonS3 = getS3Client(); final GetObjectRequest getObjectRequest = new GetObjectRequest(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()); final S3Object s3Object = amazonS3.getObject(getObjectRequest); final String targetFile = StringUtils.removeEnd(targetDirectory, File.separator) + File.separator + s3Object.getKey(); FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), new File(targetFile)); LOGGER.info("S3 object downloaded to file: " + targetFile); }
From source file:io.druid.firehose.s3.StaticS3FirehoseFactory.java
License:Apache License
@Override protected InputStream openObjectStream(S3ObjectSummary object) throws IOException { try {// w ww.j a v a 2 s . c om // Get data of the given object and open an input stream final S3Object s3Object = s3Client.getObject(object.getBucketName(), object.getKey()); if (s3Object == null) { throw new ISE("Failed to get an s3 object for bucket[%s] and key[%s]", object.getBucketName(), object.getKey()); } return s3Object.getObjectContent(); } catch (AmazonS3Exception e) { throw new IOException(e); } }
From source file:io.druid.firehose.s3.StaticS3FirehoseFactory.java
License:Apache License
@Override protected InputStream openObjectStream(S3ObjectSummary object, long start) throws IOException { final GetObjectRequest request = new GetObjectRequest(object.getBucketName(), object.getKey()); request.setRange(start);// w ww . java 2 s.c o m try { final S3Object s3Object = s3Client.getObject(request); if (s3Object == null) { throw new ISE("Failed to get an s3 object for bucket[%s], key[%s], and start[%d]", object.getBucketName(), object.getKey(), start); } return s3Object.getObjectContent(); } catch (AmazonS3Exception e) { throw new IOException(e); } }
From source file:io.druid.storage.s3.S3TimestampVersionedDataFinder.java
License:Apache License
/** * Gets the key with the most recently modified timestamp. * `pattern` is evaluated against the entire key AFTER the path given in `uri`. * The substring `pattern` is matched against will have a leading `/` removed. * For example `s3://some_bucket/some_prefix/some_key` with a URI of `s3://some_bucket/some_prefix` will match against `some_key`. * `s3://some_bucket/some_prefixsome_key` with a URI of `s3://some_bucket/some_prefix` will match against `some_key` * `s3://some_bucket/some_prefix//some_key` with a URI of `s3://some_bucket/some_prefix` will match against `/some_key` * * @param uri The URI of in the form of `s3://some_bucket/some_key` * @param pattern The pattern matcher to determine if a *key* is of interest, or `null` to match everything. * * @return A URI to the most recently modified object which matched the pattern. */// w w w. ja v a 2 s .c o m @Override public URI getLatestVersion(final URI uri, final @Nullable Pattern pattern) { try { return RetryUtils.retry(() -> { final S3Coords coords = new S3Coords(checkURI(uri)); long mostRecent = Long.MIN_VALUE; URI latest = null; final Iterator<S3ObjectSummary> objectSummaryIterator = S3Utils.objectSummaryIterator(s3Client, coords.bucket, coords.path, MAX_LISTING_KEYS); while (objectSummaryIterator.hasNext()) { final S3ObjectSummary objectSummary = objectSummaryIterator.next(); String keyString = objectSummary.getKey().substring(coords.path.length()); if (keyString.startsWith("/")) { keyString = keyString.substring(1); } if (pattern != null && !pattern.matcher(keyString).matches()) { continue; } final long latestModified = objectSummary.getLastModified().getTime(); if (latestModified >= mostRecent) { mostRecent = latestModified; latest = new URI(StringUtils.format("s3://%s/%s", objectSummary.getBucketName(), objectSummary.getKey())); } } return latest; }, shouldRetryPredicate(), DEFAULT_RETRY_COUNT); } catch (Exception e) { throw Throwables.propagate(e); } }
From source file:io.druid.storage.s3.S3Utils.java
License:Apache License
/** * Gets a single {@link S3ObjectSummary} from s3. Since this method might return a wrong object if there are multiple * objects that match the given key, this method should be used only when it's guaranteed that the given key is unique * in the given bucket.//from w ww. j a va2s . c o m * * @param s3Client s3 client * @param bucket s3 bucket * @param key unique key for the object to be retrieved */ public static S3ObjectSummary getSingleObjectSummary(AmazonS3 s3Client, String bucket, String key) { final ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucket).withPrefix(key) .withMaxKeys(1); final ListObjectsV2Result result = s3Client.listObjectsV2(request); if (result.getKeyCount() == 0) { throw new ISE("Cannot find object for bucket[%s] and key[%s]", bucket, key); } final S3ObjectSummary objectSummary = result.getObjectSummaries().get(0); if (!objectSummary.getBucketName().equals(bucket) || !objectSummary.getKey().equals(key)) { throw new ISE("Wrong object[%s] for bucket[%s] and key[%s]", objectSummary, bucket, key); } return objectSummary; }
From source file:io.konig.camel.aws.s3.DeleteObjectConsumer.java
License:Apache License
protected Queue<Exchange> createExchanges(List<S3ObjectSummary> s3ObjectSummaries) { if (LOG.isTraceEnabled()) { LOG.trace("Received {} messages in this poll", s3ObjectSummaries.size()); }/*from ww w . j a va2 s .c o m*/ Collection<S3Object> s3Objects = new ArrayList<>(); Queue<Exchange> answer = new LinkedList<Exchange>(); try { for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) { S3Object s3Object = getAmazonS3Client().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()); s3Objects.add(s3Object); Exchange exchange = getEndpoint().createExchange(s3Object); answer.add(exchange); } } catch (Throwable e) { LOG.warn("Error getting S3Object due: " + e.getMessage(), e); // ensure all previous gathered s3 objects are closed // if there was an exception creating the exchanges in this batch s3Objects.forEach(IOHelper::close); throw e; } return answer; }