List of usage examples for com.amazonaws.services.s3.model ObjectMetadata getContentLength
public long getContentLength()
From source file:net.solarnetwork.node.backup.s3.SdkS3Client.java
License:Open Source License
@Override public S3ObjectReference putObject(String key, InputStream in, ObjectMetadata objectMetadata) throws IOException { AmazonS3 client = getClient();//from w w w . jav a 2 s. com try { PutObjectRequest req = new PutObjectRequest(bucketName, key, in, objectMetadata); client.putObject(req); return new S3ObjectReference(key, objectMetadata.getContentLength(), objectMetadata.getLastModified()); } catch (AmazonServiceException e) { log.warn("AWS error: {}; HTTP code {}; AWS code {}; type {}; request ID {}", e.getMessage(), e.getStatusCode(), e.getErrorCode(), e.getErrorType(), e.getRequestId()); throw new RemoteServiceException("Error putting S3 object at " + key, e); } catch (AmazonClientException e) { log.debug("Error communicating with AWS: {}", e.getMessage()); throw new IOException("Error communicating with AWS", e); } }
From source file:ohnosequences.ivy.S3Resource.java
License:Apache License
private void initalizeResource() { try {//from www . j ava 2s .co m // System.out.println("trying to resolve bucket=" + bucket + " key=" + key); ObjectMetadata metadata = s3Repo.getS3Client().getObjectMetadata(bucket, key); this.exists = true; this.contentLength = metadata.getContentLength(); this.lastModified = metadata.getLastModified().getTime(); } catch (AmazonServiceException e) { this.exists = false; this.contentLength = 0; this.lastModified = 0; this.name = ""; } }
From source file:org.apache.beam.sdk.io.aws.s3.S3FileSystem.java
License:Apache License
@VisibleForTesting
MatchResult matchNonGlobPath(S3ResourceId path) {
ObjectMetadata s3Metadata;
try {// w w w .j av a2s .c o m
s3Metadata = getObjectMetadata(path);
} catch (AmazonClientException e) {
if (e instanceof AmazonS3Exception && ((AmazonS3Exception) e).getStatusCode() == 404) {
return MatchResult.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException());
}
return MatchResult.create(MatchResult.Status.ERROR, new IOException(e));
}
return MatchResult.create(MatchResult.Status.OK,
ImmutableList.of(createBeamMetadata(
path.withSize(s3Metadata.getContentLength()).withLastModified(s3Metadata.getLastModified()),
Strings.nullToEmpty(s3Metadata.getContentEncoding()))));
}
From source file:org.apache.beam.sdk.io.aws.s3.S3FileSystem.java
License:Apache License
@VisibleForTesting void copy(S3ResourceId sourcePath, S3ResourceId destinationPath) throws IOException { try {//from w w w . j a v a 2 s.c o m ObjectMetadata sourceObjectMetadata = getObjectMetadata(sourcePath); if (sourceObjectMetadata.getContentLength() < MAX_COPY_OBJECT_SIZE_BYTES) { atomicCopy(sourcePath, destinationPath, sourceObjectMetadata); } else { multipartCopy(sourcePath, destinationPath, sourceObjectMetadata); } } catch (AmazonClientException e) { throw new IOException(e); } }
From source file:org.apache.beam.sdk.io.aws.s3.S3FileSystem.java
License:Apache License
@VisibleForTesting CompleteMultipartUploadResult multipartCopy(S3ResourceId sourcePath, S3ResourceId destinationPath, ObjectMetadata sourceObjectMetadata) throws AmazonClientException { InitiateMultipartUploadRequest initiateUploadRequest = new InitiateMultipartUploadRequest( destinationPath.getBucket(), destinationPath.getKey()).withStorageClass(options.getS3StorageClass()) .withObjectMetadata(sourceObjectMetadata); initiateUploadRequest.setSSECustomerKey(options.getSSECustomerKey()); InitiateMultipartUploadResult initiateUploadResult = amazonS3.get() .initiateMultipartUpload(initiateUploadRequest); final String uploadId = initiateUploadResult.getUploadId(); List<PartETag> eTags = new ArrayList<>(); final long objectSize = sourceObjectMetadata.getContentLength(); // extra validation in case a caller calls directly S3FileSystem.multipartCopy // without using S3FileSystem.copy in the future if (objectSize == 0) { final CopyPartRequest copyPartRequest = new CopyPartRequest() .withSourceBucketName(sourcePath.getBucket()).withSourceKey(sourcePath.getKey()) .withDestinationBucketName(destinationPath.getBucket()) .withDestinationKey(destinationPath.getKey()).withUploadId(uploadId).withPartNumber(1); copyPartRequest.setSourceSSECustomerKey(options.getSSECustomerKey()); copyPartRequest.setDestinationSSECustomerKey(options.getSSECustomerKey()); CopyPartResult copyPartResult = amazonS3.get().copyPart(copyPartRequest); eTags.add(copyPartResult.getPartETag()); } else {// w w w . j a v a 2s .c o m long bytePosition = 0; Integer uploadBufferSizeBytes = options.getS3UploadBufferSizeBytes(); // Amazon parts are 1-indexed, not zero-indexed. for (int partNumber = 1; bytePosition < objectSize; partNumber++) { final CopyPartRequest copyPartRequest = new CopyPartRequest() .withSourceBucketName(sourcePath.getBucket()).withSourceKey(sourcePath.getKey()) .withDestinationBucketName(destinationPath.getBucket()) .withDestinationKey(destinationPath.getKey()).withUploadId(uploadId) .withPartNumber(partNumber).withFirstByte(bytePosition) .withLastByte(Math.min(objectSize - 1, bytePosition + uploadBufferSizeBytes - 1)); copyPartRequest.setSourceSSECustomerKey(options.getSSECustomerKey()); copyPartRequest.setDestinationSSECustomerKey(options.getSSECustomerKey()); CopyPartResult copyPartResult = amazonS3.get().copyPart(copyPartRequest); eTags.add(copyPartResult.getPartETag()); bytePosition += uploadBufferSizeBytes; } } CompleteMultipartUploadRequest completeUploadRequest = new CompleteMultipartUploadRequest() .withBucketName(destinationPath.getBucket()).withKey(destinationPath.getKey()) .withUploadId(uploadId).withPartETags(eTags); return amazonS3.get().completeMultipartUpload(completeUploadRequest); }
From source file:org.apache.camel.component.aws.s3.S3Endpoint.java
License:Apache License
public Exchange createExchange(ExchangePattern pattern, S3Object s3Object) { LOG.trace("Getting object with key [{}] from bucket [{}]...", s3Object.getKey(), s3Object.getBucketName()); ObjectMetadata objectMetadata = s3Object.getObjectMetadata(); LOG.trace("Got object [{}]", s3Object); Exchange exchange = new DefaultExchange(this, pattern); Message message = exchange.getIn();//from w ww .j av a 2s .c o m message.setBody(s3Object.getObjectContent()); message.setHeader(S3Constants.KEY, s3Object.getKey()); message.setHeader(S3Constants.BUCKET_NAME, s3Object.getBucketName()); message.setHeader(S3Constants.E_TAG, objectMetadata.getETag()); message.setHeader(S3Constants.LAST_MODIFIED, objectMetadata.getLastModified()); message.setHeader(S3Constants.VERSION_ID, objectMetadata.getVersionId()); message.setHeader(S3Constants.CONTENT_TYPE, objectMetadata.getContentType()); message.setHeader(S3Constants.CONTENT_MD5, objectMetadata.getContentMD5()); message.setHeader(S3Constants.CONTENT_LENGTH, objectMetadata.getContentLength()); message.setHeader(S3Constants.CONTENT_ENCODING, objectMetadata.getContentEncoding()); message.setHeader(S3Constants.CONTENT_DISPOSITION, objectMetadata.getContentDisposition()); message.setHeader(S3Constants.CACHE_CONTROL, objectMetadata.getCacheControl()); return exchange; }
From source file:org.apache.druid.storage.s3.S3TaskLogs.java
License:Apache License
private Optional<ByteSource> streamTaskFile(final long offset, String taskKey) throws IOException { try {//from w ww. j a va 2s. c om final ObjectMetadata objectMetadata = service.getObjectMetadata(config.getS3Bucket(), taskKey); return Optional.of(new ByteSource() { @Override public InputStream openStream() throws IOException { try { final long start; final long end = objectMetadata.getContentLength() - 1; if (offset > 0 && offset < objectMetadata.getContentLength()) { start = offset; } else if (offset < 0 && (-1 * offset) < objectMetadata.getContentLength()) { start = objectMetadata.getContentLength() + offset; } else { start = 0; } final GetObjectRequest request = new GetObjectRequest(config.getS3Bucket(), taskKey) .withMatchingETagConstraint(objectMetadata.getETag()).withRange(start, end); return service.getObject(request).getObjectContent(); } catch (AmazonServiceException e) { throw new IOException(e); } } }); } catch (AmazonS3Exception e) { if (404 == e.getStatusCode() || "NoSuchKey".equals(e.getErrorCode()) || "NoSuchBucket".equals(e.getErrorCode())) { return Optional.absent(); } else { throw new IOE(e, "Failed to stream logs from: %s", taskKey); } } }
From source file:org.apache.flink.fs.s3.common.writer.S3Committer.java
License:Apache License
@Override public void commitAfterRecovery() throws IOException { if (totalLength > 0L) { LOG.info("Trying to commit after recovery {} with MPU ID {}", objectName, uploadId); try {// w ww . ja va2s .c o m s3uploader.commitMultiPartUpload(objectName, uploadId, parts, totalLength, new AtomicInteger()); } catch (IOException e) { LOG.info("Failed to commit after recovery {} with MPU ID {}. " + "Checking if file was committed before...", objectName, uploadId); LOG.trace("Exception when committing:", e); try { ObjectMetadata metadata = s3uploader.getObjectMetadata(objectName); if (totalLength != metadata.getContentLength()) { String message = String.format("Inconsistent result for object %s: conflicting lengths. " + "Recovered committer for upload %s indicates %s bytes, present object is %s bytes", objectName, uploadId, totalLength, metadata.getContentLength()); LOG.warn(message); throw new IOException(message, e); } } catch (FileNotFoundException fnf) { LOG.warn("Object {} not existing after failed recovery commit with MPU ID {}", objectName, uploadId); throw new IOException(String.format( "Recovering commit failed for object %s. " + "Object does not exist and MultiPart Upload %s is not valid.", objectName, uploadId), e); } } } else { LOG.debug("No data to commit for file: {}", objectName); } }
From source file:org.apache.hadoop.fs.s3a.S3AFileSystem.java
License:Apache License
/** * Return a file status object that represents the path. * @param f The path we want information from * @return a FileStatus object// w w w .j av a 2 s.com * @throws java.io.FileNotFoundException when the path does not exist; * IOException see specific implementation */ public S3AFileStatus getFileStatus(Path f) throws IOException { String key = pathToKey(f); LOG.info("Getting path status for " + f + " (" + key + ")"); if (!key.isEmpty()) { try { ObjectMetadata meta = s3.getObjectMetadata(bucket, key); statistics.incrementReadOps(1); if (objectRepresentsDirectory(key, meta.getContentLength())) { if (LOG.isDebugEnabled()) { LOG.debug("Found exact file: fake directory"); } return new S3AFileStatus(true, true, f.makeQualified(uri, workingDir)); } else { if (LOG.isDebugEnabled()) { LOG.debug("Found exact file: normal file"); } return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()), f.makeQualified(uri, workingDir)); } } catch (AmazonServiceException e) { if (e.getStatusCode() != 404) { printAmazonServiceException(e); throw e; } } catch (AmazonClientException e) { printAmazonClientException(e); throw e; } // Necessary? if (!key.endsWith("/")) { try { String newKey = key + "/"; ObjectMetadata meta = s3.getObjectMetadata(bucket, newKey); statistics.incrementReadOps(1); if (objectRepresentsDirectory(newKey, meta.getContentLength())) { if (LOG.isDebugEnabled()) { LOG.debug("Found file (with /): fake directory"); } return new S3AFileStatus(true, true, f.makeQualified(uri, workingDir)); } else { LOG.warn("Found file (with /): real file? should not happen: " + key); return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()), f.makeQualified(uri, workingDir)); } } catch (AmazonServiceException e) { if (e.getStatusCode() != 404) { printAmazonServiceException(e); throw e; } } catch (AmazonClientException e) { printAmazonClientException(e); throw e; } } } try { if (!key.isEmpty() && !key.endsWith("/")) { key = key + "/"; } ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(bucket); request.setPrefix(key); request.setDelimiter("/"); request.setMaxKeys(1); ObjectListing objects = s3.listObjects(request); statistics.incrementReadOps(1); if (objects.getCommonPrefixes().size() > 0 || objects.getObjectSummaries().size() > 0) { if (LOG.isDebugEnabled()) { LOG.debug("Found path as directory (with /): " + objects.getCommonPrefixes().size() + "/" + objects.getObjectSummaries().size()); for (S3ObjectSummary summary : objects.getObjectSummaries()) { LOG.debug("Summary: " + summary.getKey() + " " + summary.getSize()); } for (String prefix : objects.getCommonPrefixes()) { LOG.debug("Prefix: " + prefix); } } return new S3AFileStatus(true, false, f.makeQualified(uri, workingDir)); } } catch (AmazonServiceException e) { if (e.getStatusCode() != 404) { printAmazonServiceException(e); throw e; } } catch (AmazonClientException e) { printAmazonClientException(e); throw e; } if (LOG.isDebugEnabled()) { LOG.debug("Not Found: " + f); } throw new FileNotFoundException("No such file or directory: " + f); }
From source file:org.apache.hadoop.fs.s3r.S3RFileSystem.java
License:Apache License
/** * Return a file status object that represents the path. * @param f The path we want information from * @return a FileStatus object//w w w . j a v a2 s . c o m * @throws FileNotFoundException when the path does not exist; * IOException see specific implementation */ public S3RFileStatus getFileStatus(Path f) throws IOException { String key = pathToKey(f); if (LOG.isDebugEnabled()) { LOG.debug("Getting path status for " + f + " (" + key + ")"); } if (!key.isEmpty()) { try { ObjectMetadata meta = s3.getObjectMetadata(bucket, key); statistics.incrementReadOps(1); if (objectRepresentsDirectory(key, meta.getContentLength())) { if (LOG.isDebugEnabled()) { LOG.debug("Found exact file: fake directory"); } return new S3RFileStatus(true, true, f.makeQualified(uri, workingDir)); } else { if (LOG.isDebugEnabled()) { LOG.debug("Found exact file: normal file"); } return new S3RFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()), f.makeQualified(uri, workingDir), getDefaultBlockSize(f.makeQualified(uri, workingDir))); } } catch (AmazonServiceException e) { if (e.getStatusCode() != 404) { printAmazonServiceException(e); throw e; } } catch (AmazonClientException e) { printAmazonClientException(e); throw e; } // Necessary? if (!key.endsWith("/")) { try { String newKey = key + "/"; ObjectMetadata meta = s3.getObjectMetadata(bucket, newKey); statistics.incrementReadOps(1); if (objectRepresentsDirectory(newKey, meta.getContentLength())) { if (LOG.isDebugEnabled()) { LOG.debug("Found file (with /): fake directory"); } return new S3RFileStatus(true, true, f.makeQualified(uri, workingDir)); } else { LOG.warn("Found file (with /): real file? should not happen: {}", key); return new S3RFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()), f.makeQualified(uri, workingDir), getDefaultBlockSize(f.makeQualified(uri, workingDir))); } } catch (AmazonServiceException e) { if (e.getStatusCode() != 404) { printAmazonServiceException(e); throw e; } } catch (AmazonClientException e) { printAmazonClientException(e); throw e; } } } try { if (!key.isEmpty() && !key.endsWith("/")) { key = key + "/"; } ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(bucket); request.setPrefix(key); request.setDelimiter("/"); request.setMaxKeys(1); ObjectListing objects = s3.listObjects(request); statistics.incrementReadOps(1); if (!objects.getCommonPrefixes().isEmpty() || objects.getObjectSummaries().size() > 0) { if (LOG.isDebugEnabled()) { LOG.debug("Found path as directory (with /): " + objects.getCommonPrefixes().size() + "/" + objects.getObjectSummaries().size()); for (S3ObjectSummary summary : objects.getObjectSummaries()) { LOG.debug("Summary: " + summary.getKey() + " " + summary.getSize()); } for (String prefix : objects.getCommonPrefixes()) { LOG.debug("Prefix: " + prefix); } } return new S3RFileStatus(true, false, f.makeQualified(uri, workingDir)); } } catch (AmazonServiceException e) { if (e.getStatusCode() != 404) { printAmazonServiceException(e); throw e; } } catch (AmazonClientException e) { printAmazonClientException(e); throw e; } if (LOG.isDebugEnabled()) { LOG.debug("Not Found: " + f); } throw new FileNotFoundException("No such file or directory: " + f); }