Example usage for com.amazonaws.services.s3.model ObjectMetadata getContentLength

List of usage examples for com.amazonaws.services.s3.model ObjectMetadata getContentLength

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. ObjectMetadata
  6. getContentLength

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ObjectMetadata getContentLength.

Prototype

public long getContentLength()

Source Link

Document

<p> Gets the Content-Length HTTP header indicating the size of the associated object in bytes.

Usage

From source file:gov.cdc.sdp.cbr.aphl.AphlS3Producer.java

License:Apache License

public void processMultiPart(final Exchange exchange) throws Exception {
    File filePayload = null;//from w  ww.  j  ava 2 s .  c om
    Object obj = exchange.getIn().getMandatoryBody();
    // Need to check if the message body is WrappedFile
    if (obj instanceof WrappedFile) {
        obj = ((WrappedFile<?>) obj).getFile();
    }
    if (obj instanceof File) {
        filePayload = (File) obj;
    } else {
        LOG.error("aphl-s3: MultiPart upload requires a File input.");
        throw new InvalidArgumentException("aphl-s3: MultiPart upload requires a File input.");
    }

    ObjectMetadata objectMetadata = determineMetadata(exchange);
    if (objectMetadata.getContentLength() == 0) {
        objectMetadata.setContentLength(filePayload.length());
    }

    final String keyName = determineKey(exchange);
    final InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(
            getConfiguration().getBucketName(), keyName, objectMetadata);

    String storageClass = determineStorageClass(exchange);
    if (storageClass != null) {
        initRequest.setStorageClass(StorageClass.fromValue(storageClass));
    }

    String cannedAcl = exchange.getIn().getHeader(S3Constants.CANNED_ACL, String.class);
    if (cannedAcl != null) {
        CannedAccessControlList objectAcl = CannedAccessControlList.valueOf(cannedAcl);
        initRequest.setCannedACL(objectAcl);
    }

    AccessControlList acl = exchange.getIn().getHeader(S3Constants.ACL, AccessControlList.class);
    if (acl != null) {
        // note: if cannedacl and acl are both specified the last one will
        // be used. refer to
        // PutObjectRequest#setAccessControlList for more details
        initRequest.setAccessControlList(acl);
    }

    LOG.trace("Initiating multipart upload ...");

    final InitiateMultipartUploadResult initResponse = getEndpoint().getS3Client()
            .initiateMultipartUpload(initRequest);
    final long contentLength = objectMetadata.getContentLength();
    final List<PartETag> partETags = new ArrayList<PartETag>();
    long partSize = getConfiguration().getPartSize();
    CompleteMultipartUploadResult uploadResult = null;

    long filePosition = 0;

    try {
        for (int part = 1; filePosition < contentLength; part++) {
            partSize = Math.min(partSize, contentLength - filePosition);

            UploadPartRequest uploadRequest = new UploadPartRequest()
                    .withBucketName(getConfiguration().getBucketName()).withKey(keyName)
                    .withUploadId(initResponse.getUploadId()).withPartNumber(part).withFileOffset(filePosition)
                    .withFile(filePayload).withPartSize(partSize);

            partETags.add(getEndpoint().getS3Client().uploadPart(uploadRequest).getPartETag());

            filePosition += partSize;
        }
        CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(
                getConfiguration().getBucketName(), keyName, initResponse.getUploadId(), partETags);

        uploadResult = getEndpoint().getS3Client().completeMultipartUpload(compRequest);

    } catch (Exception exception) {
        LOG.error("Multi-part upload failed, aborting", exception);
        getEndpoint().getS3Client().abortMultipartUpload(new AbortMultipartUploadRequest(
                getConfiguration().getBucketName(), keyName, initResponse.getUploadId()));
        throw exception;
    }

    Message message = getMessageForResponse(exchange);
    message.setHeader(S3Constants.E_TAG, uploadResult.getETag());
    if (uploadResult.getVersionId() != null) {
        message.setHeader(S3Constants.VERSION_ID, uploadResult.getVersionId());
    }

    if (getConfiguration().isDeleteAfterWrite() && filePayload != null) {
        FileUtil.deleteFile(filePayload);
    }
}

From source file:io.druid.storage.s3.S3TaskLogs.java

License:Apache License

@Override
public Optional<ByteSource> streamTaskLog(final String taskid, final long offset) throws IOException {
    final String taskKey = getTaskLogKey(taskid);

    try {/* www.  j  av a  2s  .com*/
        final ObjectMetadata objectMetadata = service.getObjectMetadata(config.getS3Bucket(), taskKey);

        return Optional.of(new ByteSource() {
            @Override
            public InputStream openStream() throws IOException {
                try {
                    final long start;
                    final long end = objectMetadata.getContentLength() - 1;

                    if (offset > 0 && offset < objectMetadata.getContentLength()) {
                        start = offset;
                    } else if (offset < 0 && (-1 * offset) < objectMetadata.getContentLength()) {
                        start = objectMetadata.getContentLength() + offset;
                    } else {
                        start = 0;
                    }

                    final GetObjectRequest request = new GetObjectRequest(config.getS3Bucket(), taskKey)
                            .withMatchingETagConstraint(objectMetadata.getETag()).withRange(start, end);

                    return service.getObject(request).getObjectContent();
                } catch (AmazonServiceException e) {
                    throw new IOException(e);
                }
            }
        });
    } catch (AmazonS3Exception e) {
        if (404 == e.getStatusCode() || "NoSuchKey".equals(e.getErrorCode())
                || "NoSuchBucket".equals(e.getErrorCode())) {
            return Optional.absent();
        } else {
            throw new IOE(e, "Failed to stream logs from: %s", taskKey);
        }
    }
}

From source file:io.druid.storage.s3.S3Utils.java

License:Apache License

public static boolean isDirectoryPlaceholder(String key, ObjectMetadata objectMetadata) {
    // Recognize "standard" directory place-holder indications used by
    // Amazon's AWS Console and Panic's Transmit.
    if (key.endsWith("/") && objectMetadata.getContentLength() == 0) {
        return true;
    }/*from  w w  w  .  j a v  a2 s  .  c  om*/
    // Recognize s3sync.rb directory placeholders by MD5/ETag value.
    if ("d66759af42f282e1ba19144df2d405d0".equals(objectMetadata.getETag())) {
        return true;
    }
    // Recognize place-holder objects created by the Google Storage console
    // or S3 Organizer Firefox extension.
    if (key.endsWith("_$folder$") && objectMetadata.getContentLength() == 0) {
        return true;
    }

    // We don't use JetS3t APIs anymore, but the below check is still needed for backward compatibility.

    // Recognize legacy JetS3t directory place-holder objects, only gives
    // accurate results if an object's metadata is populated.
    if (objectMetadata.getContentLength() == 0
            && MIMETYPE_JETS3T_DIRECTORY.equals(objectMetadata.getContentType())) {
        return true;
    }
    return false;
}

From source file:io.konig.camel.aws.s3.DeleteObjectEndpoint.java

License:Apache License

public Exchange createExchange(ExchangePattern pattern, final S3Object s3Object) {
    LOG.trace("Getting object with key [{}] from bucket [{}]...", s3Object.getKey(), s3Object.getBucketName());

    ObjectMetadata objectMetadata = s3Object.getObjectMetadata();

    LOG.trace("Got object [{}]", s3Object);

    Exchange exchange = super.createExchange(pattern);
    Message message = exchange.getIn();//from  w  w w .  j  av a 2 s  .  c o m

    if (configuration.isIncludeBody()) {
        message.setBody(s3Object.getObjectContent());
    } else {
        message.setBody(null);
    }

    message.setHeader(S3Constants.KEY, s3Object.getKey());
    message.setHeader(S3Constants.BUCKET_NAME, s3Object.getBucketName());
    message.setHeader(S3Constants.E_TAG, objectMetadata.getETag());
    message.setHeader(S3Constants.LAST_MODIFIED, objectMetadata.getLastModified());
    message.setHeader(S3Constants.VERSION_ID, objectMetadata.getVersionId());
    message.setHeader(S3Constants.CONTENT_TYPE, objectMetadata.getContentType());
    message.setHeader(S3Constants.CONTENT_MD5, objectMetadata.getContentMD5());
    message.setHeader(S3Constants.CONTENT_LENGTH, objectMetadata.getContentLength());
    message.setHeader(S3Constants.CONTENT_ENCODING, objectMetadata.getContentEncoding());
    message.setHeader(S3Constants.CONTENT_DISPOSITION, objectMetadata.getContentDisposition());
    message.setHeader(S3Constants.CACHE_CONTROL, objectMetadata.getCacheControl());
    message.setHeader(S3Constants.S3_HEADERS, objectMetadata.getRawMetadata());
    message.setHeader(S3Constants.SERVER_SIDE_ENCRYPTION, objectMetadata.getSSEAlgorithm());
    message.setHeader(S3Constants.USER_METADATA, objectMetadata.getUserMetadata());
    message.setHeader(S3Constants.EXPIRATION_TIME, objectMetadata.getExpirationTime());
    message.setHeader(S3Constants.REPLICATION_STATUS, objectMetadata.getReplicationStatus());
    message.setHeader(S3Constants.STORAGE_CLASS, objectMetadata.getStorageClass());

    /**
    * If includeBody != true, it is safe to close the object here. If
    * includeBody == true, the caller is responsible for closing the stream
    * and object once the body has been fully consumed. As of 2.17, the
    * consumer does not close the stream or object on commit.
    */
    if (!configuration.isIncludeBody()) {
        IOHelper.close(s3Object);
    } else {
        if (configuration.isAutocloseBody()) {
            exchange.addOnCompletion(new SynchronizationAdapter() {
                @Override
                public void onDone(Exchange exchange) {
                    IOHelper.close(s3Object);
                }
            });
        }
    }

    return exchange;
}

From source file:io.konig.camel.aws.s3.DeleteObjectProducer.java

License:Apache License

public void processMultiPart(final Exchange exchange) throws Exception {
    File filePayload = null;/*from   w  w w .  j  a va2 s  .c o m*/
    Object obj = exchange.getIn().getMandatoryBody();
    // Need to check if the message body is WrappedFile
    if (obj instanceof WrappedFile) {
        obj = ((WrappedFile<?>) obj).getFile();
    }
    if (obj instanceof File) {
        filePayload = (File) obj;
    } else {
        throw new InvalidArgumentException("aws-s3: MultiPart upload requires a File input.");
    }

    ObjectMetadata objectMetadata = determineMetadata(exchange);
    if (objectMetadata.getContentLength() == 0) {
        objectMetadata.setContentLength(filePayload.length());
    }

    final String keyName = determineKey(exchange);
    final InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(
            getConfiguration().getBucketName(), keyName, objectMetadata);

    String storageClass = determineStorageClass(exchange);
    if (storageClass != null) {
        initRequest.setStorageClass(StorageClass.fromValue(storageClass));
    }

    String cannedAcl = exchange.getIn().getHeader(S3Constants.CANNED_ACL, String.class);
    if (cannedAcl != null) {
        CannedAccessControlList objectAcl = CannedAccessControlList.valueOf(cannedAcl);
        initRequest.setCannedACL(objectAcl);
    }

    AccessControlList acl = exchange.getIn().getHeader(S3Constants.ACL, AccessControlList.class);
    if (acl != null) {
        // note: if cannedacl and acl are both specified the last one will
        // be used. refer to
        // PutObjectRequest#setAccessControlList for more details
        initRequest.setAccessControlList(acl);
    }

    if (getConfiguration().isUseAwsKMS()) {
        SSEAwsKeyManagementParams keyManagementParams;
        if (ObjectHelper.isNotEmpty(getConfiguration().getAwsKMSKeyId())) {
            keyManagementParams = new SSEAwsKeyManagementParams(getConfiguration().getAwsKMSKeyId());
        } else {
            keyManagementParams = new SSEAwsKeyManagementParams();
        }
        initRequest.setSSEAwsKeyManagementParams(keyManagementParams);
    }

    LOG.trace("Initiating multipart upload [{}] from exchange [{}]...", initRequest, exchange);

    final InitiateMultipartUploadResult initResponse = getEndpoint().getS3Client()
            .initiateMultipartUpload(initRequest);
    final long contentLength = objectMetadata.getContentLength();
    final List<PartETag> partETags = new ArrayList<PartETag>();
    long partSize = getConfiguration().getPartSize();
    CompleteMultipartUploadResult uploadResult = null;

    long filePosition = 0;

    try {
        for (int part = 1; filePosition < contentLength; part++) {
            partSize = Math.min(partSize, contentLength - filePosition);

            UploadPartRequest uploadRequest = new UploadPartRequest()
                    .withBucketName(getConfiguration().getBucketName()).withKey(keyName)
                    .withUploadId(initResponse.getUploadId()).withPartNumber(part).withFileOffset(filePosition)
                    .withFile(filePayload).withPartSize(partSize);

            LOG.trace("Uploading part [{}] for {}", part, keyName);
            partETags.add(getEndpoint().getS3Client().uploadPart(uploadRequest).getPartETag());

            filePosition += partSize;
        }
        CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(
                getConfiguration().getBucketName(), keyName, initResponse.getUploadId(), partETags);

        uploadResult = getEndpoint().getS3Client().completeMultipartUpload(compRequest);

    } catch (Exception e) {
        getEndpoint().getS3Client().abortMultipartUpload(new AbortMultipartUploadRequest(
                getConfiguration().getBucketName(), keyName, initResponse.getUploadId()));
        throw e;
    }

    Message message = getMessageForResponse(exchange);
    message.setHeader(S3Constants.E_TAG, uploadResult.getETag());
    if (uploadResult.getVersionId() != null) {
        message.setHeader(S3Constants.VERSION_ID, uploadResult.getVersionId());
    }

    if (getConfiguration().isDeleteAfterWrite() && filePayload != null) {
        FileUtil.deleteFile(filePayload);
    }
}

From source file:io.minio.awssdk.tests.S3TestUtils.java

License:Apache License

long retrieveObjectMetadata(String bucketName, String keyName, SSECustomerKey sseKey) {
    GetObjectMetadataRequest getMetadataRequest = new GetObjectMetadataRequest(bucketName, keyName)
            .withSSECustomerKey(sseKey);
    ObjectMetadata objectMetadata = s3Client.getObjectMetadata(getMetadataRequest);
    return objectMetadata.getContentLength();
}

From source file:it.polimi.modaclouds.cpimlibrary.blobmng.AmazonBlobManager.java

License:Apache License

@Override
public CloudDownloadBlob downloadBlob(String fileName) {
    System.out.println("downloadBlob " + fileName + ".");
    ObjectMetadata om = s3.getObjectMetadata(bucketName, fileName);
    InputStream is = s3.getObject(bucketName, fileName).getObjectContent();
    return new CloudDownloadBlob(fileName, is, om.getContentType(), om.getContentLength(), fileName);
}

From source file:jp.inc.forrest.aws.swf.AverageCalculatorActivitiesImpl.java

License:Open Source License

@Override
public int computeDataSizeForInputData(String bucketName, String filename) {
    ObjectMetadata metadata = storage.getObjectMetadata(bucketName, filename);
    long size = metadata.getContentLength();
    return (int) size / ROW_SIZE;
}

From source file:lumbermill.internal.aws.S3ClientImpl.java

License:Apache License

/**
 *
 *//*from   w ww.  java  2s .com*/
public void put(T event, StringTemplate bucket, StringTemplate key) {
    ObjectMetadata metadata = new ObjectMetadata();
    ByteString raw = event.raw();
    metadata.setContentLength(raw.size());

    String sKey = key.format(event).get();
    String sBucket = bucket.format(event).get();

    LOGGER.trace("Uploading to s3://{}/{} with size {}", sBucket, sKey, metadata.getContentLength());
    s3Client.putObject(sBucket, sKey, new ByteArrayInputStream(raw.toByteArray()), metadata);
}

From source file:net.henryhu.roxlab2.NotePadProvider.java

License:Apache License

private ContentValues s3query(String id) {
    if (bucket == null)
        bucket = s3.createBucket(bucketName);

    Log.w("s3query()", "query id: " + id);

    ObjectListing ol = s3.listObjects(bucketName, id + "_");
    for (Object o : ol.getObjectSummaries()) {
        S3ObjectSummary sum = (S3ObjectSummary) o;
        S3Object obj = s3.getObject(bucketName, sum.getKey());

        ObjectMetadata om = obj.getObjectMetadata();

        byte[] buf = new byte[(int) om.getContentLength()];
        try {//from   w w w  .  jav a2  s.co  m
            InputStream contents = obj.getObjectContent();
            contents.read(buf);
            contents.close();
            String sbuf = new String(buf, "UTF-8");
            Map<String, String> entries = ParseInfo.getEntries(sbuf);
            ContentValues vals = new ContentValues();
            for (String key : entries.keySet()) {
                vals.put(key, entries.get(key));
            }
            Log.w("s3query()", "query succ");
            return vals;
        } catch (Exception e) {
        }
    }
    Log.w("s3query()", "query fail");
    return null;
}