Example usage for com.amazonaws.services.s3.model ObjectMetadata getContentLength

List of usage examples for com.amazonaws.services.s3.model ObjectMetadata getContentLength

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. ObjectMetadata
  6. getContentLength

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ObjectMetadata getContentLength.

Prototype

public long getContentLength()

Source Link

Document

<p> Gets the Content-Length HTTP header indicating the size of the associated object in bytes.

Usage

From source file:com.treasure_data.td_import.source.S3Source.java

License:Apache License

static List<S3ObjectSummary> getSources(AmazonS3Client client, String bucket, String basePath) {
    String prefix;//w w  w .  j  a  v  a 2 s. com
    int index = basePath.indexOf('*');
    if (index >= 0) {
        prefix = basePath.substring(0, index);
    } else {
        ObjectMetadata om = client.getObjectMetadata(bucket, basePath);
        S3ObjectSummary s3object = new S3ObjectSummary();
        s3object.setBucketName(bucket);
        s3object.setKey(basePath);
        s3object.setSize(om.getContentLength());

        return Arrays.asList(s3object);
    }

    LOG.info(String.format("list s3 files by client %s: bucket=%s, basePath=%s, prefix=%s", client, bucket,
            basePath, prefix));

    List<S3ObjectSummary> s3objects = new ArrayList<S3ObjectSummary>();
    String lastKey = prefix;
    do {
        ObjectListing listing = client.listObjects(new ListObjectsRequest(bucket, prefix, lastKey, null, 1024));
        for (S3ObjectSummary s3object : listing.getObjectSummaries()) {
            s3objects.add(s3object);
        }
        lastKey = listing.getNextMarker();
    } while (lastKey != null);

    return filterSources(s3objects, basePath);
}

From source file:com.upplication.s3fs.AmazonS3Client.java

License:Open Source License

public void multipartCopyObject(S3Path s3Source, S3Path s3Target, Long objectSize, S3MultipartOptions opts) {

    final String sourceBucketName = s3Source.getBucket();
    final String sourceObjectKey = s3Source.getKey();
    final String targetBucketName = s3Target.getBucket();
    final String targetObjectKey = s3Target.getKey();

    // Step 2: Initialize
    InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest(targetBucketName,
            targetObjectKey);//w  ww  . jav a 2  s.com

    InitiateMultipartUploadResult initResult = client.initiateMultipartUpload(initiateRequest);

    // Step 3: Save upload Id.
    String uploadId = initResult.getUploadId();

    // Get object size.
    if (objectSize == null) {
        GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(sourceBucketName,
                sourceObjectKey);
        ObjectMetadata metadataResult = client.getObjectMetadata(metadataRequest);
        objectSize = metadataResult.getContentLength(); // in bytes
    }

    final int partSize = opts.getChunkSize(objectSize);
    ExecutorService executor = S3OutputStream.getOrCreateExecutor(opts.getMaxThreads());
    List<Callable<CopyPartResult>> copyPartRequests = new ArrayList<>();

    // Step 4. create copy part requests
    long bytePosition = 0;
    for (int i = 1; bytePosition < objectSize; i++) {
        long lastPosition = bytePosition + partSize - 1 >= objectSize ? objectSize - 1
                : bytePosition + partSize - 1;

        CopyPartRequest copyRequest = new CopyPartRequest().withDestinationBucketName(targetBucketName)
                .withDestinationKey(targetObjectKey).withSourceBucketName(sourceBucketName)
                .withSourceKey(sourceObjectKey).withUploadId(uploadId).withFirstByte(bytePosition)
                .withLastByte(lastPosition).withPartNumber(i);

        copyPartRequests.add(copyPart(client, copyRequest, opts));
        bytePosition += partSize;
    }

    log.trace(
            "Starting multipart copy from: {} to {} -- uploadId={}; objectSize={}; chunkSize={}; numOfChunks={}",
            s3Source, s3Target, uploadId, objectSize, partSize, copyPartRequests.size());

    List<PartETag> etags = new ArrayList<>();
    List<Future<CopyPartResult>> responses;
    try {
        // Step 5. Start parallel parts copy
        responses = executor.invokeAll(copyPartRequests);

        // Step 6. Fetch all results
        for (Future<CopyPartResult> response : responses) {
            CopyPartResult result = response.get();
            etags.add(new PartETag(result.getPartNumber(), result.getETag()));
        }
    } catch (Exception e) {
        throw new IllegalStateException("Multipart copy reported an unexpected error -- uploadId=" + uploadId,
                e);
    }

    // Step 7. Complete copy operation
    CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(targetBucketName,
            targetObjectKey, initResult.getUploadId(), etags);

    log.trace("Completing multipart copy uploadId={}", uploadId);
    client.completeMultipartUpload(completeRequest);
}

From source file:com.upplication.s3fs.S3FileSystemProvider.java

License:Open Source License

@Override
public void copy(Path source, Path target, CopyOption... options) throws IOException {
    Preconditions.checkArgument(source instanceof S3Path, "source must be an instance of %s",
            S3Path.class.getName());
    Preconditions.checkArgument(target instanceof S3Path, "target must be an instance of %s",
            S3Path.class.getName());

    if (isSameFile(source, target)) {
        return;//from  ww  w .j av a  2  s.c  o m
    }

    S3Path s3Source = (S3Path) source;
    S3Path s3Target = (S3Path) target;
    /*
     * Preconditions.checkArgument(!s3Source.isDirectory(),
     * "copying directories is not yet supported: %s", source); // TODO
     * Preconditions.checkArgument(!s3Target.isDirectory(),
     * "copying directories is not yet supported: %s", target); // TODO
     */
    ImmutableSet<CopyOption> actualOptions = ImmutableSet.copyOf(options);
    verifySupportedOptions(EnumSet.of(StandardCopyOption.REPLACE_EXISTING), actualOptions);

    if (!actualOptions.contains(StandardCopyOption.REPLACE_EXISTING)) {
        if (exists(s3Target)) {
            throw new FileAlreadyExistsException(format("target already exists: %s", target));
        }
    }

    AmazonS3Client client = s3Source.getFileSystem().getClient();

    final ObjectMetadata sourceObjMetadata = s3Source.getFileSystem().getClient()
            .getObjectMetadata(s3Source.getBucket(), s3Source.getKey());
    final S3MultipartOptions opts = props != null ? new S3MultipartOptions<>(props) : new S3MultipartOptions();
    final int chunkSize = opts.getChunkSize();
    final long length = sourceObjMetadata.getContentLength();

    if (length <= chunkSize) {

        CopyObjectRequest copyObjRequest = new CopyObjectRequest(s3Source.getBucket(), s3Source.getKey(),
                s3Target.getBucket(), s3Target.getKey());
        if (sourceObjMetadata.getSSEAlgorithm() != null) {
            ObjectMetadata targetObjectMetadata = new ObjectMetadata();
            targetObjectMetadata.setSSEAlgorithm(sourceObjMetadata.getSSEAlgorithm());
            copyObjRequest.setNewObjectMetadata(targetObjectMetadata);
        }

        client.copyObject(copyObjRequest);
    } else {
        client.multipartCopyObject(s3Source, s3Target, length, opts);
    }
}

From source file:com.upplication.s3fs.util.S3ObjectSummaryLookup.java

License:Open Source License

/**
 * Get the {@link com.amazonaws.services.s3.model.S3ObjectSummary} that represent this Path or her first child if this path not exists
 * @param s3Path {@link com.upplication.s3fs.S3Path}
 * @return {@link com.amazonaws.services.s3.model.S3ObjectSummary}
 * @throws java.nio.file.NoSuchFileException if not found the path and any child
 *//*from  w ww .jav  a2 s .c  o m*/
public S3ObjectSummary lookup(S3Path s3Path) throws NoSuchFileException {

    /*
     * check is object summary has been cached
     */
    S3ObjectSummary summary = s3Path.fetchObjectSummary();
    if (summary != null) {
        return summary;
    }

    final AmazonS3Client client = s3Path.getFileSystem().getClient();

    /*
     * when `key` is an empty string retrieve the object meta-data of the bucket
     */
    if ("".equals(s3Path.getKey())) {
        ObjectMetadata meta = client.getObjectMetadata(s3Path.getBucket(), "");
        if (meta == null)
            throw new NoSuchFileException("s3://" + s3Path.getBucket());

        summary = new S3ObjectSummary();
        summary.setBucketName(s3Path.getBucket());
        summary.setETag(meta.getETag());
        summary.setKey(s3Path.getKey());
        summary.setLastModified(meta.getLastModified());
        summary.setSize(meta.getContentLength());
        // TODO summary.setOwner(?);
        // TODO summary.setStorageClass(?);
        return summary;
    }

    /*
     * Lookup for the object summary for the specified object key
     * by using a `listObjects` request
     */
    String marker = null;
    while (true) {
        ListObjectsRequest request = new ListObjectsRequest();
        request.setBucketName(s3Path.getBucket());
        request.setPrefix(s3Path.getKey());
        request.setMaxKeys(250);
        if (marker != null)
            request.setMarker(marker);

        ObjectListing listing = client.listObjects(request);
        List<S3ObjectSummary> results = listing.getObjectSummaries();

        if (results.isEmpty()) {
            break;
        }

        for (S3ObjectSummary item : results) {
            if (matchName(s3Path.getKey(), item)) {
                return item;
            }
        }

        if (listing.isTruncated())
            marker = listing.getNextMarker();
        else
            break;
    }

    throw new NoSuchFileException("s3://" + s3Path.getBucket() + "/" + s3Path.toString());
}

From source file:com.yahoo.ycsb.utils.connection.S3Connection.java

License:Open Source License

public byte[] read(String key) throws InterruptedException {
    //long starttime = System.currentTimeMillis();
    byte[] bytes = null;
    try {/*  w w w  .j  a  v a 2  s.c o  m*/
        GetObjectRequest getObjectRequest = null;
        GetObjectMetadataRequest getObjectMetadataRequest = null;
        if (ssecKey != null) {
            getObjectRequest = new GetObjectRequest(bucket, key).withSSECustomerKey(ssecKey);
            getObjectMetadataRequest = new GetObjectMetadataRequest(bucket, key).withSSECustomerKey(ssecKey);
        } else {
            getObjectRequest = new GetObjectRequest(bucket, key);
            getObjectMetadataRequest = new GetObjectMetadataRequest(bucket, key);
        }
        //System.out.println("Get object " + key + " from " + bucket);
        S3Object object = awsClient.getObject(getObjectRequest);
        ObjectMetadata objectMetadata = awsClient.getObjectMetadata(getObjectMetadataRequest);
        //System.out.println("Get object " + key + " from " + bucket + " OK");
        InputStream objectData = object.getObjectContent(); //consuming the stream
        // writing the stream to bytes and to results
        int sizeOfFile = (int) objectMetadata.getContentLength();
        bytes = new byte[sizeOfFile];
        int offset = 0;
        while (offset < sizeOfFile) {
            int chunk_size;

            //read in 4k chunks
            chunk_size = sizeOfFile - offset > 4096 ? 4096 : sizeOfFile - offset;

            int nr_bytes_read = objectData.read(bytes, offset, sizeOfFile - offset);
            offset = offset + nr_bytes_read;

            if (Thread.interrupted()) {
                //System.out.println("interrupt " + key);
                objectData.close();
                throw new InterruptedException();
            }
        }
        //int nr_bytes_read = objectData.read(bytes, 0, sizeOfFile);
        objectData.close();
    } catch (IOException e) {
        logger.warn("Not possible to get the object " + key);
    }
    //long endtime = System.currentTimeMillis();
    //System.out.println("ReadS3: " + key + " " + (endtime - starttime) + " " + region);
    return bytes;
}

From source file:eu.stratosphere.nephele.fs.s3.S3FileSystem.java

License:Apache License

/**
 * {@inheritDoc}//from  w  w w . j  a  v  a 2  s  .  co m
 */
@Override
public FileStatus getFileStatus(final Path f) throws IOException {

    final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f);

    // This is the S3:/// base directory
    if (!bop.hasBucket() && !bop.hasObject()) {
        return new S3FileStatus(f, 0L, true, 0L, 0L);
    }

    try {
        if (bop.hasBucket() && !bop.hasObject()) {

            final List<Bucket> buckets = this.s3Client.listBuckets();
            final Iterator<Bucket> it = buckets.iterator();

            // Iterator throw list of buckets to find out creation date
            while (it.hasNext()) {

                final Bucket bucket = it.next();
                if (bop.getBucket().equals(bucket.getName())) {

                    final long creationDate = dateToLong(bucket.getCreationDate());
                    // S3 does not track access times, so this implementation always sets it to 0
                    return new S3FileStatus(f, 0L, true, creationDate, 0L);
                }
            }

            throw new FileNotFoundException("Cannot find " + f.toUri());
        }

        try {
            final ObjectMetadata om = this.s3Client.getObjectMetadata(bop.getBucket(), bop.getObject());
            final long modificationDate = dateToLong(om.getLastModified());
            // S3 does not track access times, so this implementation always sets it to 0
            if (objectRepresentsDirectory(bop.getObject(), om.getContentLength())) {
                return new S3FileStatus(f, 0L, true, modificationDate, 0L);
            } else {
                return new S3FileStatus(f, om.getContentLength(), false, modificationDate, 0L);
            }

        } catch (AmazonServiceException e) {
            if (e.getStatusCode() == HTTP_RESOURCE_NOT_FOUND_CODE) {
                throw new FileNotFoundException("Cannot find " + f.toUri());
            } else {
                throw e;
            }
        }
    } catch (AmazonClientException e) {
        throw new IOException(StringUtils.stringifyException(e));
    }
}

From source file:eu.stratosphere.nephele.fs.s3.S3FileSystem.java

License:Apache License

/**
 * {@inheritDoc}/*from   w ww. ja v  a  2  s  .  c o m*/
 */
@Override
public FileStatus[] listStatus(final Path f) throws IOException {

    final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f);

    try {

        if (!bop.hasBucket()) {

            final List<Bucket> list = this.s3Client.listBuckets();
            final S3FileStatus[] array = new S3FileStatus[list.size()];
            final Iterator<Bucket> it = list.iterator();
            int i = 0;
            while (it.hasNext()) {
                final Bucket bucket = it.next();
                final long creationDate = dateToLong(bucket.getCreationDate());
                // S3 does not track access times, so this implementation always sets it to 0
                final S3FileStatus status = new S3FileStatus(
                        extendPath(f, bucket.getName() + S3_DIRECTORY_SEPARATOR), 0, true, creationDate, 0L);
                array[i++] = status;
            }

            return array;
        }

        if (bop.hasBucket() && !bop.hasObject()) {

            // Check if the bucket really exists
            if (!this.s3Client.doesBucketExist(bop.getBucket())) {
                throw new FileNotFoundException("Cannot find " + f.toUri());
            }

            return listBucketContent(f, bop);

        } else {

            final ObjectMetadata omd = this.s3Client.getObjectMetadata(bop.getBucket(), bop.getObject());
            if (objectRepresentsDirectory(bop.getObject(), omd.getContentLength())) {

                return listBucketContent(f, bop);

            } else {
                final S3FileStatus fileStatus = new S3FileStatus(f, omd.getContentLength(), false,
                        dateToLong(omd.getLastModified()), 0L);

                return new FileStatus[] { fileStatus };
            }

        }

    } catch (AmazonClientException e) {
        throw new IOException(StringUtils.stringifyException(e));
    }
}

From source file:eu.stratosphere.runtime.fs.s3.S3FileSystem.java

License:Apache License

@Override
public FileStatus getFileStatus(final Path f) throws IOException {

    final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f);

    // This is the S3:/// base directory
    if (!bop.hasBucket() && !bop.hasObject()) {
        return new S3FileStatus(f, 0L, true, 0L, 0L);
    }/*from   w w  w .  j a  v a  2 s  .  c om*/

    try {
        if (bop.hasBucket() && !bop.hasObject()) {

            final List<Bucket> buckets = this.s3Client.listBuckets();
            final Iterator<Bucket> it = buckets.iterator();

            // Iterator throw list of buckets to find out creation date
            while (it.hasNext()) {

                final Bucket bucket = it.next();
                if (bop.getBucket().equals(bucket.getName())) {

                    final long creationDate = dateToLong(bucket.getCreationDate());
                    // S3 does not track access times, so this implementation always sets it to 0
                    return new S3FileStatus(f, 0L, true, creationDate, 0L);
                }
            }

            throw new FileNotFoundException("Cannot find " + f.toUri());
        }

        try {
            final ObjectMetadata om = this.s3Client.getObjectMetadata(bop.getBucket(), bop.getObject());
            final long modificationDate = dateToLong(om.getLastModified());
            // S3 does not track access times, so this implementation always sets it to 0
            if (objectRepresentsDirectory(bop.getObject(), om.getContentLength())) {
                return new S3FileStatus(f, 0L, true, modificationDate, 0L);
            } else {
                return new S3FileStatus(f, om.getContentLength(), false, modificationDate, 0L);
            }

        } catch (AmazonServiceException e) {
            if (e.getStatusCode() == HTTP_RESOURCE_NOT_FOUND_CODE) {
                throw new FileNotFoundException("Cannot find " + f.toUri());
            } else {
                throw e;
            }
        }
    } catch (AmazonClientException e) {
        throw new IOException(StringUtils.stringifyException(e));
    }
}

From source file:eu.stratosphere.runtime.fs.s3.S3FileSystem.java

License:Apache License

@Override
public FileStatus[] listStatus(final Path f) throws IOException {

    final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f);

    try {// ww w .jav  a2s  . c  o  m

        if (!bop.hasBucket()) {

            final List<Bucket> list = this.s3Client.listBuckets();
            final S3FileStatus[] array = new S3FileStatus[list.size()];
            final Iterator<Bucket> it = list.iterator();
            int i = 0;
            while (it.hasNext()) {
                final Bucket bucket = it.next();
                final long creationDate = dateToLong(bucket.getCreationDate());
                // S3 does not track access times, so this implementation always sets it to 0
                final S3FileStatus status = new S3FileStatus(
                        extendPath(f, bucket.getName() + S3_DIRECTORY_SEPARATOR), 0, true, creationDate, 0L);
                array[i++] = status;
            }

            return array;
        }

        if (bop.hasBucket() && !bop.hasObject()) {

            // Check if the bucket really exists
            if (!this.s3Client.doesBucketExist(bop.getBucket())) {
                throw new FileNotFoundException("Cannot find " + f.toUri());
            }

            return listBucketContent(f, bop);

        } else {

            final ObjectMetadata omd = this.s3Client.getObjectMetadata(bop.getBucket(), bop.getObject());
            if (objectRepresentsDirectory(bop.getObject(), omd.getContentLength())) {

                return listBucketContent(f, bop);

            } else {
                final S3FileStatus fileStatus = new S3FileStatus(f, omd.getContentLength(), false,
                        dateToLong(omd.getLastModified()), 0L);

                return new FileStatus[] { fileStatus };
            }

        }

    } catch (AmazonClientException e) {
        throw new IOException(StringUtils.stringifyException(e));
    }
}

From source file:fr.ens.biologie.genomique.eoulsan.data.protocols.S3DataProtocol.java

License:LGPL

@Override
public DataFileMetadata getMetadata(final DataFile src) throws IOException {

    if (!exists(src, true)) {
        throw new FileNotFoundException("File not found: " + src);
    }//from  w w w .j  a  v a2  s.c om

    final ObjectMetadata md = new S3URL(src).getMetaData();

    final SimpleDataFileMetadata result = new SimpleDataFileMetadata();
    result.setContentLength(md.getContentLength());
    result.setLastModified(md.getLastModified().getTime());
    result.setContentType(md.getContentType());
    result.setContentEncoding(md.getContentEncoding());
    result.setDataFormat(DataFormatRegistry.getInstance().getDataFormatFromFilename(src.getName()));

    return result;
}