Example usage for com.amazonaws.services.s3.model S3ObjectSummary setSize

List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary setSize

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. S3ObjectSummary
  6. setSize

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model S3ObjectSummary setSize.

Prototype

public void setSize(long size)

Source Link

Document

Sets the size of this object in bytes.

Usage

From source file:com.treasure_data.td_import.source.S3Source.java

License:Apache License

static List<S3ObjectSummary> getSources(AmazonS3Client client, String bucket, String basePath) {
    String prefix;// www  . j a  va  2  s.  c o  m
    int index = basePath.indexOf('*');
    if (index >= 0) {
        prefix = basePath.substring(0, index);
    } else {
        ObjectMetadata om = client.getObjectMetadata(bucket, basePath);
        S3ObjectSummary s3object = new S3ObjectSummary();
        s3object.setBucketName(bucket);
        s3object.setKey(basePath);
        s3object.setSize(om.getContentLength());

        return Arrays.asList(s3object);
    }

    LOG.info(String.format("list s3 files by client %s: bucket=%s, basePath=%s, prefix=%s", client, bucket,
            basePath, prefix));

    List<S3ObjectSummary> s3objects = new ArrayList<S3ObjectSummary>();
    String lastKey = prefix;
    do {
        ObjectListing listing = client.listObjects(new ListObjectsRequest(bucket, prefix, lastKey, null, 1024));
        for (S3ObjectSummary s3object : listing.getObjectSummaries()) {
            s3objects.add(s3object);
        }
        lastKey = listing.getNextMarker();
    } while (lastKey != null);

    return filterSources(s3objects, basePath);
}

From source file:com.upplication.s3fs.util.AmazonS3ClientMock.java

License:Open Source License

/**
 * create a new S3ObjectSummary using the S3Element
 * @param elem S3Element to parse//from www .  j a  v a  2  s  . c  o  m
 * @return S3ObjectSummary
 */
private S3ObjectSummary parseToS3ObjectSummary(S3Element elem) {
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(elem.getS3Object().getBucketName());
    s3ObjectSummary.setKey(elem.getS3Object().getKey());
    s3ObjectSummary.setLastModified(elem.getS3Object().getObjectMetadata().getLastModified());
    s3ObjectSummary.setOwner(owner);
    s3ObjectSummary.setETag(elem.getS3Object().getObjectMetadata().getETag());
    s3ObjectSummary.setSize(elem.getS3Object().getObjectMetadata().getContentLength());

    return s3ObjectSummary;
}

From source file:com.upplication.s3fs.util.S3ObjectSummaryLookup.java

License:Open Source License

/**
 * Get the {@link com.amazonaws.services.s3.model.S3ObjectSummary} that represent this Path or her first child if this path not exists
 * @param s3Path {@link com.upplication.s3fs.S3Path}
 * @return {@link com.amazonaws.services.s3.model.S3ObjectSummary}
 * @throws java.nio.file.NoSuchFileException if not found the path and any child
 *///w ww  .  java  2s .  com
public S3ObjectSummary lookup(S3Path s3Path) throws NoSuchFileException {

    /*
     * check is object summary has been cached
     */
    S3ObjectSummary summary = s3Path.fetchObjectSummary();
    if (summary != null) {
        return summary;
    }

    final AmazonS3Client client = s3Path.getFileSystem().getClient();

    /*
     * when `key` is an empty string retrieve the object meta-data of the bucket
     */
    if ("".equals(s3Path.getKey())) {
        ObjectMetadata meta = client.getObjectMetadata(s3Path.getBucket(), "");
        if (meta == null)
            throw new NoSuchFileException("s3://" + s3Path.getBucket());

        summary = new S3ObjectSummary();
        summary.setBucketName(s3Path.getBucket());
        summary.setETag(meta.getETag());
        summary.setKey(s3Path.getKey());
        summary.setLastModified(meta.getLastModified());
        summary.setSize(meta.getContentLength());
        // TODO summary.setOwner(?);
        // TODO summary.setStorageClass(?);
        return summary;
    }

    /*
     * Lookup for the object summary for the specified object key
     * by using a `listObjects` request
     */
    String marker = null;
    while (true) {
        ListObjectsRequest request = new ListObjectsRequest();
        request.setBucketName(s3Path.getBucket());
        request.setPrefix(s3Path.getKey());
        request.setMaxKeys(250);
        if (marker != null)
            request.setMarker(marker);

        ObjectListing listing = client.listObjects(request);
        List<S3ObjectSummary> results = listing.getObjectSummaries();

        if (results.isEmpty()) {
            break;
        }

        for (S3ObjectSummary item : results) {
            if (matchName(s3Path.getKey(), item)) {
                return item;
            }
        }

        if (listing.isTruncated())
            marker = listing.getNextMarker();
        else
            break;
    }

    throw new NoSuchFileException("s3://" + s3Path.getBucket() + "/" + s3Path.toString());
}

From source file:org.apache.tajo.storage.s3.MockObjectListing.java

License:Apache License

private S3ObjectSummary getS3ObjectSummary(String bucketName, String key, long size) {
    S3ObjectSummary objectSummary = new S3ObjectSummary();
    objectSummary.setBucketName(bucketName);
    objectSummary.setKey(key);//w w  w  . ja v a2s .c  om
    objectSummary.setSize(size);
    return objectSummary;
}

From source file:org.elasticsearch.cloud.aws.blobstore.MockAmazonS3.java

License:Apache License

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
        throws AmazonClientException, AmazonServiceException {
    MockObjectListing list = new MockObjectListing();
    list.setTruncated(false);/*from   w ww .j a  v  a  2 s . c  om*/

    String blobName;
    String prefix = listObjectsRequest.getPrefix();

    ArrayList<S3ObjectSummary> mockObjectSummaries = new ArrayList<>();

    for (Map.Entry<String, InputStream> blob : blobs.entrySet()) {
        blobName = blob.getKey();
        S3ObjectSummary objectSummary = new S3ObjectSummary();

        if (prefix.isEmpty() || blobName.startsWith(prefix)) {
            objectSummary.setKey(blobName);

            try {
                objectSummary.setSize(getSize(blob.getValue()));
            } catch (IOException e) {
                throw new AmazonS3Exception("Object listing " + "failed for blob [" + blob.getKey() + "]");
            }

            mockObjectSummaries.add(objectSummary);
        }
    }

    list.setObjectSummaries(mockObjectSummaries);
    return list;
}

From source file:org.elasticsearch.repositories.s3.MockAmazonS3.java

License:Apache License

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
        throws AmazonClientException, AmazonServiceException {
    simulateS3SocketConnection();/*w w w . ja  v a2s  .c  om*/
    MockObjectListing list = new MockObjectListing();
    list.setTruncated(false);

    String blobName;
    String prefix = listObjectsRequest.getPrefix();

    ArrayList<S3ObjectSummary> mockObjectSummaries = new ArrayList<>();

    for (Map.Entry<String, InputStream> blob : blobs.entrySet()) {
        blobName = blob.getKey();
        S3ObjectSummary objectSummary = new S3ObjectSummary();

        if (prefix.isEmpty() || blobName.startsWith(prefix)) {
            objectSummary.setKey(blobName);

            try {
                objectSummary.setSize(getSize(blob.getValue()));
            } catch (IOException e) {
                throw new AmazonS3Exception("Object listing " + "failed for blob [" + blob.getKey() + "]");
            }

            mockObjectSummaries.add(objectSummary);
        }
    }

    list.setObjectSummaries(mockObjectSummaries);
    return list;
}

From source file:org.finra.dm.dao.impl.MockS3OperationsImpl.java

License:Apache License

/**
 * Returns a list of objects. If the bucket does not exist, returns a listing with an empty list.
 * If a prefix is specified in listObjectsRequest, only keys starting with the prefix will be returned.
 */// w  w w.j a v a 2 s  . co m
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest, AmazonS3Client s3Client) {
    LOGGER.debug("listObjects(): listObjectsRequest.getBucketName() = " + listObjectsRequest.getBucketName());

    String bucketName = listObjectsRequest.getBucketName();

    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName)) {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(
                MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    }

    ObjectListing objectListing = new ObjectListing();
    objectListing.setBucketName(bucketName);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null) {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values()) {
            String s3ObjectKey = mockS3Object.getKey();
            if (listObjectsRequest.getPrefix() == null
                    || s3ObjectKey.startsWith(listObjectsRequest.getPrefix())) {
                S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                s3ObjectSummary.setBucketName(bucketName);
                s3ObjectSummary.setKey(s3ObjectKey);
                s3ObjectSummary.setSize(mockS3Object.getData().length);

                objectListing.getObjectSummaries().add(s3ObjectSummary);
            }
        }
    }

    return objectListing;
}

From source file:org.finra.herd.dao.impl.MockS3OperationsImpl.java

License:Apache License

/**
 * {@inheritDoc}/*from ww  w .  j a va 2 s.c  o  m*/
 * <p/>
 * If the bucket does not exist, returns a listing with an empty list. If a prefix is specified in listObjectsRequest, only keys starting with the prefix
 * will be returned.
 */
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest, AmazonS3 s3Client) {
    LOGGER.debug("listObjects(): listObjectsRequest.getBucketName() = " + listObjectsRequest.getBucketName());

    String bucketName = listObjectsRequest.getBucketName();

    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName)) {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(
                MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    }

    ObjectListing objectListing = new ObjectListing();
    objectListing.setBucketName(bucketName);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null) {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values()) {
            String s3ObjectKey = mockS3Object.getKey();
            if (listObjectsRequest.getPrefix() == null
                    || s3ObjectKey.startsWith(listObjectsRequest.getPrefix())) {
                S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                s3ObjectSummary.setBucketName(bucketName);
                s3ObjectSummary.setKey(s3ObjectKey);
                s3ObjectSummary.setSize(mockS3Object.getData().length);
                s3ObjectSummary.setStorageClass(mockS3Object.getObjectMetadata() != null
                        ? mockS3Object.getObjectMetadata().getStorageClass()
                        : null);

                objectListing.getObjectSummaries().add(s3ObjectSummary);
            }
        }
    }

    return objectListing;
}

From source file:org.weakref.s3fs.util.AmazonS3ClientMock.java

License:Apache License

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
        throws AmazonClientException, AmazonServiceException {
    ObjectListing objectListing = new ObjectListing();
    Integer capacity = listObjectsRequest.getMaxKeys();
    if (capacity == null) {
        capacity = Integer.MAX_VALUE;
    }//from   w w  w .  java  2s . c o m

    Bucket bucket = find(listObjectsRequest.getBucketName());
    for (S3Element elem : objects.get(bucket)) {
        if (capacity > 0) {
            // TODO. add delimiter and marker support
            if (listObjectsRequest.getPrefix() != null
                    && elem.getS3Object().getKey().startsWith(listObjectsRequest.getPrefix())) {
                S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                s3ObjectSummary.setBucketName(elem.getS3Object().getBucketName());
                s3ObjectSummary.setKey(elem.getS3Object().getKey());
                s3ObjectSummary.setLastModified(elem.getS3Object().getObjectMetadata().getLastModified());
                s3ObjectSummary.setOwner(owner);
                s3ObjectSummary.setETag(elem.getS3Object().getObjectMetadata().getETag());
                s3ObjectSummary.setSize(elem.getS3Object().getObjectMetadata().getContentLength());
                objectListing.getObjectSummaries().add(s3ObjectSummary);
                capacity--;
            }
        }

    }

    return objectListing;
}