Example usage for com.amazonaws.services.s3.model ObjectListing getObjectSummaries

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ObjectListing getObjectSummaries.

Prototype

public List<S3ObjectSummary> getObjectSummaries() 

Source Link

Document

Gets the list of object summaries describing the objects stored in the S3 bucket.

Usage

From source file:nl.nn.adapterframework.filesystem.AmazonS3FileSystem.java

License:Apache License

@Override
public Iterator<S3Object> listFiles(String folder) throws FileSystemException {
    List<S3ObjectSummary> summaries = null;
    String prefix = folder != null ? folder + "/" : "";
    try {/*from   www. jav a2 s  .c  o m*/
        ObjectListing listing = s3Client.listObjects(bucketName, prefix);
        summaries = listing.getObjectSummaries();
        while (listing.isTruncated()) {
            listing = s3Client.listNextBatchOfObjects(listing);
            summaries.addAll(listing.getObjectSummaries());
        }
    } catch (AmazonServiceException e) {
        throw new FileSystemException("Cannot process requested action", e);
    }

    List<S3Object> list = new ArrayList<S3Object>();
    for (S3ObjectSummary summary : summaries) {
        S3Object object = new S3Object();
        ObjectMetadata metadata = new ObjectMetadata();
        metadata.setContentLength(summary.getSize());

        object.setBucketName(summary.getBucketName());
        object.setKey(summary.getKey());
        object.setObjectMetadata(metadata);
        if (!object.getKey().endsWith("/") && !(prefix.isEmpty() && object.getKey().contains("/"))) {
            list.add(object);
        }
    }

    return list.iterator();
}

---

From source file:nl.nn.adapterframework.filesystem.AmazonS3FileSystem.java

License:Apache License

@Override
public boolean folderExists(String folder) throws FileSystemException {
    ObjectListing objectListing = s3Client.listObjects(bucketName);
    Iterator<S3ObjectSummary> objIter = objectListing.getObjectSummaries().iterator();
    while (objIter.hasNext()) {
        S3ObjectSummary s3ObjectSummary = objIter.next();
        String key = s3ObjectSummary.getKey();
        if (key.endsWith("/") && key.equals(folder + "/")) {
            return true;
        }//w ww . jav a2 s  .c  o  m
    }
    return false;
}

---

From source file:ohnosequences.ivy.S3Repository.java

License:Apache License

@Override
public List<String> list(String parent) {
    try {//from  w  w  w. jav  a 2s.com
        String marker = null;
        List<String> keys = new ArrayList<String>();

        do {
            ListObjectsRequest request = new ListObjectsRequest().withBucketName(S3Utils.getBucket(parent))
                    .withPrefix(S3Utils.getKey(parent)).withDelimiter("/") // RFC 2396
                    .withMarker(marker);

            ObjectListing listing = getS3Client().listObjects(request);

            // Add "directories"
            keys.addAll(listing.getCommonPrefixes());

            // Add "files"
            for (S3ObjectSummary summary : listing.getObjectSummaries()) {
                keys.add(summary.getKey());
            }

            marker = listing.getNextMarker();
        } while (marker != null);

        return keys;
    } catch (AmazonServiceException e) {
        throw new S3RepositoryException(e);
    }
}

---

From source file:opendap.aws.s3.SimpleS3Uploader.java

License:Open Source License

public void listBucket() {
    System.out.println("- - - - - - - - - - - - - - - - - - - - - -");
    System.out.println("S3 Bucket: " + s3BucketName);
    System.out.println("Listing: ");

    long totalSize = 0;
    int totalItems = 0;

    ObjectListing objects = s3.listObjects(s3BucketName);
    do {//  w  ww . j av  a 2 s  .c  om
        for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
            System.out.println("   " + objectSummary.getKey() + " " + objectSummary.getSize() + " bytes");
            totalSize += objectSummary.getSize();
            totalItems++;
        }
        objects = s3.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());

    System.out.println("The  Amazon S3 bucket '" + s3BucketName + "'" + "contains " + totalItems
            + " objects with a total size of " + totalSize + " bytes.");

}

---

From source file:org.alanwilliamson.amazon.s3.List.java

License:Open Source License

public cfData execute(cfSession _session, cfArgStructData argStruct) throws cfmRunTimeException {

    AmazonKey amazonKey = getAmazonKey(_session, argStruct);
    AmazonS3 s3Client = getAmazonS3(amazonKey);

    String bucket = getNamedStringParam(argStruct, "bucket", null);
    String prefix = getNamedStringParam(argStruct, "prefix", "");

    if (bucket == null)
        throwException(_session, "Please specify a bucket");

    try {/*from w  w w.ja  v a  2 s.co  m*/
        // Create the results
        cfQueryResultData qD = new cfQueryResultData(new String[] { "key", "size", "modified", "etag" }, null);
        qD.setQuerySource("AmazonS3." + amazonKey.getDataSource());

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
                .withDelimiter("/").withPrefix(prefix);
        ObjectListing objectListing;

        do {
            objectListing = s3Client.listObjects(listObjectsRequest);

            java.util.List<String> prefixes = objectListing.getCommonPrefixes();

            // first add the prefixes
            for (String nextPrefix : prefixes) {
                qD.addRow(1);
                qD.setCurrentRow(qD.getSize());

                qD.setCell(1, new cfStringData(nextPrefix));
                qD.setCell(2, new cfNumberData(0));
                qD.setCell(3, cfNullData.NULL);
                qD.setCell(4, cfNullData.NULL);

            }

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {

                // don't include the prefix being listed
                if (objectSummary.getKey().equals(prefix)) {
                    continue;
                }
                qD.addRow(1);
                qD.setCurrentRow(qD.getSize());

                qD.setCell(1, new cfStringData(objectSummary.getKey()));
                qD.setCell(2, new cfNumberData(objectSummary.getSize()));
                qD.setCell(3, new cfDateData(objectSummary.getLastModified()));
                qD.setCell(4, new cfStringData(objectSummary.getETag()));
            }

            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

        return qD;
    } catch (Exception e) {
        throwException(_session, "AmazonS3: " + e.getMessage());
        return cfBooleanData.FALSE;
    }
}

---

From source file:org.apache.camel.component.aws.s3.AmazonS3ClientMock.java

License:Apache License

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
        throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknow bucket");
        ex.setStatusCode(404);//ww  w  .ja  va 2 s .c  o  m
        throw ex;
    }

    ObjectListing objectListing = new ObjectListing();
    int capacity = listObjectsRequest.getMaxKeys();

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}

---

From source file:org.apache.camel.component.aws.s3.S3Consumer.java

License:Apache License

@Override
protected int poll() throws Exception {
    // must reset for each poll
    shutdownRunningTask = null;/*from w  w  w.ja  v a 2  s .  co  m*/
    pendingExchanges = 0;

    String fileName = getConfiguration().getFileName();
    String bucketName = getConfiguration().getBucketName();
    Queue<Exchange> exchanges = null;

    if (fileName != null) {
        LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName);

        S3Object s3Object = getAmazonS3Client().getObject(new GetObjectRequest(bucketName, fileName));
        exchanges = createExchanges(s3Object);
    } else {
        LOG.trace("Queueing objects in bucket [{}]...", bucketName);

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
        listObjectsRequest.setBucketName(bucketName);
        listObjectsRequest.setPrefix(getConfiguration().getPrefix());
        listObjectsRequest.setMaxKeys(maxMessagesPerPoll);

        ObjectListing listObjects = getAmazonS3Client().listObjects(listObjectsRequest);

        if (LOG.isTraceEnabled()) {
            LOG.trace("Found {} objects in bucket [{}]...", listObjects.getObjectSummaries().size(),
                    bucketName);
        }

        exchanges = createExchanges(listObjects.getObjectSummaries());
    }
    return processBatch(CastUtils.cast(exchanges));
}

---

From source file:org.apache.camel.itest.osgi.aws.AmazonS3ClientMock.java

License:Apache License

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
        throws AmazonClientException, AmazonServiceException {
    ObjectListing objectListing = new ObjectListing();
    int capacity = listObjectsRequest.getMaxKeys();

    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());

        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }// w  w  w  .  j  a v a  2  s .  com

    return objectListing;
}

---

From source file:org.apache.hadoop.fs.s3a.S3AFileSystem.java

License:Apache License

/**
 * Renames Path src to Path dst.  Can take place on local fs
 * or remote DFS./*  ww w  . j  a  va 2 s.c  o m*/
 *
 * Warning: S3 does not support renames. This method does a copy which can take S3 some time to execute with large
 *          files and directories. Since there is no Progressable passed in, this can time out jobs.
 *
 * Note: This implementation differs with other S3 drivers. Specifically:
 *       Fails if src is a file and dst is a directory.
 *       Fails if src is a directory and dst is a file.
 *       Fails if the parent of dst does not exist or is a file.
 *       Fails if dst is a directory that is not empty.
 *
 * @param src path to be renamed
 * @param dst new path after rename
 * @throws IOException on failure
 * @return true if rename is successful
 */
public boolean rename(Path src, Path dst) throws IOException {
    LOG.info("Rename path " + src + " to " + dst);

    String srcKey = pathToKey(src);
    String dstKey = pathToKey(dst);

    if (srcKey.length() == 0 || dstKey.length() == 0) {
        LOG.info("rename: src or dst are empty");
        return false;
    }

    if (srcKey.equals(dstKey)) {
        LOG.info("rename: src and dst refer to the same file");
        return true;
    }

    S3AFileStatus srcStatus;
    try {
        srcStatus = getFileStatus(src);
    } catch (FileNotFoundException e) {
        LOG.info("rename: src not found " + src);
        return false;
    }

    S3AFileStatus dstStatus = null;
    try {
        dstStatus = getFileStatus(dst);

        if (srcStatus.isFile() && dstStatus.isDirectory()) {
            LOG.info("rename: src is a file and dst is a directory");
            return false;
        }

        if (srcStatus.isDirectory() && dstStatus.isFile()) {
            LOG.info("rename: src is a directory and dst is a file");
            return false;
        }

        if (dstStatus.isDirectory() && !dstStatus.isEmptyDirectory()) {
            return false;
        }
    } catch (FileNotFoundException e) {
        // Parent must exist
        Path parent = dst.getParent();
        if (!pathToKey(parent).isEmpty()) {
            try {
                S3AFileStatus dstParentStatus = getFileStatus(dst.getParent());
                if (!dstParentStatus.isDirectory()) {
                    return false;
                }
            } catch (FileNotFoundException e2) {
                return false;
            }
        }
    }

    // Ok! Time to start
    if (srcStatus.isFile()) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("rename: renaming file " + src + " to " + dst);
        }
        copyFile(srcKey, dstKey);
        delete(src, false);
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug("rename: renaming directory " + src + " to " + dst);
        }

        // This is a directory to directory copy
        if (!dstKey.endsWith("/")) {
            dstKey = dstKey + "/";
        }

        if (!srcKey.endsWith("/")) {
            srcKey = srcKey + "/";
        }

        List<DeleteObjectsRequest.KeyVersion> keysToDelete = new ArrayList<DeleteObjectsRequest.KeyVersion>();
        if (dstStatus != null && dstStatus.isEmptyDirectory()) {
            copyFile(srcKey, dstKey);
            statistics.incrementWriteOps(1);
            keysToDelete.add(new DeleteObjectsRequest.KeyVersion(srcKey));
        }

        ListObjectsRequest request = new ListObjectsRequest();
        request.setBucketName(bucket);
        request.setPrefix(srcKey);
        request.setMaxKeys(maxKeys);

        ObjectListing objects = s3.listObjects(request);
        statistics.incrementReadOps(1);

        while (true) {
            for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                keysToDelete.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
                String newDstKey = dstKey + summary.getKey().substring(srcKey.length());
                copyFile(summary.getKey(), newDstKey);

                if (keysToDelete.size() == MAX_ENTRIES_TO_DELETE) {
                    DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket)
                            .withKeys(keysToDelete);
                    s3.deleteObjects(deleteRequest);
                    statistics.incrementWriteOps(1);
                    keysToDelete.clear();
                }
            }

            if (objects.isTruncated()) {
                objects = s3.listNextBatchOfObjects(objects);
                statistics.incrementReadOps(1);
            } else {
                break;
            }
        }

        if (!keysToDelete.isEmpty()) {
            DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket);
            deleteRequest.setKeys(keysToDelete);
            s3.deleteObjects(deleteRequest);
            statistics.incrementWriteOps(1);
        }
    }

    if (src.getParent() != dst.getParent()) {
        deleteUnnecessaryFakeDirectories(dst.getParent());
        createFakeDirectoryIfNecessary(src.getParent());
    }
    return true;
}

---

From source file:org.apache.hadoop.fs.s3a.S3AFileSystem.java

License:Apache License

/** Delete a file.
 *
 * @param f the path to delete./*from   www  .j ava 2  s . c  o  m*/
 * @param recursive if path is a directory and set to
 * true, the directory is deleted else throws an exception. In
 * case of a file the recursive can be set to either true or false.
 * @return  true if delete is successful else false.
 * @throws IOException
 */
public boolean delete(Path f, boolean recursive) throws IOException {
    LOG.info("Delete path " + f + " - recursive " + recursive);
    S3AFileStatus status;
    try {
        status = getFileStatus(f);
    } catch (FileNotFoundException e) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Couldn't delete " + f + " - does not exist");
        }
        return false;
    }

    String key = pathToKey(f);

    if (status.isDirectory()) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("delete: Path is a directory");
        }

        if (!recursive) {
            throw new IOException("Path is a folder: " + f);
        }

        if (!key.endsWith("/")) {
            key = key + "/";
        }

        if (status.isEmptyDirectory()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Deleting fake empty directory");
            }
            s3.deleteObject(bucket, key);
            statistics.incrementWriteOps(1);
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Getting objects for directory prefix " + key + " to delete");
            }

            ListObjectsRequest request = new ListObjectsRequest();
            request.setBucketName(bucket);
            request.setPrefix(key);
            // Hopefully not setting a delimiter will cause this to find everything
            //request.setDelimiter("/");
            request.setMaxKeys(maxKeys);

            List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<DeleteObjectsRequest.KeyVersion>();
            ObjectListing objects = s3.listObjects(request);
            statistics.incrementReadOps(1);
            while (true) {
                for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                    keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Got object to delete " + summary.getKey());
                    }

                    if (keys.size() == MAX_ENTRIES_TO_DELETE) {
                        DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keys);
                        s3.deleteObjects(deleteRequest);
                        statistics.incrementWriteOps(1);
                        keys.clear();
                    }
                }

                if (objects.isTruncated()) {
                    objects = s3.listNextBatchOfObjects(objects);
                    statistics.incrementReadOps(1);
                } else {
                    break;
                }
            }

            if (!keys.isEmpty()) {
                DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keys);
                s3.deleteObjects(deleteRequest);
                statistics.incrementWriteOps(1);
            }
        }
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug("delete: Path is a file");
        }
        s3.deleteObject(bucket, key);
        statistics.incrementWriteOps(1);
    }

    createFakeDirectoryIfNecessary(f.getParent());

    return true;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• »