Example usage for com.amazonaws.services.s3.model S3ObjectSummary getSize

List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary getSize

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. S3ObjectSummary
  6. getSize

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model S3ObjectSummary getSize.

Prototype

public long getSize()

Source Link

Document

Gets the size of this object in bytes.

Usage

From source file:n3phele.agent.repohandlers.S3Large.java

License:Open Source License

@Override
public List<String> getFileList() {
    List<String> result = new ArrayList<String>();
    int starIndex = key.indexOf("*");
    int questionIndex = key.indexOf("?");
    int curlyIndex = key.indexOf("{");
    if (starIndex == -1 && questionIndex == -1 && curlyIndex == -1) {
        this.totalLength = null;
        return result; // not wildcard
    }//from   w w w .ja v a2s .  com

    if (starIndex == -1)
        starIndex = Integer.MAX_VALUE;
    if (questionIndex == -1)
        questionIndex = Integer.MAX_VALUE;
    if (curlyIndex == -1)
        curlyIndex = Integer.MAX_VALUE;

    int wildStart = Math.min(Math.min(starIndex, questionIndex), curlyIndex);

    base = key.substring(0, wildStart);
    base = base.substring(0, base.lastIndexOf("/") + 1);
    // String wild = key.substring(base.lastIndexOf("/")+1);
    boolean done = false;
    this.totalLength = 0L;
    Pattern pattern = Pattern.compile("^" + Helper.wildcardToRegex(key));
    ObjectListing listing = s3().listObjects(this.root, base);
    while (!done) {
        done = !listing.isTruncated();

        for (S3ObjectSummary f : listing.getObjectSummaries()) {
            if (pattern.matcher(f.getKey()).matches()) {
                String file = f.getKey().substring(base.length());
                result.add(file);
                this.totalLength += f.getSize();
                log.info("Adding " + file + " size " + f.getSize());
            }
        }
        if (!done)
            listing = s3().listNextBatchOfObjects(listing);
    }

    return result;
}

From source file:n3phele.storage.s3.CloudStorageImpl.java

License:Open Source License

public List<FileNode> getFileList(Repository repo, String prefix, int max) {
    List<FileNode> result = new ArrayList<FileNode>();
    Credential credential = repo.getCredential().decrypt();

    AmazonS3Client s3 = new AmazonS3Client(
            new BasicAWSCredentials(credential.getAccount(), credential.getSecret()));
    s3.setEndpoint(repo.getTarget().toString());
    Policy p = null;//  ww w. jav a2s.  c o m
    try {
        BucketPolicy bp = s3.getBucketPolicy(repo.getRoot());
        log.info("Policy text " + bp.getPolicyText());
        if (bp != null && bp.getPolicyText() != null) {
            p = PolicyHelper.parse(bp.getPolicyText());
            log.info("Policy object is " + (p == null ? null : p.toJson()));
        }
    } catch (Exception e) {
        log.log(Level.WARNING, "Policy not supported", e);
    }

    try {
        ObjectListing s3Objects = s3
                .listObjects(new ListObjectsRequest(repo.getRoot(), prefix, null, "/", max));
        if (s3Objects.getCommonPrefixes() != null) {
            for (String dirName : s3Objects.getCommonPrefixes()) {
                String name = dirName;
                if (dirName.endsWith("/")) {
                    name = dirName.substring(0, dirName.length() - 1);
                }
                boolean isPublic = isPublicFolder(p, repo.getRoot(), name);
                name = name.substring(name.lastIndexOf("/") + 1);
                FileNode folder = FileNode.newFolder(name, prefix, repo, isPublic);
                log.info("Folder:" + folder);
                result.add(folder);
            }
        }
        if (s3Objects.getObjectSummaries() != null) {
            for (S3ObjectSummary fileSummary : s3Objects.getObjectSummaries()) {
                String key = fileSummary.getKey();
                if (key != null && !key.equals(prefix)) {
                    String name = key.substring(key.lastIndexOf("/") + 1);
                    UriBuilder builder = UriBuilder.fromUri(repo.getTarget()).path(repo.getRoot());
                    if (prefix != null && !prefix.isEmpty()) {
                        builder = builder.path(prefix);
                    }
                    FileNode file = FileNode.newFile(name, prefix, repo, fileSummary.getLastModified(),
                            fileSummary.getSize(), builder.path(name).toString());
                    log.info("File:" + file);
                    result.add(file);
                }
            }
        }

    } catch (AmazonServiceException e) {
        log.log(Level.WARNING, "Service Error processing " + repo, e);
    } catch (AmazonClientException e) {
        log.log(Level.SEVERE, "Client Error processing " + repo, e);
    }
    return result;
}

From source file:net.solarnetwork.node.backup.s3.SdkS3Client.java

License:Open Source License

@Override
public Set<S3ObjectReference> listObjects(String prefix) throws IOException {
    AmazonS3 client = getClient();/*  w w  w .j av a2  s  .  c  o  m*/
    Set<S3ObjectReference> result = new LinkedHashSet<>(100);
    try {
        final ListObjectsV2Request req = new ListObjectsV2Request();
        req.setBucketName(bucketName);
        req.setMaxKeys(maximumKeysPerRequest);
        req.setPrefix(prefix);
        ListObjectsV2Result listResult;
        do {
            listResult = client.listObjectsV2(req);

            for (S3ObjectSummary objectSummary : listResult.getObjectSummaries()) {
                result.add(new S3ObjectReference(objectSummary.getKey(), objectSummary.getSize(),
                        objectSummary.getLastModified()));
            }
            req.setContinuationToken(listResult.getNextContinuationToken());
        } while (listResult.isTruncated() == true);

    } catch (AmazonServiceException e) {
        log.warn("AWS error: {}; HTTP code {}; AWS code {}; type {}; request ID {}", e.getMessage(),
                e.getStatusCode(), e.getErrorCode(), e.getErrorType(), e.getRequestId());
        throw new RemoteServiceException("Error listing S3 objects at " + prefix, e);
    } catch (AmazonClientException e) {
        log.debug("Error communicating with AWS: {}", e.getMessage());
        throw new IOException("Error communicating with AWS", e);
    }
    return result;
}

From source file:nl.nn.adapterframework.filesystem.AmazonS3FileSystem.java

License:Apache License

@Override
public Iterator<S3Object> listFiles(String folder) throws FileSystemException {
    List<S3ObjectSummary> summaries = null;
    String prefix = folder != null ? folder + "/" : "";
    try {//  w w w  .  jav a 2  s  .  co m
        ObjectListing listing = s3Client.listObjects(bucketName, prefix);
        summaries = listing.getObjectSummaries();
        while (listing.isTruncated()) {
            listing = s3Client.listNextBatchOfObjects(listing);
            summaries.addAll(listing.getObjectSummaries());
        }
    } catch (AmazonServiceException e) {
        throw new FileSystemException("Cannot process requested action", e);
    }

    List<S3Object> list = new ArrayList<S3Object>();
    for (S3ObjectSummary summary : summaries) {
        S3Object object = new S3Object();
        ObjectMetadata metadata = new ObjectMetadata();
        metadata.setContentLength(summary.getSize());

        object.setBucketName(summary.getBucketName());
        object.setKey(summary.getKey());
        object.setObjectMetadata(metadata);
        if (!object.getKey().endsWith("/") && !(prefix.isEmpty() && object.getKey().contains("/"))) {
            list.add(object);
        }
    }

    return list.iterator();
}

From source file:opendap.aws.s3.SimpleS3Uploader.java

License:Open Source License

public void listBucket() {
    System.out.println("- - - - - - - - - - - - - - - - - - - - - -");
    System.out.println("S3 Bucket: " + s3BucketName);
    System.out.println("Listing: ");

    long totalSize = 0;
    int totalItems = 0;

    ObjectListing objects = s3.listObjects(s3BucketName);
    do {/*  w w w.  java  2 s .co  m*/
        for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
            System.out.println("   " + objectSummary.getKey() + " " + objectSummary.getSize() + " bytes");
            totalSize += objectSummary.getSize();
            totalItems++;
        }
        objects = s3.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());

    System.out.println("The  Amazon S3 bucket '" + s3BucketName + "'" + "contains " + totalItems
            + " objects with a total size of " + totalSize + " bytes.");

}

From source file:org.alanwilliamson.amazon.s3.List.java

License:Open Source License

public cfData execute(cfSession _session, cfArgStructData argStruct) throws cfmRunTimeException {

    AmazonKey amazonKey = getAmazonKey(_session, argStruct);
    AmazonS3 s3Client = getAmazonS3(amazonKey);

    String bucket = getNamedStringParam(argStruct, "bucket", null);
    String prefix = getNamedStringParam(argStruct, "prefix", "");

    if (bucket == null)
        throwException(_session, "Please specify a bucket");

    try {//from w w  w . ja  v  a  2 s  . c  o  m
        // Create the results
        cfQueryResultData qD = new cfQueryResultData(new String[] { "key", "size", "modified", "etag" }, null);
        qD.setQuerySource("AmazonS3." + amazonKey.getDataSource());

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
                .withDelimiter("/").withPrefix(prefix);
        ObjectListing objectListing;

        do {
            objectListing = s3Client.listObjects(listObjectsRequest);

            java.util.List<String> prefixes = objectListing.getCommonPrefixes();

            // first add the prefixes
            for (String nextPrefix : prefixes) {
                qD.addRow(1);
                qD.setCurrentRow(qD.getSize());

                qD.setCell(1, new cfStringData(nextPrefix));
                qD.setCell(2, new cfNumberData(0));
                qD.setCell(3, cfNullData.NULL);
                qD.setCell(4, cfNullData.NULL);

            }

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {

                // don't include the prefix being listed
                if (objectSummary.getKey().equals(prefix)) {
                    continue;
                }
                qD.addRow(1);
                qD.setCurrentRow(qD.getSize());

                qD.setCell(1, new cfStringData(objectSummary.getKey()));
                qD.setCell(2, new cfNumberData(objectSummary.getSize()));
                qD.setCell(3, new cfDateData(objectSummary.getLastModified()));
                qD.setCell(4, new cfStringData(objectSummary.getETag()));
            }

            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

        return qD;
    } catch (Exception e) {
        throwException(_session, "AmazonS3: " + e.getMessage());
        return cfBooleanData.FALSE;
    }
}

From source file:org.apache.beam.sdk.io.aws.s3.S3FileSystem.java

License:Apache License

private ExpandedGlob expandGlob(S3ResourceId glob) {
    // The S3 API can list objects, filtered by prefix, but not by wildcard.
    // Here, we find the longest prefix without wildcard "*",
    // then filter the results with a regex.
    checkArgument(glob.isWildcard(), "isWildcard");
    String keyPrefix = glob.getKeyNonWildcardPrefix();
    Pattern wildcardRegexp = Pattern.compile(wildcardToRegexp(glob.getKey()));

    LOG.debug("expanding bucket {}, prefix {}, against pattern {}", glob.getBucket(), keyPrefix,
            wildcardRegexp.toString());//  w  w  w . j a v  a2 s.c  om

    ImmutableList.Builder<S3ResourceId> expandedPaths = ImmutableList.builder();
    String continuationToken = null;

    do {
        ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(glob.getBucket())
                .withPrefix(keyPrefix).withContinuationToken(continuationToken);
        ListObjectsV2Result result;
        try {
            result = amazonS3.get().listObjectsV2(request);
        } catch (AmazonClientException e) {
            return ExpandedGlob.create(glob, new IOException(e));
        }
        continuationToken = result.getNextContinuationToken();

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            // Filter against regex.
            if (wildcardRegexp.matcher(objectSummary.getKey()).matches()) {
                S3ResourceId expandedPath = S3ResourceId
                        .fromComponents(objectSummary.getBucketName(), objectSummary.getKey())
                        .withSize(objectSummary.getSize()).withLastModified(objectSummary.getLastModified());
                LOG.debug("Expanded S3 object path {}", expandedPath);
                expandedPaths.add(expandedPath);
            }
        }
    } while (continuationToken != null);

    return ExpandedGlob.create(glob, expandedPaths.build());
}

From source file:org.apache.flink.cloudsort.io.aws.AwsInput.java

License:Apache License

@Override
public List<InputSplit> list() {
    Preconditions.checkNotNull(bucket);//  ww w  .  ja  va2s  .c  o m
    Preconditions.checkNotNull(prefix);

    List<InputSplit> objectNames = new ArrayList<>();

    // this will read credentials from user's home directory
    AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());

    final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucket).withPrefix(prefix);

    ListObjectsV2Result result;
    int index = 0;
    do {
        result = s3client.listObjectsV2(req);

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            String objectName = objectSummary.getKey();
            long objectSize = objectSummary.getSize();
            objectNames.add(new InputSplit(index++, objectName, objectSize));
        }
        req.setContinuationToken(result.getNextContinuationToken());
    } while (result.isTruncated());

    return objectNames;
}

From source file:org.apache.hadoop.fs.s3a.S3AFileSystem.java

License:Apache License

/**
 * List the statuses of the files/directories in the given path if the path is
 * a directory.//from  w  w  w.j  a v a  2  s.  c o  m
 *
 * @param f given path
 * @return the statuses of the files/directories in the given patch
 * @throws FileNotFoundException when the path does not exist;
 *         IOException see specific implementation
 */
public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException {
    String key = pathToKey(f);
    LOG.info("List status for path: " + f);

    final List<FileStatus> result = new ArrayList<FileStatus>();
    final FileStatus fileStatus = getFileStatus(f);

    if (fileStatus.isDirectory()) {
        if (!key.isEmpty()) {
            key = key + "/";
        }

        ListObjectsRequest request = new ListObjectsRequest();
        request.setBucketName(bucket);
        request.setPrefix(key);
        request.setDelimiter("/");
        request.setMaxKeys(maxKeys);

        if (LOG.isDebugEnabled()) {
            LOG.debug("listStatus: doing listObjects for directory " + key);
        }

        ObjectListing objects = s3.listObjects(request);
        statistics.incrementReadOps(1);

        while (true) {
            for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, workingDir);
                // Skip over keys that are ourselves and old S3N _$folder$ files
                if (keyPath.equals(f) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Ignoring: " + keyPath);
                    }
                    continue;
                }

                if (objectRepresentsDirectory(summary.getKey(), summary.getSize())) {
                    result.add(new S3AFileStatus(true, true, keyPath));
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Adding: fd: " + keyPath);
                    }
                } else {
                    result.add(new S3AFileStatus(summary.getSize(), dateToLong(summary.getLastModified()),
                            keyPath));
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Adding: fi: " + keyPath);
                    }
                }
            }

            for (String prefix : objects.getCommonPrefixes()) {
                Path keyPath = keyToPath(prefix).makeQualified(uri, workingDir);
                if (keyPath.equals(f)) {
                    continue;
                }
                result.add(new S3AFileStatus(true, false, keyPath));
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Adding: rd: " + keyPath);
                }
            }

            if (objects.isTruncated()) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("listStatus: list truncated - getting next batch");
                }

                objects = s3.listNextBatchOfObjects(objects);
                statistics.incrementReadOps(1);
            } else {
                break;
            }
        }
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Adding: rd (not a dir): " + f);
        }
        result.add(fileStatus);
    }

    return result.toArray(new FileStatus[result.size()]);
}

From source file:org.apache.hadoop.fs.s3a.S3AFileSystem.java

License:Apache License

/**
 * Return a file status object that represents the path.
 * @param f The path we want information from
 * @return a FileStatus object/*from  w w  w  .  j  a  v a2s .c  o  m*/
 * @throws java.io.FileNotFoundException when the path does not exist;
 *         IOException see specific implementation
 */
public S3AFileStatus getFileStatus(Path f) throws IOException {
    String key = pathToKey(f);

    LOG.info("Getting path status for " + f + " (" + key + ")");

    if (!key.isEmpty()) {
        try {
            ObjectMetadata meta = s3.getObjectMetadata(bucket, key);
            statistics.incrementReadOps(1);

            if (objectRepresentsDirectory(key, meta.getContentLength())) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Found exact file: fake directory");
                }
                return new S3AFileStatus(true, true, f.makeQualified(uri, workingDir));
            } else {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Found exact file: normal file");
                }
                return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()),
                        f.makeQualified(uri, workingDir));
            }
        } catch (AmazonServiceException e) {
            if (e.getStatusCode() != 404) {
                printAmazonServiceException(e);
                throw e;
            }
        } catch (AmazonClientException e) {
            printAmazonClientException(e);
            throw e;
        }

        // Necessary?
        if (!key.endsWith("/")) {
            try {
                String newKey = key + "/";
                ObjectMetadata meta = s3.getObjectMetadata(bucket, newKey);
                statistics.incrementReadOps(1);

                if (objectRepresentsDirectory(newKey, meta.getContentLength())) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Found file (with /): fake directory");
                    }
                    return new S3AFileStatus(true, true, f.makeQualified(uri, workingDir));
                } else {
                    LOG.warn("Found file (with /): real file? should not happen: " + key);

                    return new S3AFileStatus(meta.getContentLength(), dateToLong(meta.getLastModified()),
                            f.makeQualified(uri, workingDir));
                }
            } catch (AmazonServiceException e) {
                if (e.getStatusCode() != 404) {
                    printAmazonServiceException(e);
                    throw e;
                }
            } catch (AmazonClientException e) {
                printAmazonClientException(e);
                throw e;
            }
        }
    }

    try {
        if (!key.isEmpty() && !key.endsWith("/")) {
            key = key + "/";
        }
        ListObjectsRequest request = new ListObjectsRequest();
        request.setBucketName(bucket);
        request.setPrefix(key);
        request.setDelimiter("/");
        request.setMaxKeys(1);

        ObjectListing objects = s3.listObjects(request);
        statistics.incrementReadOps(1);

        if (objects.getCommonPrefixes().size() > 0 || objects.getObjectSummaries().size() > 0) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Found path as directory (with /): " + objects.getCommonPrefixes().size() + "/"
                        + objects.getObjectSummaries().size());

                for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                    LOG.debug("Summary: " + summary.getKey() + " " + summary.getSize());
                }
                for (String prefix : objects.getCommonPrefixes()) {
                    LOG.debug("Prefix: " + prefix);
                }
            }

            return new S3AFileStatus(true, false, f.makeQualified(uri, workingDir));
        }
    } catch (AmazonServiceException e) {
        if (e.getStatusCode() != 404) {
            printAmazonServiceException(e);
            throw e;
        }
    } catch (AmazonClientException e) {
        printAmazonClientException(e);
        throw e;
    }

    if (LOG.isDebugEnabled()) {
        LOG.debug("Not Found: " + f);
    }
    throw new FileNotFoundException("No such file or directory: " + f);
}