Example usage for com.amazonaws.services.s3.model S3ObjectSummary getLastModified

List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary getLastModified

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. S3ObjectSummary
  6. getLastModified

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model S3ObjectSummary getLastModified.

Prototype

public Date getLastModified()

Source Link

Document

Gets the date when, according to Amazon S3, this object was last modified.

Usage

From source file:org.alanwilliamson.amazon.s3.List.java

License:Open Source License

public cfData execute(cfSession _session, cfArgStructData argStruct) throws cfmRunTimeException {

    AmazonKey amazonKey = getAmazonKey(_session, argStruct);
    AmazonS3 s3Client = getAmazonS3(amazonKey);

    String bucket = getNamedStringParam(argStruct, "bucket", null);
    String prefix = getNamedStringParam(argStruct, "prefix", "");

    if (bucket == null)
        throwException(_session, "Please specify a bucket");

    try {/*from www.ja v  a  2  s .  c  o  m*/
        // Create the results
        cfQueryResultData qD = new cfQueryResultData(new String[] { "key", "size", "modified", "etag" }, null);
        qD.setQuerySource("AmazonS3." + amazonKey.getDataSource());

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
                .withDelimiter("/").withPrefix(prefix);
        ObjectListing objectListing;

        do {
            objectListing = s3Client.listObjects(listObjectsRequest);

            java.util.List<String> prefixes = objectListing.getCommonPrefixes();

            // first add the prefixes
            for (String nextPrefix : prefixes) {
                qD.addRow(1);
                qD.setCurrentRow(qD.getSize());

                qD.setCell(1, new cfStringData(nextPrefix));
                qD.setCell(2, new cfNumberData(0));
                qD.setCell(3, cfNullData.NULL);
                qD.setCell(4, cfNullData.NULL);

            }

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {

                // don't include the prefix being listed
                if (objectSummary.getKey().equals(prefix)) {
                    continue;
                }
                qD.addRow(1);
                qD.setCurrentRow(qD.getSize());

                qD.setCell(1, new cfStringData(objectSummary.getKey()));
                qD.setCell(2, new cfNumberData(objectSummary.getSize()));
                qD.setCell(3, new cfDateData(objectSummary.getLastModified()));
                qD.setCell(4, new cfStringData(objectSummary.getETag()));
            }

            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

        return qD;
    } catch (Exception e) {
        throwException(_session, "AmazonS3: " + e.getMessage());
        return cfBooleanData.FALSE;
    }
}

From source file:org.apache.beam.sdk.io.aws.s3.S3FileSystem.java

License:Apache License

private ExpandedGlob expandGlob(S3ResourceId glob) {
    // The S3 API can list objects, filtered by prefix, but not by wildcard.
    // Here, we find the longest prefix without wildcard "*",
    // then filter the results with a regex.
    checkArgument(glob.isWildcard(), "isWildcard");
    String keyPrefix = glob.getKeyNonWildcardPrefix();
    Pattern wildcardRegexp = Pattern.compile(wildcardToRegexp(glob.getKey()));

    LOG.debug("expanding bucket {}, prefix {}, against pattern {}", glob.getBucket(), keyPrefix,
            wildcardRegexp.toString());/*  ww  w. ja  v a  2 s  .com*/

    ImmutableList.Builder<S3ResourceId> expandedPaths = ImmutableList.builder();
    String continuationToken = null;

    do {
        ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(glob.getBucket())
                .withPrefix(keyPrefix).withContinuationToken(continuationToken);
        ListObjectsV2Result result;
        try {
            result = amazonS3.get().listObjectsV2(request);
        } catch (AmazonClientException e) {
            return ExpandedGlob.create(glob, new IOException(e));
        }
        continuationToken = result.getNextContinuationToken();

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            // Filter against regex.
            if (wildcardRegexp.matcher(objectSummary.getKey()).matches()) {
                S3ResourceId expandedPath = S3ResourceId
                        .fromComponents(objectSummary.getBucketName(), objectSummary.getKey())
                        .withSize(objectSummary.getSize()).withLastModified(objectSummary.getLastModified());
                LOG.debug("Expanded S3 object path {}", expandedPath);
                expandedPaths.add(expandedPath);
            }
        }
    } while (continuationToken != null);

    return ExpandedGlob.create(glob, expandedPaths.build());
}

From source file:org.apache.druid.storage.s3.S3DataSegmentPuller.java

License:Apache License

private FileObject buildFileObject(final URI uri) throws AmazonServiceException {
    final S3Coords coords = new S3Coords(checkURI(uri));
    final S3ObjectSummary objectSummary = S3Utils.getSingleObjectSummary(s3Client, coords.bucket, coords.path);
    final String path = uri.getPath();

    return new FileObject() {
        S3Object s3Object = null;

        @Override// w w  w  .  ja v a  2s  .c  o m
        public URI toUri() {
            return uri;
        }

        @Override
        public String getName() {
            final String ext = Files.getFileExtension(path);
            return Files.getNameWithoutExtension(path) + (Strings.isNullOrEmpty(ext) ? "" : ("." + ext));
        }

        /**
         * Returns an input stream for a s3 object. The returned input stream is not thread-safe.
         */
        @Override
        public InputStream openInputStream() throws IOException {
            try {
                if (s3Object == null) {
                    // lazily promote to full GET
                    s3Object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey());
                }

                final InputStream in = s3Object.getObjectContent();
                final Closer closer = Closer.create();
                closer.register(in);
                closer.register(s3Object);

                return new FilterInputStream(in) {
                    @Override
                    public void close() throws IOException {
                        closer.close();
                    }
                };
            } catch (AmazonServiceException e) {
                throw new IOE(e, "Could not load S3 URI [%s]", uri);
            }
        }

        @Override
        public OutputStream openOutputStream() {
            throw new UOE("Cannot stream S3 output");
        }

        @Override
        public Reader openReader(boolean ignoreEncodingErrors) {
            throw new UOE("Cannot open reader");
        }

        @Override
        public CharSequence getCharContent(boolean ignoreEncodingErrors) {
            throw new UOE("Cannot open character sequence");
        }

        @Override
        public Writer openWriter() {
            throw new UOE("Cannot open writer");
        }

        @Override
        public long getLastModified() {
            return objectSummary.getLastModified().getTime();
        }

        @Override
        public boolean delete() {
            throw new UOE(
                    "Cannot delete S3 items anonymously. jetS3t doesn't support authenticated deletes easily.");
        }
    };
}

From source file:org.apache.druid.storage.s3.S3DataSegmentPuller.java

License:Apache License

/**
 * Returns the "version" (aka last modified timestamp) of the URI
 *
 * @param uri The URI to check the last timestamp
 *
 * @return The time in ms of the last modification of the URI in String format
 *
 * @throws IOException/* w ww  .  j  a v  a 2  s .  c om*/
 */
@Override
public String getVersion(URI uri) throws IOException {
    try {
        final S3Coords coords = new S3Coords(checkURI(uri));
        final S3ObjectSummary objectSummary = S3Utils.getSingleObjectSummary(s3Client, coords.bucket,
                coords.path);
        return StringUtils.format("%d", objectSummary.getLastModified().getTime());
    } catch (AmazonServiceException e) {
        if (S3Utils.isServiceExceptionRecoverable(e)) {
            // The recoverable logic is always true for IOException, so we want to only pass IOException if it is recoverable
            throw new IOE(e, "Could not fetch last modified timestamp from URI [%s]", uri);
        } else {
            throw new RE(e, "Error fetching last modified timestamp from URI [%s]", uri);
        }
    }
}

From source file:org.apache.druid.storage.s3.S3TimestampVersionedDataFinder.java

License:Apache License

/**
 * Gets the key with the most recently modified timestamp.
 * `pattern` is evaluated against the entire key AFTER the path given in `uri`.
 * The substring `pattern` is matched against will have a leading `/` removed.
 * For example `s3://some_bucket/some_prefix/some_key` with a URI of `s3://some_bucket/some_prefix` will match against `some_key`.
 * `s3://some_bucket/some_prefixsome_key` with a URI of `s3://some_bucket/some_prefix` will match against `some_key`
 * `s3://some_bucket/some_prefix//some_key` with a URI of `s3://some_bucket/some_prefix` will match against `/some_key`
 *
 * @param uri     The URI of in the form of `s3://some_bucket/some_key`
 * @param pattern The pattern matcher to determine if a *key* is of interest, or `null` to match everything.
 *
 * @return A URI to the most recently modified object which matched the pattern.
 */// ww w. j a  va  2 s . c  o  m
@Override
public URI getLatestVersion(final URI uri, final @Nullable Pattern pattern) {
    try {
        return RetryUtils.retry(() -> {
            final S3Coords coords = new S3Coords(checkURI(uri));
            long mostRecent = Long.MIN_VALUE;
            URI latest = null;
            final Iterator<S3ObjectSummary> objectSummaryIterator = S3Utils.objectSummaryIterator(s3Client,
                    coords.bucket, coords.path, MAX_LISTING_KEYS);
            while (objectSummaryIterator.hasNext()) {
                final S3ObjectSummary objectSummary = objectSummaryIterator.next();
                String keyString = objectSummary.getKey().substring(coords.path.length());
                if (keyString.startsWith("/")) {
                    keyString = keyString.substring(1);
                }
                if (pattern != null && !pattern.matcher(keyString).matches()) {
                    continue;
                }
                final long latestModified = objectSummary.getLastModified().getTime();
                if (latestModified >= mostRecent) {
                    mostRecent = latestModified;
                    latest = new URI(StringUtils.format("s3://%s/%s", objectSummary.getBucketName(),
                            objectSummary.getKey()));
                }
            }
            return latest;
        }, shouldRetryPredicate(), DEFAULT_RETRY_COUNT);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

From source file:org.apache.hadoop.fs.s3a.S3AFileSystem.java

License:Apache License

/**
 * List the statuses of the files/directories in the given path if the path is
 * a directory.//from   w  ww  .ja v a 2s  .  c  o m
 *
 * @param f given path
 * @return the statuses of the files/directories in the given patch
 * @throws FileNotFoundException when the path does not exist;
 *         IOException see specific implementation
 */
public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException {
    String key = pathToKey(f);
    LOG.info("List status for path: " + f);

    final List<FileStatus> result = new ArrayList<FileStatus>();
    final FileStatus fileStatus = getFileStatus(f);

    if (fileStatus.isDirectory()) {
        if (!key.isEmpty()) {
            key = key + "/";
        }

        ListObjectsRequest request = new ListObjectsRequest();
        request.setBucketName(bucket);
        request.setPrefix(key);
        request.setDelimiter("/");
        request.setMaxKeys(maxKeys);

        if (LOG.isDebugEnabled()) {
            LOG.debug("listStatus: doing listObjects for directory " + key);
        }

        ObjectListing objects = s3.listObjects(request);
        statistics.incrementReadOps(1);

        while (true) {
            for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, workingDir);
                // Skip over keys that are ourselves and old S3N _$folder$ files
                if (keyPath.equals(f) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Ignoring: " + keyPath);
                    }
                    continue;
                }

                if (objectRepresentsDirectory(summary.getKey(), summary.getSize())) {
                    result.add(new S3AFileStatus(true, true, keyPath));
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Adding: fd: " + keyPath);
                    }
                } else {
                    result.add(new S3AFileStatus(summary.getSize(), dateToLong(summary.getLastModified()),
                            keyPath));
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Adding: fi: " + keyPath);
                    }
                }
            }

            for (String prefix : objects.getCommonPrefixes()) {
                Path keyPath = keyToPath(prefix).makeQualified(uri, workingDir);
                if (keyPath.equals(f)) {
                    continue;
                }
                result.add(new S3AFileStatus(true, false, keyPath));
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Adding: rd: " + keyPath);
                }
            }

            if (objects.isTruncated()) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("listStatus: list truncated - getting next batch");
                }

                objects = s3.listNextBatchOfObjects(objects);
                statistics.incrementReadOps(1);
            } else {
                break;
            }
        }
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Adding: rd (not a dir): " + f);
        }
        result.add(fileStatus);
    }

    return result.toArray(new FileStatus[result.size()]);
}

From source file:org.apache.hadoop.fs.s3a.S3AUtils.java

License:Apache License

/**
 * Create a files status instance from a listing.
 * @param keyPath path to entry//w  ww. j a va 2s . c  o m
 * @param summary summary from AWS
 * @param blockSize block size to declare.
 * @return a status entry
 */
public static S3AFileStatus createFileStatus(Path keyPath, S3ObjectSummary summary, long blockSize) {
    if (objectRepresentsDirectory(summary.getKey(), summary.getSize())) {
        return new S3AFileStatus(true, true, keyPath);
    } else {
        return new S3AFileStatus(summary.getSize(), dateToLong(summary.getLastModified()), keyPath, blockSize);
    }
}

From source file:org.apache.hadoop.fs.s3r.S3RFileSystem.java

License:Apache License

/**
 * List the statuses of the files/directories in the given path if the path is
 * a directory.//from   w ww  . j a  v a2  s .c o  m
 *
 * @param f given path
 * @return the statuses of the files/directories in the given patch
 * @throws FileNotFoundException when the path does not exist;
 *         IOException see specific implementation
 */
public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException {
    String key = pathToKey(f);
    if (LOG.isDebugEnabled()) {
        LOG.debug("List status for path: " + f);
    }

    final List<FileStatus> result = new ArrayList<FileStatus>();
    final FileStatus fileStatus = getFileStatus(f);

    if (fileStatus.isDirectory()) {
        if (!key.isEmpty()) {
            key = key + "/";
        }

        ListObjectsRequest request = new ListObjectsRequest();
        request.setBucketName(bucket);
        request.setPrefix(key);
        request.setDelimiter("/");
        request.setMaxKeys(maxKeys);

        if (LOG.isDebugEnabled()) {
            LOG.debug("listStatus: doing listObjects for directory " + key);
        }

        ObjectListing objects = s3.listObjects(request);
        statistics.incrementReadOps(1);

        while (true) {
            for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, workingDir);
                // Skip over keys that are ourselves and old S3N _$folder$ files
                if (keyPath.equals(f) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Ignoring: " + keyPath);
                    }
                    continue;
                }

                if (objectRepresentsDirectory(summary.getKey(), summary.getSize())) {
                    result.add(new S3RFileStatus(true, true, keyPath));
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Adding: fd: " + keyPath);
                    }
                } else {
                    result.add(new S3RFileStatus(summary.getSize(), dateToLong(summary.getLastModified()),
                            keyPath, getDefaultBlockSize(f.makeQualified(uri, workingDir))));
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Adding: fi: " + keyPath);
                    }
                }
            }

            for (String prefix : objects.getCommonPrefixes()) {
                Path keyPath = keyToPath(prefix).makeQualified(uri, workingDir);
                if (keyPath.equals(f)) {
                    continue;
                }
                result.add(new S3RFileStatus(true, false, keyPath));
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Adding: rd: " + keyPath);
                }
            }

            if (objects.isTruncated()) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("listStatus: list truncated - getting next batch");
                }

                objects = s3.listNextBatchOfObjects(objects);
                statistics.incrementReadOps(1);
            } else {
                break;
            }
        }
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Adding: rd (not a dir): " + f);
        }
        result.add(fileStatus);
    }

    return result.toArray(new FileStatus[result.size()]);
}

From source file:org.apache.jackrabbit.aws.ext.ds.S3Backend.java

License:Apache License

@Override
public Set<DataIdentifier> deleteAllOlderThan(long min) throws DataStoreException {
    long start = System.currentTimeMillis();
    // S3 stores lastModified to lower boundary of timestamp in ms.
    // and hence min is reduced by 1000ms.
    min = min - 1000;//  w w w .j  a va2 s .  c  o m
    Set<DataIdentifier> deleteIdSet = new HashSet<DataIdentifier>(30);
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ObjectListing prevObjectListing = s3service.listObjects(bucket);
        while (true) {
            List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
            for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                DataIdentifier identifier = new DataIdentifier(getIdentifierName(s3ObjSumm.getKey()));
                long lastModified = s3ObjSumm.getLastModified().getTime();
                LOG.debug("Identifier [{}]'s lastModified = [{}]", identifier, lastModified);
                if (!store.isInUse(identifier) && lastModified < min) {
                    LOG.debug("add id [{}] to delete lists", s3ObjSumm.getKey());
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                    deleteIdSet.add(identifier);
                }
            }
            if (deleteList.size() > 0) {
                DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
                delObjsReq.setKeys(deleteList);
                DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq);
                if (dobjs.getDeletedObjects().size() != deleteList.size()) {
                    throw new DataStoreException(
                            "Incomplete delete object request. only  " + dobjs.getDeletedObjects().size()
                                    + " out of " + deleteList.size() + " are deleted");
                } else {
                    LOG.debug("[{}] records deleted from datastore", deleteList);
                }
            }
            if (!prevObjectListing.isTruncated()) {
                break;
            }
            prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
    LOG.info(
            "deleteAllOlderThan: min=[{}] exit. Deleted[{}] records. Number of records deleted [{}] took [{}]ms",
            new Object[] { min, deleteIdSet, deleteIdSet.size(), (System.currentTimeMillis() - start) });
    return deleteIdSet;
}

From source file:org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3Backend.java

License:Apache License

@Override
public Set<DataIdentifier> deleteAllOlderThan(long min) throws DataStoreException {
    long start = System.currentTimeMillis();
    // S3 stores lastModified to lower boundary of timestamp in ms.
    // and hence min is reduced by 1000ms.
    min = min - 1000;// ww  w.j  a  va 2  s . c  o  m
    Set<DataIdentifier> deleteIdSet = new HashSet<DataIdentifier>(30);
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ObjectListing prevObjectListing = s3service.listObjects(bucket);
        while (true) {
            List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
            for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                if (!s3ObjSumm.getKey().startsWith(META_KEY_PREFIX)) {
                    DataIdentifier identifier = new DataIdentifier(getIdentifierName(s3ObjSumm.getKey()));
                    long lastModified = s3ObjSumm.getLastModified().getTime();
                    LOG.debug("Identifier [{}]'s lastModified = [{}]", identifier, lastModified);
                    if (lastModified < min && store.confirmDelete(identifier)
                    // confirm once more that record's lastModified < min
                    //  order is important here
                            && s3service.getObjectMetadata(bucket, s3ObjSumm.getKey()).getLastModified()
                                    .getTime() < min) {

                        store.deleteFromCache(identifier);
                        LOG.debug("add id [{}] to delete lists", s3ObjSumm.getKey());
                        deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                        deleteIdSet.add(identifier);
                    }
                }
            }
            if (deleteList.size() > 0) {
                DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
                delObjsReq.setKeys(deleteList);
                DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq);
                if (dobjs.getDeletedObjects().size() != deleteList.size()) {
                    throw new DataStoreException(
                            "Incomplete delete object request. only  " + dobjs.getDeletedObjects().size()
                                    + " out of " + deleteList.size() + " are deleted");
                } else {
                    LOG.debug("[{}] records deleted from datastore", deleteList);
                }
            }
            if (!prevObjectListing.isTruncated()) {
                break;
            }
            prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
    LOG.info(
            "deleteAllOlderThan: min=[{}] exit. Deleted[{}] records. Number of records deleted [{}] took [{}]ms",
            new Object[] { min, deleteIdSet, deleteIdSet.size(), (System.currentTimeMillis() - start) });
    return deleteIdSet;
}