List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary getBucketName
public String getBucketName()
From source file:it.openutils.mgnlaws.magnolia.datastore.S3DataRecord.java
License:Open Source License
public S3DataRecord(AmazonS3 client, S3ObjectSummary summary) { this(new DataIdentifier(StringUtils.substringAfterLast(summary.getKey(), "/")), new S3LazyObject(client, summary.getBucketName(), summary.getKey())); }
From source file:jenkins.plugins.itemstorage.s3.Downloads.java
License:Open Source License
public void startDownload(TransferManager manager, File base, String pathPrefix, S3ObjectSummary summary) throws AmazonServiceException, IOException { // calculate target file name File targetFile = FileUtils.getFile(base, summary.getKey().substring(pathPrefix.length() + 1)); // if target file exists, only download it if newer if (targetFile.lastModified() < summary.getLastModified().getTime()) { // ensure directory above file exists FileUtils.forceMkdir(targetFile.getParentFile()); // Start the download Download download = manager.download(summary.getBucketName(), summary.getKey(), targetFile); // Keep for later startedDownloads.add(new Memo(download, targetFile, summary.getLastModified().getTime())); }//from w ww.j a va 2s . c o m }
From source file:jp.co.tagbangers.jgroups.S3_CLIENT_PING.java
License:Apache License
protected void readResponse(S3ObjectSummary summary, List<Address> members, Responses responses) { S3Object object = amazonS3.getObject(summary.getBucketName(), summary.getKey()); List<PingData> list;// w w w . j av a 2 s. c om try { list = read(object.getObjectContent()); if (list != null) { for (PingData data : list) { if (members == null || members.contains(data.getAddress())) { responses.addResponse(data, data.isCoord()); } if (local_addr != null && !local_addr.equals(data.getAddress())) { addDiscoveryResponseToCaches(data.getAddress(), data.getLogicalName(), data.getPhysicalAddr()); } } } } catch (Throwable e) { log.error("failed unmarshalling response", e); } }
From source file:nl.nn.adapterframework.filesystem.AmazonS3FileSystem.java
License:Apache License
@Override public Iterator<S3Object> listFiles(String folder) throws FileSystemException { List<S3ObjectSummary> summaries = null; String prefix = folder != null ? folder + "/" : ""; try {//from ww w .j a v a2 s.co m ObjectListing listing = s3Client.listObjects(bucketName, prefix); summaries = listing.getObjectSummaries(); while (listing.isTruncated()) { listing = s3Client.listNextBatchOfObjects(listing); summaries.addAll(listing.getObjectSummaries()); } } catch (AmazonServiceException e) { throw new FileSystemException("Cannot process requested action", e); } List<S3Object> list = new ArrayList<S3Object>(); for (S3ObjectSummary summary : summaries) { S3Object object = new S3Object(); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(summary.getSize()); object.setBucketName(summary.getBucketName()); object.setKey(summary.getKey()); object.setObjectMetadata(metadata); if (!object.getKey().endsWith("/") && !(prefix.isEmpty() && object.getKey().contains("/"))) { list.add(object); } } return list.iterator(); }
From source file:org.apache.beam.sdk.io.aws.s3.S3FileSystem.java
License:Apache License
private ExpandedGlob expandGlob(S3ResourceId glob) { // The S3 API can list objects, filtered by prefix, but not by wildcard. // Here, we find the longest prefix without wildcard "*", // then filter the results with a regex. checkArgument(glob.isWildcard(), "isWildcard"); String keyPrefix = glob.getKeyNonWildcardPrefix(); Pattern wildcardRegexp = Pattern.compile(wildcardToRegexp(glob.getKey())); LOG.debug("expanding bucket {}, prefix {}, against pattern {}", glob.getBucket(), keyPrefix, wildcardRegexp.toString());// www . j a v a 2 s . c o m ImmutableList.Builder<S3ResourceId> expandedPaths = ImmutableList.builder(); String continuationToken = null; do { ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(glob.getBucket()) .withPrefix(keyPrefix).withContinuationToken(continuationToken); ListObjectsV2Result result; try { result = amazonS3.get().listObjectsV2(request); } catch (AmazonClientException e) { return ExpandedGlob.create(glob, new IOException(e)); } continuationToken = result.getNextContinuationToken(); for (S3ObjectSummary objectSummary : result.getObjectSummaries()) { // Filter against regex. if (wildcardRegexp.matcher(objectSummary.getKey()).matches()) { S3ResourceId expandedPath = S3ResourceId .fromComponents(objectSummary.getBucketName(), objectSummary.getKey()) .withSize(objectSummary.getSize()).withLastModified(objectSummary.getLastModified()); LOG.debug("Expanded S3 object path {}", expandedPath); expandedPaths.add(expandedPath); } } } while (continuationToken != null); return ExpandedGlob.create(glob, expandedPaths.build()); }
From source file:org.apache.camel.component.aws.s3.S3Consumer.java
License:Apache License
protected Queue<Exchange> createExchanges(List<S3ObjectSummary> s3ObjectSummaries) { if (LOG.isTraceEnabled()) { LOG.trace("Received {} messages in this poll", s3ObjectSummaries.size()); }/*from w ww . j ava 2 s . c om*/ Queue<Exchange> answer = new LinkedList<Exchange>(); for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) { S3Object s3Object = getAmazonS3Client().getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()); Exchange exchange = getEndpoint().createExchange(s3Object); answer.add(exchange); } return answer; }
From source file:org.apache.druid.firehose.s3.StaticS3FirehoseFactory.java
License:Apache License
/** * Create an {@link URI} from the given {@link S3ObjectSummary}. The result URI is composed as below. * * <pre>/*from w ww .j a v a 2 s . c o m*/ * {@code s3://{BUCKET_NAME}/{OBJECT_KEY}} * </pre> */ private static URI toUri(S3ObjectSummary object) { final String originalAuthority = object.getBucketName(); final String originalPath = object.getKey(); final String authority = originalAuthority.endsWith("/") ? originalAuthority.substring(0, originalAuthority.length() - 1) : originalAuthority; final String path = originalPath.startsWith("/") ? originalPath.substring(1) : originalPath; return URI.create(StringUtils.format("s3://%s/%s", authority, path)); }
From source file:org.apache.druid.storage.s3.S3DataSegmentPuller.java
License:Apache License
private FileObject buildFileObject(final URI uri) throws AmazonServiceException { final S3Coords coords = new S3Coords(checkURI(uri)); final S3ObjectSummary objectSummary = S3Utils.getSingleObjectSummary(s3Client, coords.bucket, coords.path); final String path = uri.getPath(); return new FileObject() { S3Object s3Object = null; @Override//from ww w . j av a2 s .c om public URI toUri() { return uri; } @Override public String getName() { final String ext = Files.getFileExtension(path); return Files.getNameWithoutExtension(path) + (Strings.isNullOrEmpty(ext) ? "" : ("." + ext)); } /** * Returns an input stream for a s3 object. The returned input stream is not thread-safe. */ @Override public InputStream openInputStream() throws IOException { try { if (s3Object == null) { // lazily promote to full GET s3Object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); } final InputStream in = s3Object.getObjectContent(); final Closer closer = Closer.create(); closer.register(in); closer.register(s3Object); return new FilterInputStream(in) { @Override public void close() throws IOException { closer.close(); } }; } catch (AmazonServiceException e) { throw new IOE(e, "Could not load S3 URI [%s]", uri); } } @Override public OutputStream openOutputStream() { throw new UOE("Cannot stream S3 output"); } @Override public Reader openReader(boolean ignoreEncodingErrors) { throw new UOE("Cannot open reader"); } @Override public CharSequence getCharContent(boolean ignoreEncodingErrors) { throw new UOE("Cannot open character sequence"); } @Override public Writer openWriter() { throw new UOE("Cannot open writer"); } @Override public long getLastModified() { return objectSummary.getLastModified().getTime(); } @Override public boolean delete() { throw new UOE( "Cannot delete S3 items anonymously. jetS3t doesn't support authenticated deletes easily."); } }; }
From source file:org.apache.druid.storage.s3.S3TimestampVersionedDataFinder.java
License:Apache License
/** * Gets the key with the most recently modified timestamp. * `pattern` is evaluated against the entire key AFTER the path given in `uri`. * The substring `pattern` is matched against will have a leading `/` removed. * For example `s3://some_bucket/some_prefix/some_key` with a URI of `s3://some_bucket/some_prefix` will match against `some_key`. * `s3://some_bucket/some_prefixsome_key` with a URI of `s3://some_bucket/some_prefix` will match against `some_key` * `s3://some_bucket/some_prefix//some_key` with a URI of `s3://some_bucket/some_prefix` will match against `/some_key` * * @param uri The URI of in the form of `s3://some_bucket/some_key` * @param pattern The pattern matcher to determine if a *key* is of interest, or `null` to match everything. * * @return A URI to the most recently modified object which matched the pattern. *///from ww w . ja v a 2s . com @Override public URI getLatestVersion(final URI uri, final @Nullable Pattern pattern) { try { return RetryUtils.retry(() -> { final S3Coords coords = new S3Coords(checkURI(uri)); long mostRecent = Long.MIN_VALUE; URI latest = null; final Iterator<S3ObjectSummary> objectSummaryIterator = S3Utils.objectSummaryIterator(s3Client, coords.bucket, coords.path, MAX_LISTING_KEYS); while (objectSummaryIterator.hasNext()) { final S3ObjectSummary objectSummary = objectSummaryIterator.next(); String keyString = objectSummary.getKey().substring(coords.path.length()); if (keyString.startsWith("/")) { keyString = keyString.substring(1); } if (pattern != null && !pattern.matcher(keyString).matches()) { continue; } final long latestModified = objectSummary.getLastModified().getTime(); if (latestModified >= mostRecent) { mostRecent = latestModified; latest = new URI(StringUtils.format("s3://%s/%s", objectSummary.getBucketName(), objectSummary.getKey())); } } return latest; }, shouldRetryPredicate(), DEFAULT_RETRY_COUNT); } catch (Exception e) { throw new RuntimeException(e); } }
From source file:org.apache.druid.storage.s3.S3Utils.java
License:Apache License
/** * Gets a single {@link S3ObjectSummary} from s3. Since this method might return a wrong object if there are multiple * objects that match the given key, this method should be used only when it's guaranteed that the given key is unique * in the given bucket.//from w w w . j a va2s .c o m * * @param s3Client s3 client * @param bucket s3 bucket * @param key unique key for the object to be retrieved */ public static S3ObjectSummary getSingleObjectSummary(ServerSideEncryptingAmazonS3 s3Client, String bucket, String key) { final ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(bucket).withPrefix(key) .withMaxKeys(1); final ListObjectsV2Result result = s3Client.listObjectsV2(request); // Using getObjectSummaries().size() instead of getKeyCount as, in some cases // it is observed that even though the getObjectSummaries returns some data // keyCount is still zero. if (result.getObjectSummaries().size() == 0) { throw new ISE("Cannot find object for bucket[%s] and key[%s]", bucket, key); } final S3ObjectSummary objectSummary = result.getObjectSummaries().get(0); if (!objectSummary.getBucketName().equals(bucket) || !objectSummary.getKey().equals(key)) { throw new ISE("Wrong object[%s] for bucket[%s] and key[%s]", objectSummary, bucket, key); } return objectSummary; }