List of usage examples for com.amazonaws.services.s3.model ObjectMetadata getLastModified
public Date getLastModified()
From source file:eu.stratosphere.runtime.fs.s3.S3FileSystem.java
License:Apache License
@Override public FileStatus getFileStatus(final Path f) throws IOException { final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f); // This is the S3:/// base directory if (!bop.hasBucket() && !bop.hasObject()) { return new S3FileStatus(f, 0L, true, 0L, 0L); }//w w w. ja v a2 s . c o m try { if (bop.hasBucket() && !bop.hasObject()) { final List<Bucket> buckets = this.s3Client.listBuckets(); final Iterator<Bucket> it = buckets.iterator(); // Iterator throw list of buckets to find out creation date while (it.hasNext()) { final Bucket bucket = it.next(); if (bop.getBucket().equals(bucket.getName())) { final long creationDate = dateToLong(bucket.getCreationDate()); // S3 does not track access times, so this implementation always sets it to 0 return new S3FileStatus(f, 0L, true, creationDate, 0L); } } throw new FileNotFoundException("Cannot find " + f.toUri()); } try { final ObjectMetadata om = this.s3Client.getObjectMetadata(bop.getBucket(), bop.getObject()); final long modificationDate = dateToLong(om.getLastModified()); // S3 does not track access times, so this implementation always sets it to 0 if (objectRepresentsDirectory(bop.getObject(), om.getContentLength())) { return new S3FileStatus(f, 0L, true, modificationDate, 0L); } else { return new S3FileStatus(f, om.getContentLength(), false, modificationDate, 0L); } } catch (AmazonServiceException e) { if (e.getStatusCode() == HTTP_RESOURCE_NOT_FOUND_CODE) { throw new FileNotFoundException("Cannot find " + f.toUri()); } else { throw e; } } } catch (AmazonClientException e) { throw new IOException(StringUtils.stringifyException(e)); } }
From source file:eu.stratosphere.runtime.fs.s3.S3FileSystem.java
License:Apache License
@Override public FileStatus[] listStatus(final Path f) throws IOException { final S3BucketObjectPair bop = this.directoryStructure.toBucketObjectPair(f); try {/* w ww. j a va 2 s . co m*/ if (!bop.hasBucket()) { final List<Bucket> list = this.s3Client.listBuckets(); final S3FileStatus[] array = new S3FileStatus[list.size()]; final Iterator<Bucket> it = list.iterator(); int i = 0; while (it.hasNext()) { final Bucket bucket = it.next(); final long creationDate = dateToLong(bucket.getCreationDate()); // S3 does not track access times, so this implementation always sets it to 0 final S3FileStatus status = new S3FileStatus( extendPath(f, bucket.getName() + S3_DIRECTORY_SEPARATOR), 0, true, creationDate, 0L); array[i++] = status; } return array; } if (bop.hasBucket() && !bop.hasObject()) { // Check if the bucket really exists if (!this.s3Client.doesBucketExist(bop.getBucket())) { throw new FileNotFoundException("Cannot find " + f.toUri()); } return listBucketContent(f, bop); } else { final ObjectMetadata omd = this.s3Client.getObjectMetadata(bop.getBucket(), bop.getObject()); if (objectRepresentsDirectory(bop.getObject(), omd.getContentLength())) { return listBucketContent(f, bop); } else { final S3FileStatus fileStatus = new S3FileStatus(f, omd.getContentLength(), false, dateToLong(omd.getLastModified()), 0L); return new FileStatus[] { fileStatus }; } } } catch (AmazonClientException e) { throw new IOException(StringUtils.stringifyException(e)); } }
From source file:fr.ens.biologie.genomique.eoulsan.data.protocols.S3DataProtocol.java
License:LGPL
@Override public DataFileMetadata getMetadata(final DataFile src) throws IOException { if (!exists(src, true)) { throw new FileNotFoundException("File not found: " + src); }/*ww w .j a v a 2 s .c o m*/ final ObjectMetadata md = new S3URL(src).getMetaData(); final SimpleDataFileMetadata result = new SimpleDataFileMetadata(); result.setContentLength(md.getContentLength()); result.setLastModified(md.getLastModified().getTime()); result.setContentType(md.getContentType()); result.setContentEncoding(md.getContentEncoding()); result.setDataFormat(DataFormatRegistry.getInstance().getDataFormatFromFilename(src.getName())); return result; }
From source file:io.konig.camel.aws.s3.DeleteObjectEndpoint.java
License:Apache License
public Exchange createExchange(ExchangePattern pattern, final S3Object s3Object) { LOG.trace("Getting object with key [{}] from bucket [{}]...", s3Object.getKey(), s3Object.getBucketName()); ObjectMetadata objectMetadata = s3Object.getObjectMetadata(); LOG.trace("Got object [{}]", s3Object); Exchange exchange = super.createExchange(pattern); Message message = exchange.getIn();//from w w w . j a v a2s . c o m if (configuration.isIncludeBody()) { message.setBody(s3Object.getObjectContent()); } else { message.setBody(null); } message.setHeader(S3Constants.KEY, s3Object.getKey()); message.setHeader(S3Constants.BUCKET_NAME, s3Object.getBucketName()); message.setHeader(S3Constants.E_TAG, objectMetadata.getETag()); message.setHeader(S3Constants.LAST_MODIFIED, objectMetadata.getLastModified()); message.setHeader(S3Constants.VERSION_ID, objectMetadata.getVersionId()); message.setHeader(S3Constants.CONTENT_TYPE, objectMetadata.getContentType()); message.setHeader(S3Constants.CONTENT_MD5, objectMetadata.getContentMD5()); message.setHeader(S3Constants.CONTENT_LENGTH, objectMetadata.getContentLength()); message.setHeader(S3Constants.CONTENT_ENCODING, objectMetadata.getContentEncoding()); message.setHeader(S3Constants.CONTENT_DISPOSITION, objectMetadata.getContentDisposition()); message.setHeader(S3Constants.CACHE_CONTROL, objectMetadata.getCacheControl()); message.setHeader(S3Constants.S3_HEADERS, objectMetadata.getRawMetadata()); message.setHeader(S3Constants.SERVER_SIDE_ENCRYPTION, objectMetadata.getSSEAlgorithm()); message.setHeader(S3Constants.USER_METADATA, objectMetadata.getUserMetadata()); message.setHeader(S3Constants.EXPIRATION_TIME, objectMetadata.getExpirationTime()); message.setHeader(S3Constants.REPLICATION_STATUS, objectMetadata.getReplicationStatus()); message.setHeader(S3Constants.STORAGE_CLASS, objectMetadata.getStorageClass()); /** * If includeBody != true, it is safe to close the object here. If * includeBody == true, the caller is responsible for closing the stream * and object once the body has been fully consumed. As of 2.17, the * consumer does not close the stream or object on commit. */ if (!configuration.isIncludeBody()) { IOHelper.close(s3Object); } else { if (configuration.isAutocloseBody()) { exchange.addOnCompletion(new SynchronizationAdapter() { @Override public void onDone(Exchange exchange) { IOHelper.close(s3Object); } }); } } return exchange; }
From source file:n3phele.agent.repohandlers.S3Large.java
License:Open Source License
public Origin put(InputStream input, long length, String encoding) { Origin result = new Origin(source + "/" + root + "/" + key, 0, null, null); TransferManager tm = null;/* w ww. j ava 2 s .c o m*/ try { tm = new TransferManager(this.credentials); tm.getAmazonS3Client().setEndpoint(source.toString()); objectMetadata = new ObjectMetadata(); objectMetadata.setContentLength(this.length = length); this.encoding = encoding; if (encoding != null) objectMetadata.setContentType(this.encoding); log.info("Output: " + source + "/" + root + "/" + key + " Content-Type: " + encoding + "length: " + length); Upload upload = tm.upload(root, key, input, objectMetadata); upload.waitForCompletion(); // PutObjectResult object = s3().putObject(root, key, input, objectMetadata); result.setLength(length); ObjectMetadata od = s3().getObjectMetadata(root, key); result.setModified(od.getLastModified()); } catch (AmazonServiceException e) { throw e; } catch (AmazonClientException e) { throw e; } catch (InterruptedException e) { throw new AmazonClientException(e.getMessage()); } finally { try { input.close(); } catch (IOException e) { } try { tm.shutdownNow(); } catch (Exception e) { } try { s3().shutdown(); } catch (Exception e) { } } return result; }
From source file:net.solarnetwork.node.backup.s3.SdkS3Client.java
License:Open Source License
@Override public S3ObjectReference putObject(String key, InputStream in, ObjectMetadata objectMetadata) throws IOException { AmazonS3 client = getClient();//from ww w . ja v a 2 s .c om try { PutObjectRequest req = new PutObjectRequest(bucketName, key, in, objectMetadata); client.putObject(req); return new S3ObjectReference(key, objectMetadata.getContentLength(), objectMetadata.getLastModified()); } catch (AmazonServiceException e) { log.warn("AWS error: {}; HTTP code {}; AWS code {}; type {}; request ID {}", e.getMessage(), e.getStatusCode(), e.getErrorCode(), e.getErrorType(), e.getRequestId()); throw new RemoteServiceException("Error putting S3 object at " + key, e); } catch (AmazonClientException e) { log.debug("Error communicating with AWS: {}", e.getMessage()); throw new IOException("Error communicating with AWS", e); } }
From source file:ohnosequences.ivy.S3Resource.java
License:Apache License
private void initalizeResource() { try {/*from ww w.j ava2s .c o m*/ // System.out.println("trying to resolve bucket=" + bucket + " key=" + key); ObjectMetadata metadata = s3Repo.getS3Client().getObjectMetadata(bucket, key); this.exists = true; this.contentLength = metadata.getContentLength(); this.lastModified = metadata.getLastModified().getTime(); } catch (AmazonServiceException e) { this.exists = false; this.contentLength = 0; this.lastModified = 0; this.name = ""; } }
From source file:org.apache.beam.sdk.io.aws.s3.S3FileSystem.java
License:Apache License
@VisibleForTesting
MatchResult matchNonGlobPath(S3ResourceId path) {
ObjectMetadata s3Metadata;
try {/*from w w w .j av a2s. com*/
s3Metadata = getObjectMetadata(path);
} catch (AmazonClientException e) {
if (e instanceof AmazonS3Exception && ((AmazonS3Exception) e).getStatusCode() == 404) {
return MatchResult.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException());
}
return MatchResult.create(MatchResult.Status.ERROR, new IOException(e));
}
return MatchResult.create(MatchResult.Status.OK,
ImmutableList.of(createBeamMetadata(
path.withSize(s3Metadata.getContentLength()).withLastModified(s3Metadata.getLastModified()),
Strings.nullToEmpty(s3Metadata.getContentEncoding()))));
}
From source file:org.apache.camel.component.aws.s3.S3Endpoint.java
License:Apache License
public Exchange createExchange(ExchangePattern pattern, S3Object s3Object) { LOG.trace("Getting object with key [{}] from bucket [{}]...", s3Object.getKey(), s3Object.getBucketName()); ObjectMetadata objectMetadata = s3Object.getObjectMetadata(); LOG.trace("Got object [{}]", s3Object); Exchange exchange = new DefaultExchange(this, pattern); Message message = exchange.getIn();//from w w w . j ava 2 s . c o m message.setBody(s3Object.getObjectContent()); message.setHeader(S3Constants.KEY, s3Object.getKey()); message.setHeader(S3Constants.BUCKET_NAME, s3Object.getBucketName()); message.setHeader(S3Constants.E_TAG, objectMetadata.getETag()); message.setHeader(S3Constants.LAST_MODIFIED, objectMetadata.getLastModified()); message.setHeader(S3Constants.VERSION_ID, objectMetadata.getVersionId()); message.setHeader(S3Constants.CONTENT_TYPE, objectMetadata.getContentType()); message.setHeader(S3Constants.CONTENT_MD5, objectMetadata.getContentMD5()); message.setHeader(S3Constants.CONTENT_LENGTH, objectMetadata.getContentLength()); message.setHeader(S3Constants.CONTENT_ENCODING, objectMetadata.getContentEncoding()); message.setHeader(S3Constants.CONTENT_DISPOSITION, objectMetadata.getContentDisposition()); message.setHeader(S3Constants.CACHE_CONTROL, objectMetadata.getCacheControl()); return exchange; }
From source file:org.apache.druid.storage.s3.S3DataSegmentFinder.java
License:Apache License
@Override public Set<DataSegment> findSegments(String workingDirPath, boolean updateDescriptor) throws SegmentLoadingException { final Map<String, Pair<DataSegment, Long>> timestampedSegments = new HashMap<>(); try {//from w w w. j a v a 2 s .c om final Iterator<S3ObjectSummary> objectSummaryIterator = S3Utils.objectSummaryIterator(s3Client, config.getBucket(), workingDirPath.length() == 0 ? config.getBaseKey() : workingDirPath, config.getMaxListingLength()); while (objectSummaryIterator.hasNext()) { final S3ObjectSummary objectSummary = objectSummaryIterator.next(); if ("descriptor.json".equals(S3Utils.toFilename(objectSummary.getKey()))) { final String descriptorJson = objectSummary.getKey(); String indexZip = S3Utils.indexZipForSegmentPath(descriptorJson); if (S3Utils.isObjectInBucketIgnoringPermission(s3Client, config.getBucket(), indexZip)) { try (S3Object indexObject = s3Client.getObject(config.getBucket(), descriptorJson); S3ObjectInputStream is = indexObject.getObjectContent()) { final ObjectMetadata objectMetadata = indexObject.getObjectMetadata(); final DataSegment dataSegment = jsonMapper.readValue(is, DataSegment.class); log.info("Found segment [%s] located at [%s]", dataSegment.getIdentifier(), indexZip); final Map<String, Object> loadSpec = dataSegment.getLoadSpec(); if (!S3StorageDruidModule.SCHEME.equals(loadSpec.get("type")) || !indexZip.equals(loadSpec.get("key")) || !config.getBucket().equals(loadSpec.get("bucket"))) { loadSpec.put("type", S3StorageDruidModule.SCHEME); loadSpec.put("key", indexZip); loadSpec.put("bucket", config.getBucket()); if (updateDescriptor) { log.info("Updating loadSpec in descriptor.json at [%s] with new path [%s]", descriptorJson, indexObject); final ByteArrayInputStream bais = new ByteArrayInputStream( StringUtils.toUtf8(jsonMapper.writeValueAsString(dataSegment))); s3Client.putObject(config.getBucket(), descriptorJson, bais, objectMetadata); } } DataSegmentFinder.putInMapRetainingNewest(timestampedSegments, dataSegment, objectMetadata.getLastModified() == null ? 0 : objectMetadata.getLastModified().getTime()); } } else { throw new SegmentLoadingException( "index.zip didn't exist at [%s] while descriptor.json exists!?", indexZip); } } } } catch (AmazonServiceException e) { throw new SegmentLoadingException(e, "Problem interacting with S3"); } catch (IOException e) { throw new SegmentLoadingException(e, "IO exception"); } catch (Exception e) { Throwables.propagateIfInstanceOf(e, SegmentLoadingException.class); Throwables.propagate(e); } return timestampedSegments.values().stream().map(x -> x.lhs).collect(Collectors.toSet()); }