List of usage examples for com.amazonaws.services.s3.model PutObjectRequest setStorageClass
public void setStorageClass(String storageClass)
From source file:fr.eurecom.hybris.kvs.drivers.AmazonKvs.java
License:Apache License
public void put(String key, byte[] value) throws IOException { try {// ww w . java2 s . com ByteArrayInputStream bais = new ByteArrayInputStream(value); ObjectMetadata om = new ObjectMetadata(); om.setContentLength(value.length); PutObjectRequest request = new PutObjectRequest(this.rootContainer, key, bais, om); request.setStorageClass(StorageClass.ReducedRedundancy); Upload upload = this.tm.upload(request); // NB: asynchronous, returns immediately upload.waitForCompletion(); } catch (AmazonClientException | InterruptedException e) { throw new IOException(e); } }
From source file:gov.cdc.sdp.cbr.aphl.AphlS3Producer.java
License:Apache License
public void processSingleOp(final Exchange exchange) throws Exception { ObjectMetadata objectMetadata = determineMetadata(exchange); File filePayload = null;// w w w . j a v a 2 s . co m InputStream is = null; Object obj = exchange.getIn().getMandatoryBody(); PutObjectRequest putObjectRequest = null; // Need to check if the message body is WrappedFile if (obj instanceof WrappedFile) { obj = ((WrappedFile<?>) obj).getFile(); } if (obj instanceof File) { filePayload = (File) obj; is = new FileInputStream(filePayload); } else { is = exchange.getIn().getMandatoryBody(InputStream.class); } putObjectRequest = new PutObjectRequest(getConfiguration().getBucketName(), determineKey(exchange), is, objectMetadata); String storageClass = determineStorageClass(exchange); if (storageClass != null) { putObjectRequest.setStorageClass(storageClass); } String cannedAcl = exchange.getIn().getHeader(S3Constants.CANNED_ACL, String.class); if (cannedAcl != null) { CannedAccessControlList objectAcl = CannedAccessControlList.valueOf(cannedAcl); putObjectRequest.setCannedAcl(objectAcl); } AccessControlList acl = exchange.getIn().getHeader(S3Constants.ACL, AccessControlList.class); if (acl != null) { // note: if cannedacl and acl are both specified the last one will // be used. refer to // PutObjectRequest#setAccessControlList for more details putObjectRequest.setAccessControlList(acl); } PutObjectResult putObjectResult = getEndpoint().getS3Client().putObject(putObjectRequest); Message message = getMessageForResponse(exchange); message.setHeader(S3Constants.E_TAG, putObjectResult.getETag()); if (putObjectResult.getVersionId() != null) { message.setHeader(S3Constants.VERSION_ID, putObjectResult.getVersionId()); } if (getConfiguration().isDeleteAfterWrite() && filePayload != null) { // close streams IOHelper.close(putObjectRequest.getInputStream()); IOHelper.close(is); FileUtil.deleteFile(filePayload); } }
From source file:io.konig.camel.aws.s3.DeleteObjectProducer.java
License:Apache License
public void processSingleOp(final Exchange exchange) throws Exception { ObjectMetadata objectMetadata = determineMetadata(exchange); File filePayload = null;//from w ww .j a va 2s .c om InputStream is = null; ByteArrayOutputStream baos = null; Object obj = exchange.getIn().getMandatoryBody(); PutObjectRequest putObjectRequest = null; // Need to check if the message body is WrappedFile if (obj instanceof WrappedFile) { obj = ((WrappedFile<?>) obj).getFile(); } if (obj instanceof File) { filePayload = (File) obj; is = new FileInputStream(filePayload); } else { is = exchange.getIn().getMandatoryBody(InputStream.class); baos = determineLengthInputStream(is); objectMetadata.setContentLength(baos.size()); is = new ByteArrayInputStream(baos.toByteArray()); } putObjectRequest = new PutObjectRequest(getConfiguration().getBucketName(), determineKey(exchange), is, objectMetadata); String storageClass = determineStorageClass(exchange); if (storageClass != null) { putObjectRequest.setStorageClass(storageClass); } String cannedAcl = exchange.getIn().getHeader(S3Constants.CANNED_ACL, String.class); if (cannedAcl != null) { CannedAccessControlList objectAcl = CannedAccessControlList.valueOf(cannedAcl); putObjectRequest.setCannedAcl(objectAcl); } AccessControlList acl = exchange.getIn().getHeader(S3Constants.ACL, AccessControlList.class); if (acl != null) { // note: if cannedacl and acl are both specified the last one will // be used. refer to // PutObjectRequest#setAccessControlList for more details putObjectRequest.setAccessControlList(acl); } if (getConfiguration().isUseAwsKMS()) { SSEAwsKeyManagementParams keyManagementParams; if (ObjectHelper.isNotEmpty(getConfiguration().getAwsKMSKeyId())) { keyManagementParams = new SSEAwsKeyManagementParams(getConfiguration().getAwsKMSKeyId()); } else { keyManagementParams = new SSEAwsKeyManagementParams(); } putObjectRequest.setSSEAwsKeyManagementParams(keyManagementParams); } LOG.trace("Put object [{}] from exchange [{}]...", putObjectRequest, exchange); PutObjectResult putObjectResult = getEndpoint().getS3Client().putObject(putObjectRequest); LOG.trace("Received result [{}]", putObjectResult); Message message = getMessageForResponse(exchange); message.setHeader(S3Constants.E_TAG, putObjectResult.getETag()); if (putObjectResult.getVersionId() != null) { message.setHeader(S3Constants.VERSION_ID, putObjectResult.getVersionId()); } if (getConfiguration().isDeleteAfterWrite() && filePayload != null) { // close streams IOHelper.close(putObjectRequest.getInputStream()); IOHelper.close(is); FileUtil.deleteFile(filePayload); } }
From source file:org.alanwilliamson.amazon.s3.BackgroundUploader.java
License:Open Source License
private void uploadFile(Map<String, Object> jobFile) { File localFile = new File((String) jobFile.get("localpath")); if (!localFile.isFile()) { removeJobFile(jobFile);//from www .ja v a 2 s . c om callbackCfc(jobFile, false, "local file no longer exists"); cfEngine.log("AmazonS3Write.BackgroundUploader: file no longer exists=" + localFile.getName()); return; } // Setup the object data ObjectMetadata omd = new ObjectMetadata(); if (jobFile.containsKey("metadata")) omd.setUserMetadata((Map<String, String>) jobFile.get("metadata")); TransferManager tm = null; AmazonS3 s3Client = null; try { AmazonKey amazonKey = (AmazonKey) jobFile.get("amazonkey"); s3Client = new AmazonBase().getAmazonS3(amazonKey); PutObjectRequest por = new PutObjectRequest((String) jobFile.get("bucket"), (String) jobFile.get("key"), localFile); por.setMetadata(omd); por.setStorageClass((StorageClass) jobFile.get("storage")); if (jobFile.containsKey("acl")) por.setCannedAcl(amazonKey.getAmazonCannedAcl((String) jobFile.get("acl"))); if (jobFile.containsKey("aes256key")) por.setSSECustomerKey(new SSECustomerKey((String) jobFile.get("aes256key"))); if (jobFile.containsKey("customheaders")) { Map<String, String> customheaders = (Map) jobFile.get("customheaders"); Iterator<String> it = customheaders.keySet().iterator(); while (it.hasNext()) { String k = it.next(); por.putCustomRequestHeader(k, customheaders.get(k)); } } long startTime = System.currentTimeMillis(); tm = new TransferManager(s3Client); Upload upload = tm.upload(por); upload.waitForCompletion(); log(jobFile, "Uploaded; timems=" + (System.currentTimeMillis() - startTime)); removeJobFile(jobFile); callbackCfc(jobFile, true, null); if ((Boolean) jobFile.get("deletefile")) localFile.delete(); } catch (Exception e) { log(jobFile, "Failed=" + e.getMessage()); callbackCfc(jobFile, false, e.getMessage()); int retry = (Integer) jobFile.get("retry"); int attempt = (Integer) jobFile.get("attempt") + 1; if (retry == attempt) { removeJobFile(jobFile); } else { jobFile.put("attempt", attempt); jobFile.put("attemptdate", System.currentTimeMillis() + (Long) jobFile.get("retryms")); acceptFile(jobFile); } if (s3Client != null) cleanupMultiPartUploads(s3Client, (String) jobFile.get("bucket")); } finally { if (tm != null) tm.shutdownNow(true); } }
From source file:org.alanwilliamson.amazon.s3.Write.java
License:Open Source License
private void writeData(AmazonKey amazonKey, String bucket, String key, Map<String, String> metadata, StorageClass storage, String mimetype, cfData data, int retry, int retryseconds, String acl, String aes256key, Map<String, String> customheaders) throws Exception { if (mimetype == null) { if (data.getDataType() == cfData.CFBINARYDATA) mimetype = "application/unknown"; else if (cfData.isSimpleValue(data)) mimetype = "text/plain"; else// w w w . j av a 2 s . c o m mimetype = "application/json"; // Check to see if the mime type is in the metadata if (metadata != null && metadata.containsKey("Content-Type")) mimetype = metadata.get("Content-Type"); } InputStream ios = null; long size = 0; if (data.getDataType() == cfData.CFSTRINGDATA) { ios = new java.io.ByteArrayInputStream(data.getString().getBytes()); size = data.getString().length(); } else if (data.getDataType() == cfData.CFBINARYDATA) { ios = new java.io.ByteArrayInputStream(((cfBinaryData) data).getByteArray()); size = ((cfBinaryData) data).getLength(); } else { serializejson json = new serializejson(); StringBuilder out = new StringBuilder(); json.encodeJSON(out, data, false, CaseType.MAINTAIN, DateType.LONG); size = out.length(); mimetype = "application/json"; ios = new java.io.ByteArrayInputStream(out.toString().getBytes()); } // Setup the object data ObjectMetadata omd = new ObjectMetadata(); if (metadata != null) omd.setUserMetadata(metadata); omd.setContentType(mimetype); omd.setContentLength(size); AmazonS3 s3Client = getAmazonS3(amazonKey); // Let us run around the number of attempts int attempts = 0; while (attempts < retry) { try { PutObjectRequest por = new PutObjectRequest(bucket, key, ios, omd); por.setStorageClass(storage); if (aes256key != null && !aes256key.isEmpty()) por.setSSECustomerKey(new SSECustomerKey(aes256key)); if (acl != null && !acl.isEmpty()) por.setCannedAcl(amazonKey.getAmazonCannedAcl(acl)); if (customheaders != null && !customheaders.isEmpty()) { Iterator<String> it = customheaders.keySet().iterator(); while (it.hasNext()) { String k = it.next(); por.putCustomRequestHeader(k, customheaders.get(k)); } } s3Client.putObject(por); break; } catch (Exception e) { cfEngine.log("Failed: AmazonS3Write(bucket=" + bucket + "; key=" + key + "; attempt=" + (attempts + 1) + "; exception=" + e.getMessage() + ")"); attempts++; if (attempts == retry) throw e; else Thread.sleep(retryseconds * 1000); } } }
From source file:org.alanwilliamson.amazon.s3.Write.java
License:Open Source License
private void writeFile(AmazonKey amazonKey, String bucket, String key, Map<String, String> metadata, StorageClass storage, String localpath, int retry, int retryseconds, boolean deletefile, boolean background, String callback, String callbackdata, String appname, String acl, String aes256key, Map<String, String> customheaders) throws Exception { File localFile = new File(localpath); if (!localFile.isFile()) throw new Exception("The file specified does not exist: " + localpath); // Push this to the background loader to handle and return immediately if (background) { BackgroundUploader.acceptFile(amazonKey, bucket, key, metadata, storage, localpath, retry, retryseconds, deletefile, callback, callbackdata, appname, acl, aes256key, customheaders); return;/* w w w .j a va2 s .com*/ } // Setup the object data ObjectMetadata omd = new ObjectMetadata(); if (metadata != null) omd.setUserMetadata(metadata); AmazonS3 s3Client = getAmazonS3(amazonKey); // Let us run around the number of attempts int attempts = 0; while (attempts < retry) { try { PutObjectRequest por = new PutObjectRequest(bucket, key, localFile); por.setMetadata(omd); por.setStorageClass(storage); if (acl != null && !acl.isEmpty()) por.setCannedAcl(amazonKey.getAmazonCannedAcl(acl)); if (aes256key != null && !aes256key.isEmpty()) por.setSSECustomerKey(new SSECustomerKey(aes256key)); if (customheaders != null && !customheaders.isEmpty()) { Iterator<String> it = customheaders.keySet().iterator(); while (it.hasNext()) { String k = it.next(); por.putCustomRequestHeader(k, customheaders.get(k)); } } s3Client.putObject(por); break; } catch (Exception e) { cfEngine.log("Failed: AmazonS3Write(bucket=" + bucket + "key=" + key + "; file=" + localFile + "; attempt=" + (attempts + 1) + "; exception=" + e.getMessage() + ")"); attempts++; if (attempts == retry) throw e; else Thread.sleep(retryseconds * 1000); } } // delete the file now that it is a success if (deletefile) localFile.delete(); }
From source file:org.apache.camel.component.aws.s3.S3Producer.java
License:Apache License
@Override public void process(final Exchange exchange) throws Exception { ObjectMetadata objectMetadata = new ObjectMetadata(); Long contentLength = exchange.getIn().getHeader(S3Constants.CONTENT_LENGTH, Long.class); if (contentLength != null) { objectMetadata.setContentLength(contentLength); }// w w w . j av a2s .c o m String contentType = exchange.getIn().getHeader(S3Constants.CONTENT_TYPE, String.class); if (contentType != null) { objectMetadata.setContentType(contentType); } String cacheControl = exchange.getIn().getHeader(S3Constants.CACHE_CONTROL, String.class); if (cacheControl != null) { objectMetadata.setCacheControl(cacheControl); } String contentDisposition = exchange.getIn().getHeader(S3Constants.CONTENT_DISPOSITION, String.class); if (contentDisposition != null) { objectMetadata.setContentDisposition(contentDisposition); } String contentEncoding = exchange.getIn().getHeader(S3Constants.CONTENT_ENCODING, String.class); if (contentEncoding != null) { objectMetadata.setContentEncoding(contentEncoding); } String contentMD5 = exchange.getIn().getHeader(S3Constants.CONTENT_MD5, String.class); if (contentMD5 != null) { objectMetadata.setContentMD5(contentMD5); } Date lastModified = exchange.getIn().getHeader(S3Constants.LAST_MODIFIED, Date.class); if (lastModified != null) { objectMetadata.setLastModified(lastModified); } Map<String, String> userMetadata = exchange.getIn().getHeader(S3Constants.USER_METADATA, Map.class); if (userMetadata != null) { objectMetadata.setUserMetadata(userMetadata); } File filePayload = null; Object obj = exchange.getIn().getMandatoryBody(); if (obj instanceof File) { filePayload = (File) obj; } PutObjectRequest putObjectRequest = new PutObjectRequest(getConfiguration().getBucketName(), determineKey(exchange), exchange.getIn().getMandatoryBody(InputStream.class), objectMetadata); String storageClass = determineStorageClass(exchange); if (storageClass != null) { putObjectRequest.setStorageClass(storageClass); } String cannedAcl = exchange.getIn().getHeader(S3Constants.CANNED_ACL, String.class); if (cannedAcl != null) { CannedAccessControlList objectAcl = CannedAccessControlList.valueOf(cannedAcl); putObjectRequest.setCannedAcl(objectAcl); } AccessControlList acl = exchange.getIn().getHeader(S3Constants.ACL, AccessControlList.class); if (acl != null) { // note: if cannedacl and acl are both specified the last one will be used. refer to // PutObjectRequest#setAccessControlList for more details putObjectRequest.setAccessControlList(acl); } LOG.trace("Put object [{}] from exchange [{}]...", putObjectRequest, exchange); PutObjectResult putObjectResult = getEndpoint().getS3Client().putObject(putObjectRequest); LOG.trace("Received result [{}]", putObjectResult); Message message = getMessageForResponse(exchange); message.setHeader(S3Constants.E_TAG, putObjectResult.getETag()); if (putObjectResult.getVersionId() != null) { message.setHeader(S3Constants.VERSION_ID, putObjectResult.getVersionId()); } if (getConfiguration().isDeleteAfterWrite() && filePayload != null) { IOHelper.close(putObjectRequest.getInputStream()); FileUtil.deleteFile(filePayload); } }
From source file:org.apache.nifi.processors.aws.s3.PutS3Object.java
License:Apache License
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) { FlowFile flowFile = session.get();/*from www . ja v a2 s . co m*/ if (flowFile == null) { return; } final long startNanos = System.nanoTime(); final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue(); final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue(); final String cacheKey = getIdentifier() + "/" + bucket + "/" + key; final AmazonS3Client s3 = getClient(); final FlowFile ff = flowFile; final Map<String, String> attributes = new HashMap<>(); final String ffFilename = ff.getAttributes().get(CoreAttributes.FILENAME.key()); attributes.put(S3_BUCKET_KEY, bucket); attributes.put(S3_OBJECT_KEY, key); final Long multipartThreshold = context.getProperty(MULTIPART_THRESHOLD).asDataSize(DataUnit.B).longValue(); final Long multipartPartSize = context.getProperty(MULTIPART_PART_SIZE).asDataSize(DataUnit.B).longValue(); final long now = System.currentTimeMillis(); /* * If necessary, run age off for existing uploads in AWS S3 and local state */ ageoffS3Uploads(context, s3, now); /* * Then */ try { session.read(flowFile, new InputStreamCallback() { @Override public void process(final InputStream rawIn) throws IOException { try (final InputStream in = new BufferedInputStream(rawIn)) { final ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentDisposition(ff.getAttribute(CoreAttributes.FILENAME.key())); objectMetadata.setContentLength(ff.getSize()); final String contentType = context.getProperty(CONTENT_TYPE) .evaluateAttributeExpressions(ff).getValue(); if (contentType != null) { objectMetadata.setContentType(contentType); attributes.put(S3_CONTENT_TYPE, contentType); } final String expirationRule = context.getProperty(EXPIRATION_RULE_ID) .evaluateAttributeExpressions(ff).getValue(); if (expirationRule != null) { objectMetadata.setExpirationTimeRuleId(expirationRule); } final Map<String, String> userMetadata = new HashMap<>(); for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties() .entrySet()) { if (entry.getKey().isDynamic()) { final String value = context.getProperty(entry.getKey()) .evaluateAttributeExpressions(ff).getValue(); userMetadata.put(entry.getKey().getName(), value); } } final String serverSideEncryption = context.getProperty(SERVER_SIDE_ENCRYPTION).getValue(); if (!serverSideEncryption.equals(NO_SERVER_SIDE_ENCRYPTION)) { objectMetadata.setSSEAlgorithm(serverSideEncryption); attributes.put(S3_SSE_ALGORITHM, serverSideEncryption); } if (!userMetadata.isEmpty()) { objectMetadata.setUserMetadata(userMetadata); } if (ff.getSize() <= multipartThreshold) { //---------------------------------------- // single part upload //---------------------------------------- final PutObjectRequest request = new PutObjectRequest(bucket, key, in, objectMetadata); request.setStorageClass( StorageClass.valueOf(context.getProperty(STORAGE_CLASS).getValue())); final AccessControlList acl = createACL(context, ff); if (acl != null) { request.setAccessControlList(acl); } final CannedAccessControlList cannedAcl = createCannedACL(context, ff); if (cannedAcl != null) { request.withCannedAcl(cannedAcl); } try { final PutObjectResult result = s3.putObject(request); if (result.getVersionId() != null) { attributes.put(S3_VERSION_ATTR_KEY, result.getVersionId()); } if (result.getETag() != null) { attributes.put(S3_ETAG_ATTR_KEY, result.getETag()); } if (result.getExpirationTime() != null) { attributes.put(S3_EXPIRATION_ATTR_KEY, result.getExpirationTime().toString()); } if (result.getMetadata().getRawMetadata().keySet() .contains(S3_STORAGECLASS_META_KEY)) { attributes.put(S3_STORAGECLASS_ATTR_KEY, result.getMetadata() .getRawMetadataValue(S3_STORAGECLASS_META_KEY).toString()); } if (userMetadata.size() > 0) { StringBuilder userMetaBldr = new StringBuilder(); for (String userKey : userMetadata.keySet()) { userMetaBldr.append(userKey).append("=").append(userMetadata.get(userKey)); } attributes.put(S3_USERMETA_ATTR_KEY, userMetaBldr.toString()); } attributes.put(S3_API_METHOD_ATTR_KEY, S3_API_METHOD_PUTOBJECT); } catch (AmazonClientException e) { getLogger().info("Failure completing upload flowfile={} bucket={} key={} reason={}", new Object[] { ffFilename, bucket, key, e.getMessage() }); throw (e); } } else { //---------------------------------------- // multipart upload //---------------------------------------- // load or create persistent state //------------------------------------------------------------ MultipartState currentState; try { currentState = getLocalStateIfInS3(s3, bucket, cacheKey); if (currentState != null) { if (currentState.getPartETags().size() > 0) { final PartETag lastETag = currentState.getPartETags() .get(currentState.getPartETags().size() - 1); getLogger().info("Resuming upload for flowfile='{}' bucket='{}' key='{}' " + "uploadID='{}' filePosition='{}' partSize='{}' storageClass='{}' " + "contentLength='{}' partsLoaded={} lastPart={}/{}", new Object[] { ffFilename, bucket, key, currentState.getUploadId(), currentState.getFilePosition(), currentState.getPartSize(), currentState.getStorageClass().toString(), currentState.getContentLength(), currentState.getPartETags().size(), Integer.toString(lastETag.getPartNumber()), lastETag.getETag() }); } else { getLogger().info("Resuming upload for flowfile='{}' bucket='{}' key='{}' " + "uploadID='{}' filePosition='{}' partSize='{}' storageClass='{}' " + "contentLength='{}' no partsLoaded", new Object[] { ffFilename, bucket, key, currentState.getUploadId(), currentState.getFilePosition(), currentState.getPartSize(), currentState.getStorageClass().toString(), currentState.getContentLength() }); } } else { currentState = new MultipartState(); currentState.setPartSize(multipartPartSize); currentState.setStorageClass( StorageClass.valueOf(context.getProperty(STORAGE_CLASS).getValue())); currentState.setContentLength(ff.getSize()); persistLocalState(cacheKey, currentState); getLogger().info("Starting new upload for flowfile='{}' bucket='{}' key='{}'", new Object[] { ffFilename, bucket, key }); } } catch (IOException e) { getLogger().error("IOException initiating cache state while processing flow files: " + e.getMessage()); throw (e); } // initiate multipart upload or find position in file //------------------------------------------------------------ if (currentState.getUploadId().isEmpty()) { final InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest( bucket, key, objectMetadata); initiateRequest.setStorageClass(currentState.getStorageClass()); final AccessControlList acl = createACL(context, ff); if (acl != null) { initiateRequest.setAccessControlList(acl); } final CannedAccessControlList cannedAcl = createCannedACL(context, ff); if (cannedAcl != null) { initiateRequest.withCannedACL(cannedAcl); } try { final InitiateMultipartUploadResult initiateResult = s3 .initiateMultipartUpload(initiateRequest); currentState.setUploadId(initiateResult.getUploadId()); currentState.getPartETags().clear(); try { persistLocalState(cacheKey, currentState); } catch (Exception e) { getLogger().info("Exception saving cache state while processing flow file: " + e.getMessage()); throw (new ProcessException("Exception saving cache state", e)); } getLogger().info( "Success initiating upload flowfile={} available={} position={} " + "length={} bucket={} key={} uploadId={}", new Object[] { ffFilename, in.available(), currentState.getFilePosition(), currentState.getContentLength(), bucket, key, currentState.getUploadId() }); if (initiateResult.getUploadId() != null) { attributes.put(S3_UPLOAD_ID_ATTR_KEY, initiateResult.getUploadId()); } } catch (AmazonClientException e) { getLogger().info( "Failure initiating upload flowfile={} bucket={} key={} reason={}", new Object[] { ffFilename, bucket, key, e.getMessage() }); throw (e); } } else { if (currentState.getFilePosition() > 0) { try { final long skipped = in.skip(currentState.getFilePosition()); if (skipped != currentState.getFilePosition()) { getLogger().info( "Failure skipping to resume upload flowfile={} " + "bucket={} key={} position={} skipped={}", new Object[] { ffFilename, bucket, key, currentState.getFilePosition(), skipped }); } } catch (Exception e) { getLogger().info( "Failure skipping to resume upload flowfile={} bucket={} " + "key={} position={} reason={}", new Object[] { ffFilename, bucket, key, currentState.getFilePosition(), e.getMessage() }); throw (new ProcessException(e)); } } } // upload parts //------------------------------------------------------------ long thisPartSize; for (int part = currentState.getPartETags().size() + 1; currentState .getFilePosition() < currentState.getContentLength(); part++) { if (!PutS3Object.this.isScheduled()) { throw new IOException(S3_PROCESS_UNSCHEDULED_MESSAGE + " flowfile=" + ffFilename + " part=" + part + " uploadId=" + currentState.getUploadId()); } thisPartSize = Math.min(currentState.getPartSize(), (currentState.getContentLength() - currentState.getFilePosition())); UploadPartRequest uploadRequest = new UploadPartRequest().withBucketName(bucket) .withKey(key).withUploadId(currentState.getUploadId()).withInputStream(in) .withPartNumber(part).withPartSize(thisPartSize); try { UploadPartResult uploadPartResult = s3.uploadPart(uploadRequest); currentState.addPartETag(uploadPartResult.getPartETag()); currentState.setFilePosition(currentState.getFilePosition() + thisPartSize); try { persistLocalState(cacheKey, currentState); } catch (Exception e) { getLogger().info("Exception saving cache state processing flow file: " + e.getMessage()); } getLogger().info( "Success uploading part flowfile={} part={} available={} " + "etag={} uploadId={}", new Object[] { ffFilename, part, in.available(), uploadPartResult.getETag(), currentState.getUploadId() }); } catch (AmazonClientException e) { getLogger().info( "Failure uploading part flowfile={} part={} bucket={} key={} " + "reason={}", new Object[] { ffFilename, part, bucket, key, e.getMessage() }); throw (e); } } // complete multipart upload //------------------------------------------------------------ CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( bucket, key, currentState.getUploadId(), currentState.getPartETags()); try { CompleteMultipartUploadResult completeResult = s3 .completeMultipartUpload(completeRequest); getLogger().info("Success completing upload flowfile={} etag={} uploadId={}", new Object[] { ffFilename, completeResult.getETag(), currentState.getUploadId() }); if (completeResult.getVersionId() != null) { attributes.put(S3_VERSION_ATTR_KEY, completeResult.getVersionId()); } if (completeResult.getETag() != null) { attributes.put(S3_ETAG_ATTR_KEY, completeResult.getETag()); } if (completeResult.getExpirationTime() != null) { attributes.put(S3_EXPIRATION_ATTR_KEY, completeResult.getExpirationTime().toString()); } if (currentState.getStorageClass() != null) { attributes.put(S3_STORAGECLASS_ATTR_KEY, currentState.getStorageClass().toString()); } if (userMetadata.size() > 0) { StringBuilder userMetaBldr = new StringBuilder(); for (String userKey : userMetadata.keySet()) { userMetaBldr.append(userKey).append("=").append(userMetadata.get(userKey)); } attributes.put(S3_USERMETA_ATTR_KEY, userMetaBldr.toString()); } attributes.put(S3_API_METHOD_ATTR_KEY, S3_API_METHOD_MULTIPARTUPLOAD); } catch (AmazonClientException e) { getLogger().info("Failure completing upload flowfile={} bucket={} key={} reason={}", new Object[] { ffFilename, bucket, key, e.getMessage() }); throw (e); } } } } }); if (!attributes.isEmpty()) { flowFile = session.putAllAttributes(flowFile, attributes); } session.transfer(flowFile, REL_SUCCESS); final String url = s3.getResourceUrl(bucket, key); final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); session.getProvenanceReporter().send(flowFile, url, millis); getLogger().info("Successfully put {} to Amazon S3 in {} milliseconds", new Object[] { ff, millis }); try { removeLocalState(cacheKey); } catch (IOException e) { getLogger().info("Error trying to delete key {} from cache: {}", new Object[] { cacheKey, e.getMessage() }); } } catch (final ProcessException | AmazonClientException pe) { if (pe.getMessage().contains(S3_PROCESS_UNSCHEDULED_MESSAGE)) { getLogger().info(pe.getMessage()); session.rollback(); } else { getLogger().error("Failed to put {} to Amazon S3 due to {}", new Object[] { flowFile, pe }); flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); } } }
From source file:org.duracloud.s3storage.S3StorageProvider.java
License:Apache License
/** * Adds content to a hidden space.//from www .ja v a 2 s . c o m * * @param spaceId hidden spaceId * @param contentId * @param contentMimeType * @param content * @return */ public String addHiddenContent(String spaceId, String contentId, String contentMimeType, InputStream content) { log.debug("addHiddenContent(" + spaceId + ", " + contentId + ", " + contentMimeType + ")"); // Will throw if bucket does not exist String bucketName = getBucketName(spaceId); // Wrap the content in order to be able to retrieve a checksum if (contentMimeType == null || contentMimeType.equals("")) { contentMimeType = DEFAULT_MIMETYPE; } ObjectMetadata objMetadata = new ObjectMetadata(); objMetadata.setContentType(contentMimeType); PutObjectRequest putRequest = new PutObjectRequest(bucketName, contentId, content, objMetadata); putRequest.setStorageClass(DEFAULT_STORAGE_CLASS); putRequest.setCannedAcl(CannedAccessControlList.Private); try { PutObjectResult putResult = s3Client.putObject(putRequest); return putResult.getETag(); } catch (AmazonClientException e) { String err = "Could not add content " + contentId + " with type " + contentMimeType + " to S3 bucket " + bucketName + " due to error: " + e.getMessage(); throw new StorageException(err, e, NO_RETRY); } }
From source file:org.duracloud.s3storage.S3StorageProvider.java
License:Apache License
/** * {@inheritDoc}//from w w w. j a v a 2 s .c o m */ public String addContent(String spaceId, String contentId, String contentMimeType, Map<String, String> userProperties, long contentSize, String contentChecksum, InputStream content) { log.debug("addContent(" + spaceId + ", " + contentId + ", " + contentMimeType + ", " + contentSize + ", " + contentChecksum + ")"); // Will throw if bucket does not exist String bucketName = getBucketName(spaceId); // Wrap the content in order to be able to retrieve a checksum ChecksumInputStream wrappedContent = new ChecksumInputStream(content, contentChecksum); String contentEncoding = removeContentEncoding(userProperties); userProperties = removeCalculatedProperties(userProperties); if (contentMimeType == null || contentMimeType.equals("")) { contentMimeType = DEFAULT_MIMETYPE; } ObjectMetadata objMetadata = new ObjectMetadata(); objMetadata.setContentType(contentMimeType); if (contentSize > 0) { objMetadata.setContentLength(contentSize); } if (null != contentChecksum && !contentChecksum.isEmpty()) { String encodedChecksum = ChecksumUtil.convertToBase64Encoding(contentChecksum); objMetadata.setContentMD5(encodedChecksum); } if (contentEncoding != null) { objMetadata.setContentEncoding(contentEncoding); } if (userProperties != null) { for (String key : userProperties.keySet()) { String value = userProperties.get(key); if (log.isDebugEnabled()) { log.debug("[" + key + "|" + value + "]"); } objMetadata.addUserMetadata(getSpaceFree(encodeHeaderKey(key)), encodeHeaderValue(value)); } } PutObjectRequest putRequest = new PutObjectRequest(bucketName, contentId, wrappedContent, objMetadata); putRequest.setStorageClass(DEFAULT_STORAGE_CLASS); putRequest.setCannedAcl(CannedAccessControlList.Private); // Add the object String etag; try { PutObjectResult putResult = s3Client.putObject(putRequest); etag = putResult.getETag(); } catch (AmazonClientException e) { if (e instanceof AmazonS3Exception) { AmazonS3Exception s3Ex = (AmazonS3Exception) e; String errorCode = s3Ex.getErrorCode(); Integer statusCode = s3Ex.getStatusCode(); String message = MessageFormat.format( "exception putting object {0} into {1}: errorCode={2}," + " statusCode={3}, errorMessage={4}", contentId, bucketName, errorCode, statusCode, e.getMessage()); if (errorCode.equals("InvalidDigest") || errorCode.equals("BadDigest")) { log.error(message, e); String err = "Checksum mismatch detected attempting to add " + "content " + contentId + " to S3 bucket " + bucketName + ". Content was not added."; throw new ChecksumMismatchException(err, e, NO_RETRY); } else if (errorCode.equals("IncompleteBody")) { log.error(message, e); throw new StorageException("The content body was incomplete for " + contentId + " to S3 bucket " + bucketName + ". Content was not added.", e, NO_RETRY); } else if (!statusCode.equals(HttpStatus.SC_SERVICE_UNAVAILABLE) && !statusCode.equals(HttpStatus.SC_NOT_FOUND)) { log.error(message, e); } else { log.warn(message, e); } } else { String err = MessageFormat.format("exception putting object {0} into {1}: {2}", contentId, bucketName, e.getMessage()); log.error(err, e); } // Check to see if file landed successfully in S3, despite the exception etag = doesContentExistWithExpectedChecksum(bucketName, contentId, contentChecksum); if (null == etag) { String err = "Could not add content " + contentId + " with type " + contentMimeType + " and size " + contentSize + " to S3 bucket " + bucketName + " due to error: " + e.getMessage(); throw new StorageException(err, e, NO_RETRY); } } // Compare checksum String providerChecksum = getETagValue(etag); String checksum = wrappedContent.getMD5(); StorageProviderUtil.compareChecksum(providerChecksum, spaceId, contentId, checksum); return providerChecksum; }