Java tutorial
/* * Copyright 2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.springframework.integration.aws.outbound; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import org.springframework.cloud.aws.core.env.ResourceIdResolver; import org.springframework.expression.EvaluationContext; import org.springframework.expression.Expression; import org.springframework.expression.common.LiteralExpression; import org.springframework.integration.expression.ExpressionUtils; import org.springframework.integration.expression.ValueExpression; import org.springframework.integration.handler.AbstractReplyProducingMessageHandler; import org.springframework.messaging.Message; import org.springframework.messaging.MessageHandlingException; import org.springframework.messaging.MessageHeaders; import org.springframework.messaging.support.MessageBuilder; import org.springframework.util.Assert; import org.springframework.util.StreamUtils; import com.amazonaws.AmazonClientException; import com.amazonaws.event.ProgressEvent; import com.amazonaws.event.ProgressEventType; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.internal.Mimetypes; import com.amazonaws.services.s3.model.AccessControlList; import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.CopyObjectRequest; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.SetObjectAclRequest; import com.amazonaws.services.s3.transfer.ObjectMetadataProvider; import com.amazonaws.services.s3.transfer.PersistableTransfer; import com.amazonaws.services.s3.transfer.Transfer; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.internal.S3ProgressListener; import com.amazonaws.services.s3.transfer.internal.S3ProgressListenerChain; import com.amazonaws.util.Md5Utils; /** * The {@link AbstractReplyProducingMessageHandler} implementation for the Amazon S3 services. * <p> * The implementation is fully based on the {@link TransferManager} and support its {@code upload}, * {@code download} and {@code copy} operations which can be determined by the provided * or evaluated via SpEL expression at runtime {@link S3MessageHandler.Command}. * <p> * This {@link AbstractReplyProducingMessageHandler} can behave as a "one-way" (by default) or * "request-reply" component according to the {@link #produceReply} constructor argument. * <p> * The "one-way" behavior is also blocking, which is achieved with the {@link Transfer#waitForException()} * invocation. Consider to use an async upstream hand off if this blocking behavior isn't appropriate. * <p> * The "request-reply" behavior is async and the {@link Transfer} result from the {@link TransferManager} * operation is sent to the {@link #outputChannel}, assuming the transfer progress observation in the * downstream flow. * <p> * The {@link S3ProgressListener} can be supplied to track the transfer progress. * Also the listener can be populated into the returned {@link Transfer} afterwards in the downstream flow. * * <p> * For the upload operation the {@link UploadMetadataProvider} callback can be supplied to populate required * {@link ObjectMetadata} options, as for a single entry, as well as for each file in directory to upload. * <p> * For the upload operation the {@link #objectAclExpression} can be provided to {@link AmazonS3#setObjectAcl} * after the successful transfer. * The supported SpEL result types are: {@link AccessControlList} or {@link CannedAccessControlList}. * <p> * For download operation the {@code payload} must be a {@link File} instance, representing a single file * for downloaded content or directory to download all files from the S3 virtual directory. * <p> * An S3 Object {@code key} for upload and download can be determined by the provided * {@link #keyExpression} or the {@link File#getName()} is used directly. The former has precedence. * <p> * For copy operation all {@link #keyExpression}, {@link #destinationBucketExpression} and * {@link #destinationKeyExpression} are required and must not evaluate to {@code null}. * * @author Artem Bilan * * @see TransferManager */ public class S3MessageHandler extends AbstractReplyProducingMessageHandler { private final TransferManager transferManager; private final boolean produceReply; private final Expression bucketExpression; private EvaluationContext evaluationContext; private Expression keyExpression; private Expression objectAclExpression; private Expression destinationBucketExpression; private Expression destinationKeyExpression; private Expression commandExpression = new ValueExpression<>(Command.UPLOAD); private S3ProgressListener s3ProgressListener; private UploadMetadataProvider uploadMetadataProvider; private ResourceIdResolver resourceIdResolver; public S3MessageHandler(AmazonS3 amazonS3, String bucket) { this(amazonS3, bucket, false); } public S3MessageHandler(AmazonS3 amazonS3, Expression bucketExpression) { this(amazonS3, bucketExpression, false); } public S3MessageHandler(AmazonS3 amazonS3, String bucket, boolean produceReply) { this(new TransferManager(amazonS3), bucket, produceReply); Assert.notNull(amazonS3, "'amazonS3' must not be null"); } public S3MessageHandler(AmazonS3 amazonS3, Expression bucketExpression, boolean produceReply) { this(new TransferManager(amazonS3), bucketExpression, produceReply); Assert.notNull(amazonS3, "'amazonS3' must not be null"); } public S3MessageHandler(TransferManager transferManager, String bucket) { this(transferManager, bucket, false); } public S3MessageHandler(TransferManager transferManager, Expression bucketExpression) { this(transferManager, bucketExpression, false); } public S3MessageHandler(TransferManager transferManager, String bucket, boolean produceReply) { this(transferManager, new LiteralExpression(bucket), produceReply); Assert.notNull(bucket, "'bucket' must not be null"); } public S3MessageHandler(TransferManager transferManager, Expression bucketExpression, boolean produceReply) { Assert.notNull(transferManager, "'transferManager' must not be null"); Assert.notNull(bucketExpression, "'bucketExpression' must not be null"); this.transferManager = transferManager; this.bucketExpression = bucketExpression; this.produceReply = produceReply; } /** * The SpEL expression to evaluate S3 object key at runtime against {@code requestMessage}. * @param keyExpression the SpEL expression for S3 key. */ public void setKeyExpression(Expression keyExpression) { this.keyExpression = keyExpression; } /** * The SpEL expression to evaluate S3 object ACL at runtime against {@code requestMessage} * for the {@code upload} operation. * @param objectAclExpression the SpEL expression for S3 object ACL. */ public void setObjectAclExpression(Expression objectAclExpression) { this.objectAclExpression = objectAclExpression; } /** * Specify a {@link S3MessageHandler.Command} to perform against {@link TransferManager}. * @param command The {@link S3MessageHandler.Command} to use. * @see S3MessageHandler.Command */ public void setCommand(Command command) { Assert.notNull(command, "'command' must not be null"); setCommandExpression(new ValueExpression<>(command)); } /** * The SpEL expression to evaluate the command to perform on {@link TransferManager}: {@code upload}, * {@code download} or {@code copy}. * @param commandExpression the SpEL expression to evaluate the {@link TransferManager} operation. * @see Command */ public void setCommandExpression(Expression commandExpression) { Assert.notNull(commandExpression, "'commandExpression' must not be null"); this.commandExpression = commandExpression; } /** * The SpEL expression to evaluate the target S3 bucket for copy operation. * @param destinationBucketExpression the SpEL expression for destination bucket. * @see TransferManager#copy(String, String, String, String) */ public void setDestinationBucketExpression(Expression destinationBucketExpression) { this.destinationBucketExpression = destinationBucketExpression; } /** * The SpEL expression to evaluate the target S3 key for copy operation. * @param destinationKeyExpression the SpEL expression for destination key. * @see TransferManager#copy(String, String, String, String) */ public void setDestinationKeyExpression(Expression destinationKeyExpression) { this.destinationKeyExpression = destinationKeyExpression; } /** * Specify a {@link S3ProgressListener} for upload and download operations. * @param s3ProgressListener the {@link S3ProgressListener} to use. */ public void setProgressListener(S3ProgressListener s3ProgressListener) { this.s3ProgressListener = s3ProgressListener; } /** * Specify an {@link ObjectMetadata} callback to populate the metadata for upload operation, * e.g. {@code Content-MD5}, {@code Content-Type} or any other required options. * @param uploadMetadataProvider the {@link UploadMetadataProvider} to use for upload. */ public void setUploadMetadataProvider(UploadMetadataProvider uploadMetadataProvider) { this.uploadMetadataProvider = uploadMetadataProvider; } /** * Specify a {@link ResourceIdResolver} to resolve logical bucket names to physical resource ids. * @param resourceIdResolver the {@link ResourceIdResolver} to use. */ public void setResourceIdResolver(ResourceIdResolver resourceIdResolver) { this.resourceIdResolver = resourceIdResolver; } @Override protected void doInit() { Assert.notNull(this.bucketExpression, "The 'bucketExpression' must not be null"); super.doInit(); this.evaluationContext = ExpressionUtils.createStandardEvaluationContext(getBeanFactory()); } @Override protected Object handleRequestMessage(Message<?> requestMessage) { Command command = this.commandExpression.getValue(this.evaluationContext, requestMessage, Command.class); Assert.state(command != null, "'commandExpression' [" + this.commandExpression.getExpressionString() + "] cannot evaluate to null."); Transfer transfer = null; switch (command) { case UPLOAD: transfer = upload(requestMessage); break; case DOWNLOAD: transfer = download(requestMessage); break; case COPY: transfer = copy(requestMessage); break; } if (this.produceReply) { return transfer; } else { try { AmazonClientException amazonClientException = transfer.waitForException(); if (amazonClientException != null) { throw amazonClientException; } } catch (InterruptedException e) { Thread.currentThread().interrupt(); } return null; } } private Transfer upload(Message<?> requestMessage) { Object payload = requestMessage.getPayload(); String bucketName = obtainBucket(requestMessage); String key = null; if (this.keyExpression != null) { key = this.keyExpression.getValue(this.evaluationContext, requestMessage, String.class); } if (payload instanceof File && ((File) payload).isDirectory()) { File fileToUpload = (File) payload; if (key == null) { key = fileToUpload.getName(); } return this.transferManager.uploadDirectory(bucketName, key, fileToUpload, true, new MessageHeadersObjectMetadataProvider(requestMessage.getHeaders())); } else { ObjectMetadata metadata = new ObjectMetadata(); if (this.uploadMetadataProvider != null) { this.uploadMetadataProvider.populateMetadata(metadata, requestMessage); } InputStream inputStream; if (payload instanceof InputStream) { inputStream = (InputStream) payload; } else if (payload instanceof File) { File fileToUpload = (File) payload; if (key == null) { key = fileToUpload.getName(); } try { inputStream = new FileInputStream(fileToUpload); if (metadata.getContentLength() == 0) { metadata.setContentLength(fileToUpload.length()); } if (metadata.getContentType() == null) { metadata.setContentType(Mimetypes.getInstance().getMimetype(fileToUpload)); } } catch (FileNotFoundException e) { throw new AmazonClientException(e); } } else if (payload instanceof byte[]) { inputStream = new ByteArrayInputStream((byte[]) payload); } else { throw new IllegalArgumentException("Unsupported payload type: [" + payload.getClass() + "]. The only supported payloads for the upload request are " + "java.io.File, java.io.InputStream, byte[] and PutObjectRequest."); } Assert.state(key != null, "The 'keyExpression' must not be null for non-File payloads and can't evaluate to null. " + "Root object is: " + requestMessage); if (key == null) { if (this.keyExpression != null) { throw new IllegalStateException( "The 'keyExpression' [" + this.keyExpression.getExpressionString() + "] must not evaluate to null. Root object is: " + requestMessage); } else { throw new IllegalStateException("Specify a 'keyExpression' for non-java.io.File payloads"); } } if (metadata.getContentMD5() == null) { String contentMd5 = null; try { contentMd5 = Md5Utils.md5AsBase64(StreamUtils.copyToByteArray(inputStream)); if (inputStream.markSupported()) { inputStream.reset(); } metadata.setContentMD5(contentMd5); } catch (IOException e) { throw new MessageHandlingException(requestMessage, e); } } PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, inputStream, metadata); S3ProgressListener progressListener = this.s3ProgressListener; if (this.objectAclExpression != null) { Object acl = this.objectAclExpression.getValue(this.evaluationContext, requestMessage); Assert.state(acl instanceof AccessControlList || acl instanceof CannedAccessControlList, "The 'objectAclExpression' [" + this.objectAclExpression.getExpressionString() + "] must evaluate to com.amazonaws.services.s3.model.AccessControlList " + "or must evaluate to com.amazonaws.services.s3.model.CannedAccessControlList. " + "Gotten: [" + acl + "]"); SetObjectAclRequest aclRequest; if (acl instanceof AccessControlList) { aclRequest = new SetObjectAclRequest(bucketName, key, (AccessControlList) acl); } else { aclRequest = new SetObjectAclRequest(bucketName, key, (CannedAccessControlList) acl); } final SetObjectAclRequest theAclRequest = aclRequest; progressListener = new S3ProgressListener() { @Override public void onPersistableTransfer(PersistableTransfer persistableTransfer) { } @Override public void progressChanged(ProgressEvent progressEvent) { if (ProgressEventType.TRANSFER_COMPLETED_EVENT.equals(progressEvent.getEventType())) { S3MessageHandler.this.transferManager.getAmazonS3Client().setObjectAcl(theAclRequest); } } }; if (this.s3ProgressListener != null) { progressListener = new S3ProgressListenerChain(this.s3ProgressListener, progressListener); } } if (progressListener != null) { return this.transferManager.upload(putObjectRequest, progressListener); } else { return this.transferManager.upload(putObjectRequest); } } } private Transfer download(Message<?> requestMessage) { Object payload = requestMessage.getPayload(); Assert.state(payload instanceof File, "For the 'DOWNLOAD' operation the 'payload' must be of " + "'java.io.File' type, but gotten: [" + payload.getClass() + ']'); File targetFile = (File) payload; String bucket = obtainBucket(requestMessage); String key = null; if (this.keyExpression != null) { key = this.keyExpression.getValue(this.evaluationContext, requestMessage, String.class); } else { key = targetFile.getName(); } Assert.state(key != null, "The 'keyExpression' must not be null for non-File payloads and can't evaluate to null. " + "Root object is: " + requestMessage); if (targetFile.isDirectory()) { return this.transferManager.downloadDirectory(bucket, key, targetFile); } else { if (this.s3ProgressListener != null) { return this.transferManager.download(new GetObjectRequest(bucket, key), targetFile, this.s3ProgressListener); } else { return this.transferManager.download(bucket, key, targetFile); } } } private Transfer copy(Message<?> requestMessage) { String sourceBucketName = obtainBucket(requestMessage); String sourceKey = null; if (this.keyExpression != null) { sourceKey = this.keyExpression.getValue(this.evaluationContext, requestMessage, String.class); } Assert.state(sourceKey != null, "The 'keyExpression' must not be null for 'copy' operation and can't evaluate to null. " + "Root object is: " + requestMessage); String destinationBucketName = null; if (this.destinationBucketExpression != null) { destinationBucketName = this.destinationBucketExpression.getValue(this.evaluationContext, requestMessage, String.class); } if (this.resourceIdResolver != null) { destinationBucketName = this.resourceIdResolver.resolveToPhysicalResourceId(destinationBucketName); } Assert.state(destinationBucketName != null, "The 'destinationBucketExpression' must not be null for 'copy' operation and can't evaluate to null. " + "Root object is: " + requestMessage); String destinationKey = null; if (this.destinationKeyExpression != null) { destinationKey = this.destinationKeyExpression.getValue(this.evaluationContext, requestMessage, String.class); } Assert.state(destinationKey != null, "The 'destinationKeyExpression' must not be null for 'copy' operation and can't evaluate to null. " + "Root object is: " + requestMessage); CopyObjectRequest copyObjectRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destinationBucketName, destinationKey); return this.transferManager.copy(copyObjectRequest); } private String obtainBucket(Message<?> requestMessage) { String bucketName; if (this.bucketExpression instanceof LiteralExpression) { bucketName = (String) this.bucketExpression.getValue(); } else { bucketName = this.bucketExpression.getValue(this.evaluationContext, requestMessage, String.class); } Assert.state(bucketName != null, "The 'bucketExpression' [" + this.bucketExpression.getExpressionString() + "] must not evaluate to null. Root object is: " + requestMessage); if (this.resourceIdResolver != null) { bucketName = this.resourceIdResolver.resolveToPhysicalResourceId(bucketName); } return bucketName; } /** * The {@link S3MessageHandler} mode. * * @see #setCommand */ public enum Command { /** * The command to perform {@link TransferManager#upload} operation. */ UPLOAD, /** * The command to perform {@link TransferManager#download} operation. */ DOWNLOAD, /** * The command to perform {@link TransferManager#copy} operation. */ COPY } /** * The callback to populate an {@link ObjectMetadata} for upload operation. * The message can be used as a metadata source. */ public interface UploadMetadataProvider { void populateMetadata(ObjectMetadata metadata, Message<?> message); } private class MessageHeadersObjectMetadataProvider implements ObjectMetadataProvider { private final MessageHeaders messageHeaders; MessageHeadersObjectMetadataProvider(MessageHeaders messageHeaders) { this.messageHeaders = messageHeaders; } @Override public void provideObjectMetadata(File file, ObjectMetadata metadata) { if (S3MessageHandler.this.uploadMetadataProvider != null) { S3MessageHandler.this.uploadMetadataProvider.populateMetadata(metadata, MessageBuilder.createMessage(file, this.messageHeaders)); } if (metadata.getContentMD5() == null) { try { String contentMd5 = Md5Utils.md5AsBase64(file); metadata.setContentMD5(contentMd5); } catch (Exception e) { throw new AmazonClientException(e); } } } } }