Java tutorial
/* * Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. * A copy of the License is located at * * http://aws.amazon.com/apache2.0 * * or in the "license" file accompanying this file. This file is distributed * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ package com.amazonaws.services.s3; import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import org.apache.commons.logging.LogFactory; import com.amazonaws.AmazonWebServiceRequest; import com.amazonaws.services.s3.internal.MultiFileOutputStream; import com.amazonaws.services.s3.internal.PartCreationEvent; import com.amazonaws.services.s3.internal.S3DirectSpi; import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; import com.amazonaws.services.s3.model.EncryptedInitiateMultipartUploadRequest; import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; import com.amazonaws.services.s3.model.PartETag; import com.amazonaws.services.s3.model.UploadObjectRequest; import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; /** * An observer that gets notified of ciphertext file creation for the purpose of * pipelined parallel multi-part uploads of encrypted data to S3. This observer * is responsible for uploading the files to S3 via multi-part upload, including * the multi-part upload initiation, individual part uploads, and multi-part * upload completion. * <p> * This observer is designed for extension so that custom behavior can be * provided. A customer observer can be configured via * {@link UploadObjectRequest#withUploadObjectObserver(UploadObjectObserver)}. * * @see UploadObjectRequest */ public class UploadObjectObserver { private final List<Future<UploadPartResult>> futures = new ArrayList<Future<UploadPartResult>>(); private UploadObjectRequest req; private String uploadId; private S3DirectSpi s3direct; private AmazonS3 s3; private ExecutorService es; /** * Used to initialized this observer. This method is an SPI (service * provider interface) that is called from * <code>AmazonS3EncryptionClient</code>. * <p> * Implementation of this method should never block. * * @param req * the upload object request * @param s3direct * used to perform non-encrypting s3 operation via the current * instance of s3 (encryption) client * @param s3 * the current instance of s3 (encryption) client * @param es * the executor service to be used for concurrent uploads * @return this object */ @SuppressWarnings("checkstyle:hiddenfield") public UploadObjectObserver init(UploadObjectRequest req, S3DirectSpi s3direct, AmazonS3 s3, ExecutorService es) { this.req = req; this.s3direct = s3direct; this.s3 = s3; this.es = es; return this; } @SuppressWarnings("checkstyle:hiddenfield") protected InitiateMultipartUploadRequest newInitiateMultipartUploadRequest(UploadObjectRequest req) { return new EncryptedInitiateMultipartUploadRequest(req.getBucketName(), req.getKey(), req.getMetadata()) .withMaterialsDescription(req.getMaterialsDescription()) .withRedirectLocation(req.getRedirectLocation()) .withSSEAwsKeyManagementParams(req.getSSEAwsKeyManagementParams()) .withSSECustomerKey(req.getSSECustomerKey()).withStorageClass(req.getStorageClass()) .withAccessControlList(req.getAccessControlList()).withCannedACL(req.getCannedAcl()) .withGeneralProgressListener(req.getGeneralProgressListener()) .withRequestMetricCollector(req.getRequestMetricCollector()); } /** * Notified from * {@link AmazonS3EncryptionClient#uploadObject(UploadObjectRequest)} to * initiate a multi-part upload. * * @param req * the upload object request * @return the initiated multi-part uploadId */ @SuppressWarnings("checkstyle:hiddenfield") public String onUploadInitiation(UploadObjectRequest req) { final InitiateMultipartUploadResult res = s3 .initiateMultipartUpload(newInitiateMultipartUploadRequest(req)); this.uploadId = res.getUploadId(); return this.uploadId; } /** * Notified from {@link MultiFileOutputStream#fos()} when a part ready for * upload has been successfully created on disk. By default, this method * performs the following: * <ol> * <li>calls {@link #newUploadPartRequest(PartCreationEvent, File)} to * create an upload-part request for the newly created ciphertext file</li> * <li>call {@link #appendUserAgent(AmazonWebServiceRequest, String)} to * append the necessary user agent string to the request</li> * <li>and finally submit a concurrent task, which calls the method * {@link #uploadPart(UploadPartRequest)}, to be performed</li> * </ol> * <p> * To enable parallel uploads, implementation of this method should never * block. * * @param event * to represent the completion of a ciphertext file creation * which is ready for multipart upload to S3. */ public void onPartCreate(PartCreationEvent event) { final File part = event.getPart(); final UploadPartRequest reqUploadPart = newUploadPartRequest(event, part); final OnFileDelete fileDeleteObserver = event.getFileDeleteObserver(); appendUserAgent(reqUploadPart, AmazonS3EncryptionClient.USER_AGENT); futures.add(es.submit(new Callable<UploadPartResult>() { @Override public UploadPartResult call() { // Upload the ciphertext directly via the non-encrypting // s3 client try { return uploadPart(reqUploadPart); } finally { // clean up part already uploaded if (!part.delete()) { LogFactory.getLog(getClass()).debug( "Ignoring failure to delete file " + part + " which has already been uploaded"); } else { if (fileDeleteObserver != null) fileDeleteObserver.onFileDelete(null); } } } })); } /** * Notified from * {@link AmazonS3EncryptionClient#uploadObject(UploadObjectRequest)} when * all parts have been successfully uploaded to S3. This method is * responsible for finishing off the upload by making a complete multi-part * upload request to S3 with the given list of etags. * * @param partETags * all the etags returned from S3 for the previous part uploads. * * @return the completed multi-part upload result */ public CompleteMultipartUploadResult onCompletion(List<PartETag> partETags) { return s3.completeMultipartUpload( new CompleteMultipartUploadRequest(req.getBucketName(), req.getKey(), uploadId, partETags)); } /** * Notified from * {@link AmazonS3EncryptionClient#uploadObject(UploadObjectRequest)} when * failed to upload any part. This method is responsible for cancelling * ongoing uploads and aborting the multi-part upload request. */ public void onAbort() { for (Future<?> future : getFutures()) { future.cancel(true); } if (uploadId != null) { try { s3.abortMultipartUpload( new AbortMultipartUploadRequest(req.getBucketName(), req.getKey(), uploadId)); } catch (Exception e) { LogFactory.getLog(getClass()).debug("Failed to abort multi-part upload: " + uploadId, e); } } } /** * Creates and returns an upload-part request corresponding to a ciphertext * file upon a part-creation event. * * @param event * the part-creation event of the ciphertxt file. * @param part * the created ciphertext file corresponding to the upload-part */ protected UploadPartRequest newUploadPartRequest(PartCreationEvent event, final File part) { final UploadPartRequest reqUploadPart = new UploadPartRequest().withBucketName(req.getBucketName()) .withFile(part).withKey(req.getKey()).withPartNumber(event.getPartNumber()) .withPartSize(part.length()).withLastPart(event.isLastPart()).withUploadId(uploadId) .withObjectMetadata(req.getUploadPartMetadata()); return reqUploadPart; } /** * Uploads the ciphertext via the non-encrypting s3 client. * @param reqUploadPart part upload request * @return the result of the part upload when there is no exception */ protected UploadPartResult uploadPart(UploadPartRequest reqUploadPart) { // Upload the ciphertext directly via the non-encrypting // s3 client return s3direct.uploadPart(reqUploadPart); } /** * Appends the given user agent to the given request. * * @return the given request. */ protected <X extends AmazonWebServiceRequest> X appendUserAgent(X request, String userAgent) { request.getRequestClientOptions().appendUserAgent(userAgent); return request; } public List<Future<UploadPartResult>> getFutures() { return futures; } /** * Returns the request initialized via * {@link #init(UploadObjectRequest, S3DirectSpi, AmazonS3, ExecutorService)} */ protected UploadObjectRequest getRequest() { return req; } /** * Returns the upload id after the multi-part upload has been initiated via * {@link #onUploadInitiation(UploadObjectRequest)} */ protected String getUploadId() { return uploadId; } /** * Returns the <code>S3DirectSpi</code> instance initialized via * {@link #init(UploadObjectRequest, S3DirectSpi, AmazonS3, ExecutorService)} */ protected S3DirectSpi getS3DirectSpi() { return s3direct; } /** * Returns the <code>AmazonS3</code> instance initialized via * {@link #init(UploadObjectRequest, S3DirectSpi, AmazonS3, ExecutorService)} */ protected AmazonS3 getAmazonS3() { return s3; } /** * Returns the <code>ExecutorService</code> instance initialized via * {@link #init(UploadObjectRequest, S3DirectSpi, AmazonS3, ExecutorService)} */ protected ExecutorService getExecutorService() { return es; } }