Java tutorial
/* * Copyright 2010-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. * A copy of the License is located at * * http://aws.amazon.com/apache2.0 * * or in the "license" file accompanying this file. This file is distributed * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ package com.sina.cloudstorage.services.scs; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Map.Entry; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.Consts; import com.sina.cloudstorage.ClientConfiguration; import com.sina.cloudstorage.DefaultRequest; import com.sina.cloudstorage.HttpMethod; import com.sina.cloudstorage.Request; import com.sina.cloudstorage.Response; import com.sina.cloudstorage.SCSClientException; import com.sina.cloudstorage.SCSServiceException; import com.sina.cloudstorage.SCSWebServiceClient; import com.sina.cloudstorage.SCSWebServiceRequest; import com.sina.cloudstorage.SCSWebServiceResponse; import com.sina.cloudstorage.auth.AWSCredentials; import com.sina.cloudstorage.auth.AWSCredentialsProvider; import com.sina.cloudstorage.auth.ClasspathPropertiesFileCredentialsProvider; import com.sina.cloudstorage.auth.Signer; import com.sina.cloudstorage.event.ProgressEvent; import com.sina.cloudstorage.event.ProgressListener; import com.sina.cloudstorage.event.ProgressListenerCallbackExecutor; import com.sina.cloudstorage.event.ProgressReportingInputStream; import com.sina.cloudstorage.http.ExecutionContext; import com.sina.cloudstorage.http.HttpMethodName; import com.sina.cloudstorage.http.HttpResponseHandler; import com.sina.cloudstorage.internal.StaticCredentialsProvider; import com.sina.cloudstorage.services.scs.internal.BucketNameUtils; import com.sina.cloudstorage.services.scs.internal.Constants; import com.sina.cloudstorage.services.scs.internal.MD5DigestCalculatingInputStream; import com.sina.cloudstorage.services.scs.internal.Mimetypes; import com.sina.cloudstorage.services.scs.internal.RepeatableFileInputStream; import com.sina.cloudstorage.services.scs.internal.S3ErrorResponseHandler; import com.sina.cloudstorage.services.scs.internal.S3JsonResponseHandler; import com.sina.cloudstorage.services.scs.internal.S3MetadataResponseHandler; import com.sina.cloudstorage.services.scs.internal.S3ObjectResponseHandler; import com.sina.cloudstorage.services.scs.internal.S3QueryStringSigner; import com.sina.cloudstorage.services.scs.internal.S3Signer; import com.sina.cloudstorage.services.scs.internal.ServiceUtils; import com.sina.cloudstorage.services.scs.model.AccessControlList; import com.sina.cloudstorage.services.scs.model.Bucket; import com.sina.cloudstorage.services.scs.model.BucketInfo; import com.sina.cloudstorage.services.scs.model.CannedAccessControlList; import com.sina.cloudstorage.services.scs.model.CopyObjectRequest; import com.sina.cloudstorage.services.scs.model.CreateBucketRequest; import com.sina.cloudstorage.services.scs.model.DeleteBucketRequest; import com.sina.cloudstorage.services.scs.model.DeleteObjectRequest; import com.sina.cloudstorage.services.scs.model.DigestValidationInputStream; import com.sina.cloudstorage.services.scs.model.GeneratePresignedUrlRequest; import com.sina.cloudstorage.services.scs.model.GenericBucketRequest; import com.sina.cloudstorage.services.scs.model.GetBucketAclRequest; import com.sina.cloudstorage.services.scs.model.GetObjectMetadataRequest; import com.sina.cloudstorage.services.scs.model.GetObjectRequest; import com.sina.cloudstorage.services.scs.model.ListBucketsRequest; import com.sina.cloudstorage.services.scs.model.ListObjectsRequest; import com.sina.cloudstorage.services.scs.model.ObjectInfo; import com.sina.cloudstorage.services.scs.model.ObjectListing; import com.sina.cloudstorage.services.scs.model.ObjectMetadata; import com.sina.cloudstorage.services.scs.model.Permission; import com.sina.cloudstorage.services.scs.model.PutObjectRelaxRequest; import com.sina.cloudstorage.services.scs.model.PutObjectRequest; import com.sina.cloudstorage.services.scs.model.PutObjectResult; import com.sina.cloudstorage.services.scs.model.ResponseHeaderOverrides; import com.sina.cloudstorage.services.scs.model.S3Object; import com.sina.cloudstorage.services.scs.model.S3ObjectInputStream; import com.sina.cloudstorage.services.scs.model.SCSS3Exception; import com.sina.cloudstorage.services.scs.model.SetBucketAclRequest; import com.sina.cloudstorage.services.scs.model.transform.AclJsonFactory; import com.sina.cloudstorage.services.scs.model.transform.Unmarshallers; import com.sina.cloudstorage.transform.Unmarshaller; import com.sina.cloudstorage.util.BinaryUtils; import com.sina.cloudstorage.util.ContentLengthValidationInputStream; import com.sina.cloudstorage.util.DateUtils; import com.sina.cloudstorage.util.HttpUtils; import com.sina.cloudstorage.util.Md5Utils; import com.sina.cloudstorage.util.URIBuilder; /** * * */ public class SCSClient extends SCSWebServiceClient implements SCS { /** Shared logger for client events */ private static Log log = LogFactory.getLog(SCSClient.class); /** Responsible for handling error responses from all S3 service calls. */ private S3ErrorResponseHandler errorResponseHandler = new S3ErrorResponseHandler(); /** Shared response handler for operations with no response. */ private S3JsonResponseHandler<Void> voidResponseHandler = new S3JsonResponseHandler<Void>(null); /** S3 specific client configuration options */ private S3ClientOptions clientOptions = new S3ClientOptions(); /** Provider for AWS credentials. */ private AWSCredentialsProvider awsCredentialsProvider; /** * Constructs a new client to invoke service methods on Amazon S3. A * credentials provider chain will be used that searches for credentials in * this order: * <ul> * <li>Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY</li> * <li>Java System Properties - aws.accessKeyId and aws.secretKey</li> * <li>Instance Profile Credentials - delivered through the Amazon EC2 * metadata service</li> * </ul> * * <p> * If no credentials are found in the chain, this client will attempt to * work in an anonymous mode where requests aren't signed. Only a subset of * the Amazon S3 API will work with anonymous <i>(i.e. unsigned)</i> requests, * but this can prove useful in some situations. For example: * <ul> * <li>If an Amazon S3 bucket has {@link Permission#Read} permission for the * {@link GroupGrantee#AllUsers} group, anonymous clients can call * {@link #listObjects(String)} to see what objects are stored in a bucket.</li> * <li>If an object has {@link Permission#Read} permission for the * {@link GroupGrantee#AllUsers} group, anonymous clients can call * {@link #getObject(String, String)} and * {@link #getObjectMetadata(String, String)} to pull object content and * metadata.</li> * <li>If a bucket has {@link Permission#Write} permission for the * {@link GroupGrantee#AllUsers} group, anonymous clients can upload objects * to the bucket.</li> * </ul> * </p> * <p> * You can force the client to operate in an anonymous mode, and skip the credentials * provider chain, by passing in <code>null</code> for the credentials. * </p> * * @see SCSClient#SCSClient(AWSCredentials) * @see SCSClient#SCSClient(AWSCredentials, ClientConfiguration) */ public SCSClient() { this(new ClasspathPropertiesFileCredentialsProvider()); } /** * Constructs a new Amazon S3 client using the specified AWS credentials to * access Amazon S3. * * @param awsCredentials * The AWS credentials to use when making requests to Amazon S3 * with this client. * * @see SCSClient#SCSClient() * @see SCSClient#SCSClient(AWSCredentials, ClientConfiguration) */ public SCSClient(AWSCredentials awsCredentials) { this(awsCredentials, new ClientConfiguration()); } /** * Constructs a new Amazon S3 client using the specified AWS credentials and * client configuration to access Amazon S3. * * @param awsCredentials * The AWS credentials to use when making requests to Amazon S3 * with this client. * @param clientConfiguration * The client configuration options controlling how this client * connects to Amazon S3 (e.g. proxy settings, retry counts, etc). * * @see SCSClient#SCSClient() * @see SCSClient#SCSClient(AWSCredentials) */ public SCSClient(AWSCredentials awsCredentials, ClientConfiguration clientConfiguration) { super(clientConfiguration); this.awsCredentialsProvider = new StaticCredentialsProvider(awsCredentials); init(); } /** * Constructs a new Amazon S3 client using the specified AWS credentials * provider to access Amazon S3. * * @param credentialsProvider * The AWS credentials provider which will provide credentials * to authenticate requests with AWS services. */ public SCSClient(AWSCredentialsProvider credentialsProvider) { this(credentialsProvider, new ClientConfiguration()); } /** * Constructs a new Amazon S3 client using the specified AWS credentials and * client configuration to access Amazon S3. * * @param credentialsProvider * The AWS credentials provider which will provide credentials * to authenticate requests with AWS services. * @param clientConfiguration * The client configuration options controlling how this client * connects to Amazon S3 (e.g. proxy settings, retry counts, etc). */ public SCSClient(AWSCredentialsProvider credentialsProvider, ClientConfiguration clientConfiguration) { super(clientConfiguration); this.awsCredentialsProvider = credentialsProvider; init(); } protected final ExecutionContext createExecutionContext(SCSWebServiceRequest req) { return new ExecutionContext(); } /** * Constructs a new client using the specified client configuration to * access Amazon S3. A credentials provider chain will be used that searches * for credentials in this order: * <ul> * <li>Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY</li> * <li>Java System Properties - aws.accessKeyId and aws.secretKey</li> * <li>Instance Profile Credentials - delivered through the Amazon EC2 * metadata service</li> * </ul> * * <p> * If no credentials are found in the chain, this client will attempt to * work in an anonymous mode where requests aren't signed. Only a subset of * the Amazon S3 API will work with anonymous <i>(i.e. unsigned)</i> * requests, but this can prove useful in some situations. For example: * <ul> * <li>If an Amazon S3 bucket has {@link Permission#Read} permission for the * {@link GroupGrantee#AllUsers} group, anonymous clients can call * {@link #listObjects(String)} to see what objects are stored in a bucket.</li> * <li>If an object has {@link Permission#Read} permission for the * {@link GroupGrantee#AllUsers} group, anonymous clients can call * {@link #getObject(String, String)} and * {@link #getObjectMetadata(String, String)} to pull object content and * metadata.</li> * <li>If a bucket has {@link Permission#Write} permission for the * {@link GroupGrantee#AllUsers} group, anonymous clients can upload objects * to the bucket.</li> * </ul> * </p> * <p> * You can force the client to operate in an anonymous mode, and skip the * credentials provider chain, by passing in <code>null</code> for the * credentials. * </p> * * @param clientConfiguration * The client configuration options controlling how this client * connects to Amazon S3 (e.g. proxy settings, retry counts, etc). * * @see SCSClient#SCSClient(AWSCredentials) * @see SCSClient#SCSClient(AWSCredentials, ClientConfiguration) */ public SCSClient(ClientConfiguration clientConfiguration) { this(new ClasspathPropertiesFileCredentialsProvider(), clientConfiguration); } private void init() { // Because of S3's virtual host style addressing, we need to change the // default, strict hostname verification to be more lenient. // client.disableStrictHostnameVerification(); setEndpoint(Constants.S3_HOSTNAME); setEndpoint4Upload(Constants.S3_UPLOAD_HOSTNAME); // setEndpoint4Download(Constants.S3_DOWNLOAD_HOSTNAME); } /** * <p> * Override the default S3 client options for this client. * </p> * @param clientOptions * The S3 client options to use. */ public void setS3ClientOptions(S3ClientOptions clientOptions) { this.clientOptions = new S3ClientOptions(clientOptions); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#listObjects(java.lang.String) */ public ObjectListing listObjects(String bucketName) throws SCSClientException, SCSServiceException { return listObjects(new ListObjectsRequest(bucketName, null, null, null, null)); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#listObjects(java.lang.String, java.lang.String) */ public ObjectListing listObjects(String bucketName, String prefix) throws SCSClientException, SCSServiceException { return listObjects(new ListObjectsRequest(bucketName, prefix, null, null, null)); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#listObjects(com.amazonaws.services.s3.model.ListObjectsRequest) */ public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws SCSClientException, SCSServiceException { assertParameterNotNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket"); Request<ListObjectsRequest> request = createRequest(listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET); if (listObjectsRequest.getPrefix() != null) request.addParameter("prefix", listObjectsRequest.getPrefix()); if (listObjectsRequest.getMarker() != null) request.addParameter("marker", listObjectsRequest.getMarker()); if (listObjectsRequest.getDelimiter() != null) request.addParameter("delimiter", listObjectsRequest.getDelimiter()); if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString()); ObjectListing objectListing = invoke(request, new Unmarshallers.ListObjectsUnmarshaller(), listObjectsRequest.getBucketName(), null); objectListing.setBucketName(listObjectsRequest.getBucketName()); objectListing.setDelimiter(listObjectsRequest.getDelimiter()); if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) objectListing.setMaxKeys(listObjectsRequest.getMaxKeys()); return objectListing; } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#listNextBatchOfObjects(com.amazonaws.services.s3.model.S3ObjectListing) */ public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) throws SCSClientException, SCSServiceException { assertParameterNotNull(previousObjectListing, "The previous object listing parameter must be specified when listing the next batch of objects in a bucket"); if (!previousObjectListing.isTruncated()) { ObjectListing emptyListing = new ObjectListing(); emptyListing.setBucketName(previousObjectListing.getBucketName()); emptyListing.setDelimiter(previousObjectListing.getDelimiter()); emptyListing.setMarker(previousObjectListing.getNextMarker()); emptyListing.setMaxKeys(previousObjectListing.getMaxKeys()); emptyListing.setPrefix(previousObjectListing.getPrefix()); emptyListing.setTruncated(false); return emptyListing; } return listObjects(new ListObjectsRequest(previousObjectListing.getBucketName(), previousObjectListing.getPrefix(), previousObjectListing.getNextMarker(), previousObjectListing.getDelimiter(), Integer.valueOf(previousObjectListing.getMaxKeys()))); } // /* (non-Javadoc) // * @see com.amazonaws.services.s3.AmazonS3#getS3AccountOwner() // */ // public Owner getS3AccountOwner() // throws SCSClientException, SCSServiceException { // Request<ListBucketsRequest> request = createRequest(null, null, new ListBucketsRequest(), HttpMethodName.GET); // return invoke(request, new Unmarshallers.ListBucketsOwnerUnmarshaller(), null, null); // } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#listBuckets() */ public List<Bucket> listBuckets(ListBucketsRequest listBucketsRequest) throws SCSClientException, SCSServiceException { Request<ListBucketsRequest> request = createRequest(null, null, listBucketsRequest, HttpMethodName.GET); return invoke(request, new Unmarshallers.ListBucketsUnmarshaller(), null, null); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#listBuckets() */ public List<Bucket> listBuckets() throws SCSClientException, SCSServiceException { return listBuckets(new ListBucketsRequest()); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#createBucket(java.lang.String) */ public Bucket createBucket(String bucketName) throws SCSClientException, SCSServiceException { return createBucket(new CreateBucketRequest(bucketName)); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#createBucket(com.amazonaws.services.s3.model.CreateBucketRequest) */ public Bucket createBucket(CreateBucketRequest createBucketRequest) throws SCSClientException, SCSServiceException { assertParameterNotNull(createBucketRequest, "The CreateBucketRequest parameter must be specified when creating a bucket"); String bucketName = createBucketRequest.getBucketName(); assertParameterNotNull(bucketName, "The bucket name parameter must be specified when creating a bucket"); if (bucketName != null) bucketName = bucketName.trim(); BucketNameUtils.validateBucketName(bucketName); Request<CreateBucketRequest> request = createRequest(bucketName, null, createBucketRequest, HttpMethodName.PUT); // if ( createBucketRequest.getAccessControlList() != null ) { // addAclHeaders(request, createBucketRequest.getAccessControlList()); // } else if (createBucketRequest.getCannedAcl() != null) { request.addHeader(Headers.S3_CANNED_ACL, createBucketRequest.getCannedAcl().toString()); } invoke(request, voidResponseHandler, bucketName, null); return new Bucket(bucketName); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#getObjectAcl(java.lang.String, java.lang.String) */ public AccessControlList getObjectAcl(String bucketName, String key) throws SCSClientException, SCSServiceException { SCSWebServiceRequest originalRequest = new GenericBucketRequest(bucketName); Request<SCSWebServiceRequest> request = createRequest(bucketName, key, originalRequest, HttpMethodName.GET); request.addParameter("acl", null); return invoke(request, new Unmarshallers.AccessControlListUnmarshaller(), bucketName, key); } @Override public void setObjectAcl(String bucketName, String key, AccessControlList acl) throws SCSClientException, SCSServiceException { assertParameterNotNull(bucketName, "The bucket name parameter must be specified when setting an object's ACL"); assertParameterNotNull(key, "The key parameter must be specified when setting an object's ACL"); assertParameterNotNull(acl, "The ACL parameter must be specified when setting an object's ACL"); setAcl(bucketName, key, acl, new GenericBucketRequest(bucketName)); } // @Override // public void setObjectAcl(String bucketName, String key, CannedAccessControlList acl) // throws SCSClientException, SCSServiceException { // assertParameterNotNull(bucketName, "The bucket name parameter must be specified when setting an object's ACL"); // assertParameterNotNull(key, "The key parameter must be specified when setting an object's ACL"); // assertParameterNotNull(acl, "The ACL parameter must be specified when setting an object's ACL"); // // setAcl(bucketName, key, acl,new GenericBucketRequest(bucketName)); // } /* * (non-Javadoc) * @see com.sina.scs.SCS#getBucketMetadata(java.lang.String) */ public BucketInfo getBucketInfo(String bucketName) throws SCSClientException, SCSServiceException { assertParameterNotNull(bucketName, "The bucket name parameter must be specified when requesting an bucket's info"); GenericBucketRequest originalRequest = new GenericBucketRequest(bucketName); Request<GenericBucketRequest> request = createRequest(bucketName, null, originalRequest, HttpMethodName.GET); request.addParameter("meta", null); return invoke(request, new Unmarshallers.BucketInfoUnmarshaller(), bucketName, null); } /* * (non-Javadoc) * @see com.sina.scs.SCS#getObjectInfo(java.lang.String, java.lang.String) */ public ObjectInfo getObjectInfo(String bucketName, String key) throws SCSClientException, SCSServiceException { assertParameterNotNull(bucketName, "The bucket name parameter must be specified when requesting an Object's info"); assertParameterNotNull(key, "The bucket name parameter must be specified when requesting an Object's info"); GenericBucketRequest originalRequest = new GenericBucketRequest(bucketName); Request<GenericBucketRequest> request = createRequest(bucketName, key, originalRequest, HttpMethodName.GET); request.addParameter("meta", null); return invoke(request, new Unmarshallers.ObjectInfoUnmarshaller(), bucketName, key); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#getBucketAcl(java.lang.String) */ public AccessControlList getBucketAcl(String bucketName) throws SCSClientException, SCSServiceException { assertParameterNotNull(bucketName, "The bucket name parameter must be specified when requesting a bucket's ACL"); return getAcl(bucketName, null, null, null); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#getBucketAcl(com.amazonaws.services.s3.GetBucketAclRequest) */ public AccessControlList getBucketAcl(GetBucketAclRequest getBucketAclRequest) throws SCSClientException, SCSServiceException { String bucketName = getBucketAclRequest.getBucketName(); assertParameterNotNull(bucketName, "The bucket name parameter must be specified when requesting a bucket's ACL"); return getAcl(bucketName, null, null, getBucketAclRequest); } @Override public void setBucketAcl(String bucketName, AccessControlList acl) throws SCSClientException, SCSServiceException { assertParameterNotNull(bucketName, "The bucket name parameter must be specified when setting a bucket's ACL"); assertParameterNotNull(acl, "The ACL parameter must be specified when setting a bucket's ACL"); setAcl(bucketName, null, acl, new GenericBucketRequest(bucketName)); } @Override public void setBucketAcl(SetBucketAclRequest setBucketAclRequest) throws SCSClientException, SCSServiceException { String bucketName = setBucketAclRequest.getBucketName(); AccessControlList acl = setBucketAclRequest.getAcl(); // CannedAccessControlList cannedAcl = setBucketAclRequest.getCannedAcl(); assertParameterNotNull(bucketName, "The bucket name parameter must be specified when setting a bucket's ACL"); if (acl != null) { setAcl(bucketName, null, acl, setBucketAclRequest); // } else if (cannedAcl != null) { // setAcl(bucketName, null, cannedAcl, setBucketAclRequest); } else { assertParameterNotNull(null, "The ACL parameter must be specified when setting a bucket's ACL"); } } // @Override // public void setBucketAcl(String bucketName, CannedAccessControlList acl) // throws SCSClientException, SCSServiceException { // assertParameterNotNull(bucketName, "The bucket name parameter must be specified when setting a bucket's ACL"); // assertParameterNotNull(acl, "The ACL parameter must be specified when setting a bucket's ACL"); // // setAcl(bucketName, null, acl,new GenericBucketRequest(bucketName)); // } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#getObjectMetadata(java.lang.String, java.lang.String) */ public ObjectMetadata getObjectMetadata(String bucketName, String key) throws SCSClientException, SCSServiceException { return getObjectMetadata(new GetObjectMetadataRequest(bucketName, key)); } /* * (non-Javadoc) * @see com.sina.scs.AmazonS3#setObjectMetadata(com.sina.scs.model.ObjectMetadata) */ public void setObjectMetadata(String bucketName, String key, ObjectMetadata objectMetadata) throws SCSClientException, SCSServiceException { SCSWebServiceRequest originalRequest = new GenericBucketRequest(bucketName); Request<SCSWebServiceRequest> request = createRequest(bucketName, key, originalRequest, HttpMethodName.PUT); request.addParameter("meta", null); for (Entry<String, String> entry : objectMetadata.getUserMetadata().entrySet()) { String metaKey = entry.getKey(); String metaValue = entry.getValue(); if (!metaKey.startsWith("x-amz-meta-")) metaKey = "x-amz-meta-" + metaKey; request.addHeader(metaKey, metaValue); } invoke(request, voidResponseHandler, bucketName, key); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#getObjectMetadata(com.amazonaws.services.s3.model.GetObjectMetadataRequest) */ public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetadataRequest) throws SCSClientException, SCSServiceException { assertParameterNotNull(getObjectMetadataRequest, "The GetObjectMetadataRequest parameter must be specified when requesting an object's metadata"); String bucketName = getObjectMetadataRequest.getBucketName(); String key = getObjectMetadataRequest.getKey(); String versionId = getObjectMetadataRequest.getVersionId(); assertParameterNotNull(bucketName, "The bucket name parameter must be specified when requesting an object's metadata"); assertParameterNotNull(key, "The key parameter must be specified when requesting an object's metadata"); Request<GetObjectMetadataRequest> request = createRequest(bucketName, key, getObjectMetadataRequest, HttpMethodName.HEAD); if (versionId != null) request.addParameter("versionId", versionId); return invoke(request, new S3MetadataResponseHandler(), bucketName, key); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#getObject(java.lang.String, java.lang.String) */ public S3Object getObject(String bucketName, String key) throws SCSClientException, SCSServiceException { return getObject(new GetObjectRequest(bucketName, key)); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#doesBucketExist(java.lang.String) */ public boolean doesBucketExist(String bucketName) throws SCSClientException, SCSServiceException { try { listObjects(new ListObjectsRequest(bucketName, null, null, null, 1)); // it exists and the current account owns it return true; } catch (SCSServiceException ase) { /* * If we have no credentials, or we detect a problem with the * credentials we used, go ahead and throw the error so we don't * mask that problem as thinking that the bucket does exist. */ if (awsCredentialsProvider.getCredentials() == null) throw ase; if ("InvalidAccessKeyId".equalsIgnoreCase(ase.getErrorCode()) || "SignatureDoesNotMatch".equalsIgnoreCase(ase.getErrorCode())) { throw ase; } switch (ase.getStatusCode()) { case 301: // A redirect error means the bucket exists, but in another region. return true; case 403: // A permissions error means the bucket exists, but is owned by another account. return true; case 404: return false; default: throw ase; } } } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#getObject(com.amazonaws.services.s3.model.GetObjectRequest) */ public S3Object getObject(GetObjectRequest getObjectRequest) throws SCSClientException, SCSServiceException { assertParameterNotNull(getObjectRequest, "The GetObjectRequest parameter must be specified when requesting an object"); assertParameterNotNull(getObjectRequest.getBucketName(), "The bucket name parameter must be specified when requesting an object"); assertParameterNotNull(getObjectRequest.getKey(), "The key parameter must be specified when requesting an object"); Request<GetObjectRequest> request = createRequest(getObjectRequest.getBucketName(), getObjectRequest.getKey(), getObjectRequest, HttpMethodName.GET); if (getObjectRequest.getVersionId() != null) { request.addParameter("versionId", getObjectRequest.getVersionId()); } // Range if (getObjectRequest.getRange() != null) { long[] range = getObjectRequest.getRange(); request.addHeader(Headers.RANGE, "bytes=" + Long.toString(range[0]) + "-" + Long.toString(range[1])); } addResponseHeaderParameters(request, getObjectRequest.getResponseHeaders()); addDateHeader(request, Headers.GET_OBJECT_IF_MODIFIED_SINCE, getObjectRequest.getModifiedSinceConstraint()); addDateHeader(request, Headers.GET_OBJECT_IF_UNMODIFIED_SINCE, getObjectRequest.getUnmodifiedSinceConstraint()); addStringListHeader(request, Headers.GET_OBJECT_IF_MATCH, getObjectRequest.getMatchingETagConstraints()); addStringListHeader(request, Headers.GET_OBJECT_IF_NONE_MATCH, getObjectRequest.getNonmatchingETagConstraints()); /* * This is compatible with progress listener set by either the legacy * method GetObjectRequest#setProgressListener or the new method * GetObjectRequest#setGeneralProgressListener. */ ProgressListener progressListener = getObjectRequest.getGeneralProgressListener(); ProgressListenerCallbackExecutor progressListenerCallbackExecutor = ProgressListenerCallbackExecutor .wrapListener(progressListener); try { S3Object s3Object = invoke(request, new S3ObjectResponseHandler(), getObjectRequest.getBucketName(), getObjectRequest.getKey()); /* * TODO: For now, it's easiest to set there here in the client, but * we could push this back into the response handler with a * little more work. */ s3Object.setBucketName(getObjectRequest.getBucketName()); s3Object.setKey(getObjectRequest.getKey()); S3ObjectInputStream input = s3Object.getObjectContent(); if (progressListenerCallbackExecutor != null) { ProgressReportingInputStream progressReportingInputStream = new ProgressReportingInputStream(input, progressListenerCallbackExecutor); progressReportingInputStream.setFireCompletedEvent(true); input = new S3ObjectInputStream(progressReportingInputStream);//, input.getHttpRequest() fireProgressEvent(progressListenerCallbackExecutor, ProgressEvent.STARTED_EVENT_CODE); } if (getObjectRequest.getRange() == null && System.getProperty("com.amazonaws.services.s3.disableGetObjectMD5Validation") == null) { byte[] serverSideHash = null; String etag = s3Object.getObjectMetadata().getETag(); if (etag != null && ServiceUtils.isMultipartUploadETag(etag) == false) { serverSideHash = BinaryUtils.fromHex(s3Object.getObjectMetadata().getETag()); DigestValidationInputStream inputStreamWithMD5DigestValidation; try { MessageDigest digest = MessageDigest.getInstance("MD5"); inputStreamWithMD5DigestValidation = new DigestValidationInputStream(input, digest, serverSideHash); input = new S3ObjectInputStream(inputStreamWithMD5DigestValidation);//, input.getHttpRequest() } catch (NoSuchAlgorithmException e) { log.warn("No MD5 digest algorithm available. Unable to calculate " + "checksum and verify data integrity.", e); } } } else { input = new S3ObjectInputStream(new ContentLengthValidationInputStream(input, s3Object.getObjectMetadata().getContentLength())); // input.getHttpRequest()); } s3Object.setObjectContent(input); return s3Object; } catch (SCSS3Exception ase) { /* * If the request failed because one of the specified constraints * was not met (ex: matching ETag, modified since date, etc.), then * return null, so that users don't have to wrap their code in * try/catch blocks and check for this status code if they want to * use constraints. */ if (ase.getStatusCode() == 412 || ase.getStatusCode() == 304) { fireProgressEvent(progressListenerCallbackExecutor, ProgressEvent.CANCELED_EVENT_CODE); return null; } fireProgressEvent(progressListenerCallbackExecutor, ProgressEvent.FAILED_EVENT_CODE); throw ase; } } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#getObject(com.amazonaws.services.s3.model.GetObjectRequest, java.io.File) */ public ObjectMetadata getObject(final GetObjectRequest getObjectRequest, File destinationFile) throws SCSClientException, SCSServiceException { assertParameterNotNull(destinationFile, "The destination file parameter must be specified when downloading an object directly to a file"); S3Object s3Object = ServiceUtils.retryableDownloadS3ObjectToFile(destinationFile, new ServiceUtils.RetryableS3DownloadTask() { @Override public S3Object getS3ObjectStream() { return getObject(getObjectRequest); } @Override public boolean needIntegrityCheck() { return getObjectRequest.getRange() == null; } }); // getObject can return null if constraints were specified but not met if (s3Object == null) return null; return s3Object.getObjectMetadata(); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#deleteBucket(java.lang.String) */ public void deleteBucket(String bucketName) throws SCSClientException, SCSServiceException { deleteBucket(new DeleteBucketRequest(bucketName)); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#deleteBucket(com.amazonaws.services.s3.model.DeleteBucketRequest) */ public void deleteBucket(DeleteBucketRequest deleteBucketRequest) throws SCSClientException, SCSServiceException { assertParameterNotNull(deleteBucketRequest, "The DeleteBucketRequest parameter must be specified when deleting a bucket"); String bucketName = deleteBucketRequest.getBucketName(); assertParameterNotNull(bucketName, "The bucket name parameter must be specified when deleting a bucket"); Request<DeleteBucketRequest> request = createRequest(bucketName, null, deleteBucketRequest, HttpMethodName.DELETE); invoke(request, voidResponseHandler, bucketName, null); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#putObject(java.lang.String, java.lang.String, java.io.File) */ public PutObjectResult putObject(String bucketName, String key, File file) throws SCSClientException, SCSServiceException { return putObject(new PutObjectRequest(bucketName, key, file).withMetadata(new ObjectMetadata())); } /* * (non-Javadoc) * @see com.sina.scs.SCS#putObject(java.lang.String, java.lang.String, java.io.File, java.util.Map) */ public PutObjectResult putObject(String bucketName, String key, File file, Map<String, String> requestHeader) throws SCSClientException, SCSServiceException { ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setUserHeader(requestHeader); return putObject(new PutObjectRequest(bucketName, key, file).withMetadata(objectMetadata)); } /* * (non-Javadoc) * @see com.sina.scs.AmazonS3#putObjectRelax(java.lang.String, java.lang.String, java.lang.String, long) */ public PutObjectResult putObjectRelax(String bucketName, String key, String fileSha1, long fileLength) { return putObjectRelax(new PutObjectRelaxRequest(bucketName, key, fileSha1, fileLength)); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#putObject(java.lang.String, java.lang.String, java.io.InputStream, com.amazonaws.services.s3.model.S3ObjectMetadata) */ public PutObjectResult putObject(String bucketName, String key, InputStream input, ObjectMetadata metadata) throws SCSClientException, SCSServiceException { return putObject(new PutObjectRequest(bucketName, key, input, metadata)); } private PutObjectResult putObjectRelax(PutObjectRelaxRequest putObjectRelaxRequest) { assertParameterNotNull(putObjectRelaxRequest, "The putObjectRelaxRequest parameter must be specified when uploading with relax"); String bucketName = putObjectRelaxRequest.getBucketName(); String key = putObjectRelaxRequest.getKey(); String fileSha1 = putObjectRelaxRequest.getFileSha1(); long fileLength = putObjectRelaxRequest.getFileLength(); ObjectMetadata metadata = putObjectRelaxRequest.getMetadata(); if (metadata == null) metadata = new ObjectMetadata(); assertParameterNotNull(bucketName, "The bucket name parameter must be specified when uploading an object"); assertParameterNotNull(key, "The key parameter must be specified when uploading an object"); // If a file is specified for upload, we need to pull some additional // information from it to auto-configure a few options if (putObjectRelaxRequest.getFileSha1() != null) { metadata.setContentLength(0); // Only set the content type if it hasn't already been set if (metadata.getContentType() == null) { metadata.setContentType(Mimetypes.getInstance().getMimetype(key)); } metadata.setHeader("s-sina-sha1", fileSha1); metadata.setHeader("s-sina-length", fileLength); } Request<PutObjectRelaxRequest> request = createRequest(bucketName, key, putObjectRelaxRequest, HttpMethodName.PUT); request.addParameter("relax", null); if (putObjectRelaxRequest.getCannedAcl() != null) { request.addHeader(Headers.S3_CANNED_ACL, putObjectRelaxRequest.getCannedAcl().toString()); } if (metadata.getContentType() == null) { /* * Default to the "application/octet-stream" if the user hasn't * specified a content type. */ metadata.setContentType(Mimetypes.MIMETYPE_OCTET_STREAM); } populateRequestMetadata(request, metadata); ObjectMetadata returnedMetadata = null; try { returnedMetadata = invoke(request, new S3MetadataResponseHandler(), bucketName, key); } catch (SCSClientException ace) { throw ace; } PutObjectResult result = new PutObjectResult(); result.setETag(returnedMetadata.getETag()); result.setVersionId(returnedMetadata.getVersionId()); result.setServerSideEncryption(returnedMetadata.getServerSideEncryption()); result.setExpirationTime(returnedMetadata.getExpirationTime()); result.setExpirationTimeRuleId(returnedMetadata.getExpirationTimeRuleId()); return result; } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#putObject(com.amazonaws.services.s3.model.PutObjectRequest) */ public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws SCSClientException, SCSServiceException { assertParameterNotNull(putObjectRequest, "The PutObjectRequest parameter must be specified when uploading an object"); String bucketName = putObjectRequest.getBucketName(); String key = putObjectRequest.getKey(); ObjectMetadata metadata = putObjectRequest.getMetadata(); InputStream input = putObjectRequest.getInputStream(); /* * This is compatible with progress listener set by either the legacy * method PutObjectRequest#setProgressListener or the new method * PutObjectRequest#setGeneralProgressListener. */ ProgressListener progressListener = putObjectRequest.getGeneralProgressListener(); ProgressListenerCallbackExecutor progressListenerCallbackExecutor = ProgressListenerCallbackExecutor .wrapListener(progressListener); if (metadata == null) metadata = new ObjectMetadata(); assertParameterNotNull(bucketName, "The bucket name parameter must be specified when uploading an object"); assertParameterNotNull(key, "The key parameter must be specified when uploading an object"); // If a file is specified for upload, we need to pull some additional // information from it to auto-configure a few options if (putObjectRequest.getFile() != null) { File file = putObjectRequest.getFile(); // Always set the content length, even if it's already set metadata.setContentLength(file.length()); // Only set the content type if it hasn't already been set if (metadata.getContentType() == null) { metadata.setContentType(Mimetypes.getInstance().getMimetype(file)); } FileInputStream fileInputStream = null; try { fileInputStream = new FileInputStream(file); byte[] md5Hash = Md5Utils.computeMD5Hash(fileInputStream); metadata.setContentMD5(BinaryUtils.toBase64(md5Hash)); } catch (Exception e) { throw new SCSClientException("Unable to calculate MD5 hash: " + e.getMessage(), e); } finally { try { fileInputStream.close(); } catch (Exception e) { } } try { input = new RepeatableFileInputStream(file); } catch (FileNotFoundException fnfe) { throw new SCSClientException("Unable to find file to upload", fnfe); } } Request<PutObjectRequest> request = createRequest(bucketName, key, putObjectRequest, HttpMethodName.PUT); if (putObjectRequest.getAccessControlList() != null) { addAclHeaders(request, putObjectRequest.getAccessControlList()); } else if (putObjectRequest.getCannedAcl() != null) { request.addHeader(Headers.S3_CANNED_ACL, putObjectRequest.getCannedAcl().toString()); } //?? // if (putObjectRequest.getStorageClass() != null) { // request.addHeader(Headers.STORAGE_CLASS, putObjectRequest.getStorageClass()); // } // // if (putObjectRequest.getRedirectLocation() != null) { // request.addHeader(Headers.REDIRECT_LOCATION, putObjectRequest.getRedirectLocation()); // if (input == null) { // input = new ByteArrayInputStream(new byte[0]); // } // } // Use internal interface to differentiate 0 from unset. if (metadata.getRawMetadata().get(Headers.CONTENT_LENGTH) == null) { /* * There's nothing we can do except for let the HTTP client buffer * the input stream contents if the caller doesn't tell us how much * data to expect in a stream since we have to explicitly tell * Amazon S3 how much we're sending before we start sending any of * it. */ log.warn("No content length specified for stream data. " + "Stream contents will be buffered in memory and could result in " + "out of memory errors."); } if (progressListenerCallbackExecutor != null) { input = new ProgressReportingInputStream(input, progressListenerCallbackExecutor); fireProgressEvent(progressListenerCallbackExecutor, ProgressEvent.STARTED_EVENT_CODE); } // if (!input.markSupported()) { // int streamBufferSize = Constants.DEFAULT_STREAM_BUFFER_SIZE; // String bufferSizeOverride = System.getProperty("com.amazonaws.sdk.s3.defaultStreamBufferSize"); // if (bufferSizeOverride != null) { // try { // streamBufferSize = Integer.parseInt(bufferSizeOverride); // } catch (Exception e) { // log.warn("Unable to parse buffer size override from value: " + bufferSizeOverride); // } // } // // input = new RepeatableInputStream(input, streamBufferSize); // } MD5DigestCalculatingInputStream md5DigestStream = null; if (metadata.getContentMD5() == null) { /* * If the user hasn't set the content MD5, then we don't want to * buffer the whole stream in memory just to calculate it. Instead, * we can calculate it on the fly and validate it with the returned * ETag from the object upload. */ try { md5DigestStream = new MD5DigestCalculatingInputStream(input); input = md5DigestStream; } catch (NoSuchAlgorithmException e) { log.warn("No MD5 digest algorithm available. Unable to calculate " + "checksum and verify data integrity.", e); } } if (metadata.getContentType() == null) { /* * Default to the "application/octet-stream" if the user hasn't * specified a content type. */ metadata.setContentType(Mimetypes.MIMETYPE_OCTET_STREAM); } populateRequestMetadata(request, metadata); request.setContent(input); ObjectMetadata returnedMetadata = null; try { returnedMetadata = invoke(request, new S3MetadataResponseHandler(), bucketName, key); } catch (SCSClientException ace) { fireProgressEvent(progressListenerCallbackExecutor, ProgressEvent.FAILED_EVENT_CODE); throw ace; } finally { try { input.close(); } catch (Exception e) { log.warn("Unable to cleanly close input stream: " + e.getMessage(), e); } } String contentMd5 = metadata.getContentMD5(); if (md5DigestStream != null) { contentMd5 = BinaryUtils.toBase64(md5DigestStream.getMd5Digest()); } //?ETag? if (returnedMetadata != null && contentMd5 != null && returnedMetadata.getETag() != null) { byte[] clientSideHash = BinaryUtils.fromBase64(contentMd5); byte[] serverSideHash = BinaryUtils.fromHex(returnedMetadata.getETag()); if (!Arrays.equals(clientSideHash, serverSideHash)) { fireProgressEvent(progressListenerCallbackExecutor, ProgressEvent.FAILED_EVENT_CODE); throw new SCSClientException("Unable to verify integrity of data upload. " + "Client calculated content hash didn't match hash calculated by Amazon S3. " + "You may need to delete the data stored in Amazon S3."); } } fireProgressEvent(progressListenerCallbackExecutor, ProgressEvent.COMPLETED_EVENT_CODE); PutObjectResult result = new PutObjectResult(); result.setETag(returnedMetadata.getETag()); result.setVersionId(returnedMetadata.getVersionId()); result.setServerSideEncryption(returnedMetadata.getServerSideEncryption()); result.setExpirationTime(returnedMetadata.getExpirationTime()); result.setExpirationTimeRuleId(returnedMetadata.getExpirationTimeRuleId()); result.setContentMd5(contentMd5); result.setServiceSideKey(returnedMetadata.getServersideKey()); return result; } /** * TODO:??????? * Sets the acccess control headers for the request given. */ private static void addAclHeaders(Request<? extends SCSWebServiceRequest> request, AccessControlList acl) { // Set<Grant> grants = acl.getGrants(); // Map<Permission, Collection<Grantee>> grantsByPermission = new HashMap<Permission, Collection<Grantee>>(); // for ( Grant grant : grants ) { // if ( !grantsByPermission.containsKey(grant.getPermission()) ) { // grantsByPermission.put(grant.getPermission(), new LinkedList<Grantee>()); // } // grantsByPermission.get(grant.getPermission()).add(grant.getGrantee()); // } // for ( Permission permission : Permission.values() ) { // if ( grantsByPermission.containsKey(permission) ) { // Collection<Grantee> grantees = grantsByPermission.get(permission); // boolean seenOne = false; // StringBuilder granteeString = new StringBuilder(); // for ( Grantee grantee : grantees ) { // if ( !seenOne ) // seenOne = true; // else // granteeString.append(", "); // granteeString.append(grantee.getTypeIdentifier()).append("=").append("\"") // .append(grantee.getIdentifier()).append("\""); // } // request.addHeader(permission.getHeaderName(), granteeString.toString()); // } // } } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#copyObject(java.lang.String, java.lang.String, java.lang.String, java.lang.String) */ public void copyObject(String sourceBucketName, String sourceKey, String destinationBucketName, String destinationKey) throws SCSClientException, SCSServiceException { copyObject(new CopyObjectRequest(sourceBucketName, sourceKey, destinationBucketName, destinationKey)); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#copyObject(com.amazonaws.services.s3.model.CopyObjectRequest) */ public void copyObject(CopyObjectRequest copyObjectRequest) throws SCSClientException, SCSServiceException { assertParameterNotNull(copyObjectRequest.getSourceBucketName(), "The source bucket name must be specified when copying an object"); assertParameterNotNull(copyObjectRequest.getSourceKey(), "The source object key must be specified when copying an object"); assertParameterNotNull(copyObjectRequest.getDestinationBucketName(), "The destination bucket name must be specified when copying an object"); assertParameterNotNull(copyObjectRequest.getDestinationKey(), "The destination object key must be specified when copying an object"); String destinationKey = copyObjectRequest.getDestinationKey(); String destinationBucketName = copyObjectRequest.getDestinationBucketName(); Request<CopyObjectRequest> request = createRequest(destinationBucketName, destinationKey, copyObjectRequest, HttpMethodName.PUT); populateRequestWithCopyObjectParameters(request, copyObjectRequest); /* * We can't send the Content-Length header if the user specified it, * otherwise it messes up the HTTP connection when the remote server * thinks there's more data to pull. */ request.getHeaders().remove(Headers.CONTENT_LENGTH); invoke(request, voidResponseHandler, destinationBucketName, destinationKey); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#deleteObject(java.lang.String, java.lang.String) */ public void deleteObject(String bucketName, String key) throws SCSClientException, SCSServiceException { deleteObject(new DeleteObjectRequest(bucketName, key)); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#deleteObject(com.amazonaws.services.s3.DeleteObjectRequest) */ public void deleteObject(DeleteObjectRequest deleteObjectRequest) throws SCSClientException, SCSServiceException { assertParameterNotNull(deleteObjectRequest, "The delete object request must be specified when deleting an object"); assertParameterNotNull(deleteObjectRequest.getBucketName(), "The bucket name must be specified when deleting an object"); assertParameterNotNull(deleteObjectRequest.getKey(), "The key must be specified when deleting an object"); Request<DeleteObjectRequest> request = createRequest(deleteObjectRequest.getBucketName(), deleteObjectRequest.getKey(), deleteObjectRequest, HttpMethodName.DELETE); invoke(request, voidResponseHandler, deleteObjectRequest.getBucketName(), deleteObjectRequest.getKey()); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#generatePresignedUrl(java.lang.String, java.lang.String, java.util.Date) */ public URL generatePresignedUrl(String bucketName, String key, Date expiration, boolean bucketNameAsDomain) throws SCSClientException { return generatePresignedUrl(bucketName, key, expiration, HttpMethod.GET, bucketNameAsDomain); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#generatePresignedUrl(java.lang.String, java.lang.String, java.util.Date, com.amazonaws.HttpMethod) */ public URL generatePresignedUrl(String bucketName, String key, Date expiration, HttpMethod method, boolean bucketNameAsDomain) throws SCSClientException { GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, key, method); request.setExpiration(expiration); request.setBucketNameAsDomain(bucketNameAsDomain); return generatePresignedUrl(request); } /* (non-Javadoc) * @see com.amazonaws.services.s3.AmazonS3#generatePresignedUrl(com.amazonaws.services.s3.model.GeneratePresignedUrlRequest) */ public URL generatePresignedUrl(GeneratePresignedUrlRequest generatePresignedUrlRequest) throws SCSClientException { assertParameterNotNull(generatePresignedUrlRequest, "The request parameter must be specified when generating a pre-signed URL"); String bucketName = generatePresignedUrlRequest.getBucketName(); String key = generatePresignedUrlRequest.getKey(); // assertParameterNotNull(bucketName, // "The bucket name parameter must be specified when generating a pre-signed URL"); assertParameterNotNull(generatePresignedUrlRequest.getMethod(), "The HTTP method request parameter must be specified when generating a pre-signed URL"); if (generatePresignedUrlRequest.getExpiration() == null) { generatePresignedUrlRequest.setExpiration(new Date(System.currentTimeMillis() + 1000 * 60 * 15)); } HttpMethodName httpMethod = HttpMethodName.valueOf(generatePresignedUrlRequest.getMethod().toString()); // If the key starts with a slash character itself, the following method // will actually add another slash before the resource path to prevent // the HttpClient mistakenly treating the slash as a path delimiter. // For presigned request, we need to remember to remove this extra slash // before generating the URL. Request<GeneratePresignedUrlRequest> request = createRequest(bucketName, key, generatePresignedUrlRequest, httpMethod); for (Entry<String, String> entry : generatePresignedUrlRequest.getRequestParameters().entrySet()) { request.addParameter(entry.getKey(), entry.getValue()); } if (generatePresignedUrlRequest.getContentType() != null) { request.addHeader(Headers.CONTENT_TYPE, generatePresignedUrlRequest.getContentType()); } addResponseHeaderParameters(request, generatePresignedUrlRequest.getResponseHeaders()); presignRequest(request, generatePresignedUrlRequest.getMethod(), bucketName, key, generatePresignedUrlRequest.getExpiration(), null); // Remove the leading slash (if any) in the resource-path return ServiceUtils.convertRequestToUrl(request, true); } // /* (non-Javadoc) // * @see com.amazonaws.services.s3.AmazonS3#completeMultipartUpload(com.amazonaws.services.s3.model.CompleteMultipartUploadRequest) // */ // public ObjectMetadata completeMultipartUpload( // CompleteMultipartUploadRequest completeMultipartUploadRequest) // throws SCSClientException, SCSServiceException { // assertParameterNotNull(completeMultipartUploadRequest, // "The request parameter must be specified when completing a multipart upload"); // // String bucketName = completeMultipartUploadRequest.getBucketName(); // String key = completeMultipartUploadRequest.getKey(); // String uploadId = completeMultipartUploadRequest.getUploadId(); // assertParameterNotNull(bucketName, // "The bucket name parameter must be specified when completing a multipart upload"); // assertParameterNotNull(key, // "The key parameter must be specified when completing a multipart upload"); // assertParameterNotNull(uploadId, // "The upload ID parameter must be specified when completing a multipart upload"); // assertParameterNotNull(completeMultipartUploadRequest.getPartETags(), // "The part ETags parameter must be specified when completing a multipart upload"); // // Request<CompleteMultipartUploadRequest> request = createRequest(bucketName, key, completeMultipartUploadRequest, HttpMethodName.POST); // request.addParameter("uploadId", uploadId); // // byte[] json = RequestJsonFactory.convertToJsonByteArray(completeMultipartUploadRequest.getPartETags()); // request.addHeader("Content-Type", "text/plain"); // request.addHeader("Content-Length", String.valueOf(json.length)); // // request.setContent(new ByteArrayInputStream(json)); // // // ObjectMetadata returnedMetadata = invoke(request, new S3MetadataResponseHandler(), bucketName, key); // return returnedMetadata; // // } // /* // * (non-Javadoc) // * @see com.sina.scs.AmazonS3#initiateMultipartUpload(java.lang.String, java.lang.String) // */ // public InitiateMultipartUploadResult initiateMultipartUpload(String bucketName, String key) // throws SCSClientException, SCSServiceException{ // return this.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucketName, key)); // } // /* (non-Javadoc) // * @see com.amazonaws.services.s3.AmazonS3#initiateMultipartUpload(com.amazonaws.services.s3.model.InitiateMultipartUploadRequest) // */ // public InitiateMultipartUploadResult initiateMultipartUpload( // InitiateMultipartUploadRequest initiateMultipartUploadRequest) // throws SCSClientException, SCSServiceException { // assertParameterNotNull(initiateMultipartUploadRequest, // "The request parameter must be specified when initiating a multipart upload"); // // assertParameterNotNull(initiateMultipartUploadRequest.getBucketName(), // "The bucket name parameter must be specified when initiating a multipart upload"); // assertParameterNotNull(initiateMultipartUploadRequest.getKey(), // "The key parameter must be specified when initiating a multipart upload"); // // Request<InitiateMultipartUploadRequest> request = createRequest(initiateMultipartUploadRequest.getBucketName(), initiateMultipartUploadRequest.getKey(), initiateMultipartUploadRequest, HttpMethodName.POST); // request.addParameter("multipart", null); // // if ( initiateMultipartUploadRequest.getCannedACL() != null ) { // request.addHeader(Headers.S3_CANNED_ACL, initiateMultipartUploadRequest.getCannedACL().toString()); // } // // if (initiateMultipartUploadRequest.objectMetadata != null) populateRequestMetadata(request, initiateMultipartUploadRequest.objectMetadata); // // // Be careful that we don't send the object's total size as the content // // length for the InitiateMultipartUpload request. // request.getHeaders().remove(Headers.CONTENT_LENGTH); // request.addHeader(Headers.CONTENT_LENGTH,"0"); // // Set the request content to be empty (but not null) to force the runtime to pass // // any query params in the query string and not the request body, to keep S3 happy. // request.setContent(new ByteArrayInputStream(new byte[0])); // // // request.addHeader("Content-Type",Mimetypes.MIMETYPE_OCTET_STREAM); // // // @SuppressWarnings("unchecked") // ResponseHeaderHandlerChain<InitiateMultipartUploadResult> responseHandler = new ResponseHeaderHandlerChain<InitiateMultipartUploadResult>( // new Unmarshallers.InitiateMultipartUploadResultUnmarshaller(), // new ServerSideEncryptionHeaderHandler<InitiateMultipartUploadResult>()); // return invoke(request, responseHandler, // initiateMultipartUploadRequest.getBucketName(), initiateMultipartUploadRequest.getKey()); // } // /* (non-Javadoc) // * @see com.amazonaws.services.s3.AmazonS3#listParts(com.amazonaws.services.s3.model.ListPartsRequest) // */ // public PartListing listParts(ListPartsRequest listPartsRequest) // throws SCSClientException, SCSServiceException { // assertParameterNotNull(listPartsRequest, // "The request parameter must be specified when listing parts"); // // assertParameterNotNull(listPartsRequest.getBucketName(), // "The bucket name parameter must be specified when listing parts"); // assertParameterNotNull(listPartsRequest.getKey(), // "The key parameter must be specified when listing parts"); // assertParameterNotNull(listPartsRequest.getUploadId(), // "The upload ID parameter must be specified when listing parts"); // // Request<ListPartsRequest> request = createRequest(listPartsRequest.getBucketName(), listPartsRequest.getKey(), listPartsRequest, HttpMethodName.GET); // request.addParameter("uploadId", listPartsRequest.getUploadId()); // // if (listPartsRequest.getMaxParts() != null) request.addParameter("max-parts", listPartsRequest.getMaxParts().toString()); // if (listPartsRequest.getPartNumberMarker() != null) request.addParameter("part-number-marker", listPartsRequest.getPartNumberMarker().toString()); // // return invoke(request, new Unmarshallers.ListPartsResultUnmarshaller(), listPartsRequest.getBucketName(), listPartsRequest.getKey()); // } // /* (non-Javadoc) // * @see com.amazonaws.services.s3.AmazonS3#uploadPart(com.amazonaws.services.s3.model.UploadPartRequest) // */ // public UploadPartResult uploadPart(UploadPartRequest uploadPartRequest) // throws SCSClientException, SCSServiceException { // assertParameterNotNull(uploadPartRequest, // "The request parameter must be specified when uploading a part"); // // String bucketName = uploadPartRequest.getBucketName(); // String key = uploadPartRequest.getKey(); // String uploadId = uploadPartRequest.getUploadId(); // int partNumber = uploadPartRequest.getPartNumber(); // long partSize = uploadPartRequest.getPartSize(); // // assertParameterNotNull(bucketName, // "The bucket name parameter must be specified when uploading a part"); // assertParameterNotNull(key, // "The key parameter must be specified when uploading a part"); // assertParameterNotNull(uploadId, // "The upload ID parameter must be specified when uploading a part"); // assertParameterNotNull(partNumber, // "The part number parameter must be specified when uploading a part"); // assertParameterNotNull(partSize, // "The part size parameter must be specified when uploading a part"); // // Request<UploadPartRequest> request = createRequest(bucketName, key, uploadPartRequest, HttpMethodName.PUT); // request.addParameter("uploadId", uploadId); // request.addParameter("partNumber", Integer.toString(partNumber)); // // if (uploadPartRequest.getMd5Digest() != null) // request.addHeader(Headers.CONTENT_MD5, uploadPartRequest.getMd5Digest()); // // request.addHeader(Headers.CONTENT_LENGTH, Long.toString(partSize)); // // InputStream inputStream = null; // if (uploadPartRequest.getInputStream() != null) { // inputStream = uploadPartRequest.getInputStream(); // } else if (uploadPartRequest.getFile() != null) { // try { // inputStream = new InputSubstream(new RepeatableFileInputStream(uploadPartRequest.getFile()), // uploadPartRequest.getFileOffset(), partSize, true); // } catch (FileNotFoundException e) { // throw new IllegalArgumentException("The specified file doesn't exist", e); // } // } else { // throw new IllegalArgumentException("A File or InputStream must be specified when uploading part"); // } // // MD5DigestCalculatingInputStream md5DigestStream = null; // if (uploadPartRequest.getMd5Digest() == null) { // /* // * If the user hasn't set the content MD5, then we don't want to // * buffer the whole stream in memory just to calculate it. Instead, // * we can calculate it on the fly and validate it with the returned // * ETag from the object upload. // */ // try { // md5DigestStream = new MD5DigestCalculatingInputStream(inputStream); // inputStream = md5DigestStream; // } catch (NoSuchAlgorithmException e) { // log.warn("No MD5 digest algorithm available. Unable to calculate " + // "checksum and verify data integrity.", e); // } // } // // /* // * This is compatible with progress listener set by either the legacy // * method UploadPartRequest#setProgressListener or the new method // * UploadPartRequest#setGeneralProgressListener. // */ // ProgressListener progressListener = uploadPartRequest.getGeneralProgressListener(); // ProgressListenerCallbackExecutor progressListenerCallbackExecutor = ProgressListenerCallbackExecutor // .wrapListener(progressListener); // // if (progressListenerCallbackExecutor != null) { // inputStream = new ProgressReportingInputStream(inputStream, progressListenerCallbackExecutor); // fireProgressEvent(progressListenerCallbackExecutor, ProgressEvent.PART_STARTED_EVENT_CODE); // } // // try { // request.setContent(inputStream); // ObjectMetadata metadata = invoke(request, new S3MetadataResponseHandler(), bucketName, key); // // if (metadata != null && md5DigestStream != null && metadata.getETag() != null) { // String contentMd5 = BinaryUtils.toBase64(md5DigestStream.getMd5Digest()); // byte[] clientSideHash = BinaryUtils.fromBase64(contentMd5); // byte[] serverSideHash = BinaryUtils.fromHex(metadata.getETag()); // // if (!Arrays.equals(clientSideHash, serverSideHash)) { // throw new SCSClientException("Unable to verify integrity of data upload. " + // "Client calculated content hash didn't match hash calculated by Amazon S3. " + // "You may need to delete the data stored in Amazon S3."); // } // } // // fireProgressEvent(progressListenerCallbackExecutor, ProgressEvent.PART_COMPLETED_EVENT_CODE); // // UploadPartResult result = new UploadPartResult(); // result.setETag(metadata.getETag()); // result.setPartNumber(partNumber); // result.setServerSideEncryption(metadata.getServerSideEncryption()); // return result; // } catch (SCSClientException ace) { // fireProgressEvent(progressListenerCallbackExecutor, ProgressEvent.PART_FAILED_EVENT_CODE); // // // Leaving this here in case anyone is depending on it, but it's // // inconsistent with other methods which only generate one of // // COMPLETED_EVENT_CODE or FAILED_EVENT_CODE. // fireProgressEvent(progressListenerCallbackExecutor, ProgressEvent.PART_COMPLETED_EVENT_CODE); // // throw ace; // } finally { // if (inputStream != null) { // try {inputStream.close();} // catch (Exception e) {} // } // } // } /* * Private Interface */ /** * <p> * Asserts that the specified parameter value is not <code>null</code> and if it is, * throws an <code>IllegalArgumentException</code> with the specified error message. * </p> * * @param parameterValue * The parameter value being checked. * @param errorMessage * The error message to include in the IllegalArgumentException * if the specified parameter is null. */ private void assertParameterNotNull(Object parameterValue, String errorMessage) { if (parameterValue == null) throw new IllegalArgumentException(errorMessage); } // /** // * Fires a progress event with the specified event type to the specified // * listener. // * // * @param progressListenerCallbackExecutor // * The listener callback executor. // * @param eventType // * The type of event to fire. // */ // private void fireProgressEvent(final ProgressListenerCallbackExecutor progressListenerCallbackExecutor, final int eventType) { // if (progressListenerCallbackExecutor == null) return; // ProgressEvent event = new ProgressEvent(0); // event.setEventCode(eventType); // progressListenerCallbackExecutor.progressChanged(event); // } /** * <p> * Gets the Amazon S3 {@link AccessControlList} (ACL) for the specified resource. * (bucket if only the bucketName parameter is specified, otherwise the object with the * specified key in the bucket). * </p> * * @param bucketName * The name of the bucket whose ACL should be returned if the key * parameter is not specified, otherwise the bucket containing * the specified key. * @param key * The object key whose ACL should be retrieve. If not specified, * the bucket's ACL is returned. * @param versionId * The version ID of the object version whose ACL is being * retrieved. * @param originalRequest * The original, user facing request object. * * @return The S3 ACL for the specified resource. */ private AccessControlList getAcl(String bucketName, String key, String versionId, SCSWebServiceRequest originalRequest) { if (originalRequest == null) originalRequest = new GenericBucketRequest(bucketName); Request<SCSWebServiceRequest> request = createRequest(bucketName, key, originalRequest, HttpMethodName.GET); request.addParameter("acl", null); if (versionId != null) request.addParameter("versionId", versionId); return invoke(request, new Unmarshallers.AccessControlListUnmarshaller(), bucketName, key); } /** * Sets the ACL for the specified resource in S3. If only bucketName is * specified, the ACL will be applied to the bucket, otherwise if bucketName * and key are specified, the ACL will be applied to the object. * * @param bucketName * The name of the bucket containing the specified key, or if no * key is listed, the bucket whose ACL will be set. * @param key * The optional object key within the specified bucket whose ACL * will be set. If not specified, the bucket ACL will be set. * @param versionId * The version ID of the object version whose ACL is being set. * @param acl * The ACL to apply to the resource. * @param originalRequest * The original, user facing request object. */ private void setAcl(String bucketName, String key, AccessControlList acl, SCSWebServiceRequest originalRequest) { if (originalRequest == null) originalRequest = new GenericBucketRequest(bucketName); Request<SCSWebServiceRequest> request = createRequest(bucketName, key, originalRequest, HttpMethodName.PUT); request.addParameter("acl", null); byte[] aclAsJSON = new AclJsonFactory().convertToJsonByteArray(acl); request.addHeader("Content-Type", "text/plain"); request.addHeader("Content-Length", String.valueOf(aclAsJSON.length)); request.setContent(new ByteArrayInputStream(aclAsJSON)); invoke(request, voidResponseHandler, bucketName, key); } // /** // * Sets the Canned ACL for the specified resource in S3. If only bucketName // * is specified, the canned ACL will be applied to the bucket, otherwise if // * bucketName and key are specified, the canned ACL will be applied to the // * object. // * // * @param bucketName // * The name of the bucket containing the specified key, or if no // * key is listed, the bucket whose ACL will be set. // * @param key // * The optional object key within the specified bucket whose ACL // * will be set. If not specified, the bucket ACL will be set. // * @param versionId // * The version ID of the object version whose ACL is being set. // * @param cannedAcl // * The canned ACL to apply to the resource. // * @param originalRequest // * The original, user facing request object. // */ // private void setAcl(String bucketName, String key, CannedAccessControlList cannedAcl, SCSWebServiceRequest originalRequest) { // if (originalRequest == null) originalRequest = new GenericBucketRequest(bucketName); // // Request<SCSWebServiceRequest> request = createRequest(bucketName, key, originalRequest, HttpMethodName.PUT); // request.addParameter("acl", null); // request.addHeader(Headers.S3_CANNED_ACL, cannedAcl.toString()); // // invoke(request, voidResponseHandler, bucketName, key); // } protected Signer createSigner(Request<?> request, String bucketName, String key) { String resourcePath = "/" + ((bucketName != null) ? bucketName + "/" : "") + ((key != null) ? key : ""); return new S3Signer(request.getHttpMethod().toString(), resourcePath); } /** * Pre-signs the specified request, using a signature query-string * parameter. * * @param request * The request to sign. * @param methodName * The HTTP method (GET, PUT, DELETE, HEAD) for the specified * request. * @param bucketName * The name of the bucket involved in the request. If the request * is not an operation on a bucket this parameter should be null. * @param key * The object key involved in the request. If the request is not * an operation on an object, this parameter should be null. * @param expiration * The time at which the signed request is no longer valid, and * will stop working. * @param subResource * The optional sub-resource being requested as part of the * request (e.g. "location", "acl", "logging", or "torrent"). */ protected <T> void presignRequest(Request<T> request, HttpMethod methodName, String bucketName, String key, Date expiration, String subResource) { // Run any additional request handlers if present beforeRequest(request); String resourcePath = "/" + ((bucketName != null) ? bucketName + "/" : "") + ((key != null) ? HttpUtils.urlEncode(key, true) : "") + ((subResource != null) ? "?" + subResource : ""); // Make sure the resource-path for signing does not contain // any consecutive "/"s. // Note that we should also follow the same rule to escape // consecutive "/"s when generating the presigned URL. // See ServiceUtils#convertRequestToUrl(...) resourcePath = resourcePath.replaceAll("(?<=/)/", "%2F"); AWSCredentials credentials = awsCredentialsProvider.getCredentials(); SCSWebServiceRequest originalRequest = request.getOriginalRequest(); if (originalRequest != null && originalRequest.getRequestCredentials() != null) { credentials = originalRequest.getRequestCredentials(); } new S3QueryStringSigner<T>(methodName.toString(), resourcePath, expiration).sign(request, credentials); // The Amazon S3 DevPay token header is a special exception and can be safely moved // from the request's headers into the query string to ensure that it travels along // with the pre-signed URL when it's sent back to Amazon S3. if (request.getHeaders().containsKey(Headers.SECURITY_TOKEN)) { String value = request.getHeaders().get(Headers.SECURITY_TOKEN); request.addParameter(Headers.SECURITY_TOKEN, value); request.getHeaders().remove(Headers.SECURITY_TOKEN); } } private <T> void beforeRequest(Request<T> request) { // if (requestHandler2s != null) { // for (RequestHandler2 requestHandler2 : requestHandler2s) { // requestHandler2.beforeRequest(request); // } // } } /** * Converts the current endpoint set for this client into virtual addressing * style, by placing the name of the specified bucket before the S3 service * endpoint. * * @param bucketName * The name of the bucket to use in the virtual addressing style * of the returned URI. * * @return A new URI, creating from the current service endpoint URI and the * specified bucket. */ private URI convertToVirtualHostEndpoint(String bucketName, URI endpoint) { try { return new URI(endpoint.getScheme() + "://" + bucketName + "." + endpoint.getAuthority()); } catch (URISyntaxException e) { throw new IllegalArgumentException("Invalid bucket name: " + bucketName, e); } } /** * <p> * Populates the specified request object with the appropriate headers from * the {@link ObjectMetadata} object. * </p> * * @param request * The request to populate with headers. * @param metadata * The metadata containing the header information to include in * the request. */ protected static void populateRequestMetadata(Request<?> request, ObjectMetadata metadata) { Map<String, Object> rawMetadata = metadata.getRawMetadata(); if (rawMetadata != null) { for (Entry<String, Object> entry : rawMetadata.entrySet()) { request.addHeader(entry.getKey(), entry.getValue().toString()); } } Date httpExpiresDate = metadata.getHttpExpiresDate(); if (httpExpiresDate != null) { request.addHeader(Headers.EXPIRES, new DateUtils().formatRfc822Date(httpExpiresDate)); } Map<String, String> userMetadata = metadata.getUserMetadata(); if (userMetadata != null) { for (Entry<String, String> entry : userMetadata.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); if (key != null) key = key.trim(); if (value != null) value = value.trim(); request.addHeader(Headers.S3_USER_METADATA_PREFIX + key, value); } } //? Map<String, String> userHeader = metadata.getUserHeader(); if (userHeader != null) { for (Entry<String, String> entry : userHeader.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); if (key != null) key = key.trim(); if (value != null) value = value.trim(); request.addHeader(key, value); } } } // /** // * <p> // * Populates the specified request with the specified Multi-Factor // * Authentication (MFA) details. This includes the MFA header with device serial // * number and generated token. Since all requests which include the MFA // * header must be sent over HTTPS, this operation also configures the request object to // * use HTTPS instead of HTTP. // * </p> // * // * @param request // * The request to populate. // * @param mfa // * The Multi-Factor Authentication information. // */ // private void populateRequestWithMfaDetails(Request<?> request, MultiFactorAuthentication mfa) { // if (mfa == null) return; // // String endpoint = request.getEndpoint().toString(); // if (endpoint.startsWith("http://")) { // String httpsEndpoint = endpoint.replace("http://", "https://"); // request.setEndpoint(URI.create(httpsEndpoint)); // log.info("Overriding current endpoint to use HTTPS " + // "as required by S3 for requests containing an MFA header"); // } // // request.addHeader(Headers.S3_MFA, // mfa.getDeviceSerialNumber() + " " + mfa.getToken()); // } /** * <p> * Populates the specified request with the numerous options available in * <code>CopyObjectRequest</code>. * </p> * * @param request * The request to populate with headers to represent all the * options expressed in the <code>CopyObjectRequest</code> object. * @param copyObjectRequest * The object containing all the options for copying an object in * Amazon S3. */ private static void populateRequestWithCopyObjectParameters(Request<? extends SCSWebServiceRequest> request, CopyObjectRequest copyObjectRequest) { String copySourceHeader = "/" + HttpUtils.urlEncode(copyObjectRequest.getSourceBucketName(), true) + "/" + HttpUtils.urlEncode(copyObjectRequest.getSourceKey(), true); // if (copyObjectRequest.getSourceVersionId() != null) { // copySourceHeader += "?versionId=" + copyObjectRequest.getSourceVersionId(); // } request.addHeader("x-amz-copy-source", copySourceHeader); addDateHeader(request, Headers.COPY_SOURCE_IF_MODIFIED_SINCE, copyObjectRequest.getModifiedSinceConstraint()); addDateHeader(request, Headers.COPY_SOURCE_IF_UNMODIFIED_SINCE, copyObjectRequest.getUnmodifiedSinceConstraint()); addStringListHeader(request, Headers.COPY_SOURCE_IF_MATCH, copyObjectRequest.getMatchingETagConstraints()); addStringListHeader(request, Headers.COPY_SOURCE_IF_NO_MATCH, copyObjectRequest.getNonmatchingETagConstraints()); if (copyObjectRequest.getAccessControlList() != null) { addAclHeaders(request, copyObjectRequest.getAccessControlList()); } else if (copyObjectRequest.getCannedAccessControlList() != null) { request.addHeader(Headers.S3_CANNED_ACL, copyObjectRequest.getCannedAccessControlList().toString()); } ObjectMetadata newObjectMetadata = copyObjectRequest.getNewObjectMetadata(); if (newObjectMetadata != null) { request.addHeader(Headers.METADATA_DIRECTIVE, "REPLACE"); populateRequestMetadata(request, newObjectMetadata); } } /** * <p> * Adds the specified date header in RFC 822 date format to the specified * request. * This method will not add a date header if the specified date value is <code>null</code>. * </p> * * @param request * The request to add the header to. * @param header * The header name. * @param value * The header value. */ private static void addDateHeader(Request<?> request, String header, Date value) { if (value != null) { request.addHeader(header, ServiceUtils.formatRfc822Date(value)); } } /** * <p> * Adds the specified string list header, joined together separated with * commas, to the specified request. * This method will not add a string list header if the specified values * are <code>null</code> or empty. * </p> * * @param request * The request to add the header to. * @param header * The header name. * @param values * The list of strings to join together for the header value. */ private static void addStringListHeader(Request<?> request, String header, List<String> values) { if (values != null && !values.isEmpty()) { request.addHeader(header, ServiceUtils.join(values)); } } /** * <p> * Adds response headers parameters to the request given, if non-null. * </p> * * @param request * The request to add the response header parameters to. * @param responseHeaders * The full set of response headers to add, or null for none. */ private static void addResponseHeaderParameters(Request<?> request, ResponseHeaderOverrides responseHeaders) { if (responseHeaders != null) { if (responseHeaders.getCacheControl() != null) { request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CACHE_CONTROL, responseHeaders.getCacheControl()); } if (responseHeaders.getContentDisposition() != null) { request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_DISPOSITION, responseHeaders.getContentDisposition()); } if (responseHeaders.getContentEncoding() != null) { request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_ENCODING, responseHeaders.getContentEncoding()); } if (responseHeaders.getContentLanguage() != null) { request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_LANGUAGE, responseHeaders.getContentLanguage()); } if (responseHeaders.getContentType() != null) { request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_TYPE, responseHeaders.getContentType()); } if (responseHeaders.getExpires() != null) { request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_EXPIRES, responseHeaders.getExpires()); } } } /** * Returns the URL to the key in the bucket given, using the client's scheme * and endpoint. Returns null if the given bucket and key cannot be * converted to a URL. */ public String getResourceUrl(String bucketName, String key) { try { return getUrl(bucketName, key).toString(); } catch (Exception e) { return null; } } /** * Returns an URL for the object stored in the specified bucket and * key. * <p> * If the object identified by the given bucket and key has public read * permissions (ex: {@link CannedAccessControlList#PublicRead}), then this * URL can be directly accessed to retrieve the object's data. * * @param bucketName * The name of the bucket containing the object whose URL is * being requested. * @param key * The key under which the object whose URL is being requested is * stored. * * @return A unique URL for the object stored in the specified bucket and * key. */ public URL getUrl(String bucketName, String key) { Request<?> request = new DefaultRequest<Object>(Constants.S3_SERVICE_NAME); request.setHttpMethod(HttpMethodName.GET); configRequest(request, bucketName, key); return ServiceUtils.convertRequestToUrl(request); } /** * Creates and initializes a new request object for the specified S3 * resource. This method is responsible for determining the right way to * address resources. For example, bucket names that are not DNS addressable * cannot be addressed in V2, virtual host, style, and instead must use V1, * path style. The returned request object has the service name, endpoint * and resource path correctly populated. Callers can take the request, add * any additional headers or parameters, then sign and execute the request. * * @param bucketName * An optional parameter indicating the name of the bucket * containing the resource involved in the request. * @param key * An optional parameter indicating the key under which the * desired resource is stored in the specified bucket. * @param originalRequest * The original request, as created by the user. * @param httpMethod * The HTTP method to use when sending the request. * * @return A new request object, populated with endpoint, resource path, and * service name, ready for callers to populate any additional * headers or parameters, and execute. */ protected <X extends SCSWebServiceRequest> Request<X> createRequest(String bucketName, String key, X originalRequest, HttpMethodName httpMethod) { Request<X> request = new DefaultRequest<X>(originalRequest, Constants.S3_SERVICE_NAME); request.setHttpMethod(httpMethod); configRequest(request, bucketName, key); return request; } /** * Configure the given request with the specified bucket name and key. * @return the request configured */ private void configRequest(Request<?> request, String bucketName, String key) { URI theEndpoint = null; //???endpoint4Upload if (request.getOriginalRequest() instanceof PutObjectRequest) {//|| request.getOriginalRequest() instanceof UploadPartRequest) { theEndpoint = endpoint4Upload; // }else if (request.getOriginalRequest() instanceof GetObjectRequest){ // //??endpoint4Download // theEndpoint = endpoint4Download; } else if (request.getOriginalRequest() instanceof GeneratePresignedUrlRequest) { //?urlrequestisBucketNameAsDomaintrue??bucket name GeneratePresignedUrlRequest generatePresignedUrlRequest = (GeneratePresignedUrlRequest) request .getOriginalRequest(); if (generatePresignedUrlRequest.isBucketNameAsDomain()) { String domain = generatePresignedUrlRequest.getBucketName(); if (domain.contains("://") == false) { domain = clientConfiguration.getProtocol().toString() + "://" + domain; } try { theEndpoint = new URI(domain); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } request.setEndpoint(theEndpoint); request.setResourcePath(key); return; } else { theEndpoint = endpoint; } } else { theEndpoint = endpoint; } // http method GET?https? if (request.getHttpMethod() == HttpMethodName.GET && "https".equalsIgnoreCase(theEndpoint.getScheme())) { URIBuilder ub = new URIBuilder(theEndpoint); ub.setScheme("http"); try { theEndpoint = ub.build(); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } } if (!clientOptions.isPathStyleAccess() && BucketNameUtils.isDNSBucketName(bucketName) && !validIP(theEndpoint.getHost())) { request.setEndpoint(convertToVirtualHostEndpoint(bucketName, theEndpoint)); /* * If the key name starts with a slash character, in order to * prevent it being treated as a path delimiter, we need to add * another slash before the key name. * {@see com.amazonaws.http.HttpRequestFactory#createHttpRequest} */ if (key != null && key.startsWith("/")) { key = "/" + key; } request.setResourcePath(key); } else { request.setEndpoint(theEndpoint); if (bucketName != null) { request.setResourcePath(bucketName + "/" + (key != null ? key : "")); } } } private boolean validIP(String IP) { if (IP == null) { return false; } String[] tokens = IP.split("\\."); if (tokens.length != 4) { return false; } for (String token : tokens) { int tokenInt; try { tokenInt = Integer.parseInt(token); } catch (NumberFormatException ase) { return false; } if (tokenInt < 0 || tokenInt > 255) { return false; } } return true; } private <X, Y extends SCSWebServiceRequest> X invoke(Request<Y> request, Unmarshaller<X, InputStream> unmarshaller, String bucketName, String key) { return invoke(request, new S3JsonResponseHandler<X>(unmarshaller), bucketName, key); } private <X, Y extends SCSWebServiceRequest> X invoke(Request<Y> request, HttpResponseHandler<SCSWebServiceResponse<X>> responseHandler, String bucket, String key) { SCSWebServiceRequest originalRequest = request.getOriginalRequest(); ExecutionContext executionContext = createExecutionContext(originalRequest); // AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics(); // // Binds the request metrics to the current request. // request.setAWSRequestMetrics(awsRequestMetrics); // Having the ClientExecuteTime defined here is not ideal (for the // timing measurement should start as close to the top of the call // stack of the service client method as possible) // but definitely a safe compromise for S3 at least for now. // We can incrementally make it more elaborate should the need arise // for individual method. // awsRequestMetrics.startEvent(Field.ClientExecuteTime); Response<X> response = null; try { for (Entry<String, String> entry : request.getOriginalRequest().copyPrivateRequestParameters() .entrySet()) { request.addParameter(entry.getKey(), entry.getValue()); } request.addParameter("formatter", "json"); request.setTimeOffset(timeOffset); /* * The string we sign needs to include the exact headers that we * send with the request, but the client runtime layer adds the * Content-Type header before the request is sent if one isn't set, * so we have to set something here otherwise the request will fail. */ if (request.getHeaders().get("Content-Type") == null) { request.addHeader("Content-Type", //"application/json; charset=utf-8"); "application/x-www-form-urlencoded; charset=utf-8"); } AWSCredentials credentials = awsCredentialsProvider.getCredentials(); if (originalRequest.getRequestCredentials() != null) { credentials = originalRequest.getRequestCredentials(); } executionContext.setSigner(createSigner(request, bucket, key)); executionContext.setCredentials(credentials); response = client.execute(request, responseHandler, errorResponseHandler, executionContext); return response.getAwsResponse(); } finally { // endClientExecution(awsRequestMetrics, request, response); } } /** * Fires a progress event with the specified event type to the specified * listener. * * @param progressListenerCallbackExecutor * The listener callback executor. * @param eventType * The type of event to fire. */ private void fireProgressEvent(final ProgressListenerCallbackExecutor progressListenerCallbackExecutor, final int eventType) { if (progressListenerCallbackExecutor == null) return; ProgressEvent event = new ProgressEvent(0); event.setEventCode(eventType); progressListenerCallbackExecutor.progressChanged(event); } }