com.cloud.utils.S3Utils.java Source code

Java tutorial

Introduction

Here is the source code for com.cloud.utils.S3Utils.java

Source

//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
//

package com.cloud.utils;

import static com.amazonaws.Protocol.HTTP;
import static com.amazonaws.Protocol.HTTPS;
import static com.cloud.utils.StringUtils.join;
import static java.io.File.createTempFile;
import static java.lang.String.format;
import static java.lang.System.currentTimeMillis;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static java.util.Collections.unmodifiableList;
import static org.apache.commons.lang.ArrayUtils.isEmpty;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.commons.lang.StringUtils.isNotBlank;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;

import org.apache.commons.lang.ArrayUtils;
import org.apache.log4j.Logger;

import com.amazonaws.AmazonClientException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.HttpMethod;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
import com.cloud.utils.exception.CloudRuntimeException;

public final class S3Utils {

    private static final Logger LOGGER = Logger.getLogger(S3Utils.class);

    public static final String SEPARATOR = "/";

    private static final int MIN_BUCKET_NAME_LENGTH = 3;
    private static final int MAX_BUCKET_NAME_LENGTH = 63;

    private S3Utils() {
        super();
    }

    public static AmazonS3 acquireClient(final ClientOptions clientOptions) {

        final AWSCredentials credentials = new BasicAWSCredentials(clientOptions.getAccessKey(),
                clientOptions.getSecretKey());

        final ClientConfiguration configuration = new ClientConfiguration();

        if (clientOptions.isHttps() != null) {
            configuration.setProtocol(clientOptions.isHttps() == true ? HTTPS : HTTP);
        }

        if (clientOptions.getConnectionTimeout() != null) {
            configuration.setConnectionTimeout(clientOptions.getConnectionTimeout());
        }

        if (clientOptions.getMaxErrorRetry() != null) {
            configuration.setMaxErrorRetry(clientOptions.getMaxErrorRetry());
        }

        if (clientOptions.getSocketTimeout() != null) {
            configuration.setSocketTimeout(clientOptions.getSocketTimeout());
        }

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(format(
                    "Creating S3 client with configuration: [protocol: %1$s, connectionTimeOut: "
                            + "%2$s, maxErrorRetry: %3$s, socketTimeout: %4$s]",
                    configuration.getProtocol(), configuration.getConnectionTimeout(),
                    configuration.getMaxErrorRetry(), configuration.getSocketTimeout()));
        }

        final AmazonS3Client client = new AmazonS3Client(credentials, configuration);

        if (isNotBlank(clientOptions.getEndPoint())) {
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug(format("Setting the end point for S3 client %1$s to %2$s.", client,
                        clientOptions.getEndPoint()));
            }
            client.setEndpoint(clientOptions.getEndPoint());
        }

        return client;

    }

    public static void putFile(final ClientOptions clientOptions, final File sourceFile, final String bucketName,
            final String key) {

        assert clientOptions != null;
        assert sourceFile != null;
        assert !isBlank(bucketName);
        assert !isBlank(key);

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(format("Sending file %1$s as S3 object %2$s in " + "bucket %3$s", sourceFile.getName(),
                    key, bucketName));
        }

        acquireClient(clientOptions).putObject(bucketName, key, sourceFile);

    }

    public static void putObject(final ClientOptions clientOptions, final InputStream sourceStream,
            final String bucketName, final String key) {

        assert clientOptions != null;
        assert sourceStream != null;
        assert !isBlank(bucketName);
        assert !isBlank(key);

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(format("Sending stream as S3 object %1$s in " + "bucket %2$s", key, bucketName));
        }

        acquireClient(clientOptions).putObject(bucketName, key, sourceStream, null);

    }

    public static void putObject(final ClientOptions clientOptions, final PutObjectRequest req) {

        assert clientOptions != null;
        assert req != null;

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(format("Sending stream as S3 object using PutObjectRequest"));
        }

        acquireClient(clientOptions).putObject(req);

    }

    // multi-part upload file
    public static void mputFile(final ClientOptions clientOptions, final File sourceFile, final String bucketName,
            final String key) throws InterruptedException {

        assert clientOptions != null;
        assert sourceFile != null;
        assert !isBlank(bucketName);
        assert !isBlank(key);

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(format("Multipart sending file %1$s as S3 object %2$s in " + "bucket %3$s",
                    sourceFile.getName(), key, bucketName));
        }
        TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
        Upload upload = tm.upload(bucketName, key, sourceFile);
        upload.waitForCompletion();
    }

    // multi-part upload object
    public static void mputObject(final ClientOptions clientOptions, final InputStream sourceStream,
            final String bucketName, final String key) throws InterruptedException {

        assert clientOptions != null;
        assert sourceStream != null;
        assert !isBlank(bucketName);
        assert !isBlank(key);

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(format("Multipart sending stream as S3 object %1$s in " + "bucket %2$s", key, bucketName));
        }
        TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
        Upload upload = tm.upload(bucketName, key, sourceStream, null);
        upload.waitForCompletion();
    }

    // multi-part upload object
    public static void mputObject(final ClientOptions clientOptions, final PutObjectRequest req)
            throws InterruptedException {

        assert clientOptions != null;
        assert req != null;

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("Multipart sending object to S3 using PutObjectRequest");
        }
        TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
        Upload upload = tm.upload(req);
        upload.waitForCompletion();

    }

    public static void setObjectAcl(final ClientOptions clientOptions, final String bucketName, final String key,
            final CannedAccessControlList acl) {

        assert clientOptions != null;
        assert acl != null;

        acquireClient(clientOptions).setObjectAcl(bucketName, key, acl);

    }

    public static URL generatePresignedUrl(final ClientOptions clientOptions, final String bucketName,
            final String key, final Date expiration) {

        assert clientOptions != null;
        assert !isBlank(bucketName);
        assert !isBlank(key);

        return acquireClient(clientOptions).generatePresignedUrl(bucketName, key, expiration, HttpMethod.GET);

    }

    // Note that whenever S3Object is returned, client code needs to close the internal stream to avoid resource leak.
    public static S3Object getObject(final ClientOptions clientOptions, final String bucketName, final String key) {

        assert clientOptions != null;
        assert !isBlank(bucketName);
        assert !isBlank(key);

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(format("Get S3 object %1$s in " + "bucket %2$s", key, bucketName));
        }

        return acquireClient(clientOptions).getObject(bucketName, key);

    }

    @SuppressWarnings("unchecked")
    public static File getFile(final ClientOptions clientOptions, final String bucketName, final String key,
            final File targetDirectory, final FileNamingStrategy namingStrategy) {

        assert clientOptions != null;
        assert isNotBlank(bucketName);
        assert isNotBlank(key);
        assert targetDirectory != null && targetDirectory.isDirectory();
        assert namingStrategy != null;

        final AmazonS3 connection = acquireClient(clientOptions);

        File tempFile = null;
        try {

            tempFile = createTempFile(join("-", targetDirectory.getName(), currentTimeMillis(), "part"), "tmp",
                    targetDirectory);
            tempFile.deleteOnExit();

            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug(format("Downloading object %1$s from bucket %2$s to temp file %3$s", key, bucketName,
                        tempFile.getName()));
            }

            try {
                connection.getObject(new GetObjectRequest(bucketName, key), tempFile);
            } catch (AmazonClientException ex) {
                // hack to handle different ETAG format generated from RiakCS for multi-part uploaded object
                String msg = ex.getMessage();
                if (!msg.contains("verify integrity")) {
                    throw ex;
                }
            }

            final File targetFile = new File(targetDirectory, namingStrategy.determineFileName(key));
            tempFile.renameTo(targetFile);

            return targetFile;

        } catch (FileNotFoundException e) {

            throw new CloudRuntimeException(
                    format("Failed open file %1$s in order to get object %2$s from bucket %3$s.",
                            targetDirectory.getAbsoluteFile(), bucketName, key),
                    e);

        } catch (IOException e) {

            throw new CloudRuntimeException(
                    format("Unable to allocate temporary file in directory %1$s to download %2$s:%3$s from S3",
                            targetDirectory.getAbsolutePath(), bucketName, key),
                    e);

        } finally {

            if (tempFile != null) {
                tempFile.delete();
            }

        }

    }

    public static List<File> getDirectory(final ClientOptions clientOptions, final String bucketName,
            final String sourcePath, final File targetDirectory, final FileNamingStrategy namingStrategy) {

        assert clientOptions != null;
        assert isNotBlank(bucketName);
        assert isNotBlank(sourcePath);
        assert targetDirectory != null;

        final AmazonS3 connection = acquireClient(clientOptions);

        // List the objects in the source directory on S3
        final List<S3ObjectSummary> objectSummaries = listDirectory(bucketName, sourcePath, connection);
        final List<File> files = new ArrayList<File>();

        for (final S3ObjectSummary objectSummary : objectSummaries) {

            files.add(getFile(clientOptions, bucketName, objectSummary.getKey(), targetDirectory, namingStrategy));

        }

        return unmodifiableList(files);

    }

    public static List<S3ObjectSummary> getDirectory(final ClientOptions clientOptions, final String bucketName,
            final String sourcePath) {
        assert clientOptions != null;
        assert isNotBlank(bucketName);
        assert isNotBlank(sourcePath);

        final AmazonS3 connection = acquireClient(clientOptions);

        // List the objects in the source directory on S3
        return listDirectory(bucketName, sourcePath, connection);
    }

    private static List<S3ObjectSummary> listDirectory(final String bucketName, final String directory,
            final AmazonS3 client) {

        List<S3ObjectSummary> objects = new ArrayList<S3ObjectSummary>();
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix(directory + SEPARATOR);

        ObjectListing ol = client.listObjects(listObjectsRequest);
        while (ol != null && ol.isTruncated()) {
            objects.addAll(ol.getObjectSummaries());
            listObjectsRequest.setMarker(ol.getNextMarker());
            ol = client.listObjects(listObjectsRequest);
        }

        if (objects.isEmpty()) {
            return emptyList();
        }

        return unmodifiableList(objects);
    }

    public static void putDirectory(final ClientOptions clientOptions, final String bucketName,
            final File directory, final FilenameFilter fileNameFilter, final ObjectNamingStrategy namingStrategy) {

        assert clientOptions != null;
        assert isNotBlank(bucketName);
        assert directory != null && directory.isDirectory();
        assert fileNameFilter != null;
        assert namingStrategy != null;

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(
                    format("Putting directory %1$s in S3 bucket %2$s.", directory.getAbsolutePath(), bucketName));
        }

        // Determine the list of files to be sent using the passed filter ...
        final File[] files = directory.listFiles(fileNameFilter);

        if (LOGGER.isTraceEnabled()) {
            LOGGER.trace(format("Putting files (%1$s) in S3 bucket %2$s.",
                    ArrayUtils.toString(files, "no files found"), bucketName));
        }

        // Skip spinning up an S3 connection when no files will be sent ...
        if (isEmpty(files)) {
            return;
        }

        final AmazonS3 client = acquireClient(clientOptions);

        // Send the files to S3 using the passed ObjectNaming strategy to
        // determine the key ...
        for (final File file : files) {
            final String key = namingStrategy.determineKey(file);
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug(format("Putting file %1$s into bucket %2$s with key %3$s.", file.getAbsolutePath(),
                        bucketName, key));
            }
            client.putObject(bucketName, key, file);
        }

    }

    public static void deleteObject(final ClientOptions clientOptions, final String bucketName, final String key) {

        assert clientOptions != null;
        assert isNotBlank(bucketName);
        assert isNotBlank(key);

        final AmazonS3 client = acquireClient(clientOptions);

        client.deleteObject(bucketName, key);

    }

    public static void deleteDirectory(final ClientOptions clientOptions, final String bucketName,
            final String directoryName) {

        assert clientOptions != null;
        assert isNotBlank(bucketName);
        assert isNotBlank(directoryName);

        final AmazonS3 client = acquireClient(clientOptions);

        final List<S3ObjectSummary> objects = listDirectory(bucketName, directoryName, client);

        for (final S3ObjectSummary object : objects) {

            client.deleteObject(bucketName, object.getKey());

        }

        client.deleteObject(bucketName, directoryName);

    }

    public static boolean canConnect(final ClientOptions clientOptions) {

        try {

            acquireClient(clientOptions);
            return true;

        } catch (AmazonClientException e) {

            LOGGER.warn("Ignored Exception while checking connection options", e);
            return false;

        }

    }

    public static boolean doesBucketExist(final ClientOptions clientOptions, final String bucketName) {

        assert clientOptions != null;
        assert !isBlank(bucketName);

        try {

            final List<Bucket> buckets = acquireClient(clientOptions).listBuckets();

            for (Bucket bucket : buckets) {
                if (bucket.getName().equals(bucketName)) {
                    return true;
                }
            }

            return false;

        } catch (AmazonClientException e) {

            LOGGER.warn("Ignored Exception while checking bucket existence", e);
            return false;

        }

    }

    public static boolean canReadWriteBucket(final ClientOptions clientOptions, final String bucketName) {

        assert clientOptions != null;
        assert isNotBlank(bucketName);

        try {

            final AmazonS3 client = acquireClient(clientOptions);

            final String fileContent = "testing put and delete";
            final InputStream inputStream = new ByteArrayInputStream(fileContent.getBytes());
            final String key = UUID.randomUUID().toString() + ".txt";

            final ObjectMetadata metadata = new ObjectMetadata();
            metadata.setContentLength(fileContent.length());

            client.putObject(bucketName, key, inputStream, metadata);
            client.deleteObject(bucketName, key);

            return true;

        } catch (AmazonClientException e) {

            return false;

        }

    }

    public static List<String> checkClientOptions(ClientOptions clientOptions) {

        assert clientOptions != null;

        List<String> errorMessages = new ArrayList<String>();

        errorMessages.addAll(checkRequiredField("access key", clientOptions.getAccessKey()));
        errorMessages.addAll(checkRequiredField("secret key", clientOptions.getSecretKey()));

        errorMessages.addAll(checkOptionalField("connection timeout", clientOptions.getConnectionTimeout()));
        errorMessages.addAll(checkOptionalField("socket timeout", clientOptions.getSocketTimeout()));
        errorMessages.addAll(checkOptionalField("max error retries", clientOptions.getMaxErrorRetry()));

        return unmodifiableList(errorMessages);

    }

    public static List<String> checkBucketName(final String bucketLabel, final String bucket) {

        assert isNotBlank(bucketLabel);
        assert isNotBlank(bucket);

        final List<String> errorMessages = new ArrayList<String>();

        if (bucket.length() < MIN_BUCKET_NAME_LENGTH) {
            errorMessages.add(format(
                    "The length of %1$s " + "for the %2$s must have a length of at least %3$s " + "characters",
                    bucket, bucketLabel, MIN_BUCKET_NAME_LENGTH));
        }

        if (bucket.length() > MAX_BUCKET_NAME_LENGTH) {
            errorMessages.add(format("The length of %1$s " + "for the %2$s must not have a length of at greater"
                    + " than %3$s characters", bucket, bucketLabel, MAX_BUCKET_NAME_LENGTH));
        }

        return unmodifiableList(errorMessages);

    }

    private static List<String> checkOptionalField(final String fieldName, final Integer fieldValue) {
        if (fieldValue != null && fieldValue < 0) {
            return singletonList(format("The value of %1$s must " + "be greater than zero.", fieldName));
        }
        return emptyList();
    }

    private static List<String> checkRequiredField(String fieldName, String fieldValue) {
        if (isBlank(fieldValue)) {
            return singletonList(format("A %1$s must be specified.", fieldName));
        }
        return emptyList();
    }

    public interface ClientOptions {

        String getAccessKey();

        String getSecretKey();

        String getEndPoint();

        Boolean isHttps();

        Integer getConnectionTimeout();

        Integer getMaxErrorRetry();

        Integer getSocketTimeout();

    }

    public interface ObjectNamingStrategy {

        String determineKey(File file);

    }

    public interface FileNamingStrategy {

        String determineFileName(String key);

    }

}