ch.entwine.weblounge.maven.S3DeployMojo.java Source code

Java tutorial

Introduction

Here is the source code for ch.entwine.weblounge.maven.S3DeployMojo.java

Source

/*
 *  Weblounge: Web Content Management System
 *  Copyright (c) 2003 - 2011 The Weblounge Team
 *  http://entwinemedia.com/weblounge
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public License
 *  as published by the Free Software Foundation; either version 2
 *  of the License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with this program; if not, write to the Free Software Foundation
 *  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

package ch.entwine.weblounge.maven;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.AccessControlList;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.Grant;
import com.amazonaws.services.s3.model.GroupGrantee;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.Permission;
import com.amazonaws.services.s3.model.ProgressEvent;
import com.amazonaws.services.s3.model.ProgressListener;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.transfer.Transfer.TransferState;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;

import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.maven.model.FileSet;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.codehaus.plexus.util.FileUtils;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.zip.GZIPOutputStream;

/**
 * @goal deploy
 */
public class S3DeployMojo extends AbstractMojo {

    /** Number of retries */
    private static final int MAX_RETRIES = 3;

    /**
     * AWS access key used to access the S3 bucket
     * 
     * @parameter name="AWSAccessKey"
     * @required
     */
    private String awsAccessKey;

    /**
     * AWS secret key used to access the S3 bucket
     * 
     * @parameter name="AWSSecretKey"
     * @required
     */
    private String awsSecretKey;

    /**
     * Name of the destination S3 bucket.
     * 
     * @parameter
     * @required
     */
    private String bucket;

    /**
     * Prefix for the resource key.
     * 
     * @parameter
     * @required
     */
    private String keyPrefix;

    /**
     * Resources to deploy to the S3 bucket.
     * 
     * @parameter
     * @required
     */
    private FileSet resources;

    /**
     * Duration (in hours) the file remains valid (HTTP Expires header will be set
     * to now + valid). Default value is 8'760h (1 year).
     * 
     * @parameter default-value="8760"
     */
    private int valid;

    /**
     * Enable/disable GZip compression of files. Default value is 'true'.
     * 
     * @parameter default-value="true"
     */
    private boolean gzip;

    /** True if an error occurred during uploading */
    private String erroneousUpload = null;

    /**
     * 
     * {@inheritDoc}
     * 
     * @see org.apache.maven.plugin.Mojo#execute()
     */
    public void execute() throws MojoExecutionException, MojoFailureException {

        // Setup AWS S3 client
        AWSCredentials credentials = new BasicAWSCredentials(awsAccessKey, awsSecretKey);
        AmazonS3Client uploadClient = new AmazonS3Client(credentials);
        TransferManager transfers = new TransferManager(credentials);

        // Make sure key prefix does not start with a slash but has one at the
        // end
        if (keyPrefix.startsWith("/"))
            keyPrefix = keyPrefix.substring(1);
        if (!keyPrefix.endsWith("/"))
            keyPrefix = keyPrefix + "/";

        // Keep track of how much data has been transferred
        long totalBytesTransferred = 0L;
        int items = 0;
        Queue<Upload> uploads = new LinkedBlockingQueue<Upload>();

        try {
            // Check if S3 bucket exists
            getLog().debug("Checking whether bucket " + bucket + " exists");
            if (!uploadClient.doesBucketExist(bucket)) {
                getLog().error("Desired bucket '" + bucket + "' does not exist!");
                return;
            }

            getLog().debug("Collecting files to transfer from " + resources.getDirectory());
            List<File> res = getResources();
            for (File file : res) {
                // Make path of resource relative to resources directory
                String filename = file.getName();
                String extension = FilenameUtils.getExtension(filename);
                String path = file.getPath().substring(resources.getDirectory().length());
                String key = concat("/", keyPrefix, path).substring(1);

                // Delete old file version in bucket
                getLog().debug("Removing existing object at " + key);
                uploadClient.deleteObject(bucket, key);

                // Setup meta data
                ObjectMetadata meta = new ObjectMetadata();
                meta.setCacheControl("public, max-age=" + String.valueOf(valid * 3600));

                FileInputStream fis = null;
                GZIPOutputStream gzipos = null;
                final File fileToUpload;

                if (gzip && ("js".equals(extension) || "css".equals(extension))) {
                    try {
                        fis = new FileInputStream(file);
                        File gzFile = File.createTempFile(file.getName(), null);
                        gzipos = new GZIPOutputStream(new FileOutputStream(gzFile));
                        IOUtils.copy(fis, gzipos);
                        fileToUpload = gzFile;
                        meta.setContentEncoding("gzip");
                        if ("js".equals(extension))
                            meta.setContentType("text/javascript");
                        if ("css".equals(extension))
                            meta.setContentType("text/css");
                    } catch (FileNotFoundException e) {
                        getLog().error(e);
                        continue;
                    } catch (IOException e) {
                        getLog().error(e);
                        continue;
                    } finally {
                        IOUtils.closeQuietly(fis);
                        IOUtils.closeQuietly(gzipos);
                    }
                } else {
                    fileToUpload = file;
                }

                // Do a random check for existing errors before starting the next upload
                if (erroneousUpload != null)
                    break;

                // Create put object request
                long bytesToTransfer = fileToUpload.length();
                totalBytesTransferred += bytesToTransfer;
                PutObjectRequest request = new PutObjectRequest(bucket, key, fileToUpload);
                request.setProgressListener(new UploadListener(credentials, bucket, key, bytesToTransfer));
                request.setMetadata(meta);

                // Schedule put object request
                getLog().info(
                        "Uploading " + key + " (" + FileUtils.byteCountToDisplaySize((int) bytesToTransfer) + ")");
                Upload upload = transfers.upload(request);
                uploads.add(upload);
                items++;
            }
        } catch (AmazonServiceException e) {
            getLog().error("Uploading resources failed: " + e.getMessage());
        } catch (AmazonClientException e) {
            getLog().error("Uploading resources failed: " + e.getMessage());
        }

        // Wait for uploads to be finished
        String currentUpload = null;
        try {
            Thread.sleep(1000);
            getLog().info("Waiting for " + uploads.size() + " uploads to finish...");
            while (!uploads.isEmpty()) {
                Upload upload = uploads.poll();
                currentUpload = upload.getDescription().substring("Uploading to ".length());
                if (TransferState.InProgress.equals(upload.getState()))
                    getLog().debug("Waiting for upload " + currentUpload + " to finish");
                upload.waitForUploadResult();
            }
        } catch (AmazonServiceException e) {
            throw new MojoExecutionException("Error while uploading " + currentUpload);
        } catch (AmazonClientException e) {
            throw new MojoExecutionException("Error while uploading " + currentUpload);
        } catch (InterruptedException e) {
            getLog().debug("Interrupted while waiting for upload to finish");
        }

        // Check for errors that happened outside of the actual uploading
        if (erroneousUpload != null) {
            throw new MojoExecutionException("Error while uploading " + erroneousUpload);
        }

        getLog().info("Deployed " + items + " files ("
                + FileUtils.byteCountToDisplaySize((int) totalBytesTransferred) + ") to s3://" + bucket);
    }

    /**
     * @return
     * @throws MojoExecutionException
     */
    @SuppressWarnings("unchecked")
    protected List<File> getResources() throws MojoExecutionException {
        File directory = new File(resources.getDirectory());
        String includes = StringUtils.join(resources.getIncludes(), ",");
        String excludes = StringUtils.join(resources.getExcludes(), ",");
        try {
            List<File> files = FileUtils.getFiles(directory, includes, excludes);
            getLog().debug("Adding " + files.size() + " objects to the list of items to deploy");
            return files;
        } catch (IOException e) {
            throw new MojoExecutionException("Unable to get resources to deploy", e);
        }
    }

    /**
     * Concatenates the url elements with respect to leading and trailing slashes.
     * The path will always end with a trailing slash.
     * 
     * @param urlElements
     *          the path elements
     * @return the concatenated url of the two arguments
     * @throws IllegalArgumentException
     *           if less than two path elements are provided
     */
    private static String concat(String... urlElements) throws IllegalArgumentException {
        if (urlElements == null || urlElements.length < 1)
            throw new IllegalArgumentException("Prefix cannot be null or empty");
        if (urlElements.length < 2)
            throw new IllegalArgumentException("Suffix cannot be null or empty");

        StringBuffer b = new StringBuffer();
        for (String s : urlElements) {
            if (StringUtils.isBlank(s))
                throw new IllegalArgumentException("Path element cannot be null");
            String element = checkSeparator(s);
            element = removeDoubleSeparator(element);

            if (b.length() == 0) {
                b.append(element);
            } else if (b.lastIndexOf("/") < b.length() - 1 && !element.startsWith("/")) {
                b.append("/").append(element);
            } else if (b.lastIndexOf("/") == b.length() - 1 && element.startsWith("/")) {
                b.append(element.substring(1));
            } else {
                b.append(element);
            }
        }

        return b.toString();
    }

    /**
     * Checks that the path only contains the web path separator "/". If not,
     * wrong ones are replaced.
     */
    private static String checkSeparator(String path) {
        String sp = File.separator;
        if ("\\".equals(sp))
            sp = "\\\\";
        return path.replaceAll(sp, "/");
    }

    /**
     * Removes any occurrence of double separators ("//") and replaces it with
     * "/".
     * 
     * @param path
     *          the path to check
     * @return the corrected path
     */
    private static String removeDoubleSeparator(String path) {
        int protocolIndex = path.indexOf("://");
        protocolIndex += protocolIndex == -1 ? 0 : 3;
        int index = Math.max(0, protocolIndex);
        while ((index = path.indexOf("//", index)) != -1) {
            path = path.substring(0, index) + path.substring(index + 1);
        }
        return path;
    }

    /**
     * Progress listener that is monitoring upload progress of an S3 item and on
     * successful upload adjust the object's ACL to public read access.
     */
    protected class UploadListener implements ProgressListener {

        private AmazonS3Client client;
        private String bucket;
        private String key;
        private long size = 0;
        private long bytesTransferred = 0;

        /**
         * Creates a new ACL controller that will be using the client object to
         * monitor upload progress to the object identified by <code>key</code>.
         * 
         * @param credentials
         *          the amazon credentials
         * @param bucket
         *          the bucket
         * @param key
         *          the key identifying the object
         * @param size
         *          the number of bytes to upload
         */
        protected UploadListener(AWSCredentials credentials, String bucket, String key, long size) {
            this.client = new AmazonS3Client(credentials);
            this.bucket = bucket;
            this.key = key;
            this.size = size;
        }

        /**
         * {@inheritDoc}
         * 
         * @see com.amazonaws.services.s3.model.ProgressListener#progressChanged(com.amazonaws.services.s3.model.ProgressEvent)
         */
        public void progressChanged(ProgressEvent pe) {
            bytesTransferred += pe.getBytesTransfered();
            switch (pe.getEventCode()) {
            case ProgressEvent.STARTED_EVENT_CODE:
                getLog().debug("Upload of " + key + " started");
                break;
            case ProgressEvent.COMPLETED_EVENT_CODE:
                getLog().debug(
                        "Upload of '" + key + "' completed (" + bytesTransferred + " of " + size + " transferred)");
                setPublicAccess();
                break;
            case ProgressEvent.CANCELED_EVENT_CODE:
                getLog().info("Upload of " + key + " canceled");
                break;
            case ProgressEvent.FAILED_EVENT_CODE:
                getLog().warn("Upload of " + key + " failed");
                break;
            default:
                // Nothing to do
            }
        }

        /**
         * Adjusts access control to the object to public read.
         */
        private void setPublicAccess() {
            int retries = 0;
            while (retries < MAX_RETRIES) {
                try {
                    // Make sure S3 can get its act together before we bother it again
                    Thread.sleep(2000);
                    // Ask S3 to open the resource up for public read
                    client.setObjectAcl(bucket, key, CannedAccessControlList.PublicRead);
                    getLog().debug("Access control on " + key + " adjusted to public read");
                } catch (AmazonServiceException e) {
                    getLog().warn("Access control on " + key + " cannot be set: " + e.getMessage());
                } catch (AmazonClientException e) {
                    getLog().warn("Error adjusting access control on " + key + ": " + e.getMessage());
                } catch (Throwable t) {
                    getLog().warn("Error adjusting access control on " + key + ": " + t.getMessage());
                }

                // Check the object's current ACL to make sure the operation succeeded
                AccessControlList acl = client.getObjectAcl(bucket, key);
                boolean publicAccessGranted = false;
                for (Grant grant : acl.getGrants()) {
                    if (GroupGrantee.AllUsers.equals(grant.getGrantee())) {
                        publicAccessGranted = Permission.Read.equals(grant.getPermission());
                    }
                }

                if (!publicAccessGranted) {
                    retries++;
                    if (retries < MAX_RETRIES)
                        getLog().warn("Setting of access control entries on " + key + " failed, retrying");
                    else if (erroneousUpload == null) {
                        getLog().error("S3 is not responding to acl update request on " + key);
                        erroneousUpload = key;
                    }
                } else {
                    getLog().debug("Access control on " + key + " verified to be public read access");
                    if (retries > 0)
                        getLog().info("Setting of access control entries on " + key + " finally succeeded");
                    return;
                }
            }
        }

    }

}