n3phele.storage.s3.CloudStorageImpl.java Source code

Java tutorial

Introduction

Here is the source code for n3phele.storage.s3.CloudStorageImpl.java

Source

/**
 * @author Nigel Cook
 *
 * (C) Copyright 2010-2012. Nigel Cook. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 * 
 * Licensed under the terms described in LICENSE file that accompanied this code, (the "License"); you may not use this file
 * except in compliance with the License. 
 * 
 *  Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 
 *  an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 
 *  specific language governing permissions and limitations under the License.
 */

package n3phele.storage.s3;

import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.ws.rs.core.UriBuilder;

import n3phele.service.core.ForbiddenException;
import n3phele.service.core.NotFoundException;
import n3phele.service.model.core.Credential;
import n3phele.service.model.repository.FileNode;
import n3phele.service.model.repository.Repository;
import n3phele.service.model.repository.UploadSignature;
import n3phele.storage.CloudStorageInterface;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.Request;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AbstractAWSSigner;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.SigningAlgorithm;
import com.amazonaws.auth.policy.Action;
import com.amazonaws.auth.policy.Policy;
import com.amazonaws.auth.policy.Principal;
import com.amazonaws.auth.policy.Statement;
import com.amazonaws.auth.policy.Statement.Effect;
import com.amazonaws.auth.policy.actions.S3Actions;
import com.amazonaws.auth.policy.resources.S3ObjectResource;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.internal.Mimetypes;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.BucketPolicy;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.sun.jersey.core.util.Base64;

public class CloudStorageImpl implements CloudStorageInterface {
    private static Logger log = Logger.getLogger(CloudStorageImpl.class.getName());

    public boolean createBucket(Repository repo) throws ForbiddenException {
        Credential credential = repo.getCredential().decrypt();
        AmazonS3Client s3 = new AmazonS3Client(
                new BasicAWSCredentials(credential.getAccount(), credential.getSecret()));
        s3.setEndpoint(repo.getTarget().toString());
        try {
            try {
                s3.listObjects(new ListObjectsRequest(repo.getRoot(), null, null, null, 0));

                // it exists and the current account owns it
                return false;
            } catch (AmazonServiceException ase) {
                switch (ase.getStatusCode()) {
                case 403:
                    /*
                     * A permissions error means the bucket exists, but is owned by
                     * another account.
                     */
                    throw new ForbiddenException(
                            "Bucket " + repo.getRoot() + " has already been created by another user.");
                case 404:
                    Bucket bucket = s3.createBucket(repo.getRoot());
                    log.info("Bucket created " + bucket.getName());
                    return true;
                default:
                    throw ase;
                }
            }

        } catch (AmazonServiceException e) {
            log.log(Level.WARNING, "Service Error processing " + repo, e);

        } catch (AmazonClientException e) {
            log.log(Level.SEVERE, "Client Error processing " + repo, e);

        }
        return false;
    }

    public FileNode getMetadata(Repository repo, String filename) {
        FileNode f = new FileNode();
        UriBuilder result = UriBuilder.fromUri(repo.getTarget());
        result.path(repo.getRoot()).path(filename);
        f.setName(result.build().toString());
        Credential credential = repo.getCredential().decrypt();

        log.info("Get info on " + repo.getRoot() + " " + filename);
        AmazonS3Client s3 = new AmazonS3Client(
                new BasicAWSCredentials(credential.getAccount(), credential.getSecret()));
        s3.setEndpoint(repo.getTarget().toString());
        try {
            ObjectListing metadata = s3.listObjects(
                    new ListObjectsRequest().withBucketName(repo.getRoot()).withPrefix(filename).withMaxKeys(1));
            List<S3ObjectSummary> metadataList = metadata.getObjectSummaries();
            if (metadataList == null || metadataList.size() != 1) {
                throw new NotFoundException(filename);
            }
            f.setModified(metadataList.get(0).getLastModified());
            f.setSize(metadataList.get(0).getSize());

            log.info(filename + " " + f.getModified() + " " + f.getSize());
        } catch (AmazonServiceException e) {
            log.log(Level.WARNING, "Service Error processing " + f + repo, e);
            throw new NotFoundException("Retrieve " + filename + " fails " + e.toString());
        } catch (AmazonClientException e) {
            log.log(Level.SEVERE, "Client Error processing " + f + repo, e);
            throw new NotFoundException("Retrieve " + filename + " fails " + e.toString());
        }
        return f;
    }

    public boolean deleteFile(Repository repo, String filename) {
        boolean result = false;
        Credential credential = repo.getCredential().decrypt();

        AmazonS3Client s3 = new AmazonS3Client(
                new BasicAWSCredentials(credential.getAccount(), credential.getSecret()));
        s3.setEndpoint(repo.getTarget().toString());
        try {
            s3.deleteObject(repo.getRoot(), filename);
            result = true;
        } catch (AmazonServiceException e) {
            log.log(Level.WARNING, "Service Error processing " + repo, e);
        } catch (AmazonClientException e) {
            log.log(Level.SEVERE, "Client Error processing " + repo, e);
        }
        return result;
    }

    public boolean deleteFolder(Repository repo, String filename) {
        boolean result = false;
        Credential credential = repo.getCredential().decrypt();
        int retry = 3;

        setPermissions(repo, filename, false);
        if (!filename.endsWith("/")) {
            filename += "/";
        }

        AmazonS3Client s3 = new AmazonS3Client(
                new BasicAWSCredentials(credential.getAccount(), credential.getSecret()));
        s3.setEndpoint(repo.getTarget().toString());
        while (retry-- > 0) {
            try {
                ObjectListing objects = s3.listObjects(repo.getRoot(), filename);
                for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                    log.info("Delete " + repo.getRoot() + ":" + objectSummary.getKey());
                    s3.deleteObject(repo.getRoot(), objectSummary.getKey());
                }
                if (objects.isTruncated()) {
                    retry++;
                    log.info("Doing next portion");
                    continue;
                }
                result = true;
                break;
            } catch (AmazonServiceException e) {
                log.log(Level.WARNING, "Service Error processing " + repo, e);
            } catch (AmazonClientException e) {
                log.log(Level.SEVERE, "Client Error processing " + repo, e);
            }
        }
        return result;
    }

    public boolean setPermissions(Repository repo, String filename, boolean isPublic) {
        String bucket = repo.getRoot();
        Credential credential = repo.getCredential().decrypt();
        AmazonS3Client s3 = new AmazonS3Client(
                new BasicAWSCredentials(credential.getAccount(), credential.getSecret()));
        String key = new S3ObjectResource(bucket, filename).getId();
        boolean inserted = false;

        s3.setEndpoint(repo.getTarget().toString());
        try {
            List<Statement> statements = new ArrayList<Statement>();
            Policy policy = null;
            BucketPolicy bp = s3.getBucketPolicy(repo.getRoot());
            if (bp != null && bp.getPolicyText() != null) {
                log.info("Policy text " + bp.getPolicyText());
                policy = PolicyHelper.parse(bp.getPolicyText());
                log.info("Policy object is " + (policy == null ? null : policy.toJson()));

                if (policy != null) {
                    if (policy.getStatements() != null) {
                        for (Statement statement : policy.getStatements()) {
                            if (statement.getId().equals("n3phele")) {
                                List<com.amazonaws.auth.policy.Resource> resources = statement.getResources();
                                List<com.amazonaws.auth.policy.Resource> update = new ArrayList<com.amazonaws.auth.policy.Resource>();
                                if (resources != null) {
                                    for (com.amazonaws.auth.policy.Resource resource : resources) {
                                        String resourceName = resource.getId();
                                        if (resourceName.endsWith("*")) {
                                            resourceName = resourceName.substring(0, resourceName.length() - 1);
                                        }
                                        if (!(resourceName + "/").startsWith(key + "/")) {
                                            update.add(resource);
                                        } else {
                                            log.info("Removing " + resource.getId());
                                        }
                                    }
                                }
                                if (isPublic && !inserted)
                                    update.add(new S3ObjectResource(repo.getRoot(), filename + "*"));
                                if (update.size() > 0) {
                                    statement.setResources(update);
                                    statements.add(statement);
                                }
                                inserted = true;
                            } else {
                                statements.add(statement);
                            }
                        }
                    }
                    if (!inserted && isPublic) {
                        Statement statement = new Statement(Effect.Allow);
                        statement.setId("n3phele");
                        statement.setPrincipals(Arrays.asList(new Principal("*")));
                        statement.setActions(Arrays.asList((Action) S3Actions.GetObject));
                        statement.setResources(Arrays
                                .asList((com.amazonaws.auth.policy.Resource) new S3ObjectResource(repo.getRoot(),
                                        filename + "*")));
                        statements.add(statement);
                    }
                }
            }
            if (policy == null && isPublic) {
                policy = new Policy("n3phele-" + repo.getRoot());
                Statement statement = new Statement(Effect.Allow);
                statement.setId("n3phele");
                statement.setPrincipals(Arrays.asList(new Principal("*")));
                statement.setActions(Arrays.asList((Action) S3Actions.GetObject));
                statement.setResources(Arrays.asList(
                        (com.amazonaws.auth.policy.Resource) new S3ObjectResource(repo.getRoot(), filename + "*")));
                statements.add(statement);
            }
            if (policy != null) {
                if (statements.size() != 0) {
                    policy.setStatements(statements);
                    s3.setBucketPolicy(repo.getRoot(), policy.toJson());
                    log.info("Set policy " + policy.toJson());
                } else {
                    s3.deleteBucketPolicy(repo.getRoot());
                }
            }
            return true;

        } catch (AmazonServiceException e) {
            log.log(Level.WARNING, "Service Error processing " + repo, e);
        } catch (AmazonClientException e) {
            log.log(Level.SEVERE, "Client Error processing " + repo, e);
        } catch (IllegalArgumentException e) {
            log.log(Level.SEVERE, "parse error ", e);
            log.log(Level.SEVERE, "cause", e.getCause());
        }
        return false;
    }

    public boolean checkExists(Repository repo, String filename) {
        boolean result = false;
        Credential credential = repo.getCredential().decrypt();

        AmazonS3Client s3 = new AmazonS3Client(
                new BasicAWSCredentials(credential.getAccount(), credential.getSecret()));
        s3.setEndpoint(repo.getTarget().toString());
        try {
            ObjectMetadata metadata = s3.getObjectMetadata(repo.getRoot(), filename);
            log.info("Exists " + metadata.getContentType());
            return true;
        } catch (AmazonServiceException e) {
            log.log(Level.WARNING, "Service Error processing " + repo + " filename " + filename, e);
        } catch (AmazonClientException e) {
            log.log(Level.SEVERE, "Client Error processing " + repo + " filename " + filename, e);
        }
        return result;
    }

    public List<FileNode> getFileList(Repository repo, String prefix, int max) {
        List<FileNode> result = new ArrayList<FileNode>();
        Credential credential = repo.getCredential().decrypt();

        AmazonS3Client s3 = new AmazonS3Client(
                new BasicAWSCredentials(credential.getAccount(), credential.getSecret()));
        s3.setEndpoint(repo.getTarget().toString());
        Policy p = null;
        try {
            BucketPolicy bp = s3.getBucketPolicy(repo.getRoot());
            log.info("Policy text " + bp.getPolicyText());
            if (bp != null && bp.getPolicyText() != null) {
                p = PolicyHelper.parse(bp.getPolicyText());
                log.info("Policy object is " + (p == null ? null : p.toJson()));
            }
        } catch (Exception e) {
            log.log(Level.WARNING, "Policy not supported", e);
        }

        try {
            ObjectListing s3Objects = s3
                    .listObjects(new ListObjectsRequest(repo.getRoot(), prefix, null, "/", max));
            if (s3Objects.getCommonPrefixes() != null) {
                for (String dirName : s3Objects.getCommonPrefixes()) {
                    String name = dirName;
                    if (dirName.endsWith("/")) {
                        name = dirName.substring(0, dirName.length() - 1);
                    }
                    boolean isPublic = isPublicFolder(p, repo.getRoot(), name);
                    name = name.substring(name.lastIndexOf("/") + 1);
                    FileNode folder = FileNode.newFolder(name, prefix, repo, isPublic);
                    log.info("Folder:" + folder);
                    result.add(folder);
                }
            }
            if (s3Objects.getObjectSummaries() != null) {
                for (S3ObjectSummary fileSummary : s3Objects.getObjectSummaries()) {
                    String key = fileSummary.getKey();
                    if (key != null && !key.equals(prefix)) {
                        String name = key.substring(key.lastIndexOf("/") + 1);
                        UriBuilder builder = UriBuilder.fromUri(repo.getTarget()).path(repo.getRoot());
                        if (prefix != null && !prefix.isEmpty()) {
                            builder = builder.path(prefix);
                        }
                        FileNode file = FileNode.newFile(name, prefix, repo, fileSummary.getLastModified(),
                                fileSummary.getSize(), builder.path(name).toString());
                        log.info("File:" + file);
                        result.add(file);
                    }
                }
            }

        } catch (AmazonServiceException e) {
            log.log(Level.WARNING, "Service Error processing " + repo, e);
        } catch (AmazonClientException e) {
            log.log(Level.SEVERE, "Client Error processing " + repo, e);
        }
        return result;
    }

    public String getType() {
        return "S3";
    }

    @Override
    public URI getRedirectURL(Repository item, String path, String filename) {
        UriBuilder result = null;
        result = UriBuilder.fromUri(item.getTarget());
        result.path(item.getRoot()).path(path).path(filename);
        String expires = Long.toString((Calendar.getInstance().getTimeInMillis() / 1000) + 60 * 60);
        String stringToSign = "GET\n\n\n" + expires + "\n" + result.build().getPath().replace(" ", "%20");
        String signature = signS3QueryString(stringToSign, item.getCredential());

        result.queryParam("AWSAccessKeyId", item.getCredential().decrypt().getAccount());
        result.queryParam("Expires", expires);
        result.queryParam("Signature", signature);
        log.warning("Access " + result.build().getPath() + " " + result.build());
        return result.build();
    }

    @Override
    public UploadSignature getUploadSignature(Repository item, String name) {
        Calendar expires = Calendar.getInstance();
        expires.add(Calendar.MINUTE, 30);
        String acl = "private";
        String policy = getPolicy(expires, item.getRoot(), name, acl);
        String base64Policy = new String(Base64.encode(policy));
        String signature = signS3QueryString(base64Policy, item.getCredential());
        String awsId = item.getCredential().decrypt().getAccount();
        String contentType = Mimetypes.getInstance().getMimetype(name);
        UriBuilder builder = UriBuilder.fromUri(item.getTarget());
        builder.path(item.getRoot());

        UploadSignature result = new UploadSignature(name, acl, builder.build(), item.getRoot(), base64Policy,
                signature, awsId, contentType);
        return result;

    }

    /**
     * @param policy
     * @param bucket
     * @param foldername
     * @return
     */
    private static boolean isPublicFolder(Policy policy, String bucket, String foldername) {
        if (policy == null)
            return false;
        String name = new S3ObjectResource(bucket, foldername).getId();
        if (policy.getStatements() != null) {
            for (Statement statement : policy.getStatements()) {
                if (statement.getId().equals("n3phele")) {
                    List<com.amazonaws.auth.policy.Resource> resources = statement.getResources();
                    if (resources != null) {
                        for (com.amazonaws.auth.policy.Resource resource : resources) {
                            String resourceKey = resource.getId();
                            if (resourceKey.endsWith("*"))
                                resourceKey = resourceKey.substring(0, resourceKey.length() - 1);
                            if ((name + "/").startsWith(resourceKey + "/"))
                                return true;
                        }
                    }
                }
            }
        }
        return false;
    }

    public String signS3QueryString(String stringToSign, Credential credential) {
        QuerySigner signer = new QuerySigner();
        return signer.sign(stringToSign, credential.decrypt().getSecret());
    }

    private static class QuerySigner extends AbstractAWSSigner {

        /* (non-Javadoc)
         * @see com.amazonaws.auth.Signer#sign(com.amazonaws.Request, com.amazonaws.auth.AWSCredentials)
         */
        @Override
        public void sign(Request<?> arg0, AWSCredentials arg1) throws AmazonClientException {
        }

        public String sign(String stringToSign, String key) {
            return sign(stringToSign, key, SigningAlgorithm.HmacSHA1);
        }

    }

    private String getPolicy(Calendar date, String bucket, String filename, String acl) {
        String expires = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").format(date.getTime());
        return String.format(
                "{\"expiration\": \"%s\"," + "\"conditions\":[" + "{\"bucket\": \"%s\"}," + "{\"key\": \"%s\"},"
                        + "{\"success_action_status\": \"201\"}," + "{\"acl\": \"%s\"},"
                        + "[\"starts-with\", \"$Content-Type\", \"\"]," + "]" + "}",
                expires, bucket, filename, acl);
    }

}