com.shareplaylearn.models.UserItemManager.java Source code

Java tutorial

Introduction

Here is the source code for com.shareplaylearn.models.UserItemManager.java

Source

/**
 * Copyright 2015-2016 Stuart Smith
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */
package com.shareplaylearn.models;

import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.*;
import com.amazonaws.util.Base64;
import com.shareplaylearn.InternalErrorException;
import com.shareplaylearn.services.ImagePreprocessorPlugin;
import com.shareplaylearn.services.SecretsService;
import com.shareplaylearn.services.UploadPreprocessor;
import com.shareplaylearn.services.UploadPreprocessorPlugin;
import com.shareplaylearn.utilities.Exceptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.*;
import java.util.*;

/**
 * Created by stu on 9/6/15.
 * Metadata about the items the user has, and possibly
 * cached values for the items.
 * Location, type, names, etc.
 * Data here should be safe to cache in Redis (userid but no auth tokens, etc).
 * Right now, we just implicitly store the metadata as part of the item path.
 * Once we have a true metadata store, this code can be greatly simplified.
 * (in particular, the getItemList() that reconstructs metadata from the path).
 */
public class UserItemManager {
    public static class AvailableEncodings {
        public static final String BASE64 = "BASE64";
        public static final String IDENTITY = "IDENTITY";

        public static boolean isAvailable(String encoding) {
            return encoding.equals(BASE64) || encoding.equals(IDENTITY);
        }
    }

    private static final String ROOT_DIR = "/root/";

    private int totalItemQuota = Limits.DEFAULT_ITEM_QUOTA;
    private HashMap<String, Integer> itemQuota;
    private Logger log;

    private String userName;
    private String userId;
    private String userDir;

    public UserItemManager(String userName, String userId) {
        this.userName = userName;
        this.userId = userId;
        this.userDir = this.getUserDir();
        this.itemQuota = new HashMap<>();
        this.itemQuota.put(ItemSchema.IMAGE_CONTENT_TYPE, Limits.DEFAULT_ITEM_QUOTA);
        this.itemQuota.put(ItemSchema.UNKNOWN_CONTENT_TYPE, Limits.DEFAULT_ITEM_QUOTA / 2);
        this.log = LoggerFactory.getLogger(UserItemManager.class);
    }

    public Response addItem(String name, byte[] item) throws InternalErrorException {
        Response quotaCheck = this.checkQuota();
        if (quotaCheck.getStatus() != 200) {
            return quotaCheck;
        }

        List<UploadPreprocessorPlugin> uploadPreprocessorPlugins = new ArrayList<>();
        uploadPreprocessorPlugins.add(new ImagePreprocessorPlugin());
        UploadPreprocessor uploadPreprocessor = new UploadPreprocessor(uploadPreprocessorPlugins);
        Map<ItemSchema.PresentationType, byte[]> uploads = uploadPreprocessor.process(item);

        if (uploads.size() == 0) {
            throw new InternalErrorException("Upload processor returned empty upload set");
        }

        UploadPreprocessorPlugin pluginUsed = uploadPreprocessor.getProcessorPluginUsed();
        String contentType = pluginUsed.getContentType();

        for (Map.Entry<ItemSchema.PresentationType, byte[]> uploadEntry : uploads.entrySet()) {
            boolean found = false;
            ItemSchema.PresentationType presentationType = uploadEntry.getKey();
            for (ItemSchema.PresentationType type : ItemSchema.PRESENTATION_TYPES) {
                if (presentationType.equals(type)) {
                    found = true;
                }
            }
            if (found) {
                //this is a little bit of a hack, but is necessary for downloads
                //using the name of the item to work
                //OK in user agents (browsers)
                if (presentationType.equals(ItemSchema.PresentationType.PREFERRED_PRESENTATION_TYPE)
                        && pluginUsed.getPreferredFileExtension() != null
                        && pluginUsed.getPreferredFileExtension().length() > 0
                        && !name.endsWith(pluginUsed.getPreferredFileExtension())) {
                    String preferredExtension = pluginUsed.getPreferredFileExtension();
                    if (!preferredExtension.startsWith(".")) {
                        preferredExtension = "." + preferredExtension;
                    }
                    int extIndex = name.lastIndexOf(".");
                    if (extIndex > 0) {
                        this.saveItem(name.substring(0, extIndex) + preferredExtension, uploadEntry.getValue(),
                                contentType, presentationType);
                    } else {
                        this.saveItem(name + preferredExtension, uploadEntry.getValue(), contentType,
                                presentationType);
                    }
                } else {
                    this.saveItem(name, uploadEntry.getValue(), contentType, presentationType);
                }
            } else {
                log.error("Upload plugin had an entry with a presentation type of: " + presentationType
                        + " that was not found in the item types defined in the ItemSchema.");
            }
        }

        return Response.status(200).build();
    }

    public Response getItem(String contentType, ItemSchema.PresentationType presentationType, String name,
            String encoding) {
        if (encoding != null && encoding.length() > 0 && !AvailableEncodings.isAvailable(encoding)) {
            return Response.status(Response.Status.UNSUPPORTED_MEDIA_TYPE)
                    .entity("Inner Encoding Type: " + encoding + " not available").build();
        }

        AmazonS3Client s3Client = new AmazonS3Client(
                new BasicAWSCredentials(SecretsService.amazonClientId, SecretsService.amazonClientSecret));

        try {
            S3Object object = s3Client.getObject(ItemSchema.S3_BUCKET,
                    getItemLocation(name, contentType, presentationType));
            try (S3ObjectInputStream inputStream = object.getObjectContent()) {
                long contentLength = object.getObjectMetadata().getContentLength();
                if (contentLength > Limits.MAX_RETRIEVE_SIZE) {
                    throw new IOException("Object is to large: " + contentLength + " bytes.");
                }
                int bufferSize = Math.min((int) contentLength, 10 * 8192);
                byte[] buffer = new byte[bufferSize];
                ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
                int bytesRead = 0;
                int totalBytesRead = 0;
                while ((bytesRead = inputStream.read(buffer)) > 0) {
                    outputStream.write(buffer, 0, bytesRead);
                    totalBytesRead += bytesRead;
                }
                log.debug("GET in file resource read: " + totalBytesRead + " bytes.");
                if (encoding == null || encoding.length() == 0 || encoding.equals(AvailableEncodings.IDENTITY)) {
                    return Response.status(Response.Status.OK).entity(outputStream.toByteArray()).build();
                } else if (encoding.equals(AvailableEncodings.BASE64)) {
                    return Response.status(Response.Status.OK)
                            .entity(Base64.encodeAsString(outputStream.toByteArray())).build();
                } else {
                    return Response.status(Response.Status.UNSUPPORTED_MEDIA_TYPE)
                            .entity("Inner Encoding Type: " + encoding + " not available").build();
                }
            }
        } catch (Exception e) {
            StringWriter sw = new StringWriter();
            PrintWriter pw = new PrintWriter(sw);
            pw.println("\nFailed to retrieve: " + name);
            e.printStackTrace(pw);
            log.warn("Failed to retrieve: " + name);
            log.info(Exceptions.asString(e));
            return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(sw.toString()).build();
        }
    }

    /**
     * Writes items to S3, and item metadata to Redis
     */
    private void saveItem(String name, byte[] itemData, String contentType,
            ItemSchema.PresentationType presentationType) throws InternalErrorException {

        String itemLocation = this.getItemLocation(name, contentType, presentationType);
        AmazonS3Client s3Client = new AmazonS3Client(
                new BasicAWSCredentials(SecretsService.amazonClientId, SecretsService.amazonClientSecret));
        ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(itemData);
        ObjectMetadata metadata = this.makeBasicMetadata(itemData.length, false, name);
        metadata.addUserMetadata(UploadMetadataFields.CONTENT_TYPE, contentType);
        //TODO: save this metadata, along with location, to local Redis
        s3Client.putObject(ItemSchema.S3_BUCKET, itemLocation, byteArrayInputStream, metadata);
    }

    private ObjectMetadata makeBasicMetadata(int bufferLength, boolean isPublic, String itemName) {
        ObjectMetadata fileMetadata = new ObjectMetadata();
        fileMetadata.setContentEncoding(MediaType.APPLICATION_OCTET_STREAM);
        if (isPublic) {
            fileMetadata.addUserMetadata(UploadMetadataFields.PUBLIC, UploadMetadataFields.TRUE_VALUE);
        } else {
            fileMetadata.addUserMetadata(UploadMetadataFields.PUBLIC, UploadMetadataFields.FALSE_VALUE);
        }
        fileMetadata.addUserMetadata(UploadMetadataFields.DISPLAY_NAME, itemName);
        fileMetadata.setContentLength(bufferLength);
        return fileMetadata;
    }

    /**
     * @return
     */
    public List<UserItem> getItemList() {
        HashMap<String, HashMap<ItemSchema.PresentationType, List<String>>> itemLocations = getItemLocations();
        List<UserItem> itemList = new ArrayList<>();

        //on a per-user basis, then name of the item should be unique,
        //but variations of it may be stored under different presentation types
        //(e.g. preview, original)
        //there is a workaround here because we may add a file extension to some preferred items
        //we work around it by verifying a preferred item doesn't have an entry that matches without extension
        //a better fix would be a metadata store fast enough to be usable that stores the original item name
        //but this works for now.
        HashMap<String, UserItem> userItems = new HashMap<>();

        for (Map.Entry<String, HashMap<ItemSchema.PresentationType, List<String>>> items : itemLocations
                .entrySet()) {
            String contentType = items.getKey();

            for (Map.Entry<ItemSchema.PresentationType, List<String>> item : items.getValue().entrySet()) {
                ItemSchema.PresentationType presentationType = item.getKey();

                List<String> locations = item.getValue();
                for (String location : locations) {
                    String[] path = location.split("/");
                    if (path.length == 0) {
                        log.warn("Found an item path/location with no directory structure: " + location);
                        continue;
                    }
                    String name = path[path.length - 1];
                    //workaround (see above)
                    int extIndex = name.lastIndexOf(".");
                    if (extIndex > 0) {
                        name = name.substring(0, extIndex);
                    }
                    log.debug("Got a location: " + location + " for item with name: " + name + " for user: "
                            + this.userName);
                    UserItem userItem = null;
                    if (!userItems.containsKey(name)) {
                        userItems.put(name, new UserItem(contentType));
                    }
                    userItem = userItems.get(name);
                    userItem.setLocation(presentationType, location);
                    if (presentationType.equals(ItemSchema.PresentationType.PREVIEW_PRESENTATION_TYPE)) {
                        userItem.addAttr("altText", "Preview of " + name);
                    }
                }
            }
        }
        for (Map.Entry<String, UserItem> userItem : userItems.entrySet()) {
            //Note that this maps to the actual name in all cases (original, preview, preferred w/out added extension)
            //except when we add an extension to a preferred format of the item.
            userItem.getValue().addAttr(UploadMetadataFields.DISPLAY_NAME, userItem.getKey());
            itemList.add(userItem.getValue());
        }
        return itemList;
    }

    public HashMap<String, HashMap<ItemSchema.PresentationType, List<String>>> getItemLocations() {
        HashMap<String, HashMap<ItemSchema.PresentationType, List<String>>> itemLocations = new HashMap<>();

        AmazonS3Client s3Client = new AmazonS3Client(
                new BasicAWSCredentials(SecretsService.amazonClientId, SecretsService.amazonClientSecret));

        for (String contentType : ItemSchema.CONTENT_TYPES) {
            for (ItemSchema.PresentationType presentationType : ItemSchema.PRESENTATION_TYPES) {

                ObjectListing listing = s3Client.listObjects(ItemSchema.S3_BUCKET,
                        this.getItemDirectory(contentType, presentationType));

                HashSet<String> locations = getExternalItemListing(listing);
                String curDirectory = makeExternalLocation(getItemDirectory(contentType, presentationType));
                for (String location : locations) {
                    //it would be nice if s3 didn't return stuff that doesn't technically match the prefix
                    //(due to trailing /), but it looks like it might
                    if (curDirectory.endsWith(location)) {
                        log.debug("Skipping location: " + location + " because it looks like a group (folder)"
                                + ", not an object");
                        continue;
                    }
                    if (!itemLocations.containsKey(contentType)) {
                        itemLocations.put(contentType, new HashMap<>());
                    }
                    if (!itemLocations.get(contentType).containsKey(presentationType)) {
                        itemLocations.get(contentType).put(presentationType, new ArrayList<>());
                    }
                    itemLocations.get(contentType).get(presentationType).add(location);
                }
            }
        }
        return itemLocations;
    }

    private HashSet<String> getExternalItemListing(ObjectListing objectListing) {
        HashSet<String> itemLocations = new HashSet<>();
        for (S3ObjectSummary obj : objectListing.getObjectSummaries()) {
            String internalPath = obj.getKey();
            String externalPath = makeExternalLocation(internalPath);
            if (externalPath != null) {
                itemLocations.add(externalPath);
                log.debug("External path was " + externalPath);
            } else {
                log.info("External path for object list was null?");
            }
        }
        return itemLocations;
    }

    /**
     * Translates an internal S3 path to the path used in the external API
     * @param internalPath
     * @return
     */
    private String makeExternalLocation(String internalPath) {
        //"/root/" is not used in the external API, strip it off
        String[] itemPath = internalPath.split("/");
        if (itemPath.length > 2) {
            String externalPath = "";
            for (int i = 0; i < itemPath.length; ++i) {
                if (itemPath[i].equals("root") && i < 2) {
                    continue;
                }
                if (itemPath[i].trim().length() == 0) {
                    continue;
                }
                externalPath += "/";
                externalPath += itemPath[i];
            }
            return externalPath;
        }
        return null;
    }

    public String getUserDir() {
        return ROOT_DIR + this.userName + "/" + this.userId + "/";
    }

    public String getItemDirectory(String contentType, ItemSchema.PresentationType presentationType) {
        return this.userDir + contentType + "/" + presentationType + "/";
    }

    public String getItemLocation(String name, String contentType, ItemSchema.PresentationType presentationType) {
        return this.getItemDirectory(contentType, presentationType) + name;
    }

    /**
     * This is not good enough. It slows things down, and still costs money.
     * Eventually, we should have an async task that updates a local cache of
     * used storage. If the cache says your below X of the limit (think atms),
     * you're good. Once you get up close, ping Amazon every time.
     * @param objectListing
     * @param maxSize
     * @return
     */
    private Response checkObjectListingSize(ObjectListing objectListing, int maxSize) {
        if (objectListing.isTruncated() && objectListing.getMaxKeys() >= maxSize) {
            log.error("Error, too many uploads");
            return Response.status(418).entity("I'm a teapot! j/k - not enough space " + maxSize).build();
        }
        if (objectListing.getObjectSummaries().size() >= maxSize) {
            log.error("Error, too many uploads");
            return Response.status(418)
                    .entity("I'm a teapot! Er, well, at least I can't hold " + maxSize + " stuff.").build();
        }
        return Response.status(Response.Status.OK).entity("OK").build();
    }

    private Response checkQuota() {
        AmazonS3Client s3Client = new AmazonS3Client(
                new BasicAWSCredentials(SecretsService.amazonClientId, SecretsService.amazonClientSecret));
        ObjectListing curList = s3Client.listObjects(ItemSchema.S3_BUCKET, this.getUserDir());
        Response listCheck;
        if ((listCheck = this.checkObjectListingSize(curList, Limits.MAX_NUM_FILES_PER_USER))
                .getStatus() != Response.Status.OK.getStatusCode()) {
            return listCheck;
        }
        ObjectListing userList = s3Client.listObjects(ItemSchema.S3_BUCKET, "/");
        if ((listCheck = this.checkObjectListingSize(userList, Limits.MAX_TOTAL_FILES))
                .getStatus() != Response.Status.OK.getStatusCode()) {
            return listCheck;
        }
        return Response.status(Response.Status.OK).build();
    }
}