io.pravega.segmentstore.storage.impl.extendeds3.S3FileSystemImpl.java Source code

Java tutorial

Introduction

Here is the source code for io.pravega.segmentstore.storage.impl.extendeds3.S3FileSystemImpl.java

Source

/**
 * Copyright (c) 2017 Dell Inc., or its subsidiaries. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 */
package io.pravega.segmentstore.storage.impl.extendeds3;

import com.emc.object.Range;
import com.emc.object.s3.S3Exception;
import com.emc.object.s3.S3ObjectMetadata;
import com.emc.object.s3.bean.AccessControlList;
import com.emc.object.s3.bean.CompleteMultipartUploadResult;
import com.emc.object.s3.bean.CopyPartResult;
import com.emc.object.s3.bean.DeleteObjectsResult;
import com.emc.object.s3.bean.GetObjectResult;
import com.emc.object.s3.bean.ListObjectsResult;
import com.emc.object.s3.bean.PutObjectResult;
import com.emc.object.s3.bean.S3Object;
import com.emc.object.s3.request.CompleteMultipartUploadRequest;
import com.emc.object.s3.request.CopyPartRequest;
import com.emc.object.s3.request.DeleteObjectsRequest;
import com.emc.object.s3.request.PutObjectRequest;
import com.emc.object.s3.request.SetObjectAclRequest;
import io.pravega.common.io.StreamHelpers;
import java.io.ByteArrayInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Date;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import lombok.Synchronized;
import org.apache.commons.httpclient.HttpStatus;

/**
 * File system based implementation of extended S3 emulation API.
 */
public class S3FileSystemImpl extends S3ImplBase {
    private final String baseDir;
    private final ConcurrentMap<String, ConcurrentMap<Integer, CopyPartRequest>> multipartUploads = new ConcurrentHashMap<>();

    public S3FileSystemImpl(String baseDir) {
        this.baseDir = baseDir;
    }

    @Override
    public void start() throws Exception {
    }

    @Override
    public void stop() throws Exception {
    }

    @Override
    public PutObjectResult putObject(PutObjectRequest request) {

        if (request.getObjectMetadata() != null) {
            request.setObjectMetadata(null);
        }
        try {
            Path path = Paths.get(this.baseDir, request.getBucketName(), request.getKey());
            Files.createDirectories(path.getParent());
            Files.createFile(path);
        } catch (IOException e) {
            throw new S3Exception(e.getMessage(), 0, e);
        }
        PutObjectResult retVal = new PutObjectResult();
        if (request.getAcl() != null) {
            long size = 0;
            if (request.getRange() != null) {
                size = request.getRange().getLast() + 1;
            }
            aclMap.putIfAbsent(request.getKey(), new AclSize(request.getAcl(), size));
        }
        return retVal;
    }

    @Synchronized
    @Override
    public void putObject(String bucketName, String key, Range range, Object content) {

        Path path = Paths.get(this.baseDir, bucketName, key);
        try (FileChannel channel = FileChannel.open(path, StandardOpenOption.WRITE)) {

            long startOffset = range.getFirst();
            long length = range.getLast() + 1 - range.getFirst();
            do {
                long bytesTransferred = channel.transferFrom(Channels.newChannel((InputStream) content),
                        range.getFirst(), range.getLast() + 1 - range.getFirst());
                length -= bytesTransferred;
                startOffset += bytesTransferred;
            } while (length > 0);

            AclSize aclKey = aclMap.get(key);
            aclMap.put(key, aclKey.withSize(range.getLast() + 1));
        } catch (IOException e) {
            throw new S3Exception("NoObject", 404, "NoSuchKey", key);
        }
    }

    @Synchronized
    @Override
    public void setObjectAcl(String bucketName, String key, AccessControlList acl) {
        AclSize retVal = aclMap.get(key);
        if (retVal == null) {
            throw new S3Exception("NoObject", HttpStatus.SC_NOT_FOUND, "NoSuchKey", key);
        }
        aclMap.put(key, retVal.withAcl(acl));
    }

    @Synchronized
    @Override
    public void setObjectAcl(SetObjectAclRequest request) {
        AclSize retVal = aclMap.get(request.getKey());
        if (retVal == null) {
            throw new S3Exception("NoObject", HttpStatus.SC_NOT_FOUND, "NoSuchKey", request.getKey());
        }
        aclMap.put(request.getKey(), retVal.withAcl(request.getAcl()));
    }

    @Override
    public AccessControlList getObjectAcl(String bucketName, String key) {
        AclSize retVal = aclMap.get(key);
        if (retVal == null) {
            throw new S3Exception("NoObject", HttpStatus.SC_NOT_FOUND, "NoSuchKey", key);
        }
        return retVal.getAcl();
    }

    @Override
    public CopyPartResult copyPart(CopyPartRequest request) {
        Map<Integer, CopyPartRequest> partMap = multipartUploads.get(request.getKey());
        if (partMap == null) {
            throw new S3Exception("NoSuchKey", HttpStatus.SC_NOT_FOUND, "NoSuchKey", "");
        }
        partMap.put(request.getPartNumber(), request);
        CopyPartResult result = new CopyPartResult();
        result.setPartNumber(request.getPartNumber());
        result.setETag(request.getUploadId());
        return result;
    }

    @Override
    public void deleteObject(String bucketName, String key) {

    }

    @Override
    public DeleteObjectsResult deleteObject(DeleteObjectsRequest request) {
        return null;
    }

    @Override
    public ListObjectsResult listObjects(String bucketName, String prefix) {
        ListObjectsResult result = new ListObjectsResult();
        ArrayList<S3Object> list = new ArrayList<>();
        Path path = Paths.get(this.baseDir, bucketName, prefix);
        try {
            if (Files.exists(path)) {
                if (Files.isDirectory(path)) {
                    Files.list(path).forEach((file) -> {
                        addFileAsObjectToList(file, list, bucketName);
                    });
                } else {
                    addFileAsObjectToList(path, list, bucketName);
                }
            }
        } catch (IOException e) {
            throw new S3Exception("NoSuchKey", HttpStatus.SC_NOT_FOUND, "NoSuchKey", "");
        }
        result.setObjects(list);
        return result;
    }

    @Override
    public S3ObjectMetadata getObjectMetadata(String bucketName, String key) {
        S3ObjectMetadata metadata = new S3ObjectMetadata();
        AclSize data = aclMap.get(key);
        if (data == null) {
            throw new S3Exception("NoSuchKey", HttpStatus.SC_NOT_FOUND, "NoSuchKey", "");
        }
        metadata.setContentLength(data.getSize());
        Path path = Paths.get(this.baseDir, bucketName, key);
        metadata.setLastModified(new Date(path.toFile().lastModified()));
        return metadata;
    }

    @Override
    public InputStream readObjectStream(String bucketName, String key, Range range) {
        byte[] bytes = new byte[Math.toIntExact(range.getLast() + 1 - range.getFirst())];
        Path path = Paths.get(this.baseDir, bucketName, key);
        FileInputStream returnStream;
        try {
            returnStream = new FileInputStream(path.toFile());
            if (range.getFirst() != 0) {
                long bytesSkipped = 0;
                do {
                    bytesSkipped += returnStream.skip(range.getFirst());
                } while (bytesSkipped < range.getFirst());
            }
            StreamHelpers.readAll(returnStream, bytes, 0, bytes.length);
            return new ByteArrayInputStream(bytes);
        } catch (IOException e) {
            throw new S3Exception("NoSuchKey", HttpStatus.SC_NOT_FOUND, "NoSuchKey", "");
        }
    }

    //Adds a single file to the list as an object.
    private void addFileAsObjectToList(Path path, ArrayList<S3Object> list, String bucketName) {
        S3Object object = new S3Object();
        object.setKey(path.toString().replaceFirst(Paths.get(this.baseDir, bucketName).toString(), ""));
        list.add(object);
    }

    @Override
    public String initiateMultipartUpload(String bucketName, String key) {
        multipartUploads.put(key, new ConcurrentHashMap<>());
        return Integer.toString(multipartUploads.size());
    }

    @Synchronized
    @Override
    public CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request) {
        Map<Integer, CopyPartRequest> partMap = multipartUploads.get(request.getKey());
        if (partMap == null) {
            throw new S3Exception("NoSuchKey", HttpStatus.SC_NOT_FOUND, "NoSuchKey", "");
        }
        try {
            partMap.forEach((index, copyPart) -> {
                if (copyPart.getKey() != copyPart.getSourceKey()) {
                    Path sourcePath = Paths.get(this.baseDir, copyPart.getBucketName(), copyPart.getSourceKey());
                    Path targetPath = Paths.get(this.baseDir, copyPart.getBucketName(), copyPart.getKey());
                    try (FileChannel sourceChannel = FileChannel.open(sourcePath, StandardOpenOption.READ);
                            FileChannel targetChannel = FileChannel.open(targetPath, StandardOpenOption.WRITE)) {
                        targetChannel.transferFrom(sourceChannel, Files.size(targetPath),
                                copyPart.getSourceRange().getLast() + 1 - copyPart.getSourceRange().getFirst());
                        targetChannel.close();
                        AclSize aclMap = this.aclMap.get(copyPart.getKey());
                        this.aclMap.put(copyPart.getKey(), aclMap.withSize(Files.size(targetPath)));
                    } catch (IOException e) {
                        throw new S3Exception("NoSuchKey", 404, "NoSuchKey", "");
                    }
                }
            });
        } finally {
            multipartUploads.remove(request.getKey());
        }

        return new CompleteMultipartUploadResult();
    }

    @Override
    public GetObjectResult<InputStream> getObject(String bucketName, String key) {
        if (aclMap.containsKey(key)) {
            return new GetObjectResult<>();
        } else {
            return null;
        }
    }
}