com.splicemachine.storage.HNIOFileSystem.java Source code

Java tutorial

Introduction

Here is the source code for com.splicemachine.storage.HNIOFileSystem.java

Source

/*
 * Copyright 2012 - 2016 Splice Machine, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use
 * this file except in compliance with the License. You may obtain a copy of the
 * License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed
 * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 * CONDITIONS OF ANY KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 */

package com.splicemachine.storage;

import com.splicemachine.access.api.DistributedFileSystem;
import com.splicemachine.access.api.FileInfo;
import com.splicemachine.si.api.data.ExceptionFactory;
import com.splicemachine.utils.SpliceLogUtils;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.protocol.AclException;
import org.apache.log4j.Logger;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.FileAttributeView;
import java.util.Map;
import java.util.Set;

/**
 * @author Scott Fines
 *         Date: 1/21/16
 */
public class HNIOFileSystem extends DistributedFileSystem {
    private final org.apache.hadoop.fs.FileSystem fs;
    private final boolean isDistributedFS;
    private final ExceptionFactory exceptionFactory;
    private static Logger LOG = Logger.getLogger(HNIOFileSystem.class);

    public HNIOFileSystem(org.apache.hadoop.fs.FileSystem fs, ExceptionFactory ef) {
        this.fs = fs;
        this.exceptionFactory = ef;
        this.isDistributedFS = (fs instanceof org.apache.hadoop.hdfs.DistributedFileSystem);
    }

    @Override
    public void delete(Path path) throws IOException {
        delete(path, false);
    }

    @Override
    public void delete(Path path, boolean recursive) throws IOException {
        fs.delete(toHPath(path), recursive);
    }

    @Override
    public void delete(String dir, boolean recursive) throws IOException {
        if (LOG.isTraceEnabled())
            SpliceLogUtils.trace(LOG, "delete(): dir=%s, recursive=%s", dir, recursive);
        org.apache.hadoop.fs.Path p = new org.apache.hadoop.fs.Path(dir);
        boolean result = fs.delete(p, recursive);
        if (LOG.isTraceEnabled())
            SpliceLogUtils.trace(LOG, "delete(): dir=%s, recursive=%s, result=%s", dir, recursive, result);
    }

    @Override
    public void delete(String dir, String fileName, boolean recursive) throws IOException {
        if (LOG.isTraceEnabled())
            SpliceLogUtils.trace(LOG, "delete(): dir=%s, fileName=%s, recursive=%s", dir, fileName, recursive);
        org.apache.hadoop.fs.Path p = new org.apache.hadoop.fs.Path(dir, fileName);
        boolean result = fs.delete(p, recursive);
        if (LOG.isTraceEnabled())
            SpliceLogUtils.trace(LOG, "delete(): dir=%s, fileName=%s, recursive=%s, result=%s", dir, fileName,
                    recursive, result);
    }

    public String[] getExistingFiles(String dir, String filePattern) throws IOException {
        FileStatus[] statuses = fs.globStatus(new org.apache.hadoop.fs.Path(dir, filePattern));
        String[] files = new String[statuses.length];
        int index = 0;
        for (FileStatus status : statuses) {
            if (status != null && status.getPath() != null)
                files[index++] = status.getPath().getName();
        }
        return files;
    }

    @Override
    public Path getPath(String directory, String fileName) {
        return Paths.get(directory, fileName);
    }

    @Override
    public Path getPath(String fullPath) {
        return Paths.get(fullPath);
    }

    @Override
    public FileInfo getInfo(String filePath) throws IOException {
        org.apache.hadoop.fs.Path f = new org.apache.hadoop.fs.Path(filePath);
        ContentSummary contentSummary;
        try {
            contentSummary = fs.getContentSummary(f);
        } catch (IOException ioe) {
            LOG.error(
                    "Unexpected error getting content summary. We ignore it for now, but you should probably check it out:",
                    ioe);
            contentSummary = new ContentSummary(0L, 0L, 0L);

        }
        return new HFileInfo(f, contentSummary);
    }

    @Override
    public String getScheme() {
        return fs.getScheme();
    }

    @Override
    public FileSystem newFileSystem(URI uri, Map<String, ?> env) throws IOException {
        throw new UnsupportedOperationException("IMPLEMENT");
    }

    @Override
    public FileSystem getFileSystem(URI uri) {
        throw new UnsupportedOperationException("IMPLEMENT");
    }

    @Override
    public Path getPath(URI uri) {
        return Paths.get(uri);
    }

    @Override
    public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options,
            FileAttribute<?>... attrs) throws IOException {
        throw new UnsupportedOperationException();
    }

    @Override
    public DirectoryStream<Path> newDirectoryStream(Path dir, DirectoryStream.Filter<? super Path> filter)
            throws IOException {
        throw new UnsupportedOperationException();
    }

    @Override
    public OutputStream newOutputStream(Path path, OpenOption... options) throws IOException {
        return fs.create(toHPath(path));
    }

    @Override
    public OutputStream newOutputStream(String dir, String fileName, OpenOption... options) throws IOException {
        org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(dir, fileName);
        return fs.create(path);
    }

    @Override
    public OutputStream newOutputStream(String fullPath, OpenOption... options) throws IOException {
        org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(fullPath);
        return fs.create(path);
    }

    @Override
    public void createDirectory(Path dir, FileAttribute<?>... attrs) throws IOException {
        org.apache.hadoop.fs.Path f = toHPath(dir);
        if (LOG.isTraceEnabled())
            SpliceLogUtils.trace(LOG, "createDirectory(): path=%s", f);
        try {
            FileStatus fileStatus = fs.getFileStatus(f);
            throw new FileAlreadyExistsException(dir.toString());
        } catch (FileNotFoundException fnfe) {
            fs.mkdirs(f);
        }
    }

    @Override
    public boolean createDirectory(Path path, boolean errorIfExists) throws IOException {
        org.apache.hadoop.fs.Path f = toHPath(path);
        if (LOG.isTraceEnabled())
            SpliceLogUtils.trace(LOG, "createDirectory(): path=%s", f);
        try {
            FileStatus fileStatus = fs.getFileStatus(f);
            return !errorIfExists && fileStatus.isDirectory();
        } catch (FileNotFoundException fnfe) {
            return fs.mkdirs(f);
        }
    }

    @Override
    public boolean createDirectory(String fullPath, boolean errorIfExists) throws IOException {
        boolean isTrace = LOG.isTraceEnabled();
        if (isTrace)
            SpliceLogUtils.trace(LOG, "createDirectory(): path string=%s", fullPath);
        org.apache.hadoop.fs.Path f = new org.apache.hadoop.fs.Path(fullPath);
        if (isTrace)
            SpliceLogUtils.trace(LOG, "createDirectory(): hdfs path=%s", f);
        try {
            FileStatus fileStatus = fs.getFileStatus(f);
            if (isTrace)
                SpliceLogUtils.trace(LOG, "createDirectory(): file status=%s", fileStatus);
            return !errorIfExists && fileStatus.isDirectory();
        } catch (FileNotFoundException fnfe) {
            if (isTrace)
                SpliceLogUtils.trace(LOG, "createDirectory(): directory not found so we will create it: %s", f);
            boolean created = fs.mkdirs(f);
            if (isTrace)
                SpliceLogUtils.trace(LOG, "createDirectory(): created=%s", created);
            return created;
        }
    }

    @Override
    public void touchFile(Path path) throws IOException {
        if (!fs.createNewFile(toHPath(path))) {
            throw new FileAlreadyExistsException(path.toString());
        }
    }

    @Override
    public void touchFile(String dir, String fileName) throws IOException {
        org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(dir, fileName);
        if (!fs.createNewFile(path)) {
            throw new FileAlreadyExistsException(path.toString());
        }
    }

    @Override
    public void concat(Path target, Path... sources) throws IOException {
        org.apache.hadoop.fs.Path[] srcPaths = new org.apache.hadoop.fs.Path[sources.length];
        for (int i = 0; i < sources.length; i++) {
            srcPaths[i] = new org.apache.hadoop.fs.Path(sources[i].getParent().toString(),
                    sources[i].getFileName().toString());
        }
        org.apache.hadoop.fs.Path targetPath = new org.apache.hadoop.fs.Path(target.getParent().toString(),
                target.getFileName().toString());

        if (isDistributedFS) {
            fs.concat(targetPath, srcPaths);
        } else {
            for (org.apache.hadoop.fs.Path src : srcPaths) {
                fs.copyFromLocalFile(true, false, src, targetPath);
            }
        }
    }

    @Override
    public void copy(Path source, Path target, CopyOption... options) throws IOException {
        throw new UnsupportedOperationException("IMPLEMENT");
    }

    @Override
    public void move(Path source, Path target, CopyOption... options) throws IOException {
        throw new UnsupportedOperationException("IMPLEMENT");
    }

    @Override
    public boolean isSameFile(Path path, Path path2) throws IOException {
        return path.equals(path2);
    }

    @Override
    public boolean isHidden(Path path) throws IOException {
        return false;
    }

    @Override
    public FileStore getFileStore(Path path) throws IOException {
        throw new UnsupportedOperationException("IMPLEMENT");
    }

    @Override
    public void checkAccess(Path path, AccessMode... modes) throws IOException {
        throw new UnsupportedOperationException("IMPLEMENT");
    }

    @Override
    public <V extends FileAttributeView> V getFileAttributeView(Path path, Class<V> type, LinkOption... options) {
        throw new UnsupportedOperationException("IMPLEMENT");
    }

    @Override
    public <A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options)
            throws IOException {
        throw new UnsupportedOperationException("IMPLEMENT");
    }

    @Override
    public Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options)
            throws IOException {
        throw new UnsupportedOperationException("IMPLEMENT");
    }

    @Override
    public void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException {
        throw new UnsupportedOperationException("IMPLEMENT");
    }

    /* *************************************************************************************/
    /*private helper methods*/
    private org.apache.hadoop.fs.Path toHPath(Path path) {
        return new org.apache.hadoop.fs.Path(path.toUri());
    }

    private class HFileInfo implements FileInfo {
        private final boolean isDir;
        private final AclStatus aclStatus;
        private final boolean isReadable;
        private final boolean isWritable;
        private org.apache.hadoop.fs.Path path;
        private ContentSummary contentSummary;

        public HFileInfo(org.apache.hadoop.fs.Path path, ContentSummary contentSummary) throws IOException {
            this.path = path;
            this.contentSummary = contentSummary;
            this.isDir = fs.isDirectory(path);
            AclStatus aclS;
            try {
                aclS = fs.getAclStatus(path);
            } catch (UnsupportedOperationException | AclException e) { // Runtime Exception for RawFS
                aclS = new AclStatus.Builder().owner("unknown").group("unknown").build();
            } catch (Exception e) {
                e = exceptionFactory.processRemoteException(e); //strip any multi-retry errors out
                //noinspection ConstantConditions
                if (e instanceof UnsupportedOperationException || e instanceof AclException) {
                    /*
                     * Some Filesystems don't support aclStatus. In that case,
                     * we replace it with our own ACL status object
                     */
                    aclS = new AclStatus.Builder().owner("unknown").group("unknown").build();
                } else {
                    /*
                     * the remaining errors are of the category of FileNotFound,UnresolvedPath,
                     * etc. These are environmental, so we should throw up here.
                     */
                    throw new IOException(e);
                }
            }
            this.aclStatus = aclS;
            boolean readable;
            try {
                fs.access(path, FsAction.READ);
                readable = true;
            } catch (IOException ioe) {
                readable = false;
            }
            boolean writable;
            try {
                fs.access(path, FsAction.WRITE);
                writable = true;
            } catch (IOException ioe) {
                writable = false;
            }
            this.isReadable = readable;
            this.isWritable = writable;
        }

        @Override
        public String fileName() {
            return path.getName();
        }

        @Override
        public String fullPath() {
            return path.toString();
        }

        @Override
        public boolean isDirectory() {
            return isDir;
        }

        @Override
        public long fileCount() {
            return contentSummary.getFileCount();
        }

        @Override
        public long spaceConsumed() {
            return contentSummary.getSpaceConsumed();
        }

        @Override
        public long size() {
            return contentSummary.getLength();
        }

        @Override
        public boolean isReadable() {
            return isReadable;
        }

        @Override
        public String getUser() {
            return aclStatus.getOwner();
        }

        @Override
        public String getGroup() {
            return aclStatus.getGroup();
        }

        @Override
        public boolean isWritable() {
            return isWritable;
        }

        @Override
        public String toSummary() { // FileUtils.byteCountToDisplaySize
            StringBuilder sb = new StringBuilder();
            sb.append(this.isDirectory() ? "Directory = " : "File = ").append(fullPath());
            sb.append("\nFile Count = ").append(contentSummary.getFileCount());
            sb.append("\nSize = ").append(FileUtils.byteCountToDisplaySize(this.size()));
            // Not important to display here, but keep it around in case.
            // For import we only care about the actual file size, not space consumed.
            // if (this.spaceConsumed() != this.size())
            //     sb.append("\nSpace Consumed = ").append(FileUtils.byteCountToDisplaySize(this.spaceConsumed()));
            return sb.toString();
        }

        @Override
        public boolean exists() {
            try {
                return fs.exists(path);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
    }
}