org.apache.hadoop.io.IOUtils.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.io.IOUtils.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.io;

import java.io.*;
import java.lang.reflect.Constructor;
import java.net.Socket;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.file.DirectoryStream;
import java.nio.file.DirectoryIteratorException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.logging.Log;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.util.Shell;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;

/**
 * An utility class for I/O related functionality. 
 */
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class IOUtils {
    public static final Logger LOG = LoggerFactory.getLogger(IOUtils.class);

    /**
     * Copies from one stream to another.
     *
     * @param in InputStrem to read from
     * @param out OutputStream to write to
     * @param buffSize the size of the buffer 
     * @param close whether or not close the InputStream and 
     * OutputStream at the end. The streams are closed in the finally clause.  
     */
    public static void copyBytes(InputStream in, OutputStream out, int buffSize, boolean close) throws IOException {
        try {
            copyBytes(in, out, buffSize);
            if (close) {
                out.close();
                out = null;
                in.close();
                in = null;
            }
        } finally {
            if (close) {
                closeStream(out);
                closeStream(in);
            }
        }
    }

    /**
     * Copies from one stream to another.
     * 
     * @param in InputStrem to read from
     * @param out OutputStream to write to
     * @param buffSize the size of the buffer 
     */
    public static void copyBytes(InputStream in, OutputStream out, int buffSize) throws IOException {
        PrintStream ps = out instanceof PrintStream ? (PrintStream) out : null;
        byte buf[] = new byte[buffSize];
        int bytesRead = in.read(buf);
        while (bytesRead >= 0) {
            out.write(buf, 0, bytesRead);
            if ((ps != null) && ps.checkError()) {
                throw new IOException("Unable to write to output stream.");
            }
            bytesRead = in.read(buf);
        }
    }

    /**
     * Copies from one stream to another. <strong>closes the input and output streams 
     * at the end</strong>.
     *
     * @param in InputStrem to read from
     * @param out OutputStream to write to
     * @param conf the Configuration object 
     */
    public static void copyBytes(InputStream in, OutputStream out, Configuration conf) throws IOException {
        copyBytes(in, out, conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT), true);
    }

    /**
     * Copies from one stream to another.
     *
     * @param in InputStream to read from
     * @param out OutputStream to write to
     * @param conf the Configuration object
     * @param close whether or not close the InputStream and 
     * OutputStream at the end. The streams are closed in the finally clause.
     */
    public static void copyBytes(InputStream in, OutputStream out, Configuration conf, boolean close)
            throws IOException {
        copyBytes(in, out, conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT), close);
    }

    /**
     * Copies count bytes from one stream to another.
     *
     * @param in InputStream to read from
     * @param out OutputStream to write to
     * @param count number of bytes to copy
     * @param close whether to close the streams
     * @throws IOException if bytes can not be read or written
     */
    public static void copyBytes(InputStream in, OutputStream out, long count, boolean close) throws IOException {
        byte buf[] = new byte[4096];
        long bytesRemaining = count;
        int bytesRead;

        try {
            while (bytesRemaining > 0) {
                int bytesToRead = (int) (bytesRemaining < buf.length ? bytesRemaining : buf.length);

                bytesRead = in.read(buf, 0, bytesToRead);
                if (bytesRead == -1)
                    break;

                out.write(buf, 0, bytesRead);
                bytesRemaining -= bytesRead;
            }
            if (close) {
                out.close();
                out = null;
                in.close();
                in = null;
            }
        } finally {
            if (close) {
                closeStream(out);
                closeStream(in);
            }
        }
    }

    /**
     * Utility wrapper for reading from {@link InputStream}. It catches any errors
     * thrown by the underlying stream (either IO or decompression-related), and
     * re-throws as an IOException.
     * 
     * @param is - InputStream to be read from
     * @param buf - buffer the data is read into
     * @param off - offset within buf
     * @param len - amount of data to be read
     * @return number of bytes read
     */
    public static int wrappedReadForCompressedData(InputStream is, byte[] buf, int off, int len)
            throws IOException {
        try {
            return is.read(buf, off, len);
        } catch (IOException ie) {
            throw ie;
        } catch (Throwable t) {
            throw new IOException("Error while reading compressed data", t);
        }
    }

    /**
     * Reads len bytes in a loop.
     *
     * @param in InputStream to read from
     * @param buf The buffer to fill
     * @param off offset from the buffer
     * @param len the length of bytes to read
     * @throws IOException if it could not read requested number of bytes 
     * for any reason (including EOF)
     */
    public static void readFully(InputStream in, byte[] buf, int off, int len) throws IOException {
        int toRead = len;
        while (toRead > 0) {
            int ret = in.read(buf, off, toRead);
            if (ret < 0) {
                throw new IOException("Premature EOF from inputStream");
            }
            toRead -= ret;
            off += ret;
        }
    }

    /**
     * Similar to readFully(). Skips bytes in a loop.
     * @param in The InputStream to skip bytes from
     * @param len number of bytes to skip.
     * @throws IOException if it could not skip requested number of bytes 
     * for any reason (including EOF)
     */
    public static void skipFully(InputStream in, long len) throws IOException {
        long amt = len;
        while (amt > 0) {
            long ret = in.skip(amt);
            if (ret == 0) {
                // skip may return 0 even if we're not at EOF.  Luckily, we can 
                // use the read() method to figure out if we're at the end.
                int b = in.read();
                if (b == -1) {
                    throw new EOFException(
                            "Premature EOF from inputStream after " + "skipping " + (len - amt) + " byte(s).");
                }
                ret = 1;
            }
            amt -= ret;
        }
    }

    /**
     * Close the Closeable objects and <b>ignore</b> any {@link Throwable} or
     * null pointers. Must only be used for cleanup in exception handlers.
     *
     * @param log the log to record problems to at debug level. Can be null.
     * @param closeables the objects to close
     * @deprecated use {@link #cleanupWithLogger(Logger, java.io.Closeable...)}
     * instead
     */
    @Deprecated
    public static void cleanup(Log log, java.io.Closeable... closeables) {
        for (java.io.Closeable c : closeables) {
            if (c != null) {
                try {
                    c.close();
                } catch (Throwable e) {
                    if (log != null && log.isDebugEnabled()) {
                        log.debug("Exception in closing " + c, e);
                    }
                }
            }
        }
    }

    /**
     * Close the Closeable objects and <b>ignore</b> any {@link Throwable} or
     * null pointers. Must only be used for cleanup in exception handlers.
     *
     * @param logger the log to record problems to at debug level. Can be null.
     * @param closeables the objects to close
     */
    public static void cleanupWithLogger(Logger logger, java.io.Closeable... closeables) {
        for (java.io.Closeable c : closeables) {
            if (c != null) {
                try {
                    c.close();
                } catch (Throwable e) {
                    if (logger != null) {
                        logger.debug("Exception in closing {}", c, e);
                    }
                }
            }
        }
    }

    /**
     * Closes the stream ignoring {@link Throwable}.
     * Must only be called in cleaning up from exception handlers.
     *
     * @param stream the Stream to close
     */
    public static void closeStream(java.io.Closeable stream) {
        if (stream != null) {
            cleanupWithLogger(null, stream);
        }
    }

    /**
     * Closes the streams ignoring {@link Throwable}.
     * Must only be called in cleaning up from exception handlers.
     *
     * @param streams the Streams to close
     */
    public static void closeStreams(java.io.Closeable... streams) {
        if (streams != null) {
            cleanupWithLogger(null, streams);
        }
    }

    /**
     * Closes the socket ignoring {@link IOException}
     *
     * @param sock the Socket to close
     */
    public static void closeSocket(Socket sock) {
        if (sock != null) {
            try {
                sock.close();
            } catch (IOException ignored) {
                LOG.debug("Ignoring exception while closing socket", ignored);
            }
        }
    }

    /**
     * The /dev/null of OutputStreams.
     */
    public static class NullOutputStream extends OutputStream {
        @Override
        public void write(byte[] b, int off, int len) throws IOException {
        }

        @Override
        public void write(int b) throws IOException {
        }
    }

    /**
     * Write a ByteBuffer to a WritableByteChannel, handling short writes.
     * 
     * @param bc               The WritableByteChannel to write to
     * @param buf              The input buffer
     * @throws IOException     On I/O error
     */
    public static void writeFully(WritableByteChannel bc, ByteBuffer buf) throws IOException {
        do {
            bc.write(buf);
        } while (buf.remaining() > 0);
    }

    /**
     * Write a ByteBuffer to a FileChannel at a given offset, 
     * handling short writes.
     * 
     * @param fc               The FileChannel to write to
     * @param buf              The input buffer
     * @param offset           The offset in the file to start writing at
     * @throws IOException     On I/O error
     */
    public static void writeFully(FileChannel fc, ByteBuffer buf, long offset) throws IOException {
        do {
            offset += fc.write(buf, offset);
        } while (buf.remaining() > 0);
    }

    /**
     * Return the complete list of files in a directory as strings.<p/>
     *
     * This is better than File#listDir because it does not ignore IOExceptions.
     *
     * @param dir              The directory to list.
     * @param filter           If non-null, the filter to use when listing
     *                         this directory.
     * @return                 The list of files in the directory.
     *
     * @throws IOException     On I/O error
     */
    public static List<String> listDirectory(File dir, FilenameFilter filter) throws IOException {
        ArrayList<String> list = new ArrayList<String>();
        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir.toPath())) {
            for (Path entry : stream) {
                Path fileName = entry.getFileName();
                if (fileName != null) {
                    String fileNameStr = fileName.toString();
                    if ((filter == null) || filter.accept(dir, fileNameStr)) {
                        list.add(fileNameStr);
                    }
                }
            }
        } catch (DirectoryIteratorException e) {
            throw e.getCause();
        }
        return list;
    }

    /**
     * Ensure that any writes to the given file is written to the storage device
     * that contains it. This method opens channel on given File and closes it
     * once the sync is done.<br>
     * Borrowed from Uwe Schindler in LUCENE-5588
     * @param fileToSync the file to fsync
     */
    public static void fsync(File fileToSync) throws IOException {
        if (!fileToSync.exists()) {
            throw new FileNotFoundException("File/Directory " + fileToSync.getAbsolutePath() + " does not exist");
        }
        boolean isDir = fileToSync.isDirectory();

        // HDFS-13586, FileChannel.open fails with AccessDeniedException
        // for any directory, ignore.
        if (isDir && Shell.WINDOWS) {
            return;
        }

        // If the file is a directory we have to open read-only, for regular files
        // we must open r/w for the fsync to have an effect. See
        // http://blog.httrack.com/blog/2013/11/15/
        // everything-you-always-wanted-to-know-about-fsync/
        try (FileChannel channel = FileChannel.open(fileToSync.toPath(),
                isDir ? StandardOpenOption.READ : StandardOpenOption.WRITE)) {
            fsync(channel, isDir);
        }
    }

    /**
     * Ensure that any writes to the given file is written to the storage device
     * that contains it. This method opens channel on given File and closes it
     * once the sync is done.
     * Borrowed from Uwe Schindler in LUCENE-5588
     * @param channel Channel to sync
     * @param isDir if true, the given file is a directory (Channel should be
     *          opened for read and ignore IOExceptions, because not all file
     *          systems and operating systems allow to fsync on a directory)
     * @throws IOException
     */
    public static void fsync(FileChannel channel, boolean isDir) throws IOException {
        try {
            channel.force(true);
        } catch (IOException ioe) {
            if (isDir) {
                assert !(Shell.LINUX || Shell.MAC) : "On Linux and MacOSX fsyncing a directory"
                        + " should not throw IOException, we just don't want to rely"
                        + " on that in production (undocumented)" + ". Got: " + ioe;
                // Ignore exception if it is a directory
                return;
            }
            // Throw original exception
            throw ioe;
        }
    }

    /**
     * Takes an IOException, file/directory path, and method name and returns an
     * IOException with the input exception as the cause and also include the
     * file,method details. The new exception provides the stack trace of the
     * place where the exception is thrown and some extra diagnostics
     * information.
     *
     * Return instance of same exception if exception class has a public string
     * constructor; Otherwise return an PathIOException.
     * InterruptedIOException and PathIOException are returned unwrapped.
     *
     * @param path file/directory path
     * @param methodName method name
     * @param exception the caught exception.
     * @return an exception to throw
     */
    public static IOException wrapException(final String path, final String methodName,
            final IOException exception) {

        if (exception instanceof InterruptedIOException || exception instanceof PathIOException) {
            return exception;
        } else {
            String msg = String.format("Failed with %s while processing file/directory :[%s] in " + "method:[%s]",
                    exception.getClass().getName(), path, methodName);
            try {
                return wrapWithMessage(exception, msg);
            } catch (Exception ex) {
                // For subclasses which have no (String) constructor throw IOException
                // with wrapped message

                return new PathIOException(path, exception);
            }
        }
    }

    @SuppressWarnings("unchecked")
    private static <T extends IOException> T wrapWithMessage(final T exception, final String msg) throws T {
        Class<? extends Throwable> clazz = exception.getClass();
        try {
            Constructor<? extends Throwable> ctor = clazz.getConstructor(String.class);
            Throwable t = ctor.newInstance(msg);
            return (T) (t.initCause(exception));
        } catch (Throwable e) {
            LOG.warn("Unable to wrap exception of type " + clazz + ": it has no (String) constructor", e);
            throw exception;
        }
    }

    /**
     * Reads a DataInput until EOF and returns a byte array.  Make sure not to
     * pass in an infinite DataInput or this will never return.
     *
     * @param in A DataInput
     * @return a byte array containing the data from the DataInput
     * @throws IOException on I/O error, other than EOF
     */
    public static byte[] readFullyToByteArray(DataInput in) throws IOException {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        try {
            while (true) {
                baos.write(in.readByte());
            }
        } catch (EOFException eof) {
            // finished reading, do nothing
        }
        return baos.toByteArray();
    }
}