tachyon.client.file.FileSystemUtils.java Source code

Java tutorial

Introduction

Here is the source code for tachyon.client.file.FileSystemUtils.java

Source

/*
 * Licensed to the University of California, Berkeley under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package tachyon.client.file;

import java.io.IOException;
import java.io.OutputStream;
import java.util.concurrent.TimeUnit;

import javax.annotation.concurrent.ThreadSafe;

import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.io.Closer;

import tachyon.Constants;
import tachyon.TachyonURI;
import tachyon.client.ClientContext;
import tachyon.client.ReadType;
import tachyon.client.file.options.OpenFileOptions;
import tachyon.client.file.options.SetAttributeOptions;
import tachyon.conf.TachyonConf;
import tachyon.exception.FileDoesNotExistException;
import tachyon.exception.TachyonException;
import tachyon.underfs.UnderFileSystem;
import tachyon.util.CommonUtils;

/**
 * Collection of utility methods to handle with {@link FileSystem} related objects
 */
@ThreadSafe
public final class FileSystemUtils {
    private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE);

    // prevent instantiation
    private FileSystemUtils() {
    }

    /**
     * Shortcut for {@code waitCompleted(fs, uri, -1, TimeUnit.MILLISECONDS)}, i.e., wait for an
     * indefinite amount of time. Note that if a file is never completed, the thread will block
     * forever, so use with care.
     *
     * @param fs a {@link FileSystem} instance
     * @param uri the URI of the file on which the thread should wait
     * @return true if the file is complete when this method returns and false if the method timed out
     *         before the file was complete.
     * @throws IOException in case there are problems contacting the Tachyon master for the file
     *         status
     * @throws TachyonException if a Tachyon Exception occurs
     * @throws InterruptedException if the thread receives an interrupt while waiting for file
     *         completion
     * @see #waitCompleted(FileSystem, TachyonURI, long, TimeUnit)
     */
    public static boolean waitCompleted(FileSystem fs, TachyonURI uri)
            throws IOException, TachyonException, InterruptedException {
        return FileSystemUtils.waitCompleted(fs, uri, -1, TimeUnit.MILLISECONDS);
    }

    /**
     * Wait for a file to be marked as completed.
     * <p/>
     * The calling thread will block for <i>at most</i> {@code timeout} time units (as specified via
     * {@code tunit} or until the file is reported as complete by the master. The method will return
     * the last known completion status of the file (hence, false only if the method has timed out). A
     * zero value on the {@code timeout} parameter will make the calling thread check once and return;
     * a negative value will make it block indefinitely. Note that, in this last case, if a file is
     * never completed, the thread will block forever, so use with care.
     * <p/>
     * Note that the file whose uri is specified, might not exist at the moment this method this call.
     * The method will deliberately block anyway for the specified amount of time, waiting for the
     * file to be created and eventually completed. Note also that the file might be moved or deleted
     * while it is waited upon. In such cases the method will throw the a {@link TachyonException}
     * with the appropriate {@link tachyon.exception.TachyonExceptionType}
     * <p/>
     * <i>IMPLEMENTATION NOTES</i> This method is implemented by periodically polling the master about
     * the file status. The polling period is controlled by the
     * {@link Constants#USER_FILE_WAITCOMPLETED_POLL_MS} java property and defaults to a generous 1
     * second.
     *
     * @param fs an instance of {@link FileSystem}
     * @param uri the URI of the file whose completion status is to be watied for
     * @param timeout maximum time the calling thread should be blocked on this call
     * @param tunit the @{link TimeUnit} instance describing the {@code timeout} parameter
     * @return true if the file is complete when this method returns and false if the method timed out
     *         before the file was complete.
     * @throws IOException in case there are problems contacting the Tachyonmaster for the file status
     * @throws TachyonException if a Tachyon Exception occurs
     * @throws InterruptedException if the thread receives an interrupt while waiting for file
     *         completion
     */
    public static boolean waitCompleted(final FileSystem fs, final TachyonURI uri, final long timeout,
            final TimeUnit tunit) throws IOException, TachyonException, InterruptedException {

        final long deadline = System.currentTimeMillis() + tunit.toMillis(timeout);
        final long pollPeriod = ClientContext.getConf().getLong(Constants.USER_FILE_WAITCOMPLETED_POLL_MS);
        boolean completed = false;
        long timeleft = deadline - System.currentTimeMillis();
        long toSleep = 0;

        while (!completed && (timeout <= 0 || timeleft > 0)) {

            if (!fs.exists(uri)) {
                LOG.debug("The file {} being waited upon does not exist yet. Waiting for it to be " + "created.",
                        uri);
            } else {
                completed = fs.getStatus(uri).isCompleted();
            }

            if (timeout == 0) {
                return completed;
            } else if (!completed) {
                if (timeout < 0 || timeleft > pollPeriod) {
                    toSleep = pollPeriod;
                } else {
                    toSleep = timeleft;
                }

                CommonUtils.sleepMs(LOG, toSleep, true);
                timeleft = deadline - System.currentTimeMillis();
            }
        }

        return completed;
    }

    /**
     * Persist the given file to the under file system.
     *
     * @param fs {@link FileSystem} to carry out Tachyon operations
     * @param uri the uri of the file to persist
     * @param status the status info of the file
     * @param tachyonConf TachyonConf object
     * @return the size of the file persisted
     * @throws IOException if an I/O error occurs
     * @throws FileDoesNotExistException if the given file does not exist
     * @throws TachyonException if an unexpected Tachyon error occurs
     */
    public static long persistFile(FileSystem fs, TachyonURI uri, URIStatus status, TachyonConf tachyonConf)
            throws IOException, FileDoesNotExistException, TachyonException {
        // TODO(manugoyal) move this logic to the worker, as it deals with the under file system
        Closer closer = Closer.create();
        long ret;
        try {
            OpenFileOptions options = OpenFileOptions.defaults().setReadType(ReadType.NO_CACHE);
            FileInStream in = closer.register(fs.openFile(uri, options));
            TachyonURI dstPath = new TachyonURI(status.getUfsPath());
            UnderFileSystem ufs = UnderFileSystem.get(dstPath.toString(), tachyonConf);
            String parentPath = dstPath.getParent().toString();
            if (!ufs.exists(parentPath) && !ufs.mkdirs(parentPath, true)) {
                throw new IOException("Failed to create " + parentPath);
            }
            OutputStream out = closer.register(ufs.create(dstPath.getPath()));
            ret = IOUtils.copyLarge(in, out);
        } catch (Exception e) {
            throw closer.rethrow(e);
        } finally {
            closer.close();
        }
        // Tell the master to mark the file as persisted
        fs.setAttribute(uri, SetAttributeOptions.defaults().setPersisted(true));
        return ret;
    }
}