Java tutorial
/* * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 * (the "License"). You may not use this work except in compliance with the License, which is * available at www.apache.org/licenses/LICENSE-2.0 * * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, * either express or implied, as more fully set forth in the License. * * See the NOTICE file distributed with this work for information regarding copyright ownership. */ package alluxio.client.file; import alluxio.AlluxioURI; import alluxio.Configuration; import alluxio.Constants; import alluxio.PropertyKey; import alluxio.client.ReadType; import alluxio.client.file.options.OpenFileOptions; import alluxio.client.file.options.SetAttributeOptions; import alluxio.exception.AlluxioException; import alluxio.exception.FileDoesNotExistException; import alluxio.security.authorization.Permission; import alluxio.underfs.UnderFileSystem; import alluxio.underfs.options.CreateOptions; import alluxio.underfs.options.MkdirsOptions; import alluxio.util.CommonUtils; import com.google.common.io.Closer; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.OutputStream; import java.util.concurrent.TimeUnit; import javax.annotation.concurrent.ThreadSafe; /** * Collection of utility methods to handle with {@link FileSystem} related objects. */ @ThreadSafe public final class FileSystemUtils { private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE); // prevent instantiation private FileSystemUtils() { } /** * Shortcut for {@code waitCompleted(fs, uri, -1, TimeUnit.MILLISECONDS)}, i.e., wait for an * indefinite amount of time. Note that if a file is never completed, the thread will block * forever, so use with care. * * @param fs a {@link FileSystem} instance * @param uri the URI of the file on which the thread should wait * @return true if the file is complete when this method returns and false if the method timed out * before the file was complete. * @throws IOException in case there are problems contacting the Alluxio master for the file * status * @throws AlluxioException if an Alluxio Exception occurs * @throws InterruptedException if the thread receives an interrupt while waiting for file * completion * @see #waitCompleted(FileSystem, AlluxioURI, long, TimeUnit) */ public static boolean waitCompleted(FileSystem fs, AlluxioURI uri) throws IOException, AlluxioException, InterruptedException { return FileSystemUtils.waitCompleted(fs, uri, -1, TimeUnit.MILLISECONDS); } /** * Waits for a file to be marked as completed. * <p/> * The calling thread will block for <i>at most</i> {@code timeout} time units (as specified via * {@code tunit} or until the file is reported as complete by the master. The method will return * the last known completion status of the file (hence, false only if the method has timed out). A * zero value on the {@code timeout} parameter will make the calling thread check once and return; * a negative value will make it block indefinitely. Note that, in this last case, if a file is * never completed, the thread will block forever, so use with care. * <p/> * Note that the file whose uri is specified, might not exist at the moment this method this call. * The method will deliberately block anyway for the specified amount of time, waiting for the * file to be created and eventually completed. Note also that the file might be moved or deleted * while it is waited upon. In such cases the method will throw the a {@link AlluxioException} * <p/> * <i>IMPLEMENTATION NOTES</i> This method is implemented by periodically polling the master about * the file status. The polling period is controlled by the * {@link PropertyKey#USER_FILE_WAITCOMPLETED_POLL_MS} java property and defaults to a generous 1 * second. * * @param fs an instance of {@link FileSystem} * @param uri the URI of the file whose completion status is to be watied for * @param timeout maximum time the calling thread should be blocked on this call * @param tunit the @{link TimeUnit} instance describing the {@code timeout} parameter * @return true if the file is complete when this method returns and false if the method timed out * before the file was complete. * @throws IOException in case there are problems contacting the Alluxio Master * @throws AlluxioException if an Alluxio exception occurs * @throws InterruptedException if the thread receives an interrupt while waiting for file * completion */ public static boolean waitCompleted(final FileSystem fs, final AlluxioURI uri, final long timeout, final TimeUnit tunit) throws IOException, AlluxioException, InterruptedException { final long deadline = System.currentTimeMillis() + tunit.toMillis(timeout); final long pollPeriod = Configuration.getLong(PropertyKey.USER_FILE_WAITCOMPLETED_POLL_MS); boolean completed = false; long timeleft = deadline - System.currentTimeMillis(); while (!completed && (timeout <= 0 || timeleft > 0)) { if (!fs.exists(uri)) { LOG.debug("The file {} being waited upon does not exist yet. Waiting for it to be " + "created.", uri); } else { completed = fs.getStatus(uri).isCompleted(); } if (timeout == 0) { return completed; } else if (!completed) { long toSleep; if (timeout < 0 || timeleft > pollPeriod) { toSleep = pollPeriod; } else { toSleep = timeleft; } CommonUtils.sleepMs(LOG, toSleep); timeleft = deadline - System.currentTimeMillis(); } } return completed; } /** * Persists the given file to the under file system. * * @param fs {@link FileSystem} to carry out Alluxio operations * @param uri the uri of the file to persist * @param status the status info of the file * @return the size of the file persisted * @throws IOException if an I/O error occurs * @throws FileDoesNotExistException if the given file does not exist * @throws AlluxioException if an unexpected Alluxio error occurs */ public static long persistFile(FileSystem fs, AlluxioURI uri, URIStatus status) throws IOException, FileDoesNotExistException, AlluxioException { // TODO(manugoyal) move this logic to the worker, as it deals with the under file system Closer closer = Closer.create(); long ret; try { OpenFileOptions options = OpenFileOptions.defaults().setReadType(ReadType.NO_CACHE); FileInStream in = closer.register(fs.openFile(uri, options)); AlluxioURI dstPath = new AlluxioURI(status.getUfsPath()); UnderFileSystem ufs = UnderFileSystem.get(dstPath.toString()); String parentPath = dstPath.getParent().toString(); if (!ufs.exists(parentPath)) { URIStatus parentStatus = fs.getStatus(uri.getParent()); Permission parentPerm = new Permission(parentStatus.getOwner(), parentStatus.getGroup(), (short) parentStatus.getMode()); MkdirsOptions parentMkdirsOptions = new MkdirsOptions().setCreateParent(true) .setPermission(parentPerm); if (!ufs.mkdirs(parentPath, parentMkdirsOptions)) { throw new IOException("Failed to create " + parentPath); } } URIStatus uriStatus = fs.getStatus(uri); Permission perm = new Permission(uriStatus.getOwner(), uriStatus.getGroup(), (short) uriStatus.getMode()); OutputStream out = closer .register(ufs.create(dstPath.toString(), new CreateOptions().setPermission(perm))); ret = IOUtils.copyLarge(in, out); } catch (Exception e) { throw closer.rethrow(e); } finally { closer.close(); } // Tell the master to mark the file as persisted fs.setAttribute(uri, SetAttributeOptions.defaults().setPersisted(true)); return ret; } }