Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache98.hadoop.fs; import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.IdentityHashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.ServiceLoader; import java.util.Set; import java.util.Stack; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache98.hadoop.classification.InterfaceAudience; import org.apache98.hadoop.classification.InterfaceStability; import org.apache98.hadoop.conf.Configuration; import org.apache98.hadoop.conf.Configured; import org.apache98.hadoop.fs.Options.ChecksumOpt; import org.apache98.hadoop.fs.Options.Rename; import org.apache98.hadoop.fs.permission.FsPermission; import org.apache98.hadoop.hdfs.DistributedFileSystem; import org.apache98.hadoop.io.MultipleIOException; import org.apache98.hadoop.io.Text; import org.apache98.hadoop.net.NetUtils; import org.apache98.hadoop.security.AccessControlException; import org.apache98.hadoop.security.Credentials; import org.apache98.hadoop.security.SecurityUtil; import org.apache98.hadoop.security.UserGroupInformation; import org.apache98.hadoop.security.token.Token; import org.apache98.hadoop.util.DataChecksum; import org.apache98.hadoop.util.Progressable; import org.apache98.hadoop.util.ReflectionUtils; import org.apache98.hadoop.util.ShutdownHookManager; import com.google.common.annotations.VisibleForTesting; /**************************************************************** * An abstract base class for a fairly generic filesystem. It * may be implemented as a distributed filesystem, or as a "local" * one that reflects the locally-connected disk. The local version * exists for small Hadoop instances and for testing. * * <p> * * All user code that may potentially use the Hadoop Distributed File System should be written to use a FileSystem * object. The Hadoop DFS is a multi-machine system that appears as a single disk. It's useful because of its fault * tolerance and potentially very large capacity. * * <p> * The local implementation is {@link LocalFileSystem} and distributed implementation is DistributedFileSystem. *****************************************************************/ @InterfaceAudience.Public @InterfaceStability.Stable public abstract class FileSystem extends Configured implements Closeable { public static final String FS_DEFAULT_NAME_KEY = CommonConfigurationKeys.FS_DEFAULT_NAME_KEY; public static final String DEFAULT_FS = CommonConfigurationKeys.FS_DEFAULT_NAME_DEFAULT; public static final Log LOG = LogFactory.getLog(FileSystem.class); /** * Priority of the FileSystem shutdown hook. */ public static final int SHUTDOWN_HOOK_PRIORITY = 10; /** FileSystem cache */ static final Cache CACHE = new Cache(); final private static PathFilter DEFAULT_FILTER = new PathFilter() { @Override public boolean accept(Path file) { return true; } }; /** * Close all cached filesystems. Be sure those filesystems are not * used anymore. * * @throws IOException */ public static void closeAll() throws IOException { CACHE.closeAll(); } /** * Close all cached filesystems for a given UGI. Be sure those filesystems * are not used anymore. * * @param ugi * user group info to close * @throws IOException */ public static void closeAllForUGI(UserGroupInformation ugi) throws IOException { CACHE.closeAll(ugi); } /** * create a file with the provided permission * The permission of the file is set to be the provided permission as in * setPermission, not permission&~umask * * It is implemented using two RPCs. It is understood that it is inefficient, * but the implementation is thread-safe. The other option is to change the * value of umask in configuration to be 0, but it is not thread-safe. * * @param fs * file system handle * @param file * the name of the file to be created * @param permission * the permission of the file * @return an output stream * @throws IOException */ public static FSDataOutputStream create(FileSystem fs, Path file, FsPermission permission) throws IOException { // create the file with default permission FSDataOutputStream out = fs.create(file); // set its permission to the supplied one fs.setPermission(file, permission); return out; } /** * Returns the configured filesystem implementation. * * @param conf * the configuration to use */ public static FileSystem get(Configuration conf) throws IOException { return get(getDefaultUri(conf), conf); } /** * Returns the FileSystem for this URI's scheme and authority. The scheme * of the URI determines a configuration property name, <tt>fs.<i>scheme</i>.class</tt> whose value names the * FileSystem class. * The entire URI is passed to the FileSystem instance's initialize method. */ public static FileSystem get(URI uri, Configuration conf) throws IOException { String scheme = uri.getScheme(); String authority = uri.getAuthority(); if (scheme == null && authority == null) { // use default FS return get(conf); } if (scheme != null && authority == null) { // no authority URI defaultUri = getDefaultUri(conf); if (scheme.equals(defaultUri.getScheme()) // if scheme matches default && defaultUri.getAuthority() != null) { // & default has authority return get(defaultUri, conf); // return default } } String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme); if (conf.getBoolean(disableCacheName, false)) { return createFileSystem(uri, conf); } return CACHE.get(uri, conf); } /** * Get a filesystem instance based on the uri, the passed * configuration and the user * * @param uri * of the filesystem * @param conf * the configuration to use * @param user * to perform the get as * @return the filesystem instance * @throws IOException * @throws InterruptedException */ public static FileSystem get(final URI uri, final Configuration conf, final String user) throws IOException, InterruptedException { String ticketCachePath = conf.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH); UserGroupInformation ugi = UserGroupInformation.getBestUGI(ticketCachePath, user); return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { @Override public FileSystem run() throws IOException { return get(uri, conf); } }); } /** * Get the default filesystem URI from a configuration. * * @param conf * the configuration to use * @return the uri of the default filesystem */ public static URI getDefaultUri(Configuration conf) { return URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, DEFAULT_FS))); } /** * Get the local file system. * * @param conf * the configuration to configure the file system with * @return a LocalFileSystem */ public static LocalFileSystem getLocal(Configuration conf) throws IOException { return (LocalFileSystem) get(LocalFileSystem.NAME, conf); } /** @deprecated call #get(URI,Configuration) instead. */ @Deprecated public static FileSystem getNamed(String name, Configuration conf) throws IOException { return get(URI.create(fixName(name)), conf); } /** * create a directory with the provided permission * The permission of the directory is set to be the provided permission as in * setPermission, not permission&~umask * * @see #create(FileSystem, Path, FsPermission) * * @param fs * file system handle * @param dir * the name of the directory to be created * @param permission * the permission of the directory * @return true if the directory creation succeeds; false otherwise * @throws IOException */ public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission) throws IOException { // create the directory using the default permission boolean result = fs.mkdirs(dir); // set its permission to be the supplied one fs.setPermission(dir, permission); return result; } /** * Returns a unique configured filesystem implementation. * This always returns a new FileSystem object. * * @param conf * the configuration to use */ public static FileSystem newInstance(Configuration conf) throws IOException { return newInstance(getDefaultUri(conf), conf); } /** * Returns the FileSystem for this URI's scheme and authority. The scheme * of the URI determines a configuration property name, <tt>fs.<i>scheme</i>.class</tt> whose value names the * FileSystem class. * The entire URI is passed to the FileSystem instance's initialize method. * This always returns a new FileSystem object. */ public static FileSystem newInstance(URI uri, Configuration conf) throws IOException { String scheme = uri.getScheme(); String authority = uri.getAuthority(); if (scheme == null) { // no scheme: use default FS return newInstance(conf); } if (authority == null) { // no authority URI defaultUri = getDefaultUri(conf); if (scheme.equals(defaultUri.getScheme()) // if scheme matches default && defaultUri.getAuthority() != null) { // & default has authority return newInstance(defaultUri, conf); // return default } } return CACHE.getUnique(uri, conf); } /** * Returns the FileSystem for this URI's scheme and authority and the * passed user. Internally invokes {@link #newInstance(URI, Configuration)} * * @param uri * of the filesystem * @param conf * the configuration to use * @param user * to perform the get as * @return filesystem instance * @throws IOException * @throws InterruptedException */ public static FileSystem newInstance(final URI uri, final Configuration conf, final String user) throws IOException, InterruptedException { String ticketCachePath = conf.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH); UserGroupInformation ugi = UserGroupInformation.getBestUGI(ticketCachePath, user); return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { @Override public FileSystem run() throws IOException { return newInstance(uri, conf); } }); } /** * Get a unique local file system object * * @param conf * the configuration to configure the file system with * @return a LocalFileSystem * This always returns a new FileSystem object. */ public static LocalFileSystem newInstanceLocal(Configuration conf) throws IOException { return (LocalFileSystem) newInstance(LocalFileSystem.NAME, conf); } /** * Set the default filesystem URI in a configuration. * * @param conf * the configuration to alter * @param uri * the new default filesystem uri */ public static void setDefaultUri(Configuration conf, URI uri) { conf.set(FS_DEFAULT_NAME_KEY, uri.toString()); } protected static FileSystem getFSofPath(final Path absOrFqPath, final Configuration conf) throws UnsupportedFileSystemException, IOException { absOrFqPath.checkNotSchemeWithRelative(); absOrFqPath.checkNotRelative(); // Uses the default file system if not fully qualified return get(absOrFqPath.toUri(), conf); } /** * This method adds a file system for testing so that we can find it later. It * is only for testing. * * @param uri * the uri to store it under * @param conf * the configuration to store it under * @param fs * the file system to store * @throws IOException */ static void addFileSystemForTesting(URI uri, Configuration conf, FileSystem fs) throws IOException { CACHE.map.put(new Cache.Key(uri, conf), fs); } /** * Update old-format filesystem names, for back-compatibility. This should * eventually be replaced with a checkName() method that throws an exception * for old-format names. */ private static String fixName(String name) { // convert old-format name to new-format name if (name.equals("local")) { // "local" is now "file:///". LOG.warn("\"local\" is a deprecated filesystem name." + " Use \"file:///\" instead."); name = "file:///"; } else if (name.indexOf('/') == -1) { // unqualified is "hdfs://" LOG.warn( "\"" + name + "\" is a deprecated filesystem name." + " Use \"hdfs://" + name + "/\" instead."); name = "hdfs://" + name; } return name; } /** The key this instance is stored under in the cache. */ private Cache.Key key; /** Recording statistics per a FileSystem class */ private static final Map<Class<? extends FileSystem>, Statistics> statisticsTable = new IdentityHashMap<Class<? extends FileSystem>, Statistics>(); // making it volatile to be able to do a double checked locking private volatile static boolean FILE_SYSTEMS_LOADED = false; private static final Map<String, Class<? extends FileSystem>> SERVICE_FILE_SYSTEMS = new HashMap<String, Class<? extends FileSystem>>(); // Symlinks are temporarily disabled - see Hadoop-10020 private static boolean symlinkEnabled = false; private static Configuration conf = null; /** * Reset all statistics for all file systems */ public static synchronized void clearStatistics() { for (Statistics stat : statisticsTable.values()) { stat.reset(); } } @Deprecated @VisibleForTesting public static void enableSymlinks() { symlinkEnabled = true; } /** * Return the FileSystem classes that have Statistics */ public static synchronized List<Statistics> getAllStatistics() { return new ArrayList<Statistics>(statisticsTable.values()); } public static Class<? extends FileSystem> getFileSystemClass(String scheme, Configuration conf) throws IOException { if (!FILE_SYSTEMS_LOADED) { loadFileSystems(); } Class<? extends FileSystem> clazz = null; if (conf != null) { clazz = (Class<? extends FileSystem>) conf.getClass("fs." + scheme + ".impl", null); } if (clazz == null) { clazz = SERVICE_FILE_SYSTEMS.get(scheme); } if (clazz == null) { throw new IOException("No FileSystem for scheme: " + scheme); } return clazz; } /** * Get the Map of Statistics object indexed by URI Scheme. * * @return a Map having a key as URI scheme and value as Statistics object * @deprecated use {@link #getAllStatistics} instead */ @Deprecated public static synchronized Map<String, Statistics> getStatistics() { Map<String, Statistics> result = new HashMap<String, Statistics>(); for (Statistics stat : statisticsTable.values()) { result.put(stat.getScheme(), stat); } return result; } /** * Get the statistics for a particular file system * * @param cls * the class to lookup * @return a statistics object */ public static synchronized Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) { Statistics result = statisticsTable.get(cls); if (result == null) { result = new Statistics(scheme); statisticsTable.put(cls, result); } return result; } @Deprecated @VisibleForTesting public static boolean isSymlinksEnabled() { if (conf == null) { Configuration conf = new Configuration(); symlinkEnabled = conf.getBoolean("test.SymlinkEnabledForTesting", false); } return symlinkEnabled; } /** * Print all statistics for all file systems */ public static synchronized void printStatistics() throws IOException { for (Map.Entry<Class<? extends FileSystem>, Statistics> pair : statisticsTable.entrySet()) { System.out.println(" FileSystem " + pair.getKey().getName() + ": " + pair.getValue()); } } /** * Set the default filesystem URI in a configuration. * * @param conf * the configuration to alter * @param uri * the new default filesystem uri */ public static void setDefaultUri(Configuration conf, String uri) { setDefaultUri(conf, URI.create(fixName(uri))); } private static FileSystem createFileSystem(URI uri, Configuration conf) throws IOException { Class<?> clazz = DistributedFileSystem.class; // getFileSystemClass(uri.getScheme(), conf); if (clazz == null) { throw new IOException("No FileSystem for scheme: " + uri.getScheme()); } FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, conf); fs.initialize(uri, conf); return fs; } private static void loadFileSystems() { synchronized (FileSystem.class) { if (!FILE_SYSTEMS_LOADED) { ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class); for (FileSystem fs : serviceLoader) { SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass()); } FILE_SYSTEMS_LOADED = true; } } } /** * The statistics for this file system. */ protected Statistics statistics; // ///////////////////////////////////////////////////////////// // FileSystem // ///////////////////////////////////////////////////////////// /** * A cache of files that should be deleted when filsystem is closed * or the JVM is exited. */ private Set<Path> deleteOnExit = new TreeSet<Path>(); boolean resolveSymlinks; protected FileSystem() { super(null); } /** * Obtain all delegation tokens used by this FileSystem that are not * already present in the given Credentials. Existing tokens will neither * be verified as valid nor having the given renewer. Missing tokens will * be acquired and added to the given Credentials. * * Default Impl: works for simple fs with its own token * and also for an embedded fs whose tokens are those of its * children file system (i.e. the embedded fs has not tokens of its * own). * * @param renewer * the user allowed to renew the delegation tokens * @param credentials * cache in which to add new delegation tokens * @return list of new delegation tokens * @throws IOException */ @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" }) public Token<?>[] addDelegationTokens(final String renewer, Credentials credentials) throws IOException { if (credentials == null) { credentials = new Credentials(); } final List<Token<?>> tokens = new ArrayList<Token<?>>(); collectDelegationTokens(renewer, credentials, tokens); return tokens.toArray(new Token<?>[tokens.size()]); } /** * Append to an existing file (optional operation). * Same as append(f, getConf().getInt("io.file.buffer.size", 4096), null) * * @param f * the existing file to be appended. * @throws IOException */ public FSDataOutputStream append(Path f) throws IOException { return append(f, getConf().getInt("io.file.buffer.size", 4096), null); } /** * Append to an existing file (optional operation). * Same as append(f, bufferSize, null). * * @param f * the existing file to be appended. * @param bufferSize * the size of the buffer to be used. * @throws IOException */ public FSDataOutputStream append(Path f, int bufferSize) throws IOException { return append(f, bufferSize, null); } /** * Append to an existing file (optional operation). * * @param f * the existing file to be appended. * @param bufferSize * the size of the buffer to be used. * @param progress * for reporting progress if it is not null. * @throws IOException */ public abstract FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException; /** * Cancel the deletion of the path when the FileSystem is closed * * @param f * the path to cancel deletion */ public boolean cancelDeleteOnExit(Path f) { synchronized (deleteOnExit) { return deleteOnExit.remove(f); } } /** * No more filesystem operations are needed. Will * release any held locks. */ @Override public void close() throws IOException { // delete all files that were marked as delete-on-exit. processDeleteOnExit(); CACHE.remove(this.key, this); } /** * Called when we're all done writing to the target. A local FS will * do nothing, because we've written to exactly the right place. A remote * FS will copy the contents of tmpLocalFile to the correct target at * fsOutputFile. * * @param fsOutputFile * path of output file * @param tmpLocalFile * path to local tmp file */ public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException { moveFromLocalFile(tmpLocalFile, fsOutputFile); } /** * Concat existing files together. * * @param trg * the path to the target destination. * @param psrcs * the paths to the sources to use for the concatenation. * @throws IOException */ public void concat(final Path trg, final Path[] psrcs) throws IOException { throw new UnsupportedOperationException( "Not implemented by the " + getClass().getSimpleName() + " FileSystem implementation"); } /** * The src file is on the local disk. Add it to FS at * the given dst name. * delSrc indicates if the source should be removed * * @param delSrc * whether to delete the src * @param overwrite * whether to overwrite an existing file * @param src * path * @param dst * path */ public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, Path dst) throws IOException { Configuration conf = getConf(); FileUtil.copy(getLocal(conf), src, this, dst, delSrc, overwrite, conf); } /** * The src files are on the local disk. Add it to FS at * the given dst name. * delSrc indicates if the source should be removed * * @param delSrc * whether to delete the src * @param overwrite * whether to overwrite an existing file * @param srcs * array of paths which are source * @param dst * path */ public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs, Path dst) throws IOException { Configuration conf = getConf(); FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite, conf); } /** * The src file is on the local disk. Add it to FS at * the given dst name. * delSrc indicates if the source should be removed * * @param delSrc * whether to delete the src * @param src * path * @param dst * path */ public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { copyFromLocalFile(delSrc, true, src, dst); } /** * The src file is on the local disk. Add it to FS at * the given dst name and the source is kept intact afterwards * * @param src * path * @param dst * path */ public void copyFromLocalFile(Path src, Path dst) throws IOException { copyFromLocalFile(false, src, dst); } /** * The src file is under FS, and the dst is on the local disk. * Copy it from FS control to the local dst name. * delSrc indicates if the src will be removed or not. * * @param delSrc * whether to delete the src * @param src * path * @param dst * path */ public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { copyToLocalFile(delSrc, src, dst, false); } /** * The src file is under FS, and the dst is on the local disk. Copy it from FS * control to the local dst name. delSrc indicates if the src will be removed * or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem * as local file system or not. RawLocalFileSystem is non crc file system.So, * It will not create any crc files at local. * * @param delSrc * whether to delete the src * @param src * path * @param dst * path * @param useRawLocalFileSystem * whether to use RawLocalFileSystem as local file system or not. * * @throws IOException * - if any IO error */ public void copyToLocalFile(boolean delSrc, Path src, Path dst, boolean useRawLocalFileSystem) throws IOException { Configuration conf = getConf(); FileSystem local = null; if (useRawLocalFileSystem) { local = getLocal(conf).getRawFileSystem(); } else { local = getLocal(conf); } FileUtil.copy(this, src, local, dst, delSrc, conf); } /** * The src file is under FS, and the dst is on the local disk. * Copy it from FS control to the local dst name. * * @param src * path * @param dst * path */ public void copyToLocalFile(Path src, Path dst) throws IOException { copyToLocalFile(false, src, dst); } /** * Create an FSDataOutputStream at the indicated Path. * Files are overwritten by default. * * @param f * the file to create */ public FSDataOutputStream create(Path f) throws IOException { return create(f, true); } /** * Create an FSDataOutputStream at the indicated Path. * * @param f * the file to create * @param overwrite * if a file with this name already exists, then if true, * the file will be overwritten, and if false an exception will be thrown. */ public FSDataOutputStream create(Path f, boolean overwrite) throws IOException { return create(f, overwrite, getConf().getInt("io.file.buffer.size", 4096), getDefaultReplication(f), getDefaultBlockSize(f)); } /** * Create an FSDataOutputStream at the indicated Path. * * @param f * the file name to create * @param overwrite * if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize * the size of the buffer to be used. */ public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize) throws IOException { return create(f, overwrite, bufferSize, getDefaultReplication(f), getDefaultBlockSize(f)); } /** * Create an FSDataOutputStream at the indicated Path with write-progress * reporting. * * @param f * the path of the file to open * @param overwrite * if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize * the size of the buffer to be used. */ public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, Progressable progress) throws IOException { return create(f, overwrite, bufferSize, getDefaultReplication(f), getDefaultBlockSize(f), progress); } /** * Create an FSDataOutputStream at the indicated Path. * * @param f * the file name to open * @param overwrite * if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize * the size of the buffer to be used. * @param replication * required block replication for the file. */ public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize) throws IOException { return create(f, overwrite, bufferSize, replication, blockSize, null); } /** * Create an FSDataOutputStream at the indicated Path with write-progress * reporting. * * @param f * the file name to open * @param overwrite * if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize * the size of the buffer to be used. * @param replication * required block replication for the file. */ public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return this.create(f, FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(getConf())), overwrite, bufferSize, replication, blockSize, progress); } /** * Create an FSDataOutputStream at the indicated Path with write-progress * reporting. * * @param f * the file name to open * @param permission * @param overwrite * if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize * the size of the buffer to be used. * @param replication * required block replication for the file. * @param blockSize * @param progress * @throws IOException * @see #setPermission(Path, FsPermission) */ public abstract FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException; /** * Create an FSDataOutputStream at the indicated Path with write-progress * reporting. * * @param f * the file name to open * @param permission * @param flags * {@link CreateFlag}s to use for this stream. * @param bufferSize * the size of the buffer to be used. * @param replication * required block replication for the file. * @param blockSize * @param progress * @throws IOException * @see #setPermission(Path, FsPermission) */ public FSDataOutputStream create(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return create(f, permission, flags, bufferSize, replication, blockSize, progress, null); } /** * Create an FSDataOutputStream at the indicated Path with a custom * checksum option * * @param f * the file name to open * @param permission * @param flags * {@link CreateFlag}s to use for this stream. * @param bufferSize * the size of the buffer to be used. * @param replication * required block replication for the file. * @param blockSize * @param progress * @param checksumOpt * checksum parameter. If null, the values * found in conf will be used. * @throws IOException * @see #setPermission(Path, FsPermission) */ public FSDataOutputStream create(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt) throws IOException { // Checksum options are ignored by default. The file systems that // implement checksum need to override this method. The full // support is currently only available in DFS. return create(f, permission, flags.contains(CreateFlag.OVERWRITE), bufferSize, replication, blockSize, progress); } /** * Create an FSDataOutputStream at the indicated Path with write-progress * reporting. * Files are overwritten by default. * * @param f * the file to create * @param progress * to report progress */ public FSDataOutputStream create(Path f, Progressable progress) throws IOException { return create(f, true, getConf().getInt("io.file.buffer.size", 4096), getDefaultReplication(f), getDefaultBlockSize(f), progress); } /** * Create an FSDataOutputStream at the indicated Path. * Files are overwritten by default. * * @param f * the file to create * @param replication * the replication factor */ public FSDataOutputStream create(Path f, short replication) throws IOException { return create(f, true, getConf().getInt("io.file.buffer.size", 4096), replication, getDefaultBlockSize(f)); } /** * Create an FSDataOutputStream at the indicated Path with write-progress * reporting. * Files are overwritten by default. * * @param f * the file to create * @param replication * the replication factor * @param progress * to report progress */ public FSDataOutputStream create(Path f, short replication, Progressable progress) throws IOException { return create(f, true, getConf().getInt(CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY, CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT), replication, getDefaultBlockSize(f), progress); } /** * Creates the given Path as a brand-new zero-length file. If * create fails, or if it already existed, return false. * * @param f * path to use for create */ public boolean createNewFile(Path f) throws IOException { if (exists(f)) { return false; } else { create(f, false, getConf().getInt("io.file.buffer.size", 4096)).close(); return true; } } /** * Opens an FSDataOutputStream at the indicated Path with write-progress * reporting. Same as create(), except fails if parent directory doesn't * already exist. * * @param f * the file name to open * @param overwrite * if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize * the size of the buffer to be used. * @param replication * required block replication for the file. * @param blockSize * @param progress * @throws IOException * @see #setPermission(Path, FsPermission) * @deprecated API only for 0.20-append */ @Deprecated public FSDataOutputStream createNonRecursive(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return this.createNonRecursive(f, FsPermission.getFileDefault(), overwrite, bufferSize, replication, blockSize, progress); } /** * Opens an FSDataOutputStream at the indicated Path with write-progress * reporting. Same as create(), except fails if parent directory doesn't * already exist. * * @param f * the file name to open * @param permission * @param overwrite * if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize * the size of the buffer to be used. * @param replication * required block replication for the file. * @param blockSize * @param progress * @throws IOException * @see #setPermission(Path, FsPermission) * @deprecated API only for 0.20-append */ @Deprecated public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return createNonRecursive(f, permission, overwrite ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE) : EnumSet.of(CreateFlag.CREATE), bufferSize, replication, blockSize, progress); } /** * Opens an FSDataOutputStream at the indicated Path with write-progress * reporting. Same as create(), except fails if parent directory doesn't * already exist. * * @param f * the file name to open * @param permission * @param flags * {@link CreateFlag}s to use for this stream. * @param bufferSize * the size of the buffer to be used. * @param replication * required block replication for the file. * @param blockSize * @param progress * @throws IOException * @see #setPermission(Path, FsPermission) * @deprecated API only for 0.20-append */ @Deprecated public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { throw new IOException("createNonRecursive unsupported for this filesystem " + this.getClass()); } /** * Create a snapshot with a default name. * * @param path * The directory where snapshots will be taken. * @return the snapshot path. */ public final Path createSnapshot(Path path) throws IOException { return createSnapshot(path, null); } /** * Create a snapshot * * @param path * The directory where snapshots will be taken. * @param snapshotName * The name of the snapshot * @return the snapshot path. */ public Path createSnapshot(Path path, String snapshotName) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support createSnapshot"); } /** * See {@link FileContext#createSymlink(Path, Path, boolean)} */ public void createSymlink(final Path target, final Path link, final boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, IOException { // Supporting filesystems should override this method throw new UnsupportedOperationException("Filesystem does not support symlinks!"); } /** * Delete a file * * @deprecated Use {@link #delete(Path, boolean)} instead. */ @Deprecated public boolean delete(Path f) throws IOException { return delete(f, true); } /** * Delete a file. * * @param f * the path to delete. * @param recursive * if path is a directory and set to * true, the directory is deleted else throws an exception. In * case of a file the recursive can be set to either true or false. * @return true if delete is successful else false. * @throws IOException */ public abstract boolean delete(Path f, boolean recursive) throws IOException; /** * Mark a path to be deleted when FileSystem is closed. * When the JVM shuts down, * all FileSystem objects will be closed automatically. * Then, * the marked path will be deleted as a result of closing the FileSystem. * * The path has to exist in the file system. * * @param f * the path to delete. * @return true if deleteOnExit is successful, otherwise false. * @throws IOException */ public boolean deleteOnExit(Path f) throws IOException { if (!exists(f)) { return false; } synchronized (deleteOnExit) { deleteOnExit.add(f); } return true; } /** * Delete a snapshot of a directory * * @param path * The directory that the to-be-deleted snapshot belongs to * @param snapshotName * The name of the snapshot */ public void deleteSnapshot(Path path, String snapshotName) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support deleteSnapshot"); } /** * Check if exists. * * @param f * source file */ public boolean exists(Path f) throws IOException { try { return getFileStatus(f) != null; } catch (FileNotFoundException e) { return false; } } /** * Get the block size for a particular file. * * @param f * the filename * @return the number of bytes in a block */ /** @deprecated Use getFileStatus() instead */ @Deprecated public long getBlockSize(Path f) throws IOException { return getFileStatus(f).getBlockSize(); } /** * Get a canonical service name for this file system. The token cache is * the only user of the canonical service name, and uses it to lookup this * filesystem's service tokens. * If file system provides a token of its own then it must have a canonical * name, otherwise canonical name can be null. * * Default Impl: If the file system has child file systems * (such as an embedded file system) then it is assumed that the fs has no * tokens of its own and hence returns a null name; otherwise a service * name is built using Uri and port. * * @return a service string that uniquely identifies this file system, null * if the filesystem does not implement tokens * @see SecurityUtil#buildDTServiceName(URI, int) */ @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" }) public String getCanonicalServiceName() { return (getChildFileSystems() == null) ? SecurityUtil.buildDTServiceName(getUri(), getDefaultPort()) : null; } /** * Get all the immediate child FileSystems embedded in this FileSystem. * It does not recurse and get grand children. If a FileSystem * has multiple child FileSystems, then it should return a unique list * of those FileSystems. Default is to return null to signify no children. * * @return FileSystems used by this FileSystem */ @InterfaceAudience.LimitedPrivate({ "HDFS" }) @VisibleForTesting public FileSystem[] getChildFileSystems() { return null; } /** * Return the {@link ContentSummary} of a given {@link Path}. * * @param f * path to use */ public ContentSummary getContentSummary(Path f) throws IOException { FileStatus status = getFileStatus(f); if (status.isFile()) { // f is a file return new ContentSummary(status.getLen(), 1, 0); } // f is a directory long[] summary = { 0, 0, 1 }; for (FileStatus s : listStatus(f)) { ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) : new ContentSummary(s.getLen(), 1, 0); summary[0] += c.getLength(); summary[1] += c.getFileCount(); summary[2] += c.getDirectoryCount(); } return new ContentSummary(summary[0], summary[1], summary[2]); } /** * Return the number of bytes that large input files should be optimally * be split into to minimize i/o time. * * @deprecated use {@link #getDefaultBlockSize(Path)} instead */ @Deprecated public long getDefaultBlockSize() { // default to 32MB: large enough to minimize the impact of seeks return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024); } /** * Return the number of bytes that large input files should be optimally * be split into to minimize i/o time. The given path will be used to * locate the actual filesystem. The full path does not have to exist. * * @param f * path of file * @return the default block size for the path's filesystem */ public long getDefaultBlockSize(Path f) { return getDefaultBlockSize(); } /** * Get the default replication. * * @deprecated use {@link #getDefaultReplication(Path)} instead */ @Deprecated public short getDefaultReplication() { return 1; } /** * Get the default replication for a path. The given path will be used to * locate the actual filesystem. The full path does not have to exist. * * @param path * of the file * @return default replication for the path's filesystem */ public short getDefaultReplication(Path path) { return getDefaultReplication(); } /** * Get a new delegation token for this file system. * This is an internal method that should have been declared protected * but wasn't historically. * Callers should use {@link #addDelegationTokens(String, Credentials)} * * @param renewer * the account name that is allowed to renew the token. * @return a new delegation token * @throws IOException */ @InterfaceAudience.Private() public Token<?> getDelegationToken(String renewer) throws IOException { return null; } /** * Return an array containing hostnames, offset and size of * portions of the given file. For a nonexistent * file or regions, null will be returned. * * This call is most helpful with DFS, where it returns * hostnames of machines that contain the given file. * * The FileSystem will simply return an elt containing 'localhost'. * * @param file * FilesStatus to get data from * @param start * offset into the given file * @param len * length for which to get locations for */ public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException { if (file == null) { return null; } if (start < 0 || len < 0) { throw new IllegalArgumentException("Invalid start or len parameter"); } if (file.getLen() <= start) { return new BlockLocation[0]; } String[] name = { "localhost:50010" }; String[] host = { "localhost" }; return new BlockLocation[] { new BlockLocation(name, host, 0, file.getLen()) }; } /** * Return an array containing hostnames, offset and size of * portions of the given file. For a nonexistent * file or regions, null will be returned. * * This call is most helpful with DFS, where it returns * hostnames of machines that contain the given file. * * The FileSystem will simply return an elt containing 'localhost'. * * @param p * path is used to identify an FS since an FS could have * another FS that it could be delegating the call to * @param start * offset into the given file * @param len * length for which to get locations for */ public BlockLocation[] getFileBlockLocations(Path p, long start, long len) throws IOException { if (p == null) { throw new NullPointerException(); } FileStatus file = getFileStatus(p); return getFileBlockLocations(file, start, len); } /** * Get the checksum of a file. * * @param f * The file path * @return The file checksum. The default return value is null, * which indicates that no checksum algorithm is implemented * in the corresponding FileSystem. */ public FileChecksum getFileChecksum(Path f) throws IOException { return null; } /** * See {@link FileContext#getFileLinkStatus(Path)} */ public FileStatus getFileLinkStatus(final Path f) throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException, IOException { // Supporting filesystems should override this method return getFileStatus(f); } /** * Return a file status object that represents the path. * * @param f * The path we want information from * @return a FileStatus object * @throws FileNotFoundException * when the path does not exist; * IOException see specific implementation */ public abstract FileStatus getFileStatus(Path f) throws IOException; /** * Return the current user's home directory in this filesystem. * The default implementation returns "/user/$USER/". */ public Path getHomeDirectory() { return this.makeQualified(new Path("/user/" + System.getProperty("user.name"))); } /** The number of bytes in a file. */ /** @deprecated Use getFileStatus() instead */ @Deprecated public long getLength(Path f) throws IOException { return getFileStatus(f).getLen(); } /** * See {@link FileContext#getLinkTarget(Path)} */ public Path getLinkTarget(Path f) throws IOException { // Supporting filesystems should override this method throw new UnsupportedOperationException("Filesystem does not support symlinks!"); } /** @deprecated call #getUri() instead. */ @Deprecated public String getName() { return getUri().toString(); } /** * Get replication. * * @deprecated Use getFileStatus() instead * @param src * file name * @return file replication * @throws IOException */ @Deprecated public short getReplication(Path src) throws IOException { return getFileStatus(src).getReplication(); } /** * Return the protocol scheme for the FileSystem. * <p/> * This implementation throws an <code>UnsupportedOperationException</code>. * * @return the protocol scheme for the FileSystem. */ public String getScheme() { throw new UnsupportedOperationException( "Not implemented by the " + getClass().getSimpleName() + " FileSystem implementation"); } /** * Return a set of server default configuration values * * @return server default configuration values * @throws IOException * @deprecated use {@link #getServerDefaults(Path)} instead */ @Deprecated public FsServerDefaults getServerDefaults() throws IOException { Configuration conf = getConf(); // CRC32 is chosen as default as it is available in all // releases that support checksum. // The client trash configuration is ignored. return new FsServerDefaults(getDefaultBlockSize(), conf.getInt("io.bytes.per.checksum", 512), 64 * 1024, getDefaultReplication(), conf.getInt("io.file.buffer.size", 4096), false, CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT, DataChecksum.Type.CRC32); } /** * Return a set of server default configuration values * * @param p * path is used to identify an FS since an FS could have * another FS that it could be delegating the call to * @return server default configuration values * @throws IOException */ public FsServerDefaults getServerDefaults(Path p) throws IOException { return getServerDefaults(); } /** * Returns a status object describing the use and capacity of the * file system. If the file system has multiple partitions, the * use and capacity of the root partition is reflected. * * @return a FsStatus object * @throws IOException * see specific implementation */ public FsStatus getStatus() throws IOException { return getStatus(null); } /** * Returns a status object describing the use and capacity of the * file system. If the file system has multiple partitions, the * use and capacity of the partition pointed to by the specified * path is reflected. * * @param p * Path for which status should be obtained. null means * the default partition. * @return a FsStatus object * @throws IOException * see specific implementation */ public FsStatus getStatus(Path p) throws IOException { return new FsStatus(Long.MAX_VALUE, 0, Long.MAX_VALUE); } /** Returns a URI whose scheme and authority identify this FileSystem. */ public abstract URI getUri(); /** Return the total size of all files in the filesystem. */ public long getUsed() throws IOException { long used = 0; FileStatus[] files = listStatus(new Path("/")); for (FileStatus file : files) { used += file.getLen(); } return used; } /** * Get the current working directory for the given file system * * @return the directory pathname */ public abstract Path getWorkingDirectory(); /** * <p> * Return all the files that match filePattern and are not checksum files. Results are sorted by their names. * * <p> * A filename pattern is composed of <i>regular</i> characters and <i>special pattern matching</i> characters, which * are: * * <dl> * <dd> * <dl> * <p> * <dt> <tt> ? </tt> * <dd>Matches any single character. * * <p> * <dt> <tt> * </tt> * <dd>Matches zero or more characters. * * <p> * <dt> <tt> [<i>abc</i>] </tt> * <dd>Matches a single character from character set <tt>{<i>a,b,c</i>}</tt>. * * <p> * <dt> <tt> [<i>a</i>-<i>b</i>] </tt> * <dd>Matches a single character from the character range <tt>{<i>a...b</i>}</tt>. Note that character * <tt><i>a</i></tt> must be lexicographically less than or equal to character <tt><i>b</i></tt>. * * <p> * <dt> <tt> [^<i>a</i>] </tt> * <dd>Matches a single character that is not from character set or range <tt>{<i>a</i>}</tt>. Note that the * <tt>^</tt> character must occur immediately to the right of the opening bracket. * * <p> * <dt> <tt> \<i>c</i> </tt> * <dd>Removes (escapes) any special meaning of character <i>c</i>. * * <p> * <dt> <tt> {ab,cd} </tt> * <dd>Matches a string from the string set <tt>{<i>ab, cd</i>} </tt> * * <p> * <dt> <tt> {ab,c{de,fh}} </tt> * <dd>Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt> * * </dl> * </dd> * </dl> * * @param pathPattern * a regular expression specifying a pth pattern * * @return an array of paths that match the path pattern * @throws IOException */ public FileStatus[] globStatus(Path pathPattern) throws IOException { return globStatus(pathPattern, DEFAULT_FILTER); } /** * Return an array of FileStatus objects whose path names match pathPattern * and is accepted by the user-supplied path filter. Results are sorted by * their path names. * Return null if pathPattern has no glob and the path does not exist. * Return an empty array if pathPattern has a glob and no path matches it. * * @param pathPattern * a regular expression specifying the path pattern * @param filter * a user-supplied path filter * @return an array of FileStatus objects * @throws IOException * if any I/O error occurs when fetching file status */ public FileStatus[] globStatus(Path pathPattern, PathFilter filter) throws IOException { String filename = pathPattern.toUri().getPath(); List<FileStatus> allMatches = null; List<String> filePatterns = GlobExpander.expand(filename); for (String filePattern : filePatterns) { Path path = new Path(filePattern.isEmpty() ? Path.CUR_DIR : filePattern); List<FileStatus> matches = globStatusInternal(path, filter); if (matches != null) { if (allMatches == null) { allMatches = matches; } else { allMatches.addAll(matches); } } } FileStatus[] results = null; if (allMatches != null) { results = allMatches.toArray(new FileStatus[allMatches.size()]); } else if (filePatterns.size() > 1) { // no matches with multiple expansions is a non-matching glob results = new FileStatus[0]; } return results; } /** * Called after a new FileSystem instance is constructed. * * @param name * a uri whose authority section names the host, port, etc. * for this FileSystem * @param conf * the configuration */ public void initialize(URI name, Configuration conf) throws IOException { statistics = getStatistics(name.getScheme(), getClass()); resolveSymlinks = conf.getBoolean(CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT); } /** * True iff the named path is a directory. * Note: Avoid using this method. Instead reuse the FileStatus * returned by getFileStatus() or listStatus() methods. * * @param f * path to check */ public boolean isDirectory(Path f) throws IOException { try { return getFileStatus(f).isDirectory(); } catch (FileNotFoundException e) { return false; // f does not exist } } /** * True iff the named path is a regular file. * Note: Avoid using this method. Instead reuse the FileStatus * returned by getFileStatus() or listStatus() methods. * * @param f * path to check */ public boolean isFile(Path f) throws IOException { try { return getFileStatus(f).isFile(); } catch (FileNotFoundException e) { return false; // f does not exist } } /** * @return an iterator over the corrupt files under the given path * (may contain duplicates if a file has more than one corrupt block) * @throws IOException */ public RemoteIterator<Path> listCorruptFileBlocks(Path path) throws IOException { throw new UnsupportedOperationException( getClass().getCanonicalName() + " does not support" + " listCorruptFileBlocks"); } /** * List the statuses and block locations of the files in the given path. * * If the path is a directory, * if recursive is false, returns files in the directory; * if recursive is true, return files in the subtree rooted at the path. * If the path is a file, return the file's status and block locations. * * @param f * is the path * @param recursive * if the subdirectories need to be traversed recursively * * @return an iterator that traverses statuses of the files * * @throws FileNotFoundException * when the path does not exist; * IOException see specific implementation */ public RemoteIterator<LocatedFileStatus> listFiles(final Path f, final boolean recursive) throws FileNotFoundException, IOException { return new RemoteIterator<LocatedFileStatus>() { private Stack<RemoteIterator<LocatedFileStatus>> itors = new Stack<RemoteIterator<LocatedFileStatus>>(); private RemoteIterator<LocatedFileStatus> curItor = listLocatedStatus(f); private LocatedFileStatus curFile; @Override public boolean hasNext() throws IOException { while (curFile == null) { if (curItor.hasNext()) { handleFileStat(curItor.next()); } else if (!itors.empty()) { curItor = itors.pop(); } else { return false; } } return true; } @Override public LocatedFileStatus next() throws IOException { if (hasNext()) { LocatedFileStatus result = curFile; curFile = null; return result; } throw new java.util.NoSuchElementException("No more entry in " + f); } /** * Process the input stat. * If it is a file, return the file stat. * If it is a directory, traverse the directory if recursive is true; * ignore it if recursive is false. * * @param stat * input status * @throws IOException * if any IO error occurs */ private void handleFileStat(LocatedFileStatus stat) throws IOException { if (stat.isFile()) { // file curFile = stat; } else if (recursive) { // directory itors.push(curItor); curItor = listLocatedStatus(stat.getPath()); } } }; } /** * List the statuses of the files/directories in the given path if the path is * a directory. * Return the file's status and block locations If the path is a file. * * If a returned status is a file, it contains the file's block locations. * * @param f * is the path * * @return an iterator that traverses statuses of the files/directories * in the given path * * @throws FileNotFoundException * If <code>f</code> does not exist * @throws IOException * If an I/O error occurred */ public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f) throws FileNotFoundException, IOException { return listLocatedStatus(f, DEFAULT_FILTER); } /** * List the statuses of the files/directories in the given path if the path is * a directory. * * @param f * given path * @return the statuses of the files/directories in the given patch * @throws FileNotFoundException * when the path does not exist; * IOException see specific implementation */ public abstract FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException; /** * Filter files/directories in the given path using the user-supplied path * filter. * * @param f * a path name * @param filter * the user-supplied path filter * @return an array of FileStatus objects for the files under the given path * after applying the filter * @throws FileNotFoundException * when the path does not exist; * IOException see specific implementation */ public FileStatus[] listStatus(Path f, PathFilter filter) throws FileNotFoundException, IOException { ArrayList<FileStatus> results = new ArrayList<FileStatus>(); listStatus(results, f, filter); return results.toArray(new FileStatus[results.size()]); } /** * Filter files/directories in the given list of paths using default * path filter. * * @param files * a list of paths * @return a list of statuses for the files under the given paths after * applying the filter default Path filter * @throws FileNotFoundException * when the path does not exist; * IOException see specific implementation */ public FileStatus[] listStatus(Path[] files) throws FileNotFoundException, IOException { return listStatus(files, DEFAULT_FILTER); } /** * Filter files/directories in the given list of paths using user-supplied * path filter. * * @param files * a list of paths * @param filter * the user-supplied path filter * @return a list of statuses for the files under the given paths after * applying the filter * @throws FileNotFoundException * when the path does not exist; * IOException see specific implementation */ public FileStatus[] listStatus(Path[] files, PathFilter filter) throws FileNotFoundException, IOException { ArrayList<FileStatus> results = new ArrayList<FileStatus>(); for (int i = 0; i < files.length; i++) { listStatus(results, files[i], filter); } return results.toArray(new FileStatus[results.size()]); } /** * Make sure that a path specifies a FileSystem. * * @param path * to use */ public Path makeQualified(Path path) { checkPath(path); return path.makeQualified(this.getUri(), this.getWorkingDirectory()); } /** * Call {@link #mkdirs(Path, FsPermission)} with default permission. */ public boolean mkdirs(Path f) throws IOException { return mkdirs(f, FsPermission.getDirDefault()); } /** * Make the given file and all non-existent parents into * directories. Has the semantics of Unix 'mkdir -p'. * Existence of the directory hierarchy is not an error. * * @param f * path to create * @param permission * to apply to f */ public abstract boolean mkdirs(Path f, FsPermission permission) throws IOException; /** * The src file is on the local disk. Add it to FS at * the given dst name, removing the source afterwards. * * @param src * path * @param dst * path */ public void moveFromLocalFile(Path src, Path dst) throws IOException { copyFromLocalFile(true, src, dst); } /** * The src files is on the local disk. Add it to FS at * the given dst name, removing the source afterwards. * * @param srcs * path * @param dst * path */ public void moveFromLocalFile(Path[] srcs, Path dst) throws IOException { copyFromLocalFile(true, true, srcs, dst); } /** * The src file is under FS, and the dst is on the local disk. * Copy it from FS control to the local dst name. * Remove the source afterwards * * @param src * path * @param dst * path */ public void moveToLocalFile(Path src, Path dst) throws IOException { copyToLocalFile(true, src, dst); } /** * Opens an FSDataInputStream at the indicated Path. * * @param f * the file to open */ public FSDataInputStream open(Path f) throws IOException { return open(f, getConf().getInt("io.file.buffer.size", 4096)); } /** * Opens an FSDataInputStream at the indicated Path. * * @param f * the file name to open * @param bufferSize * the size of the buffer to be used. */ public abstract FSDataInputStream open(Path f, int bufferSize) throws IOException; /** * Renames Path src to Path dst. Can take place on local fs * or remote DFS. * * @param src * path to be renamed * @param dst * new path after rename * @throws IOException * on failure * @return true if rename is successful */ public abstract boolean rename(Path src, Path dst) throws IOException; /** * Rename a snapshot * * @param path * The directory path where the snapshot was taken * @param snapshotOldName * Old name of the snapshot * @param snapshotNewName * New name of the snapshot * @throws IOException */ public void renameSnapshot(Path path, String snapshotOldName, String snapshotNewName) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support renameSnapshot"); } /** * Return the fully-qualified path of path f resolving the path * through any symlinks or mount point * * @param p * path to be resolved * @return fully qualified path * @throws FileNotFoundException */ public Path resolvePath(final Path p) throws IOException { checkPath(p); return getFileStatus(p).getPath(); } /** * Set owner of a path (i.e. a file or a directory). * The parameters username and groupname cannot both be null. * * @param p * The path * @param username * If it is null, the original username remains unchanged. * @param groupname * If it is null, the original groupname remains unchanged. */ public void setOwner(Path p, String username, String groupname) throws IOException { } /** * Set permission of a path. * * @param p * @param permission */ public void setPermission(Path p, FsPermission permission) throws IOException { } /** * Set replication for an existing file. * * @param src * file name * @param replication * new replication * @throws IOException * @return true if successful; * false if file does not exist or is a directory */ public boolean setReplication(Path src, short replication) throws IOException { return true; } /** * Set access time of a file * * @param p * The path * @param mtime * Set the modification time of this file. * The number of milliseconds since Jan 1, 1970. * A value of -1 means that this call should not set modification time. * @param atime * Set the access time of this file. * The number of milliseconds since Jan 1, 1970. * A value of -1 means that this call should not set access time. */ public void setTimes(Path p, long mtime, long atime) throws IOException { } /** * Set the verify checksum flag. This is only applicable if the * corresponding FileSystem supports checksum. By default doesn't do anything. * * @param verifyChecksum */ public void setVerifyChecksum(boolean verifyChecksum) { // doesn't do anything } /** * Set the current working directory for the given file system. All relative * paths will be resolved relative to it. * * @param new_dir */ public abstract void setWorkingDirectory(Path new_dir); /** * Set the write checksum flag. This is only applicable if the * corresponding FileSystem supports checksum. By default doesn't do anything. * * @param writeChecksum */ public void setWriteChecksum(boolean writeChecksum) { // doesn't do anything } /** * Returns a local File that the user can write output to. The caller * provides both the eventual FS target name and the local working * file. If the FS is local, we write directly into the target. If * the FS is remote, we write into the tmp local area. * * @param fsOutputFile * path of output file * @param tmpLocalFile * path of local tmp file */ public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException { return tmpLocalFile; } /** * See {@link AbstractFileSystem#supportsSymlinks()} */ public boolean supportsSymlinks() { return false; } /** * Canonicalize the given URI. * * This is filesystem-dependent, but may for example consist of * canonicalizing the hostname using DNS and adding the default * port if not specified. * * The default implementation simply fills in the default port if * not specified and if the filesystem has a default port. * * @return URI * @see NetUtils#getCanonicalUri(URI, int) */ protected URI canonicalizeUri(URI uri) { if (uri.getPort() == -1 && getDefaultPort() > 0) { // reconstruct the uri with the default port set try { uri = new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), getDefaultPort(), uri.getPath(), uri.getQuery(), uri.getFragment()); } catch (URISyntaxException e) { // Should never happen! throw new AssertionError("Valid URI became unparseable: " + uri); } } return uri; } /** * Check that a Path belongs to this FileSystem. * * @param path * to check */ protected void checkPath(Path path) { URI uri = path.toUri(); String thatScheme = uri.getScheme(); if (thatScheme == null) { return; } URI thisUri = getCanonicalUri(); String thisScheme = thisUri.getScheme(); // authority and scheme are not case sensitive if (thisScheme.equalsIgnoreCase(thatScheme)) {// schemes match String thisAuthority = thisUri.getAuthority(); String thatAuthority = uri.getAuthority(); if (thatAuthority == null && // path's authority is null thisAuthority != null) { // fs has an authority URI defaultUri = getDefaultUri(getConf()); if (thisScheme.equalsIgnoreCase(defaultUri.getScheme())) { uri = defaultUri; // schemes match, so use this uri instead } else { uri = null; // can't determine auth of the path } } if (uri != null) { // canonicalize uri before comparing with this fs uri = canonicalizeUri(uri); thatAuthority = uri.getAuthority(); if (thisAuthority == thatAuthority || // authorities match (thisAuthority != null && thisAuthority.equalsIgnoreCase(thatAuthority))) { return; } } } throw new IllegalArgumentException("Wrong FS: " + path + ", expected: " + this.getUri()); } /** * See {@link FileContext#fixRelativePart} */ protected Path fixRelativePart(Path p) { if (p.isUriPathAbsolute()) { return p; } else { return new Path(getWorkingDirectory(), p); } } /** * Return a canonicalized form of this FileSystem's URI. * * The default implementation simply calls {@link #canonicalizeUri(URI)} on the filesystem's own URI, so subclasses * typically only need to * implement that method. * * @see #canonicalizeUri(URI) */ protected URI getCanonicalUri() { return canonicalizeUri(getUri()); } /** * Get the default port for this file system. * * @return the default port or 0 if there isn't one */ protected int getDefaultPort() { return 0; } /** * Note: with the new FilesContext class, getWorkingDirectory() * will be removed. * The working directory is implemented in FilesContext. * * Some file systems like LocalFileSystem have an initial workingDir * that we use as the starting workingDir. For other file systems * like HDFS there is no built in notion of an initial workingDir. * * @return if there is built in notion of workingDir then it * is returned; else a null is returned. */ protected Path getInitialWorkingDirectory() { return null; } /** * Listing a directory * The returned results include its block location if it is a file * The results are filtered by the given path filter * * @param f * a path * @param filter * a path filter * @return an iterator that traverses statuses of the files/directories * in the given path * @throws FileNotFoundException * if <code>f</code> does not exist * @throws IOException * if any I/O error occurred */ protected RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f, final PathFilter filter) throws FileNotFoundException, IOException { return new RemoteIterator<LocatedFileStatus>() { private final FileStatus[] stats = listStatus(f, filter); private int i = 0; @Override public boolean hasNext() { return i < stats.length; } @Override public LocatedFileStatus next() throws IOException { if (!hasNext()) { throw new NoSuchElementException("No more entry in " + f); } FileStatus result = stats[i++]; BlockLocation[] locs = result.isFile() ? getFileBlockLocations(result.getPath(), 0, result.getLen()) : null; return new LocatedFileStatus(result, locs); } }; } /* * . * This create has been added to support the FileContext that processes * the permission * with umask before calling this method. * This a temporary method added to support the transition from FileSystem * to FileContext for user applications. */ @Deprecated protected FSDataOutputStream primitiveCreate(Path f, FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize, short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt) throws IOException { boolean pathExists = exists(f); CreateFlag.validate(f, pathExists, flag); // Default impl assumes that permissions do not matter and // nor does the bytesPerChecksum hence // calling the regular create is good enough. // FSs that implement permissions should override this. if (pathExists && flag.contains(CreateFlag.APPEND)) { return append(f, bufferSize, progress); } return this.create(f, absolutePermission, flag.contains(CreateFlag.OVERWRITE), bufferSize, replication, blockSize, progress); } /** * This version of the mkdirs method assumes that the permission is absolute. * It has been added to support the FileContext that processes the permission * with umask before calling this method. * This a temporary method added to support the transition from FileSystem * to FileContext for user applications. */ @Deprecated protected boolean primitiveMkdir(Path f, FsPermission absolutePermission) throws IOException { // Default impl is to assume that permissions do not matter and hence // calling the regular mkdirs is good enough. // FSs that implement permissions should override this. return this.mkdirs(f, absolutePermission); } /** * This version of the mkdirs method assumes that the permission is absolute. * It has been added to support the FileContext that processes the permission * with umask before calling this method. * This a temporary method added to support the transition from FileSystem * to FileContext for user applications. */ @Deprecated protected void primitiveMkdir(Path f, FsPermission absolutePermission, boolean createParent) throws IOException { if (!createParent) { // parent must exist. // since the this.mkdirs makes parent dirs automatically // we must throw exception if parent does not exist. final FileStatus stat = getFileStatus(f.getParent()); if (stat == null) { throw new FileNotFoundException("Missing parent:" + f); } if (!stat.isDirectory()) { throw new ParentNotDirectoryException("parent is not a dir"); } // parent does exist - go ahead with mkdir of leaf } // Default impl is to assume that permissions do not matter and hence // calling the regular mkdirs is good enough. // FSs that implement permissions should override this. if (!this.mkdirs(f, absolutePermission)) { throw new IOException("mkdir of " + f + " failed"); } } /** * Delete all files that were marked as delete-on-exit. This recursively * deletes all files in the specified paths. */ protected void processDeleteOnExit() { synchronized (deleteOnExit) { for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) { Path path = iter.next(); try { if (exists(path)) { delete(path, true); } } catch (IOException e) { LOG.info("Ignoring failure to deleteOnExit for path " + path); } iter.remove(); } } } /** * Renames Path src to Path dst * <ul> * <li * <li>Fails if src is a file and dst is a directory. * <li>Fails if src is a directory and dst is a file. * <li>Fails if the parent of dst does not exist or is a file. * </ul> * <p> * If OVERWRITE option is not passed as an argument, rename fails if the dst already exists. * <p> * If OVERWRITE option is passed as an argument, rename overwrites the dst if it is a file or an empty directory. * Rename fails if dst is a non-empty directory. * <p> * Note that atomicity of rename is dependent on the file system implementation. Please refer to the file system * documentation for details. This default implementation is non atomic. * <p> * This method is deprecated since it is a temporary method added to support the transition from FileSystem to * FileContext for user applications. * * @param src * path to be renamed * @param dst * new path after rename * @throws IOException * on failure */ @Deprecated protected void rename(final Path src, final Path dst, final Rename... options) throws IOException { // Default implementation final FileStatus srcStatus = getFileLinkStatus(src); if (srcStatus == null) { throw new FileNotFoundException("rename source " + src + " not found."); } boolean overwrite = false; if (null != options) { for (Rename option : options) { if (option == Rename.OVERWRITE) { overwrite = true; } } } FileStatus dstStatus; try { dstStatus = getFileLinkStatus(dst); } catch (IOException e) { dstStatus = null; } if (dstStatus != null) { if (srcStatus.isDirectory() != dstStatus.isDirectory()) { throw new IOException( "Source " + src + " Destination " + dst + " both should be either file or directory"); } if (!overwrite) { throw new FileAlreadyExistsException("rename destination " + dst + " already exists."); } // Delete the destination that is a file or an empty directory if (dstStatus.isDirectory()) { FileStatus[] list = listStatus(dst); if (list != null && list.length != 0) { throw new IOException("rename cannot overwrite non empty destination directory " + dst); } } delete(dst, false); } else { final Path parent = dst.getParent(); final FileStatus parentStatus = getFileStatus(parent); if (parentStatus == null) { throw new FileNotFoundException("rename destination parent " + parent + " not found."); } if (!parentStatus.isDirectory()) { throw new ParentNotDirectoryException("rename destination parent " + parent + " is a file."); } } if (!rename(src, dst)) { throw new IOException("rename from " + src + " to " + dst + " failed."); } } /** * See {@link AbstractFileSystem#getLinkTarget(Path)} */ protected Path resolveLink(Path f) throws IOException { // Supporting filesystems should override this method throw new UnsupportedOperationException("Filesystem does not support symlinks!"); } /** * Recursively obtain the tokens for this FileSystem and all descended * FileSystems as determined by getChildFileSystems(). * * @param renewer * the user allowed to renew the delegation tokens * @param credentials * cache in which to add the new delegation tokens * @param tokens * list in which to add acquired tokens * @throws IOException */ private void collectDelegationTokens(final String renewer, final Credentials credentials, final List<Token<?>> tokens) throws IOException { final String serviceName = getCanonicalServiceName(); // Collect token of the this filesystem and then of its embedded children if (serviceName != null) { // fs has token, grab it final Text service = new Text(serviceName); Token<?> token = credentials.getToken(service); if (token == null) { token = getDelegationToken(renewer); if (token != null) { tokens.add(token); credentials.addToken(service, token); } } } // Now collect the tokens from the children final FileSystem[] children = getChildFileSystems(); if (children != null) { for (final FileSystem fs : children) { fs.collectDelegationTokens(renewer, credentials, tokens); } } } // sort gripes because FileStatus Comparable isn't parameterized... @SuppressWarnings("unchecked") private List<FileStatus> globStatusInternal(Path pathPattern, PathFilter filter) throws IOException { boolean patternHasGlob = false; // pathPattern has any globs List<FileStatus> matches = new ArrayList<FileStatus>(); // determine starting point int level = 0; String baseDir = Path.CUR_DIR; if (pathPattern.isAbsolute()) { level = 1; // need to skip empty item at beginning of split list baseDir = Path.SEPARATOR; } // parse components and determine if it's a glob String[] components = null; GlobFilter[] filters = null; String filename = pathPattern.toUri().getPath(); if (!filename.isEmpty() && !Path.SEPARATOR.equals(filename)) { components = filename.split(Path.SEPARATOR); filters = new GlobFilter[components.length]; for (int i = level; i < components.length; i++) { filters[i] = new GlobFilter(components[i]); patternHasGlob |= filters[i].hasPattern(); } if (!patternHasGlob) { baseDir = unquotePathComponent(filename); components = null; // short through to filter check } } // seed the parent directory path, return if it doesn't exist try { matches.add(getFileStatus(new Path(baseDir))); } catch (FileNotFoundException e) { return patternHasGlob ? matches : null; } // skip if there are no components other than the basedir if (components != null) { // iterate through each path component for (int i = level; (i < components.length) && !matches.isEmpty(); i++) { List<FileStatus> children = new ArrayList<FileStatus>(); for (FileStatus match : matches) { // don't look for children in a file matched by a glob if (!match.isDirectory()) { continue; } try { if (filters[i].hasPattern()) { // get all children matching the filter FileStatus[] statuses = listStatus(match.getPath(), filters[i]); children.addAll(Arrays.asList(statuses)); } else { // the component does not have a pattern String component = unquotePathComponent(components[i]); Path child = new Path(match.getPath(), component); children.add(getFileStatus(child)); } } catch (FileNotFoundException e) { // don't care } } matches = children; } } // remove anything that didn't match the filter if (!matches.isEmpty()) { Iterator<FileStatus> iter = matches.iterator(); while (iter.hasNext()) { if (!filter.accept(iter.next().getPath())) { iter.remove(); } } } // no final paths, if there were any globs return empty list if (matches.isEmpty()) { return patternHasGlob ? matches : null; } Collections.sort(matches); return matches; } /* * Filter files/directories in the given path using the user-supplied path * filter. Results are added to the given array <code>results</code>. */ private void listStatus(ArrayList<FileStatus> results, Path f, PathFilter filter) throws FileNotFoundException, IOException { FileStatus listing[] = listStatus(f); if (listing == null) { throw new IOException("Error accessing " + f); } for (int i = 0; i < listing.length; i++) { if (filter.accept(listing[i].getPath())) { results.add(listing[i]); } } } /** * The glob filter builds a regexp per path component. If the component * does not contain a shell metachar, then it falls back to appending the * raw string to the list of built up paths. This raw path needs to have * the quoting removed. Ie. convert all occurances of "\X" to "X" * * @param name * of the path component * @return the unquoted path component */ private String unquotePathComponent(String name) { return name.replaceAll("\\\\(.)", "$1"); } public static final class Statistics { private final String scheme; private AtomicLong bytesRead = new AtomicLong(); private AtomicLong bytesWritten = new AtomicLong(); private AtomicInteger readOps = new AtomicInteger(); private AtomicInteger largeReadOps = new AtomicInteger(); private AtomicInteger writeOps = new AtomicInteger(); /** * Copy constructor. * * @param st * The input Statistics object which is cloned. */ public Statistics(Statistics st) { this.scheme = st.scheme; this.bytesRead = new AtomicLong(st.bytesRead.longValue()); this.bytesWritten = new AtomicLong(st.bytesWritten.longValue()); } public Statistics(String scheme) { this.scheme = scheme; } /** * Get the total number of bytes read * * @return the number of bytes */ public long getBytesRead() { return bytesRead.get(); } /** * Get the total number of bytes written * * @return the number of bytes */ public long getBytesWritten() { return bytesWritten.get(); } /** * Get the number of large file system read operations such as list files * under a large directory * * @return number of large read operations */ public int getLargeReadOps() { return largeReadOps.get(); } /** * Get the number of file system read operations such as list files * * @return number of read operations */ public int getReadOps() { return readOps.get() + largeReadOps.get(); } /** * Get the uri scheme associated with this statistics object. * * @return the schema associated with this set of statistics */ public String getScheme() { return scheme; } /** * Get the number of file system write operations such as create, append * rename etc. * * @return number of write operations */ public int getWriteOps() { return writeOps.get(); } /** * Increment the bytes read in the statistics * * @param newBytes * the additional bytes read */ public void incrementBytesRead(long newBytes) { bytesRead.getAndAdd(newBytes); } /** * Increment the bytes written in the statistics * * @param newBytes * the additional bytes written */ public void incrementBytesWritten(long newBytes) { bytesWritten.getAndAdd(newBytes); } /** * Increment the number of large read operations * * @param count * number of large read operations */ public void incrementLargeReadOps(int count) { largeReadOps.getAndAdd(count); } /** * Increment the number of read operations * * @param count * number of read operations */ public void incrementReadOps(int count) { readOps.getAndAdd(count); } /** * Increment the number of write operations * * @param count * number of write operations */ public void incrementWriteOps(int count) { writeOps.getAndAdd(count); } /** * Reset the counts of bytes to 0. */ public void reset() { bytesWritten.set(0); bytesRead.set(0); } @Override public String toString() { return bytesRead + " bytes read, " + bytesWritten + " bytes written, " + readOps + " read ops, " + largeReadOps + " large read ops, " + writeOps + " write ops"; } } /** Caching FileSystem objects */ static class Cache { private final ClientFinalizer clientFinalizer = new ClientFinalizer(); private final Map<Key, FileSystem> map = new HashMap<Key, FileSystem>(); private final Set<Key> toAutoClose = new HashSet<Key>(); /** A variable that makes all objects in the cache unique */ private static AtomicLong unique = new AtomicLong(1); synchronized void closeAll() throws IOException { closeAll(false); } /** * Close all FileSystem instances in the Cache. * * @param onlyAutomatic * only close those that are marked for automatic closing */ synchronized void closeAll(boolean onlyAutomatic) throws IOException { List<IOException> exceptions = new ArrayList<IOException>(); // Make a copy of the keys in the map since we'll be modifying // the map while iterating over it, which isn't safe. List<Key> keys = new ArrayList<Key>(); keys.addAll(map.keySet()); for (Key key : keys) { final FileSystem fs = map.get(key); if (onlyAutomatic && !toAutoClose.contains(key)) { continue; } // remove from cache remove(key, fs); if (fs != null) { try { fs.close(); } catch (IOException ioe) { exceptions.add(ioe); } } } if (!exceptions.isEmpty()) { throw MultipleIOException.createIOException(exceptions); } } synchronized void closeAll(UserGroupInformation ugi) throws IOException { List<FileSystem> targetFSList = new ArrayList<FileSystem>(); // Make a pass over the list and collect the filesystems to close // we cannot close inline since close() removes the entry from the Map for (Map.Entry<Key, FileSystem> entry : map.entrySet()) { final Key key = entry.getKey(); final FileSystem fs = entry.getValue(); if (ugi.equals(key.ugi) && fs != null) { targetFSList.add(fs); } } List<IOException> exceptions = new ArrayList<IOException>(); // now make a pass over the target list and close each for (FileSystem fs : targetFSList) { try { fs.close(); } catch (IOException ioe) { exceptions.add(ioe); } } if (!exceptions.isEmpty()) { throw MultipleIOException.createIOException(exceptions); } } FileSystem get(URI uri, Configuration conf) throws IOException { Key key = new Key(uri, conf); return getInternal(uri, conf, key); } /** The objects inserted into the cache using this method are all unique */ FileSystem getUnique(URI uri, Configuration conf) throws IOException { Key key = new Key(uri, conf, unique.getAndIncrement()); return getInternal(uri, conf, key); } synchronized void remove(Key key, FileSystem fs) { if (map.containsKey(key) && fs == map.get(key)) { map.remove(key); toAutoClose.remove(key); } } private FileSystem getInternal(URI uri, Configuration conf, Key key) throws IOException { FileSystem fs; synchronized (this) { fs = map.get(key); } if (fs != null) { return fs; } fs = createFileSystem(uri, conf); synchronized (this) { // refetch the lock again FileSystem oldfs = map.get(key); if (oldfs != null) { // a file system is created while lock is releasing fs.close(); // close the new file system return oldfs; // return the old file system } // now insert the new file system into the map if (map.isEmpty() && !ShutdownHookManager.get().isShutdownInProgress()) { ShutdownHookManager.get().addShutdownHook(clientFinalizer, SHUTDOWN_HOOK_PRIORITY); } fs.key = key; map.put(key, fs); if (conf.getBoolean("fs.automatic.close", true)) { toAutoClose.add(key); } return fs; } } /** FileSystem.Cache.Key */ static class Key { static boolean isEqual(Object a, Object b) { return a == b || (a != null && a.equals(b)); } final String scheme; final String authority; final UserGroupInformation ugi; final long unique; // an artificial way to make a key unique Key(URI uri, Configuration conf) throws IOException { this(uri, conf, 0); } Key(URI uri, Configuration conf, long unique) throws IOException { scheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase(); authority = uri.getAuthority() == null ? "" : uri.getAuthority().toLowerCase(); this.unique = unique; this.ugi = UserGroupInformation.getCurrentUser(); } @Override public boolean equals(Object obj) { if (obj == this) { return true; } if (obj != null && obj instanceof Key) { Key that = (Key) obj; return isEqual(this.scheme, that.scheme) && isEqual(this.authority, that.authority) && isEqual(this.ugi, that.ugi) && (this.unique == that.unique); } return false; } @Override public int hashCode() { return (scheme + authority).hashCode() + ugi.hashCode() + (int) unique; } @Override public String toString() { return "(" + ugi.toString() + ")@" + scheme + "://" + authority; } } private class ClientFinalizer implements Runnable { @Override public synchronized void run() { try { closeAll(true); } catch (IOException e) { LOG.info("FileSystem.Cache.closeAll() threw an exception:\n" + e); } } } } }