Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.lucene.util; import java.io.BufferedReader; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.nio.channels.FileChannel; import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.FileStore; import java.nio.file.FileVisitResult; import java.nio.file.FileVisitor; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FileSwitchDirectory; import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.store.RAMDirectory; /** This class emulates the new Java 7 "Try-With-Resources" statement. * Remove once Lucene is on Java 7. * @lucene.internal */ public final class IOUtils { /** * UTF-8 charset string. * <p>Where possible, use {@link StandardCharsets#UTF_8} instead, * as using the String constant may slow things down. * @see StandardCharsets#UTF_8 */ public static final String UTF_8 = StandardCharsets.UTF_8.name(); private IOUtils() { } // no instance /** * Closes all given <tt>Closeable</tt>s. Some of the * <tt>Closeable</tt>s may be null; they are * ignored. After everything is closed, the method either * throws the first exception it hit while closing, or * completes normally if there were no exceptions. * * @param objects * objects to call <tt>close()</tt> on */ public static void close(Closeable... objects) throws IOException { close(Arrays.asList(objects)); } /** * Closes all given <tt>Closeable</tt>s. * @see #close(Closeable...) */ public static void close(Iterable<? extends Closeable> objects) throws IOException { Throwable th = null; for (Closeable object : objects) { try { if (object != null) { object.close(); } } catch (Throwable t) { th = useOrSuppress(th, t); } } if (th != null) { throw rethrowAlways(th); } } /** * Closes all given <tt>Closeable</tt>s, suppressing all thrown exceptions. * Some of the <tt>Closeable</tt>s may be null, they are ignored. * * @param objects * objects to call <tt>close()</tt> on */ public static void closeWhileHandlingException(Closeable... objects) { closeWhileHandlingException(Arrays.asList(objects)); } /** * Closes all given <tt>Closeable</tt>s, suppressing all thrown non {@link VirtualMachineError} exceptions. * Even if a {@link VirtualMachineError} is thrown all given closeable are closed. * @see #closeWhileHandlingException(Closeable...) */ public static void closeWhileHandlingException(Iterable<? extends Closeable> objects) { VirtualMachineError firstError = null; Throwable firstThrowable = null; for (Closeable object : objects) { try { if (object != null) { object.close(); } } catch (VirtualMachineError e) { firstError = useOrSuppress(firstError, e); } catch (Throwable t) { firstThrowable = useOrSuppress(firstThrowable, t); } } if (firstError != null) { // we ensure that we bubble up any errors. We can't recover from these but need to make sure they are // bubbled up. if a non-VMError is thrown we also add the suppressed exceptions to it. if (firstThrowable != null) { firstError.addSuppressed(firstThrowable); } throw firstError; } } /** * Wrapping the given {@link InputStream} in a reader using a {@link CharsetDecoder}. * Unlike Java's defaults this reader will throw an exception if your it detects * the read charset doesn't match the expected {@link Charset}. * <p> * Decoding readers are useful to load configuration files, stopword lists or synonym files * to detect character set problems. However, it's not recommended to use as a common purpose * reader. * * @param stream the stream to wrap in a reader * @param charSet the expected charset * @return a wrapping reader */ public static Reader getDecodingReader(InputStream stream, Charset charSet) { final CharsetDecoder charSetDecoder = charSet.newDecoder().onMalformedInput(CodingErrorAction.REPORT) .onUnmappableCharacter(CodingErrorAction.REPORT); return new BufferedReader(new InputStreamReader(stream, charSetDecoder)); } /** * Opens a Reader for the given resource using a {@link CharsetDecoder}. * Unlike Java's defaults this reader will throw an exception if your it detects * the read charset doesn't match the expected {@link Charset}. * <p> * Decoding readers are useful to load configuration files, stopword lists or synonym files * to detect character set problems. However, it's not recommended to use as a common purpose * reader. * @param clazz the class used to locate the resource * @param resource the resource name to load * @param charSet the expected charset * @return a reader to read the given file * */ public static Reader getDecodingReader(Class<?> clazz, String resource, Charset charSet) throws IOException { InputStream stream = null; boolean success = false; try { stream = clazz.getResourceAsStream(resource); final Reader reader = getDecodingReader(stream, charSet); success = true; return reader; } finally { if (!success) { IOUtils.close(stream); } } } /** * Deletes all given files, suppressing all thrown IOExceptions. * <p> * Note that the files should not be null. */ public static void deleteFilesIgnoringExceptions(Directory dir, Collection<String> files) { for (String name : files) { try { dir.deleteFile(name); } catch (Throwable ignored) { // ignore } } } public static void deleteFilesIgnoringExceptions(Directory dir, String... files) { deleteFilesIgnoringExceptions(dir, Arrays.asList(files)); } /** * Deletes all given file names. Some of the * file names may be null; they are * ignored. After everything is deleted, the method either * throws the first exception it hit while deleting, or * completes normally if there were no exceptions. * * @param dir Directory to delete files from * @param names file names to delete */ public static void deleteFiles(Directory dir, Collection<String> names) throws IOException { Throwable th = null; for (String name : names) { if (name != null) { try { dir.deleteFile(name); } catch (Throwable t) { th = useOrSuppress(th, t); } } } if (th != null) { throw rethrowAlways(th); } } /** * Deletes all given files, suppressing all thrown IOExceptions. * <p> * Some of the files may be null, if so they are ignored. */ public static void deleteFilesIgnoringExceptions(Path... files) { deleteFilesIgnoringExceptions(Arrays.asList(files)); } /** * Deletes all given files, suppressing all thrown IOExceptions. * <p> * Some of the files may be null, if so they are ignored. */ public static void deleteFilesIgnoringExceptions(Collection<? extends Path> files) { for (Path name : files) { if (name != null) { try { Files.delete(name); } catch (Throwable ignored) { // ignore } } } } /** * Deletes all given <tt>Path</tt>s, if they exist. Some of the * <tt>File</tt>s may be null; they are * ignored. After everything is deleted, the method either * throws the first exception it hit while deleting, or * completes normally if there were no exceptions. * * @param files files to delete */ public static void deleteFilesIfExist(Path... files) throws IOException { deleteFilesIfExist(Arrays.asList(files)); } /** * Deletes all given <tt>Path</tt>s, if they exist. Some of the * <tt>File</tt>s may be null; they are * ignored. After everything is deleted, the method either * throws the first exception it hit while deleting, or * completes normally if there were no exceptions. * * @param files files to delete */ public static void deleteFilesIfExist(Collection<? extends Path> files) throws IOException { Throwable th = null; for (Path file : files) { try { if (file != null) { Files.deleteIfExists(file); } } catch (Throwable t) { th = useOrSuppress(th, t); } } if (th != null) { throw rethrowAlways(th); } } /** * Deletes one or more files or directories (and everything underneath it). * * @throws IOException if any of the given files (or their subhierarchy files in case * of directories) cannot be removed. */ public static void rm(Path... locations) throws IOException { LinkedHashMap<Path, Throwable> unremoved = rm(new LinkedHashMap<Path, Throwable>(), locations); if (!unremoved.isEmpty()) { StringBuilder b = new StringBuilder( "Could not remove the following files (in the order of attempts):\n"); for (Map.Entry<Path, Throwable> kv : unremoved.entrySet()) { b.append(" ").append(kv.getKey().toAbsolutePath()).append(": ").append(kv.getValue()) .append("\n"); } throw new IOException(b.toString()); } } private static LinkedHashMap<Path, Throwable> rm(final LinkedHashMap<Path, Throwable> unremoved, Path... locations) { if (locations != null) { for (Path location : locations) { // TODO: remove this leniency! if (location != null && Files.exists(location)) { try { Files.walkFileTree(location, new FileVisitor<Path>() { @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { return FileVisitResult.CONTINUE; } @Override public FileVisitResult postVisitDirectory(Path dir, IOException impossible) throws IOException { assert impossible == null; try { Files.delete(dir); } catch (IOException e) { unremoved.put(dir, e); } return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { try { Files.delete(file); } catch (IOException exc) { unremoved.put(file, exc); } return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { if (exc != null) { unremoved.put(file, exc); } return FileVisitResult.CONTINUE; } }); } catch (IOException impossible) { throw new AssertionError("visitor threw exception", impossible); } } } } return unremoved; } /** * This utility method takes a previously caught (non-null) * {@code Throwable} and rethrows either the original argument * if it was a subclass of the {@code IOException} or an * {@code RuntimeException} with the cause set to the argument. * * <p>This method <strong>never returns any value</strong>, even though it declares * a return value of type {@link Error}. The return value declaration * is very useful to let the compiler know that the code path following * the invocation of this method is unreachable. So in most cases the * invocation of this method will be guarded by an {@code if} and * used together with a {@code throw} statement, as in: * </p> * <pre>{@code * if (t != null) throw IOUtils.rethrowAlways(t) * } * </pre> * * @param th The throwable to rethrow, <strong>must not be null</strong>. * @return This method always results in an exception, it never returns any value. * See method documentation for detailsa and usage example. * @throws IOException if the argument was an instance of IOException * @throws RuntimeException with the {@link RuntimeException#getCause()} set * to the argument, if it was not an instance of IOException. */ public static Error rethrowAlways(Throwable th) throws IOException, RuntimeException { if (th == null) { throw new AssertionError("rethrow argument must not be null."); } if (th instanceof IOException) { throw (IOException) th; } if (th instanceof RuntimeException) { throw (RuntimeException) th; } if (th instanceof Error) { throw (Error) th; } throw new RuntimeException(th); } /** * Rethrows the argument as {@code IOException} or {@code RuntimeException} * if it's not null. * * @deprecated This method is deprecated in favor of {@link #rethrowAlways}. Code should * be updated to {@link #rethrowAlways} and guarded with an additional null-argument check * (because {@link #rethrowAlways} is not accepting null arguments). */ @Deprecated public static void reThrow(Throwable th) throws IOException { if (th != null) { throw rethrowAlways(th); } } /** * @deprecated This method is deprecated in favor of {@link #rethrowAlways}. Code should * be updated to {@link #rethrowAlways} and guarded with an additional null-argument check * (because {@link #rethrowAlways} is not accepting null arguments). */ @Deprecated public static void reThrowUnchecked(Throwable th) { if (th != null) { if (th instanceof Error) { throw (Error) th; } if (th instanceof RuntimeException) { throw (RuntimeException) th; } throw new RuntimeException(th); } } /** * Ensure that any writes to the given file is written to the storage device that contains it. * @param fileToSync the file to fsync * @param isDir if true, the given file is a directory (we open for read and ignore IOExceptions, * because not all file systems and operating systems allow to fsync on a directory) */ public static void fsync(Path fileToSync, boolean isDir) throws IOException { // If the file is a directory we have to open read-only, for regular files we must open r/w for the fsync to have an effect. // See http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/ if (isDir && Constants.WINDOWS) { // opening a directory on Windows fails, directories can not be fsynced there if (Files.exists(fileToSync) == false) { // yet do not suppress trying to fsync directories that do not exist throw new NoSuchFileException(fileToSync.toString()); } return; } try (final FileChannel file = FileChannel.open(fileToSync, isDir ? StandardOpenOption.READ : StandardOpenOption.WRITE)) { try { file.force(true); } catch (final IOException e) { if (isDir) { assert (Constants.LINUX || Constants.MAC_OS_X) == false : "On Linux and MacOSX fsyncing a directory should not throw IOException, " + "we just don't want to rely on that in production (undocumented). Got: " + e; // Ignore exception if it is a directory return; } // Throw original exception throw e; } } } /** If the dir is an {@link FSDirectory} or wraps one via possibly * nested {@link FilterDirectory} or {@link FileSwitchDirectory}, * this returns {@link #spins(Path)} for the wrapped directory, * else, true. * * @throws IOException if {@code path} does not exist. * * @lucene.internal */ public static boolean spins(Directory dir) throws IOException { dir = FilterDirectory.unwrap(dir); if (dir instanceof FileSwitchDirectory) { FileSwitchDirectory fsd = (FileSwitchDirectory) dir; // Spinning is contagious: return spins(fsd.getPrimaryDir()) || spins(fsd.getSecondaryDir()); } else if (dir instanceof RAMDirectory) { return false; } else if (dir instanceof FSDirectory) { return spins(((FSDirectory) dir).getDirectory()); } else { return true; } } /** Rough Linux-only heuristics to determine whether the provided * {@code Path} is backed by spinning storage. For example, this * returns false if the disk is a solid-state disk. * * @param path a location to check which must exist. the mount point will be determined from this location. * @return false if the storage is non-rotational (e.g. an SSD), or true if it is spinning or could not be determined * @throws IOException if {@code path} does not exist. * * @lucene.internal */ public static boolean spins(Path path) throws IOException { // resolve symlinks (this will throw exception if the path does not exist) path = path.toRealPath(); // Super cowboy approach, but seems to work! if (!Constants.LINUX) { return true; // no detection } try { return spinsLinux(path); } catch (Exception exc) { // our crazy heuristics can easily trigger SecurityException, AIOOBE, etc ... return true; } } // following methods are package-private for testing ONLY // note: requires a real or fake linux filesystem! static boolean spinsLinux(Path path) throws IOException { FileStore store = getFileStore(path); // if fs type is tmpfs, it doesn't spin. // this won't have a corresponding block device if ("tmpfs".equals(store.type())) { return false; } // get block device name String devName = store.name(); // not a device (e.g. NFS server) if (!devName.startsWith("/")) { return true; } // resolve any symlinks to real block device (e.g. LVM) // /dev/sda0 -> sda0 // /devices/XXX -> sda0 devName = path.getRoot().resolve(devName).toRealPath().getFileName().toString(); // now try to find the longest matching device folder in /sys/block // (that starts with our dev name): Path sysinfo = path.getRoot().resolve("sys").resolve("block"); Path devsysinfo = null; int matchlen = 0; try (DirectoryStream<Path> stream = Files.newDirectoryStream(sysinfo)) { for (Path device : stream) { String name = device.getFileName().toString(); if (name.length() > matchlen && devName.startsWith(name)) { devsysinfo = device; matchlen = name.length(); } } } if (devsysinfo == null) { return true; // give up } // read first byte from rotational, it's a 1 if it spins. Path rotational = devsysinfo.resolve("queue").resolve("rotational"); try (InputStream stream = Files.newInputStream(rotational)) { return stream.read() == '1'; } } // Files.getFileStore(Path) useless here! // don't complain, just try it yourself static FileStore getFileStore(Path path) throws IOException { FileStore store = Files.getFileStore(path); String mount = getMountPoint(store); // find the "matching" FileStore from system list, it's the one we want, but only return // that if it's unambiguous (only one matching): FileStore sameMountPoint = null; for (FileStore fs : path.getFileSystem().getFileStores()) { if (mount.equals(getMountPoint(fs))) { if (sameMountPoint == null) { sameMountPoint = fs; } else { // more than one filesystem has the same mount point; something is wrong! // fall back to crappy one we got from Files.getFileStore return store; } } } if (sameMountPoint != null) { // ok, we found only one, use it: return sameMountPoint; } else { // fall back to crappy one we got from Files.getFileStore return store; } } // these are hacks that are not guaranteed, may change across JVM versions, etc. static String getMountPoint(FileStore store) { String desc = store.toString(); int index = desc.lastIndexOf(" ("); if (index != -1) { return desc.substring(0, index); } else { return desc; } } /** * Returns the second throwable if the first is null otherwise adds the second as suppressed to the first * and returns it. */ public static <T extends Throwable> T useOrSuppress(T first, T second) { if (first == null) { return second; } else { first.addSuppressed(second); } return first; } /** * Applies the consumer to all non-null elements in the collection even if an exception is thrown. The first exception * thrown by the consumer is re-thrown and subsequent exceptions are suppressed. */ public static <T> void applyToAll(Collection<T> collection, IOConsumer<T> consumer) throws IOException { IOUtils.close(collection.stream().filter(Objects::nonNull) .map(t -> (Closeable) () -> consumer.accept(t))::iterator); } /** * An IO operation with a single input. * @see java.util.function.Consumer */ @FunctionalInterface public interface IOConsumer<T> { /** * Performs this operation on the given argument. */ void accept(T input) throws IOException; } }