Java tutorial
// Copyright 2014 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.vfs; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.base.Splitter; import com.google.common.base.Throwables; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ForwardingListenableFuture; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.google.common.util.concurrent.Uninterruptibles; import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.util.Preconditions; import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; /** * Implementation of a subset of UNIX-style file globbing, expanding "*" and "?" as wildcards, but * not [a-z] ranges. * * <p><code>**</code> gets special treatment in include patterns. If it is used as a complete path * segment it matches the filenames in subdirectories recursively. * * <p>Importantly, note that the glob matches are in an unspecified order. */ public final class UnixGlob { private UnixGlob() { } private static List<Path> globInternal(Path base, Collection<String> patterns, boolean excludeDirectories, Predicate<Path> dirPred, boolean checkForInterruption, FilesystemCalls syscalls, ThreadPoolExecutor threadPool) throws IOException, InterruptedException { GlobVisitor visitor = (threadPool == null) ? new GlobVisitor(checkForInterruption) : new GlobVisitor(threadPool, checkForInterruption); return visitor.glob(base, patterns, excludeDirectories, dirPred, syscalls); } private static long globInternalAndReturnNumGlobTasksForTesting(Path base, Collection<String> patterns, boolean excludeDirectories, Predicate<Path> dirPred, boolean checkForInterruption, FilesystemCalls syscalls, ThreadPoolExecutor threadPool) throws IOException, InterruptedException { GlobVisitor visitor = (threadPool == null) ? new GlobVisitor(checkForInterruption) : new GlobVisitor(threadPool, checkForInterruption); visitor.glob(base, patterns, excludeDirectories, dirPred, syscalls); return visitor.getNumGlobTasksForTesting(); } private static Future<List<Path>> globAsyncInternal(Path base, Collection<String> patterns, boolean excludeDirectories, Predicate<Path> dirPred, FilesystemCalls syscalls, boolean checkForInterruption, ThreadPoolExecutor threadPool) { Preconditions.checkNotNull(threadPool, "%s %s", base, patterns); return new GlobVisitor(threadPool, checkForInterruption).globAsync(base, patterns, excludeDirectories, dirPred, syscalls); } /** * Checks that each pattern is valid, splits it into segments and checks * that each segment contains only valid wildcards. * * @return list of segment arrays */ private static List<String[]> checkAndSplitPatterns(Collection<String> patterns) { List<String[]> list = Lists.newArrayListWithCapacity(patterns.size()); for (String pattern : patterns) { String error = checkPatternForError(pattern); if (error != null) { throw new IllegalArgumentException(error + " (in glob pattern '" + pattern + "')"); } Iterable<String> segments = Splitter.on('/').split(pattern); list.add(Iterables.toArray(segments, String.class)); } return list; } /** * @return whether or not {@code pattern} contains illegal characters */ public static String checkPatternForError(String pattern) { if (pattern.isEmpty()) { return "pattern cannot be empty"; } if (pattern.charAt(0) == '/') { return "pattern cannot be absolute"; } for (int i = 0; i < pattern.length(); i++) { char c = pattern.charAt(i); switch (c) { case '(': case ')': case '{': case '}': case '[': case ']': return "illegal character '" + c + "'"; } } Iterable<String> segments = Splitter.on('/').split(pattern); for (String segment : segments) { if (segment.isEmpty()) { return "empty segment not permitted"; } if (segment.equals(".") || segment.equals("..")) { return "segment '" + segment + "' not permitted"; } if (segment.contains("**") && !segment.equals("**")) { return "recursive wildcard must be its own segment"; } } return null; } /** * Calls {@link #matches(String, String, Cache) matches(pattern, str, null)} */ public static boolean matches(String pattern, String str) { return matches(pattern, str, null); } /** * Returns whether {@code str} matches the glob pattern {@code pattern}. This * method may use the {@code patternCache} to speed up the matching process. * * @param pattern a glob pattern * @param str the string to match * @param patternCache a cache from patterns to compiled Pattern objects, or * {@code null} to skip caching */ public static boolean matches(String pattern, String str, Cache<String, Pattern> patternCache) { if (pattern.length() == 0 || str.length() == 0) { return false; } // Common case: ** if (pattern.equals("**")) { return true; } // Common case: * if (pattern.equals("*")) { return true; } // If a filename starts with '.', this char must be matched explicitly. if (str.charAt(0) == '.' && pattern.charAt(0) != '.') { return false; } // Common case: *.xyz if (pattern.charAt(0) == '*' && pattern.lastIndexOf('*') == 0) { return str.endsWith(pattern.substring(1)); } // Common case: xyz* int lastIndex = pattern.length() - 1; // The first clause of this if statement is unnecessary, but is an // optimization--charAt runs faster than indexOf. if (pattern.charAt(lastIndex) == '*' && pattern.indexOf('*') == lastIndex) { return str.startsWith(pattern.substring(0, lastIndex)); } Pattern regex = patternCache == null ? null : patternCache.getIfPresent(pattern); if (regex == null) { regex = makePatternFromWildcard(pattern); if (patternCache != null) { patternCache.put(pattern, regex); } } return regex.matcher(str).matches(); } /** * Returns a regular expression implementing a matcher for "pattern", in which * "*" and "?" are wildcards. * * <p>e.g. "foo*bar?.java" -> "foo.*bar.\\.java" */ private static Pattern makePatternFromWildcard(String pattern) { StringBuilder regexp = new StringBuilder(); for (int i = 0, len = pattern.length(); i < len; i++) { char c = pattern.charAt(i); switch (c) { case '*': int toIncrement = 0; if (len > i + 1 && pattern.charAt(i + 1) == '*') { // The pattern '**' is interpreted to match 0 or more directory separators, not 1 or // more. We skip the next * and then find a trailing/leading '/' and get rid of it. toIncrement = 1; if (len > i + 2 && pattern.charAt(i + 2) == '/') { // We have '**/' -- skip the '/'. toIncrement = 2; } else if (len == i + 2 && i > 0 && pattern.charAt(i - 1) == '/') { // We have '/**' -- remove the '/'. regexp.delete(regexp.length() - 1, regexp.length()); } } regexp.append(".*"); i += toIncrement; break; case '?': regexp.append('.'); break; //escape the regexp special characters that are allowed in wildcards case '^': case '$': case '|': case '+': case '{': case '}': case '[': case ']': case '\\': case '.': regexp.append('\\'); regexp.append(c); break; default: regexp.append(c); break; } } return Pattern.compile(regexp.toString()); } /** * Filesystem calls required for glob(). */ public interface FilesystemCalls { /** * Get directory entries and their types. */ Collection<Dirent> readdir(Path path, Symlinks symlinks) throws IOException; /** * Return the stat() for the given path, or null. */ FileStatus statNullable(Path path, Symlinks symlinks); } public static FilesystemCalls DEFAULT_SYSCALLS = new FilesystemCalls() { @Override public Collection<Dirent> readdir(Path path, Symlinks symlinks) throws IOException { return path.readdir(symlinks); } @Override public FileStatus statNullable(Path path, Symlinks symlinks) { return path.statNullable(symlinks); } }; public static final AtomicReference<FilesystemCalls> DEFAULT_SYSCALLS_REF = new AtomicReference<>( DEFAULT_SYSCALLS); public static Builder forPath(Path path) { return new Builder(path); } /** * Builder class for UnixGlob. * * */ public static class Builder { private Path base; private List<String> patterns; private boolean excludeDirectories; private Predicate<Path> pathFilter; private ThreadPoolExecutor threadPool; private AtomicReference<? extends FilesystemCalls> syscalls = new AtomicReference<>(DEFAULT_SYSCALLS); /** * Creates a glob builder with the given base path. */ public Builder(Path base) { this.base = base; this.patterns = Lists.newArrayList(); this.excludeDirectories = false; this.pathFilter = Predicates.alwaysTrue(); } /** * Adds a pattern to include to the glob builder. * * <p>For a description of the syntax of the patterns, see {@link UnixGlob}. */ public Builder addPattern(String pattern) { this.patterns.add(pattern); return this; } /** * Adds a pattern to include to the glob builder. * * <p>For a description of the syntax of the patterns, see {@link UnixGlob}. */ public Builder addPatterns(String... patterns) { Collections.addAll(this.patterns, patterns); return this; } /** * Adds a pattern to include to the glob builder. * * <p>For a description of the syntax of the patterns, see {@link UnixGlob}. */ public Builder addPatterns(Collection<String> patterns) { this.patterns.addAll(patterns); return this; } /** * Sets the FilesystemCalls interface to use on this glob(). */ public Builder setFilesystemCalls(AtomicReference<? extends FilesystemCalls> syscalls) { this.syscalls = (syscalls == null) ? new AtomicReference<FilesystemCalls>(DEFAULT_SYSCALLS) : syscalls; return this; } /** * If set to true, directories are not returned in the glob result. */ public Builder setExcludeDirectories(boolean excludeDirectories) { this.excludeDirectories = excludeDirectories; return this; } /** * Sets the threadpool to use for parallel glob evaluation. * If unset, evaluation is done in-thread. */ public Builder setThreadPool(ThreadPoolExecutor pool) { this.threadPool = pool; return this; } /** * If set, the given predicate is called for every directory * encountered. If it returns false, the corresponding item is not * returned in the output and directories are not traversed either. */ public Builder setDirectoryFilter(Predicate<Path> pathFilter) { this.pathFilter = pathFilter; return this; } /** * Executes the glob. */ public List<Path> glob() throws IOException { try { return globInternal(base, patterns, excludeDirectories, pathFilter, false, syscalls.get(), threadPool); } catch (InterruptedException e) { // cannot happen, since we told globInternal not to throw throw new IllegalStateException(e); } } /** * Executes the glob and returns the result. * * @throws InterruptedException if the thread is interrupted. */ public List<Path> globInterruptible() throws IOException, InterruptedException { return globInternal(base, patterns, excludeDirectories, pathFilter, true, syscalls.get(), threadPool); } @VisibleForTesting public long globInterruptibleAndReturnNumGlobTasksForTesting() throws IOException, InterruptedException { return globInternalAndReturnNumGlobTasksForTesting(base, patterns, excludeDirectories, pathFilter, true, syscalls.get(), threadPool); } /** * Executes the glob asynchronously. {@link #setThreadPool} must have been called already with a * non-null argument. * * @param checkForInterrupt if the returned future may throw InterruptedException. */ public Future<List<Path>> globAsync(boolean checkForInterrupt) { return globAsyncInternal(base, patterns, excludeDirectories, pathFilter, syscalls.get(), checkForInterrupt, threadPool); } } /** * Adapts the result of the glob visitation as a Future. */ private static class GlobFuture extends ForwardingListenableFuture<List<Path>> { private final GlobVisitor visitor; private final boolean checkForInterrupt; private final SettableFuture<List<Path>> delegate = SettableFuture.create(); public GlobFuture(GlobVisitor visitor, boolean interruptible) { this.visitor = visitor; this.checkForInterrupt = interruptible; } @Override public List<Path> get() throws InterruptedException, ExecutionException { return checkForInterrupt ? super.get() : Uninterruptibles.getUninterruptibly(delegate()); } @Override protected ListenableFuture<List<Path>> delegate() { return delegate; } public void setException(IOException exception) { delegate.setException(exception); } public void set(List<Path> paths) { delegate.set(paths); } @Override public boolean cancel(boolean mayInterruptIfRunning) { // Best-effort interrupt of the in-flight visitation. visitor.cancel(); return true; } public void markCanceled() { super.cancel(true); } } /** * GlobVisitor executes a glob using parallelism, which is useful when * the glob() requires many readdir() calls on high latency filesystems. */ private static final class GlobVisitor { // These collections are used across workers and must therefore be thread-safe. private final Collection<Path> results = Sets.newConcurrentHashSet(); private final Cache<String, Pattern> cache = CacheBuilder.newBuilder() .build(new CacheLoader<String, Pattern>() { @Override public Pattern load(String wildcard) { return makePatternFromWildcard(wildcard); } }); private final GlobFuture result; private final ThreadPoolExecutor executor; private final AtomicLong totalOps = new AtomicLong(0); private final AtomicLong pendingOps = new AtomicLong(0); private final AtomicReference<IOException> failure = new AtomicReference<>(); private volatile boolean canceled = false; GlobVisitor(ThreadPoolExecutor executor, boolean failFastOnInterrupt) { this.executor = executor; this.result = new GlobFuture(this, failFastOnInterrupt); } GlobVisitor(boolean failFastOnInterrupt) { this(null, failFastOnInterrupt); } /** * Performs wildcard globbing: returns the list of filenames that match any of * {@code patterns} relative to {@code base}. Directories are traversed if and only if they * match {@code dirPred}. The predicate is also called for the root of the traversal. The order * of the returned list is unspecified. * * <p>Patterns may include "*" and "?", but not "[a-z]". * * <p><code>**</code> gets special treatment in include patterns. If it is * used as a complete path segment it matches the filenames in * subdirectories recursively. * * @throws IllegalArgumentException if any glob pattern * {@linkplain #checkPatternForError(String) contains errors} or if any include pattern * segment contains <code>**</code> but not equal to it. */ public List<Path> glob(Path base, Collection<String> patterns, boolean excludeDirectories, Predicate<Path> dirPred, FilesystemCalls syscalls) throws IOException, InterruptedException { try { return globAsync(base, patterns, excludeDirectories, dirPred, syscalls).get(); } catch (ExecutionException e) { Throwable cause = e.getCause(); Throwables.propagateIfPossible(cause, IOException.class); throw new RuntimeException(e); } } private static boolean isRecursivePattern(String pattern) { return "**".equals(pattern); } /** * Same as {@link #glob}, except does so asynchronously and returns a {@link Future} for the * result. */ public Future<List<Path>> globAsync(Path base, Collection<String> patterns, boolean excludeDirectories, Predicate<Path> dirPred, FilesystemCalls syscalls) { FileStatus baseStat = syscalls.statNullable(base, Symlinks.FOLLOW); if (baseStat == null || patterns.isEmpty()) { return Futures.immediateFuture(Collections.<Path>emptyList()); } List<String[]> splitPatterns = checkAndSplitPatterns(patterns); // We do a dumb loop, even though it will likely duplicate logical work (note that the // physical filesystem operations are cached). In order to optimize, we would need to keep // track of which patterns shared sub-patterns and which did not (for example consider the // glob [*/*.java, sub/*.java, */*.txt]). pendingOps.incrementAndGet(); try { for (String[] splitPattern : splitPatterns) { boolean containsRecursivePattern = false; for (String pattern : splitPattern) { if (isRecursivePattern(pattern)) { containsRecursivePattern = true; break; } } GlobTaskContext context = containsRecursivePattern ? new RecursiveGlobTaskContext(splitPattern, excludeDirectories, dirPred, syscalls) : new GlobTaskContext(splitPattern, excludeDirectories, dirPred, syscalls); context.queueGlob(base, baseStat.isDirectory(), 0); } } finally { decrementAndCheckDone(); } return result; } /** Should only be called by link {@GlobTaskContext}. */ private void queueGlob(final Path base, final boolean baseIsDir, final int idx, final GlobTaskContext context) { enqueue(new Runnable() { @Override public void run() { Profiler.instance().startTask(ProfilerTask.VFS_GLOB, this); try { reallyGlob(base, baseIsDir, idx, context); } catch (IOException e) { failure.set(e); } finally { Profiler.instance().completeTask(ProfilerTask.VFS_GLOB); } } @Override public String toString() { return String.format("%s glob(include=[%s], exclude_directories=%s)", base.getPathString(), "\"" + Joiner.on("\", \"").join(context.patternParts) + "\"", context.excludeDirectories); } }); } protected void enqueue(final Runnable r) { totalOps.incrementAndGet(); pendingOps.incrementAndGet(); Runnable wrapped = new Runnable() { @Override public void run() { try { if (!canceled && failure.get() == null) { r.run(); } } finally { decrementAndCheckDone(); } } }; if (executor == null) { wrapped.run(); } else { executor.execute(wrapped); } } private long getNumGlobTasksForTesting() { return totalOps.get(); } protected void cancel() { this.canceled = true; } private void decrementAndCheckDone() { if (pendingOps.decrementAndGet() == 0) { // We get to 0 iff we are done all the relevant work. This is because we always increment // the pending ops count as we're enqueuing, and don't decrement until the task is complete // (which includes accounting for any additional tasks that one enqueues). if (canceled) { result.markCanceled(); } else if (failure.get() != null) { result.setException(failure.get()); } else { result.set(ImmutableList.copyOf(results)); } } } /** A context for evaluating all the subtasks of a single top-level glob task. */ private class GlobTaskContext { private final String[] patternParts; private final boolean excludeDirectories; private final Predicate<Path> dirPred; private final FilesystemCalls syscalls; GlobTaskContext(String[] patternParts, boolean excludeDirectories, Predicate<Path> dirPred, FilesystemCalls syscalls) { this.patternParts = patternParts; this.excludeDirectories = excludeDirectories; this.dirPred = dirPred; this.syscalls = syscalls; } protected void queueGlob(Path base, boolean baseIsDir, int patternIdx) { GlobVisitor.this.queueGlob(base, baseIsDir, patternIdx, this); } } /** * A special implementation of {@link GlobTaskContext} that dedupes glob subtasks. Our naive * implementation of recursive patterns means there are multiple ways to enqueue the same * logical subtask. */ private class RecursiveGlobTaskContext extends GlobTaskContext { private class GlobTask { private final Path base; private final int patternIdx; private GlobTask(Path base, int patternIdx) { this.base = base; this.patternIdx = patternIdx; } @Override public boolean equals(Object obj) { if (!(obj instanceof GlobTask)) { return false; } GlobTask other = (GlobTask) obj; return base.equals(other.base) && patternIdx == other.patternIdx; } @Override public int hashCode() { return Objects.hash(base, patternIdx); } } private final Set<GlobTask> visitedGlobSubTasks = Sets.newConcurrentHashSet(); private RecursiveGlobTaskContext(String[] patternParts, boolean excludeDirectories, Predicate<Path> dirPred, FilesystemCalls syscalls) { super(patternParts, excludeDirectories, dirPred, syscalls); } @Override protected void queueGlob(Path base, boolean baseIsDir, int patternIdx) { if (visitedGlobSubTasks.add(new GlobTask(base, patternIdx))) { // This is a unique glob task. For example of how duplicates can arise, consider: // glob(['**/foo.txt']) // with the only file being // a/foo.txt // // there are two ways to reach a/foo.txt: one by recursively globbing 'foo.txt' in the // subdirectory 'a', and another other by recursively globbing '**/foo.txt' in the // subdirectory 'a'. super.queueGlob(base, baseIsDir, patternIdx); } } } /** * Expressed in Haskell: * <pre> * reallyGlob base [] = { base } * reallyGlob base [x:xs] = union { reallyGlob(f, xs) | f results "base/x" } * </pre> */ private void reallyGlob(Path base, boolean baseIsDir, int idx, GlobTaskContext context) throws IOException { if (baseIsDir && !context.dirPred.apply(base)) { return; } if (idx == context.patternParts.length) { // Base case. if (!(context.excludeDirectories && baseIsDir)) { results.add(base); } return; } if (!baseIsDir) { // Nothing to find here. return; } final String pattern = context.patternParts[idx]; // ** is special: it can match nothing at all. // For example, x/** matches x, **/y matches y, and x/**/y matches x/y. final boolean isRecursivePattern = isRecursivePattern(pattern); if (isRecursivePattern) { context.queueGlob(base, baseIsDir, idx + 1); } if (!pattern.contains("*") && !pattern.contains("?")) { // We do not need to do a readdir in this case, just a stat. Path child = base.getChild(pattern); FileStatus status = context.syscalls.statNullable(child, Symlinks.FOLLOW); if (status == null || (!status.isDirectory() && !status.isFile())) { // The file is a dangling symlink, fifo, does not exist, etc. return; } boolean childIsDir = status.isDirectory(); context.queueGlob(child, childIsDir, idx + 1); return; } Collection<Dirent> dents = context.syscalls.readdir(base, Symlinks.FOLLOW); for (Dirent dent : dents) { Dirent.Type type = dent.getType(); if (type == Dirent.Type.UNKNOWN) { // The file is a dangling symlink, fifo, etc. continue; } boolean childIsDir = (type == Dirent.Type.DIRECTORY); String text = dent.getName(); Path child = base.getChild(text); if (isRecursivePattern) { // Recurse without shifting the pattern. if (childIsDir) { context.queueGlob(child, childIsDir, idx); } } if (matches(pattern, text, cache)) { // Recurse and consume one segment of the pattern. if (childIsDir) { context.queueGlob(child, childIsDir, idx + 1); } else { // Instead of using an async call, just repeat the base case above. if (idx + 1 == context.patternParts.length) { results.add(child); } } } } } } }