List of usage examples for java.nio.file Files walkFileTree
public static Path walkFileTree(Path start, Set<FileVisitOption> options, int maxDepth, FileVisitor<? super Path> visitor) throws IOException
From source file:gov.noaa.pfel.coastwatch.util.FileVisitorDNLS.java
/** * This is a convenience method for using this class. * <p>This works with Amazon AWS S3 bucket URLs. Internal /'s in the keys will be * treated as folder separators. If there aren't any /'s, all the keys will * be in the root directory./*from ww w. j av a 2s. c o m*/ * * @param tDir The starting directory, with \\ or /, with or without trailing slash. * The resulting directoryPA will contain dirs with matching slashes and trailing slash. * @param tPathRegex a regex to constrain which subdirs to include. * This is ignored if recursive is false. * null or "" is treated as .* (i.e., match everything). * @param tDirectoriesToo if true, each directory name will get its own rows * in the results. * @return a table with columns with DIRECTORY, NAME, LASTMODIFIED, and SIZE columns. * LASTMODIFIED and SIZE are LongArrays -- For directories when the values * are otherwise unknown, the value will be Long.MAX_VALUE. * If directoriesToo=true, the original dir won't be included and any * directory's file NAME will be "". * @throws IOException if trouble */ public static Table oneStep(String tDir, String tFileNameRegex, boolean tRecursive, String tPathRegex, boolean tDirectoriesToo) throws IOException { long time = System.currentTimeMillis(); //is tDir an http URL? if (tDir.matches(FileVisitorDNLS.HTTP_REGEX)) { //Is it an S3 bucket with "files"? //If testing a "dir", url should have a trailing slash. Matcher matcher = AWS_S3_PATTERN.matcher(File2.addSlash(tDir)); //force trailing slash if (matcher.matches()) { //http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html //If files have file-system-like names, e.g., // http://bucketname.s3.amazonaws.com/dir1/dir2/fileName.ext) // http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_NorESM1-M_209601-209912.nc // you still can't request just dir2 info because they aren't directories. // They are just object keys with internal slashes. //So specify prefix in request. Table table = makeEmptyTable(); StringArray directoryPA = (StringArray) table.getColumn(DIRECTORY); StringArray namePA = (StringArray) table.getColumn(NAME); LongArray lastModifiedPA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizePA = (LongArray) table.getColumn(SIZE); String bucketName = matcher.group(1); String prefix = matcher.group(2); String baseURL = tDir.substring(0, matcher.start(2)); AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from AWS S3 at" + "\nURL=" + tDir); //"\nbucket=" + bucketName + " prefix=" + prefix); //I wanted to generate lastMod for dir based on lastMod of files //but it would be inconsistent for different requests (recursive, fileNameRegex). //so just a set of dir names. HashSet<String> dirHashSet = new HashSet(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName) .withPrefix(prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyFullName = objectSummary.getKey(); String keyDir = File2.getDirectory(baseURL + keyFullName); String keyName = File2.getNameAndExtension(keyFullName); if (debugMode) String2.log( "keyFullName=" + keyFullName + "\nkeyDir=" + keyDir + "\n tDir=" + tDir); if (keyDir.startsWith(tDir) && //it should (tRecursive || keyDir.length() == tDir.length())) { //store this dir if (tDirectoriesToo) { //S3 only returns object keys. I must infer/collect directories. //Store this dir and parents back to tDir. String choppedKeyDir = keyDir; while (choppedKeyDir.length() >= tDir.length()) { if (!dirHashSet.add(choppedKeyDir)) break; //hash set already had this, so will already have parents //chop off last subdirectory choppedKeyDir = File2.getDirectory( choppedKeyDir.substring(0, choppedKeyDir.length() - 1)); //remove trailing / } } //store this file's information //Sometimes directories appear as files are named "" with size=0. //I don't store those as files. if (debugMode) String2.log("keyName=" + keyFullName + "\n tFileNameRegex=" + tFileNameRegex + " matches=" + keyName.matches(tFileNameRegex)); if (keyName.length() > 0 && keyName.matches(tFileNameRegex)) { directoryPA.add(keyDir); namePA.add(keyName); lastModifiedPA.add(objectSummary.getLastModified().getTime()); //epoch millis sizePA.add(objectSummary.getSize()); //long } } } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); //add directories to the table if (tDirectoriesToo) { Iterator<String> it = dirHashSet.iterator(); while (it.hasNext()) { directoryPA.add(it.next()); namePA.add(""); lastModifiedPA.add(Long.MAX_VALUE); sizePA.add(Long.MAX_VALUE); } } table.leftToRightSortIgnoreCase(2); return table; } catch (AmazonServiceException ase) { throw new IOException("AmazonServiceException: " + ase.getErrorType() + " ERROR, HTTP Code=" + ase.getStatusCode() + ": " + ase.getMessage(), ase); } catch (AmazonClientException ace) { throw new IOException(ace.getMessage(), ace); } } //HYRAX before THREDDS //http://dods.jpl.nasa.gov/opendap/ocean_wind/ccmp/L3.5a/data/flk/1988/ matcher = HYRAX_PATTERN.matcher(tDir); if (matcher.matches()) { try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from Hyrax at" + "\nURL=" + tDir); Table table = makeEmptyTable(); StringArray directoryPA = (StringArray) table.getColumn(DIRECTORY); StringArray namePA = (StringArray) table.getColumn(NAME); LongArray lastModifiedPA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizePA = (LongArray) table.getColumn(SIZE); DoubleArray lastModDA = new DoubleArray(); addToHyraxUrlList(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo, namePA, lastModDA, sizePA); lastModifiedPA.append(lastModDA); int n = namePA.size(); for (int i = 0; i < n; i++) { String fn = namePA.get(i); directoryPA.add(File2.getDirectory(fn)); namePA.set(i, File2.getNameAndExtension(fn)); } table.leftToRightSortIgnoreCase(2); return table; } catch (Throwable t) { throw new IOException(t.getMessage(), t); } } //THREDDS matcher = THREDDS_PATTERN.matcher(tDir); if (matcher.matches()) { try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from THREDDS at" + "\nURL=" + tDir); Table table = makeEmptyTable(); StringArray directoryPA = (StringArray) table.getColumn(DIRECTORY); StringArray namePA = (StringArray) table.getColumn(NAME); LongArray lastModifiedPA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizePA = (LongArray) table.getColumn(SIZE); DoubleArray lastModDA = new DoubleArray(); addToThreddsUrlList(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo, namePA, lastModDA, sizePA); lastModifiedPA.append(lastModDA); int n = namePA.size(); for (int i = 0; i < n; i++) { String fn = namePA.get(i); directoryPA.add(File2.getDirectory(fn)); namePA.set(i, File2.getNameAndExtension(fn)); } table.leftToRightSortIgnoreCase(2); return table; } catch (Throwable t) { throw new IOException(t.getMessage(), t); } } //default: Apache-style WAF try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from Apache-style WAF at" + "\nURL=" + tDir); Table table = makeEmptyTable(); StringArray directorySA = (StringArray) table.getColumn(DIRECTORY); StringArray nameSA = (StringArray) table.getColumn(NAME); LongArray lastModLA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizeLA = (LongArray) table.getColumn(SIZE); addToWAFUrlList(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo, directorySA, nameSA, lastModLA, sizeLA); table.leftToRightSortIgnoreCase(2); return table; } catch (Throwable t) { throw new IOException(t.getMessage(), t); } } //local files //follow symbolic links: https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileVisitor.html //But this doesn't follow Windows symbolic link .lnk's: // http://bugs.java.com/bugdatabase/view_bug.do?bug_id=4237760 FileVisitorDNLS fv = new FileVisitorDNLS(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo); EnumSet<FileVisitOption> opts = EnumSet.of(FileVisitOption.FOLLOW_LINKS); Files.walkFileTree(FileSystems.getDefault().getPath(tDir), opts, //follow symbolic links Integer.MAX_VALUE, //maxDepth fv); fv.table.leftToRightSortIgnoreCase(2); if (verbose) String2.log("FileVisitorDNLS.oneStep(local) finished successfully. n=" + fv.directoryPA.size() + " time=" + (System.currentTimeMillis() - time) + "ms"); return fv.table; }
From source file:org.craftercms.studio.impl.v1.repository.git.GitContentRepository.java
private void copyContentFromBlueprint(String blueprint, String site) { Path siteRepoPath = Paths.get(rootPath, "sites", site); Path blueprintPath = Paths.get(rootPath, "global-configuration", "blueprints", blueprint); EnumSet<FileVisitOption> opts = EnumSet.of(FileVisitOption.FOLLOW_LINKS); TreeCopier tc = new TreeCopier(blueprintPath, siteRepoPath); try {/*from w ww . j a v a2s. co m*/ Files.walkFileTree(blueprintPath, opts, Integer.MAX_VALUE, tc); } catch (IOException err) { logger.error("Error copping files from blueprint", err); } }
From source file:com.bytelightning.opensource.pokerface.PokerFace.java
/** * If requested by the user, this method walks the script directory discovering, loading, compiling, and initialing an .js javascript files it finds in the specified directory or it's children. * @param baseScriptDirectory The contents of this directory should be structured in the same layout as the url's we wish to interfere with. * @param watchScriptDirectory If true, a watch will be placed on <code>baseScriptDirectory</code> and any javascript file modifications (cud) will be dynamically rebuilt and reflected in the running server. * @return True if all scripts were successfully loaded. *//*from w w w. ja v a 2 s .com*/ protected boolean configureScripts(final List<Path> jsLibs, final HierarchicalConfiguration scriptConfig, final Path baseScriptDirectory, boolean watchScriptDirectory) { // Our unit test has verified that CompiledScripts can produce objects (endpoints) that can be executed from ANY thread (and even concurrently execute immutable methods). // However we have not validated that Nashorn can compile *and* recompile scripts from multiple threads. //TODO: Write unit test to see if we can use all available processors to compile discovered javascript files. ScriptCompilationExecutor = Executors.newSingleThreadScheduledExecutor(); // This is done to make sure the engine is allocated in the same thread that will be doing the compiling. Callable<Boolean> compileScriptsTask = new Callable<Boolean>() { @Override public Boolean call() { Nashorn = new ScriptEngineManager().getEngineByName("nashorn"); if (jsLibs != null) for (Path lib : jsLibs) if (!loadScriptLibrary(lib)) return false; // Recursively discover javascript files, compile, load, and setup any that are found. EnumSet<FileVisitOption> opts = EnumSet.of(FileVisitOption.FOLLOW_LINKS); try { Files.walkFileTree(baseScriptDirectory, opts, Integer.MAX_VALUE, new SimpleFileVisitor<Path>() { @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { if (Files.isDirectory(dir) && dir.getFileName().toString().startsWith("#")) return FileVisitResult.SKIP_SUBTREE; return super.preVisitDirectory(dir, attrs); } @Override public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException { if (Files.isRegularFile(path)) { if (path.toString().toLowerCase().endsWith(".js")) { MakeJavaScriptEndPointDescriptor(baseScriptDirectory, path, scriptConfig, new NewEndpointSetupCallback()); } } return FileVisitResult.CONTINUE; } }); } catch (IOException e) { Logger.error("Unable recursively load scripts", e); return false; } return true; } }; // Walk the root directory recursively compiling all discovered javascript files (does not return until all endpoint files have been setup). try { if (!ScriptCompilationExecutor.submit(compileScriptsTask).get()) return false; } catch (Throwable e) { Logger.error("Unable to compile scripts", e); return false; } if (watchScriptDirectory) { try { // Establish a watch on the root ScriptDirectoryWatcher.establishWatch(baseScriptDirectory, new DirectoryWatchEventListener() { // Internal Callable task to load, compile, and initialize a javascript file endpoint. final class CreateEndpointTask implements Callable<Void> { public CreateEndpointTask(Path file, EndpointSetupCompleteCallback callback) { this.file = file; this.callback = callback; } private final Path file; private final EndpointSetupCompleteCallback callback; @Override public Void call() { MakeJavaScriptEndPointDescriptor(baseScriptDirectory, file, scriptConfig, callback); return null; } } // Internal Callable task that gives us the ability to schedule a delayed unload of a deleted or obsoleted endpoint. // By delaying for a period of time longer than twice the socket timeout, we can safely call the endpoint's teardown method. final class DecommisionEndpointTask implements Callable<Void> { private DecommisionEndpointTask(ScriptObjectMirror endpoint) { this.endpoint = endpoint; } private final ScriptObjectMirror endpoint; @Override public Void call() { if (endpoint.hasMember("teardown")) endpoint.callMember("teardown"); return null; } } /** * Called by the WatchService when the contents of the script directory have changed. */ @Override public void onWatchEvent(Path watchDir, final Path oldFile, final Path newFile, FileChangeType change) { if (change == FileChangeType.eRenamed) { // If it was changed to something that does *not* end .js then it should no longer be considered an endpoint. if (oldFile.toString().toLowerCase().endsWith(".js")) if (!newFile.toString().toLowerCase().endsWith(".js")) change = FileChangeType.eDeleted; } if (change == FileChangeType.eModified || change == FileChangeType.eRenamed) { // Decommission the obsolete and load the update. try { assert newFile.toString().toLowerCase().endsWith(".js"); // Will be true because of the 'rename' check at the top of this method. ScriptCompilationExecutor .submit(new CreateEndpointTask(newFile, new NewEndpointSetupCallback() { @Override public ScriptObjectMirror setupComplete(JavaScriptEndPoint endpoint) { ScriptObjectMirror old = super.setupComplete(endpoint); assert old != null; // Yeah, it's hincky, but it won't be in use this long after we remove it from the Map. ScriptCompilationExecutor.schedule(new DecommisionEndpointTask(old), 6, TimeUnit.MINUTES); return null; } })); } catch (Throwable e) { Logger.error("Unable to compile modified script found at " + newFile.toAbsolutePath().toString(), e); } } else if (change == FileChangeType.eCreated) { // This is the easy one. If a javascript file was created, load it. if (newFile.toString().toLowerCase().endsWith(".js")) { try { ScriptCompilationExecutor.submit( new CreateEndpointTask(newFile, new NewEndpointSetupCallback())); } catch (Throwable e) { Logger.error("Unable to compile new script found at " + newFile.toAbsolutePath().toString(), e); } } } else if (change == FileChangeType.eDeleted) { // Endpoint should be decommisioned. if (oldFile.toString().toLowerCase().endsWith(".js")) { String uriKey = FileToUriKey(baseScriptDirectory, oldFile); ScriptObjectMirror desc = scripts.remove(uriKey); if (desc != null) { // Yeah, it's hincky, but it won't be in use this long after we remove it from the Map. ScriptCompilationExecutor.schedule(new DecommisionEndpointTask(desc), 6, TimeUnit.MINUTES); } } } } }); } catch (IOException e) { Logger.error("Unable to establish a real time watch on the script directory.", e); } } else // Not watching for changes, so we are done with the Executor. ScriptCompilationExecutor.shutdown(); return true; }
From source file:org.craftercms.studio.impl.v1.repository.git.GitContentRepository.java
/** * bootstrap the repository// w ww . j a va 2 s . co m */ public void bootstrap() throws Exception { Path globalConfigFolder = Paths.get(rootPath, "global-configuration"); boolean bootstrapCheck = Files.exists(globalConfigFolder); if (bootstrapEnabled && !bootstrapCheck) { try { logger.error("Bootstrapping repository for Crafter CMS"); Files.createDirectories(globalConfigFolder); } catch (Exception alreadyExistsErr) { // do nothing. } try { Path globalConfigRepoPath = Paths.get(globalConfigFolder.toAbsolutePath().toString(), ".git"); Repository repository = FileRepositoryBuilder.create(globalConfigRepoPath.toFile()); repository.create(); } catch (IOException e) { logger.error("Error while creating global configuration repository", e); } String bootstrapFolderPath = this.ctx.getRealPath(File.separator + "gitrepo-bootstrap"); Path source = java.nio.file.FileSystems.getDefault().getPath(bootstrapFolderPath); logger.info("Bootstrapping with baseline @ " + source.toFile().toString()); Path target = Paths.get(rootPath); TreeCopier tc = new TreeCopier(source, target); EnumSet<FileVisitOption> opts = EnumSet.of(FileVisitOption.FOLLOW_LINKS); Files.walkFileTree(source, opts, Integer.MAX_VALUE, tc); try { Repository globalConfigRepo = getGlobalConfigurationRepositoryInstance(); Git git = new Git(globalConfigRepo); Status status = git.status().call(); if (status.hasUncommittedChanges() || !status.isClean()) { DirCache dirCache = git.add().addFilepattern(".").call(); RevCommit commit = git.commit().setMessage("initial content").call(); String tmp = commit.getId().toString(); } } catch (IOException | GitAPIException err) { logger.error("error creating initial commit for global configuration", err); } } }
From source file:org.tinymediamanager.core.movie.tasks.MovieUpdateDatasourceTask2.java
public static HashSet<Path> getAllFilesRecursive(Path folder, int deep) { folder = folder.toAbsolutePath();//from w ww . ja va 2 s .co m AllFilesRecursive visitor = new AllFilesRecursive(); try { Files.walkFileTree(folder, EnumSet.of(FileVisitOption.FOLLOW_LINKS), deep, visitor); } catch (IOException e) { // can not happen, since we overrided visitFileFailed, which throws no // exception ;) } return visitor.fFound; }
From source file:org.tinymediamanager.core.movie.tasks.MovieUpdateDatasourceTask2.java
public void searchAndParse(Path datasource, Path folder, int deep) { folder = folder.toAbsolutePath();// ww w . j a va 2s. com SearchAndParseVisitor visitor = new SearchAndParseVisitor(datasource); try { Files.walkFileTree(folder, EnumSet.of(FileVisitOption.FOLLOW_LINKS), deep, visitor); } catch (IOException e) { // can not happen, since we override visitFileFailed, which throws no // exception ;) } }
From source file:com.facebook.buck.jvm.java.DefaultJavaLibraryIntegrationTest.java
private ImmutableList<Path> getAllFilesInPath(Path path) throws IOException { List<Path> allFiles = new ArrayList<>(); Files.walkFileTree(path, ImmutableSet.of(), Integer.MAX_VALUE, new SimpleFileVisitor<Path>() { @Override//from w w w . j a v a 2 s .c o m public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { allFiles.add(file); return super.visitFile(file, attrs); } }); return ImmutableList.copyOf(allFiles); }