Java tutorial
/** * Copyright 2005 The Apache Software Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.nutch.admin.management; import java.io.IOException; import java.util.LinkedList; import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.mapred.FileOutputFormat; import org.apache.nutch.crawl.CrawlDatum; import org.apache.nutch.parse.ParseOutputFormat; public class FileUtil { private static Log LOG = LogFactory.getLog(FileUtil.class); private static class RunningPathFilter implements PathFilter { public boolean accept(Path file) { String name = file.getName().toLowerCase(); return name.endsWith("running"); } } private static class DirectoryPathFilter implements PathFilter { private FileSystem fFileSystem; public DirectoryPathFilter(FileSystem fileSystem) { this.fFileSystem = fileSystem; } public boolean accept(Path file) { boolean ret = false; try { ret = this.fFileSystem.isDirectory(file); } catch (IOException e) { LOG.warn(e.toString()); } return ret; } } public static long size(Path folder, Configuration configuration) throws IOException { FileSystem fileSystem = FileSystem.get(configuration); // Path[] files = fileSystem.listPaths(folder); FileStatus[] filestatuses = fileSystem.listStatus(folder); int len = filestatuses.length; Path[] files = new Path[len]; for (int i = 0; i < len; i++) { files[i] = filestatuses[i].getPath(); } long size = 0; for (int i = 0; files != null && i < files.length; i++) { Path file = files[i]; if (fileSystem.isDirectory(file)) { size = size + size(file, configuration); } size = size + fileSystem.getLength(file); } return size + fileSystem.getLength(folder); } /** * @return true if fetch.done exists */ public static boolean isFetched(Path segment, Configuration configuration) throws IOException { //return exists(configuration, segment, "fetch.done"); FileSystem fs = FileSystem.get(configuration); return fs.exists(new Path(segment, CrawlDatum.FETCH_DIR_NAME)); } /** * @return true if invert.done exists */ public static boolean isInverted(Path segment, Configuration configuration) throws IOException { return exists(configuration, segment, "invert.done"); //FileSystem fs = FileSystem.get(configuration); //return fs.exists(new Path(segment, CrawlDatum.)); } /** * @return true if parse.done exists */ public static boolean isParsed(Path segment, Configuration configuration) throws IOException { //return exists(configuration, segment, "parse.done"); FileSystem fs = FileSystem.get(configuration); return fs.exists(new Path(segment, CrawlDatum.PARSE_DIR_NAME)); } /** * @return true if parse.done exists */ public static boolean isIndexed(Path segment, Configuration configuration) throws IOException { FileSystem system = FileSystem.get(configuration); // Path[] files = system.listPaths(new Path(segment, "index")); FileStatus[] filestatuses = system.listStatus(new Path(segment, "index")); int len = filestatuses.length; Path[] files = new Path[len]; for (int i = 0; i < len; i++) { files[i] = filestatuses[i].getPath(); } boolean ret = false; for (int i = 0; i < files.length; i++) { //e.g. file = part-00000 Path file = files[i]; if (system.isDirectory(file) && file.getName().startsWith("part-")) { ret = exists(configuration, file, "index.done"); if (!ret) { break; } } } return ret; } /** * @return true if parse.done exists */ public static boolean isInjected(Path instanceFolder, Configuration configuration) throws IOException { Path crawlDir = new Path(configuration.get("crawl.dir")); return exists(configuration, crawlDir, "crawldb"); } /** * @return true if search.done exists */ public static boolean isReadyToSearch(Path segment, Configuration configuration) throws IOException { return exists(configuration, segment, "search.done"); } /** * @return true if fileName in folder exists */ private static boolean exists(Configuration configuration, Path folder, String fileName) throws IOException { FileSystem fileSystem = FileSystem.get(configuration); return fileSystem.exists(new Path(folder, fileName)); } /** * @return true if parse.done exists */ public static List<String> getRunningFiles(Path folder, Configuration configuration) throws IOException { FileSystem fileSystem = FileSystem.get(configuration); // Path[] files = fileSystem.listPaths(folder, new RunningPathFilter()); FileStatus[] filestatuses = fileSystem.listStatus(folder, new RunningPathFilter()); int len = filestatuses.length; Path[] files = new Path[len]; for (int i = 0; i < len; i++) { files[i] = filestatuses[i].getPath(); } List<String> list = new LinkedList<String>(); for (int i = 0; i < files.length; i++) { Path file = files[i]; list.add(file.getName()); } return list; } /** * @return folders in this folder */ public static Path[] listFolders(Path folder, Configuration configuration) throws IOException { FileSystem system = FileSystem.get(configuration); // return system.listPaths(folder, new DirectoryPathFilter(system)); FileStatus[] filestatuses = system.listStatus(folder, new DirectoryPathFilter(system)); int len = filestatuses.length; Path[] files = new Path[len]; for (int i = 0; i < len; i++) { files[i] = filestatuses[i].getPath(); } return files; } }