Java tutorial
/* * Cloud9: A MapReduce Library for Hadoop * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You may * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. */ package edu.umd.cloud9.webgraph.driver; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import edu.umd.cloud9.webgraph.BuildReverseWebGraph; import edu.umd.cloud9.webgraph.BuildWebGraph; import edu.umd.cloud9.webgraph.CollectHostnames; import edu.umd.cloud9.webgraph.ComputeWeight; import edu.umd.cloud9.webgraph.ClueExtractLinks; import edu.umd.cloud9.webgraph.DriverUtil; /** * <p> * Main driver program for extracting the web graph, reverse web graph, and * lines of anchor text. Command-line arguments are as follows: * </p> * * <ul> * <li>[-input collection-base-path]: the base path to the collection, collection file * will be searched recursively</li> * <li>[-output output-base-path]: the base path under which the output would be stored</li> * <li>[-docno docno-mapping-file]: the path to the docno-mapping.dat file</li> * <li>[-begin from-segment]: starting segment number</li> * <li>[-end to-segment]: ending segment number</li> * <li>[-il]: include internal links (i.e., links within a domain)</li> * <li>[-caw]: compute the default weights for lines of external anchor text</li> * <li>[-normalizer normalizer] A normalizer class used to normalize the lines of anchor * text, must extend edu.umd.cloud9.webgraph.normalize.AnchorTextNormalizer.</li> * </ul> * * <p> * The starting and ending segments will correspond to paths * <code>/[collection-base-path]/en.XX</code> to * <code>/[collection-base-path]/en.YY</code>. * </p> * * <p> * The default weight used in this program was originally proposed by Metzler * et. al in the following paper: <br /> * * D. Metzler, J. Novak, H. Cui, and S. Reddy. Building enriched document * representations using aggregated anchor text. <i>In Proc. 32nd Annual * International ACM SIGIR Conference on Research and Development in Information * Retrieval</i>, pages 219{226, New York, NY, USA, 2009. ACM. * </p> * * @author Nima Asadi * */ public class ClueWebDriver extends Configured implements Tool { private static int printUsage() { System.out.println("\nusage:" + "-input collection-path" + "-output output-base" + "-docno userSpecifiedDocnoMappingFile " + "-begin begin_segment" + "-end end_segment" + "[-il] " + "[-caw] " + "-normalizer normalizerClass"); System.out.println("Help:"); System.out.println("[" + DriverUtil.CL_INPUT + " collection-path]\n\tinput directory"); System.out.println("[" + DriverUtil.CL_OUTPUT + " output-base]\n\toutput directory"); System.out.println(DriverUtil.CL_BEGIN_SEGMENT + " begin_segment: First segment to process."); System.out.println(DriverUtil.CL_END_SEGMENT + " end_segment: Last segment to process."); System.out.println(DriverUtil.CL_DOCNO_MAPPING + " docno mapping file."); System.out.println(DriverUtil.CL_INCLUDE_INTERNAL_LINKS + "\n\tinclude internal links, without this" + "option we will not include internal links"); System.out.println(DriverUtil.CL_COMPUTE_WEIGHTS + "\n\tcompute default anchor weights, without this " + "option we will not compute default anchor weights"); System.out.println(DriverUtil.CL_NORMALIZER + " normalizerClass\n\ta normalizer class" + " used to normalize the lines of anchor text," + " must extend edu.umd.cloud9.webgraph.normalize.AnchorTextNormalizer."); System.out.println(); ToolRunner.printGenericCommandUsage(System.out); return -1; } public int run(String[] args) throws Exception { if (args.length < 6) { printUsage(); return -1; } Configuration conf = getConf(); String inputArg = DriverUtil.argValue(args, DriverUtil.CL_INPUT); final String inputBase = inputArg.endsWith("/") ? inputArg : inputArg + "/"; String outputArg = DriverUtil.argValue(args, DriverUtil.CL_OUTPUT); final String outputBase = outputArg.endsWith("/") ? outputArg : outputArg + "/"; final String docnoMapping = DriverUtil.argValue(args, DriverUtil.CL_DOCNO_MAPPING); final int fromSegment = Integer.parseInt(DriverUtil.argValue(args, DriverUtil.CL_BEGIN_SEGMENT)); final int toSegment = Integer.parseInt(DriverUtil.argValue(args, DriverUtil.CL_END_SEGMENT)); final boolean includeInternalLinks = DriverUtil.argExists(args, DriverUtil.CL_INCLUDE_INTERNAL_LINKS); final boolean computeAnchorWeights = DriverUtil.argExists(args, DriverUtil.CL_COMPUTE_WEIGHTS); final String normalizer = DriverUtil.argValue(args, DriverUtil.CL_NORMALIZER); conf.setInt("Cloud9.Mappers", 2000); conf.setInt("Cloud9.Reducers", DriverUtil.DEFAULT_REDUCERS); conf.set("Cloud9.DocnoMappingFile", docnoMapping); conf.setBoolean("Cloud9.IncludeInternalLinks", includeInternalLinks); conf.set("Cloud9.AnchorTextNormalizer", normalizer); // Extract link information for each segment separately for (int i = fromSegment; i <= toSegment; i++) { String inputPath = inputBase + "en." + (i == 10 ? "10" : ("0" + i)); String outputPath = outputBase + DriverUtil.OUTPUT_EXTRACT_LINKS + "/en." + (i == 10 ? "10" : ("0" + i)); conf.set("Cloud9.InputPath", inputPath); conf.set("Cloud9.OutputPath", outputPath); int r = new ClueExtractLinks(conf).run(); if (r != 0) { return -1; } } // Construct the reverse web graph (i.e., collect incoming link // information) String inputPath = ""; for (int i = fromSegment; i < toSegment; i++) { inputPath += outputBase + DriverUtil.OUTPUT_EXTRACT_LINKS + "/en.0" + i + "/,"; } if (toSegment == 10) { inputPath += outputBase + DriverUtil.OUTPUT_EXTRACT_LINKS + "/en.10/"; } else { inputPath += outputBase + DriverUtil.OUTPUT_EXTRACT_LINKS + "/en.0" + toSegment + "/"; } String outputPath = outputBase + DriverUtil.OUTPUT_REVERSE_WEBGRAPH + "/"; conf.set("Cloud9.InputPath", inputPath); conf.set("Cloud9.OutputPath", outputPath); conf.setInt("Cloud9.Mappers", 1); conf.setInt("Cloud9.Reducers", DriverUtil.DEFAULT_REDUCERS * (toSegment - fromSegment + 1)); int r = new BuildReverseWebGraph(conf).run(); if (r != 0) { return -1; } // Construct the web graph inputPath = outputBase + DriverUtil.OUTPUT_REVERSE_WEBGRAPH + "/"; outputPath = outputBase + DriverUtil.OUTPUT_WEBGRAPH + "/"; conf.set("Cloud9.InputPath", inputPath); conf.set("Cloud9.OutputPath", outputPath); conf.setInt("Cloud9.Mappers", 1); conf.setInt("Cloud9.Reducers", DriverUtil.DEFAULT_REDUCERS * (toSegment - fromSegment + 1)); r = new BuildWebGraph(conf).run(); if (r != 0) { return -1; } if (computeAnchorWeights) { // Propagating domain names in order to compute anchor weights inputPath = outputBase + DriverUtil.OUTPUT_WEBGRAPH + "/"; outputPath = outputBase + DriverUtil.OUTPUT_HOST_NAMES + "/"; conf.set("Cloud9.InputPath", inputPath); conf.set("Cloud9.OutputPath", outputPath); conf.setInt("Cloud9.Mappers", 1); conf.setInt("Cloud9.Reducers", DriverUtil.DEFAULT_REDUCERS * (toSegment - fromSegment + 1)); r = new CollectHostnames(conf).run(); if (r != 0) { return -1; } // Compute the weights inputPath = outputBase + DriverUtil.OUTPUT_REVERSE_WEBGRAPH + "/," + outputBase + DriverUtil.OUTPUT_HOST_NAMES + "/"; outputPath = outputBase + DriverUtil.OUTPUT_WEGIHTED_REVERSE_WEBGRAPH + "/"; conf.set("Cloud9.InputPath", inputPath); conf.set("Cloud9.OutputPath", outputPath); conf.setInt("Cloud9.Mappers", 1); conf.setInt("Cloud9.Reducers", DriverUtil.DEFAULT_REDUCERS * (toSegment - fromSegment + 1)); r = new ComputeWeight(conf).run(); if (r != 0) { return -1; } } return 0; } public static void main(String[] args) throws Exception { int res = ToolRunner.run(new Configuration(), new ClueWebDriver(), args); } }