Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.nutch.crawl; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.lang.invoke.MethodHandles; import java.net.URLDecoder; import java.text.SimpleDateFormat; import java.util.HashMap; import java.util.Map; import java.util.Random; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.CounterGroup; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.apache.nutch.metadata.Nutch; import org.apache.nutch.util.NutchConfiguration; import org.apache.nutch.util.NutchJob; import org.apache.nutch.util.NutchTool; import org.apache.nutch.util.TimingUtil; import org.apache.nutch.util.URLUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Generic deduplicator which groups fetched URLs with the same digest and marks * all of them as duplicate except the one with the highest score (based on the * score in the crawldb, which is not necessarily the same as the score * indexed). If two (or more) documents have the same score, then the document * with the latest timestamp is kept. If the documents have the same timestamp * then the one with the shortest URL is kept. The documents marked as duplicate * can then be deleted with the command CleaningJob. ***/ public class DeduplicationJob extends NutchTool implements Tool { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final static Text urlKey = new Text("_URLTEMPKEY_"); private final static String DEDUPLICATION_GROUP_MODE = "deduplication.group.mode"; private final static String DEDUPLICATION_COMPARE_ORDER = "deduplication.compare.order"; public static class DBFilter extends Mapper<Text, CrawlDatum, BytesWritable, CrawlDatum> { private String groupMode; @Override public void setup(Mapper<Text, CrawlDatum, BytesWritable, CrawlDatum>.Context context) { Configuration arg0 = context.getConfiguration(); groupMode = arg0.get(DEDUPLICATION_GROUP_MODE); } @Override public void map(Text key, CrawlDatum value, Context context) throws IOException, InterruptedException { if (value.getStatus() == CrawlDatum.STATUS_DB_FETCHED || value.getStatus() == CrawlDatum.STATUS_DB_NOTMODIFIED) { // || value.getStatus() ==CrawlDatum.STATUS_DB_GONE){ byte[] signature = value.getSignature(); if (signature == null) return; String url = key.toString(); BytesWritable sig = null; byte[] data; switch (groupMode) { case "none": sig = new BytesWritable(signature); break; case "host": byte[] host = URLUtil.getHost(url).getBytes(); data = new byte[signature.length + host.length]; System.arraycopy(signature, 0, data, 0, signature.length); System.arraycopy(host, 0, data, signature.length, host.length); sig = new BytesWritable(data); break; case "domain": byte[] domain = URLUtil.getDomainName(url).getBytes(); data = new byte[signature.length + domain.length]; System.arraycopy(signature, 0, data, 0, signature.length); System.arraycopy(domain, 0, data, signature.length, domain.length); sig = new BytesWritable(data); break; } // add the URL as a temporary MD value.getMetaData().put(urlKey, key); // reduce on the signature optionally grouped on host or domain or not at all context.write(sig, value); } } } public static class DedupReducer extends Reducer<BytesWritable, CrawlDatum, Text, CrawlDatum> { private String[] compareOrder; @Override public void setup(Reducer<BytesWritable, CrawlDatum, Text, CrawlDatum>.Context context) { Configuration conf = context.getConfiguration(); compareOrder = conf.get(DEDUPLICATION_COMPARE_ORDER).split(","); } private void writeOutAsDuplicate(CrawlDatum datum, Context context) throws IOException, InterruptedException { datum.setStatus(CrawlDatum.STATUS_DB_DUPLICATE); Text key = (Text) datum.getMetaData().remove(urlKey); context.getCounter("DeduplicationJobStatus", "Documents marked as duplicate").increment(1); context.write(key, datum); } @Override public void reduce(BytesWritable key, Iterable<CrawlDatum> values, Context context) throws IOException, InterruptedException { CrawlDatum existingDoc = null; for (CrawlDatum newDoc : values) { if (existingDoc == null) { existingDoc = new CrawlDatum(); existingDoc.set(newDoc); continue; } CrawlDatum duplicate = getDuplicate(existingDoc, newDoc); if (duplicate != null) { writeOutAsDuplicate(duplicate, context); if (duplicate == existingDoc) { // keep new existingDoc.set(newDoc); } } } } private CrawlDatum getDuplicate(CrawlDatum existingDoc, CrawlDatum newDoc) throws IOException { for (int i = 0; i < compareOrder.length; i++) { switch (compareOrder[i]) { case "score": // compare based on score if (existingDoc.getScore() < newDoc.getScore()) { return existingDoc; } else if (existingDoc.getScore() > newDoc.getScore()) { // mark new one as duplicate return newDoc; } break; case "fetchTime": // same score? delete the one which is oldest if (existingDoc.getFetchTime() > newDoc.getFetchTime()) { // mark new one as duplicate return newDoc; } else if (existingDoc.getFetchTime() < newDoc.getFetchTime()) { // mark existing one as duplicate return existingDoc; } break; case "httpsOverHttp": // prefer https:// over http:// if URLs are identical except for the // protocol String url1 = existingDoc.getMetaData().get(urlKey).toString(); String url2 = newDoc.getMetaData().get(urlKey).toString(); if (url1.startsWith("https://") && url2.startsWith("http://") && url1.substring(8).equals(url2.substring(7))) { // existingDoc with https://, mark newDoc as duplicate return newDoc; } else if (url2.startsWith("https://") && url1.startsWith("http://") && url2.substring(8).equals(url1.substring(7))) { // newDoc with https://, mark existingDoc as duplicate return existingDoc; } break; case "urlLength": // same time? keep the one which has the shortest URL String urlExisting; String urlnewDoc; try { urlExisting = URLDecoder.decode(existingDoc.getMetaData().get(urlKey).toString(), "UTF8"); urlnewDoc = URLDecoder.decode(newDoc.getMetaData().get(urlKey).toString(), "UTF8"); } catch (UnsupportedEncodingException e) { LOG.error("Error decoding: " + urlKey); throw new IOException("UnsupportedEncodingException for " + urlKey); } if (urlExisting.length() < urlnewDoc.length()) { // mark new one as duplicate return newDoc; } else if (urlExisting.length() > urlnewDoc.length()) { // mark existing one as duplicate return existingDoc; } break; } } return null; // no decision possible } } /** Combine multiple new entries for a url. */ public static class StatusUpdateReducer extends Reducer<Text, CrawlDatum, Text, CrawlDatum> { @Override public void setup(Reducer<Text, CrawlDatum, Text, CrawlDatum>.Context context) { } private CrawlDatum old = new CrawlDatum(); private CrawlDatum duplicate = new CrawlDatum(); @Override public void reduce(Text key, Iterable<CrawlDatum> values, Context context) throws IOException, InterruptedException { boolean duplicateSet = false; for (CrawlDatum val : values) { if (val.getStatus() == CrawlDatum.STATUS_DB_DUPLICATE) { duplicate.set(val); duplicateSet = true; } else { old.set(val); } } // keep the duplicate if there is one if (duplicateSet) { context.write(key, duplicate); return; } // no duplicate? keep old one then context.write(key, old); } } public int run(String[] args) throws IOException { if (args.length < 1) { System.err.println( "Usage: DeduplicationJob <crawldb> [-group <none|host|domain>] [-compareOrder <score>,<fetchTime>,<httpsOverHttp>,<urlLength>]"); return 1; } String group = "none"; Path crawlDb = new Path(args[0]); String compareOrder = "score,fetchTime,urlLength"; for (int i = 1; i < args.length; i++) { if (args[i].equals("-group")) group = args[++i]; if (args[i].equals("-compareOrder")) { compareOrder = args[++i]; if (compareOrder.indexOf("score") == -1 || compareOrder.indexOf("fetchTime") == -1 || compareOrder.indexOf("urlLength") == -1) { System.err .println("DeduplicationJob: compareOrder must contain score, fetchTime and urlLength."); return 1; } } } SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); long start = System.currentTimeMillis(); LOG.info("DeduplicationJob: starting at " + sdf.format(start)); Path tempDir = new Path(crawlDb, "dedup-temp-" + Integer.toString(new Random().nextInt(Integer.MAX_VALUE))); Job job = NutchJob.getInstance(getConf()); Configuration conf = job.getConfiguration(); job.setJobName("Deduplication on " + crawlDb); conf.set(DEDUPLICATION_GROUP_MODE, group); conf.set(DEDUPLICATION_COMPARE_ORDER, compareOrder); job.setJarByClass(DeduplicationJob.class); FileInputFormat.addInputPath(job, new Path(crawlDb, CrawlDb.CURRENT_NAME)); job.setInputFormatClass(SequenceFileInputFormat.class); FileOutputFormat.setOutputPath(job, tempDir); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setMapOutputKeyClass(BytesWritable.class); job.setMapOutputValueClass(CrawlDatum.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(CrawlDatum.class); job.setMapperClass(DBFilter.class); job.setReducerClass(DedupReducer.class); FileSystem fs = tempDir.getFileSystem(getConf()); try { boolean success = job.waitForCompletion(true); if (!success) { String message = "Crawl job did not succeed, job status:" + job.getStatus().getState() + ", reason: " + job.getStatus().getFailureInfo(); LOG.error(message); fs.delete(tempDir, true); throw new RuntimeException(message); } CounterGroup g = job.getCounters().getGroup("DeduplicationJobStatus"); if (g != null) { Counter counter = g.findCounter("Documents marked as duplicate"); long dups = counter.getValue(); LOG.info("Deduplication: " + (int) dups + " documents marked as duplicates"); } } catch (IOException | InterruptedException | ClassNotFoundException e) { LOG.error("DeduplicationJob: " + StringUtils.stringifyException(e)); fs.delete(tempDir, true); return -1; } // merge with existing crawl db if (LOG.isInfoEnabled()) { LOG.info("Deduplication: Updating status of duplicate urls into crawl db."); } Job mergeJob = CrawlDb.createJob(getConf(), crawlDb); FileInputFormat.addInputPath(mergeJob, tempDir); mergeJob.setReducerClass(StatusUpdateReducer.class); mergeJob.setJarByClass(DeduplicationJob.class); fs = crawlDb.getFileSystem(getConf()); Path outPath = FileOutputFormat.getOutputPath(job); Path lock = CrawlDb.lock(getConf(), crawlDb, false); try { boolean success = mergeJob.waitForCompletion(true); if (!success) { String message = "Crawl job did not succeed, job status:" + mergeJob.getStatus().getState() + ", reason: " + mergeJob.getStatus().getFailureInfo(); LOG.error(message); fs.delete(tempDir, true); NutchJob.cleanupAfterFailure(outPath, lock, fs); throw new RuntimeException(message); } } catch (IOException | InterruptedException | ClassNotFoundException e) { LOG.error("DeduplicationMergeJob: " + StringUtils.stringifyException(e)); fs.delete(tempDir, true); NutchJob.cleanupAfterFailure(outPath, lock, fs); return -1; } CrawlDb.install(mergeJob, crawlDb); // clean up fs.delete(tempDir, true); long end = System.currentTimeMillis(); LOG.info("Deduplication finished at " + sdf.format(end) + ", elapsed: " + TimingUtil.elapsedTime(start, end)); return 0; } public static void main(String[] args) throws Exception { int result = ToolRunner.run(NutchConfiguration.create(), new DeduplicationJob(), args); System.exit(result); } @Override public Map<String, Object> run(Map<String, Object> args, String crawlId) throws Exception { Map<String, Object> results = new HashMap<>(); String[] arg = new String[1]; String crawldb; if (args.containsKey(Nutch.ARG_CRAWLDB)) { crawldb = (String) args.get(Nutch.ARG_CRAWLDB); } else { crawldb = crawlId + "/crawldb"; } arg[0] = crawldb; int res = run(arg); results.put(Nutch.VAL_RESULT, Integer.toString(res)); return results; } }