Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.nutch.indexer.replace; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.nutch.crawl.CrawlDatum; import org.apache.nutch.crawl.Inlinks; import org.apache.nutch.indexer.IndexingException; import org.apache.nutch.indexer.IndexingFilter; import org.apache.nutch.indexer.NutchDocument; import org.apache.nutch.indexer.NutchField; import org.apache.nutch.parse.Parse; /** * Do pattern replacements on selected field contents prior to indexing. * * To use this plugin, add <code>index-replace</code> to your * <code>plugin.includes</code>. Example: * * <pre> * <property> * <name>plugin.includes</name> * <value>protocol-(http)|urlfilter-regex|parse-(html|tika|metatags)|index-(basic|anchor|metadata|replace)|urlnormalizer-(pass|regex|basic)|indexer-solr</value> * </property> * </pre> * * And then add the <code>index.replace.regexp</code> property to * <code>conf/nutch-site.xml</code>. This contains a list of replacement * instructions per field name, one per line. eg. * * <pre> * fieldname=/regexp/replacement/[flags] * </pre> * * <pre> * <property> * <name>index.replace.regexp</name> * <value> * hostmatch=.*\\.com * title=/search/replace/2 * </value> * </property> * </pre> * * <code>hostmatch=</code> and <code>urlmatch=</code> lines indicate the match * pattern for a host or url. The field replacements that follow this line will * apply only to pages from the matching host or url. Replacements run in the * order specified. Field names may appear multiple times if multiple * replacements are needed. * * The property format is defined in greater detail in * <code>conf/nutch-default.xml</code>. * * @author Peter Ciuffetti * @see <a * href="https://issues.apache.org/jira/browse/NUTCH-2058">NUTCH-2058</a> */ public class ReplaceIndexer implements IndexingFilter { private static final Log LOG = LogFactory.getLog(ReplaceIndexer.class.getName()); /** Special field name signifying the start of a host-specific match set */ private static final String HOSTMATCH = "hostmatch"; /** Special field name signifying the start of a url-specific match set */ private static final String URLMATCH = "urlmatch"; private static Map<Pattern, List<FieldReplacer>> FIELDREPLACERS_BY_HOST = new LinkedHashMap<Pattern, List<FieldReplacer>>(); private static Map<Pattern, List<FieldReplacer>> FIELDREPLACERS_BY_URL = new LinkedHashMap<Pattern, List<FieldReplacer>>(); private static Pattern LINE_SPLIT = Pattern.compile("(^.+$)+", Pattern.MULTILINE); private static Pattern NAME_VALUE_SPLIT = Pattern.compile("(.*?)=(.*)"); private Configuration conf; /** * {@inheritDoc} */ public void setConf(Configuration conf) { this.conf = conf; FIELDREPLACERS_BY_HOST.clear(); FIELDREPLACERS_BY_URL.clear(); String value = conf.get("index.replace.regexp", null); if (value != null) { LOG.debug("Parsing index.replace.regexp property"); this.parseConf(value); } } /** * {@inheritDoc} */ public Configuration getConf() { return this.conf; } /** * Parse the property value into a set of maps that store a list of * replacements by field for each host and url configured into the property. * * @param propertyValue */ private void parseConf(String propertyValue) { if (propertyValue == null || propertyValue.trim().length() == 0) { return; } // At the start, all replacements apply globally to every host. Pattern hostPattern = Pattern.compile(".*"); Pattern urlPattern = null; // Split the property into lines Matcher lineMatcher = LINE_SPLIT.matcher(propertyValue); while (lineMatcher.find()) { String line = lineMatcher.group(); if (line != null && line.length() > 0) { // Split the line into field and value Matcher nameValueMatcher = NAME_VALUE_SPLIT.matcher(line.trim()); if (nameValueMatcher.find()) { String fieldName = nameValueMatcher.group(1).trim(); String value = nameValueMatcher.group(2); if (fieldName != null && value != null) { // Check if the field name is one of our special cases. if (HOSTMATCH.equals(fieldName)) { urlPattern = null; try { hostPattern = Pattern.compile(value); } catch (PatternSyntaxException pse) { LOG.error("hostmatch pattern " + value + " does not compile: " + pse.getMessage()); // Deactivate this invalid match set by making it match no host. hostPattern = Pattern.compile("willnotmatchanyhost"); } } else if (URLMATCH.equals(fieldName)) { try { urlPattern = Pattern.compile(value); } catch (PatternSyntaxException pse) { LOG.error("urlmatch pattern " + value + " does not compile: " + pse.getMessage()); // Deactivate this invalid match set by making it match no url. urlPattern = Pattern.compile("willnotmatchanyurl"); } } else if (value.length() > 3) { String toFieldName = fieldName; // If the fieldname has a colon, this indicates a different target // field. if (fieldName.indexOf(':') > 0) { toFieldName = fieldName.substring(fieldName.indexOf(':') + 1); fieldName = fieldName.substring(0, fieldName.indexOf(':')); } String sep = value.substring(0, 1); // Divide the value into pattern / replacement / flags. value = value.substring(1); if (!value.contains(sep)) { LOG.error("Pattern '" + line + "', not parseable. Missing separator " + sep); continue; } String pattern = value.substring(0, value.indexOf(sep)); value = value.substring(pattern.length() + 1); String replacement = value; if (value.contains(sep)) { replacement = value.substring(0, value.indexOf(sep)); } int flags = 0; if (value.length() > replacement.length() + 1) { value = value.substring(replacement.length() + 1).trim(); try { flags = Integer.parseInt(value); } catch (NumberFormatException e) { LOG.error("Pattern " + line + ", has invalid flags component"); continue; } } Integer iFlags = (flags > 0) ? new Integer(flags) : null; // Make a FieldReplacer out of these params. FieldReplacer fr = new FieldReplacer(fieldName, toFieldName, pattern, replacement, iFlags); // Add this field replacer to the list for this host or URL. if (urlPattern != null) { List<FieldReplacer> lfp = FIELDREPLACERS_BY_URL.get(urlPattern); if (lfp == null) { lfp = new ArrayList<FieldReplacer>(); } lfp.add(fr); FIELDREPLACERS_BY_URL.put(urlPattern, lfp); } else { List<FieldReplacer> lfp = FIELDREPLACERS_BY_HOST.get(hostPattern); if (lfp == null) { lfp = new ArrayList<FieldReplacer>(); } lfp.add(fr); FIELDREPLACERS_BY_HOST.put(hostPattern, lfp); } } } } } } } /** * {@inheritDoc} */ public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { if (doc != null) { if (FIELDREPLACERS_BY_HOST.size() > 0) { this.doReplace(doc, "host", FIELDREPLACERS_BY_HOST); } if (FIELDREPLACERS_BY_URL.size() > 0) { this.doReplace(doc, "url", FIELDREPLACERS_BY_URL); } } return doc; } /** * Iterates through the replacement map provided, to update the fields in the * Nutch Document. * * @param doc * the document we are modifying * @param keyName * either "host" or "url" -- the field that determines the * replacement set used * @param replaceMap * the list of FieldReplacers that applies to this keyName. */ private void doReplace(NutchDocument doc, String keyName, Map<Pattern, List<FieldReplacer>> replaceMap) { if (doc == null || replaceMap.size() == 0) { return; } Collection<String> docFieldNames = doc.getFieldNames(); NutchField keyField = doc.getField(keyName); if (keyField == null) { // This document doesn't have the key field; no work to do. return; } List<Object> keyFieldValues = keyField.getValues(); if (keyFieldValues.size() == 0) { // This document doesn't have any values for the key field; no work to do. return; } // For every value of the keyField (one expected) for (Object oKeyFieldValue : keyFieldValues) { if (oKeyFieldValue != null && oKeyFieldValue instanceof java.lang.String) { String keyFieldValue = (String) oKeyFieldValue; // For each pattern that we have a replacement list for... for (Map.Entry<Pattern, List<FieldReplacer>> entries : replaceMap.entrySet()) { // If this key is a match for a replacement set... if (entries.getKey().matcher(keyFieldValue).find()) { // For each field we will replace for this key... for (FieldReplacer fp : entries.getValue()) { String fieldName = fp.getFieldName(); // Does this document contain the FieldReplacer's field? if (docFieldNames.contains(fieldName)) { NutchField docField = doc.getField(fieldName); List<Object> fieldValues = docField.getValues(); ArrayList<String> newFieldValues = new ArrayList<String>(); // For each value of the field, match against our // replacer... for (Object oFieldValue : fieldValues) { if (oFieldValue != null && oFieldValue instanceof java.lang.String) { String fieldValue = (String) oFieldValue; String newValue = fp.replace(fieldValue); newFieldValues.add(newValue); } } // Remove the target field and add our replaced values. String targetFieldName = fp.getToFieldName(); doc.removeField(targetFieldName); for (String newFieldValue : newFieldValues) { doc.add(targetFieldName, newFieldValue); } } } } } } } } }