Java tutorial
/* * Ivory: A Hadoop toolkit for web-scale information retrieval * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You may * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. */ package ivory.core.preprocess; import it.unimi.dsi.sux4j.mph.TwoStepsLcpMonotoneMinimalPerfectHashFunction; import it.unimi.dsi.util.FrontCodedStringList; import it.unimi.dsi.util.ShiftAddXorSignedStringMap; import ivory.core.Constants; import ivory.core.RetrievalEnvironment; import ivory.core.data.dictionary.DictionaryTransformationStrategy; import ivory.core.util.QuickSort; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.ObjectOutputStream; import java.util.Iterator; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; import org.apache.log4j.Logger; import com.google.common.collect.Lists; import edu.umd.cloud9.io.pair.PairOfIntLong; import edu.umd.cloud9.util.PowerTool; public class BuildDictionary extends PowerTool { private static final Logger LOG = Logger.getLogger(BuildDictionary.class); protected static enum Terms { Total } private static class MyReducer extends Reducer<Text, PairOfIntLong, NullWritable, NullWritable> { private FSDataOutputStream termsOut, idsOut, idsToTermOut, dfByTermOut, cfByTermOut, dfByIntOut, cfByIntOut; private int numTerms; private int[] seqNums = null; private int[] dfs = null; private long[] cfs = null; private int curKeyIndex = 0; private String[] terms; @Override public void setup(Reducer<Text, PairOfIntLong, NullWritable, NullWritable>.Context context) throws IOException { LOG.info("Starting setup."); Configuration conf = context.getConfiguration(); FileSystem fs = FileSystem.get(conf); RetrievalEnvironment env = new RetrievalEnvironment(conf.get(Constants.IndexPath), fs); numTerms = conf.getInt(Constants.CollectionTermCount, 0); terms = new String[numTerms]; seqNums = new int[numTerms]; dfs = new int[numTerms]; cfs = new long[numTerms]; termsOut = fs.create(new Path(env.getIndexTermsData()), true); //termsOut.writeInt(numTerms); idsOut = fs.create(new Path(env.getIndexTermIdsData()), true); idsOut.writeInt(numTerms); idsToTermOut = fs.create(new Path(env.getIndexTermIdMappingData()), true); idsToTermOut.writeInt(numTerms); dfByTermOut = fs.create(new Path(env.getDfByTermData()), true); dfByTermOut.writeInt(numTerms); cfByTermOut = fs.create(new Path(env.getCfByTermData()), true); cfByTermOut.writeInt(numTerms); dfByIntOut = fs.create(new Path(env.getDfByIntData()), true); dfByIntOut.writeInt(numTerms); cfByIntOut = fs.create(new Path(env.getCfByIntData()), true); cfByIntOut.writeInt(numTerms); LOG.info("Finished setup."); } @Override public void reduce(Text key, Iterable<PairOfIntLong> values, Context context) throws IOException, InterruptedException { String term = key.toString(); Iterator<PairOfIntLong> iter = values.iterator(); PairOfIntLong p = iter.next(); int df = p.getLeftElement(); long cf = p.getRightElement(); WritableUtils.writeVInt(dfByTermOut, df); WritableUtils.writeVLong(cfByTermOut, cf); if (iter.hasNext()) { throw new RuntimeException("More than one record for term: " + term); } //termsOut.writeUTF(term); terms[curKeyIndex] = term; seqNums[curKeyIndex] = curKeyIndex; dfs[curKeyIndex] = -df; cfs[curKeyIndex] = cf; curKeyIndex++; context.getCounter(Terms.Total).increment(1); } @Override public void cleanup(Reducer<Text, PairOfIntLong, NullWritable, NullWritable>.Context context) throws IOException { LOG.info("Starting cleanup."); if (curKeyIndex != numTerms) { throw new RuntimeException( "Total expected Terms: " + numTerms + ", Total observed terms: " + curKeyIndex + "!"); } // Sort based on df and change seqNums accordingly. QuickSort.quicksortWithSecondary(seqNums, dfs, cfs, 0, numTerms - 1); // Write sorted dfs and cfs by int here. for (int i = 0; i < numTerms; i++) { WritableUtils.writeVInt(dfByIntOut, -dfs[i]); WritableUtils.writeVLong(cfByIntOut, cfs[i]); } cfs = null; // Encode the sorted dfs into ids ==> df values erased and become ids instead. Note that first // term id is 1. for (int i = 0; i < numTerms; i++) { dfs[i] = i + 1; } // Write current seq nums to be index into the term array. for (int i = 0; i < numTerms; i++) idsToTermOut.writeInt(seqNums[i]); // Sort on seqNums to get the right writing order. QuickSort.quicksort(dfs, seqNums, 0, numTerms - 1); for (int i = 0; i < numTerms; i++) { idsOut.writeInt(dfs[i]); } ByteArrayOutputStream bytesOut; ObjectOutputStream objOut; byte[] bytes; List<String> termList = Lists.newArrayList(terms); FrontCodedStringList frontcodedList = new FrontCodedStringList(termList, 8, true); bytesOut = new ByteArrayOutputStream(); objOut = new ObjectOutputStream(bytesOut); objOut.writeObject(frontcodedList); objOut.close(); bytes = bytesOut.toByteArray(); termsOut.writeInt(bytes.length); termsOut.write(bytes); ShiftAddXorSignedStringMap dict = new ShiftAddXorSignedStringMap(termList.iterator(), new TwoStepsLcpMonotoneMinimalPerfectHashFunction<CharSequence>(termList, new DictionaryTransformationStrategy(true))); bytesOut = new ByteArrayOutputStream(); objOut = new ObjectOutputStream(bytesOut); objOut.writeObject(dict); objOut.close(); bytes = bytesOut.toByteArray(); termsOut.writeInt(bytes.length); termsOut.write(bytes); termsOut.close(); idsOut.close(); idsToTermOut.close(); dfByTermOut.close(); cfByTermOut.close(); dfByIntOut.close(); cfByIntOut.close(); LOG.info("Finished cleanup."); } } public static final String[] RequiredParameters = { Constants.CollectionName, Constants.IndexPath }; public String[] getRequiredParameters() { return RequiredParameters; } public BuildDictionary(Configuration conf) { super(conf); } public int runTool() throws Exception { Configuration conf = getConf(); FileSystem fs = FileSystem.get(conf); String indexPath = conf.get(Constants.IndexPath); String collectionName = conf.get(Constants.CollectionName); LOG.info("PowerTool: " + BuildDictionary.class.getCanonicalName()); LOG.info(String.format(" - %s: %s", Constants.CollectionName, collectionName)); LOG.info(String.format(" - %s: %s", Constants.IndexPath, indexPath)); RetrievalEnvironment env = new RetrievalEnvironment(indexPath, fs); if (!fs.exists(new Path(indexPath))) { LOG.error("index path doesn't existing: skipping!"); return 0; } if (fs.exists(new Path(env.getIndexTermsData())) && fs.exists(new Path(env.getIndexTermIdsData())) && fs.exists(new Path(env.getIndexTermIdMappingData())) && fs.exists(new Path(env.getDfByTermData())) && fs.exists(new Path(env.getCfByTermData())) && fs.exists(new Path(env.getDfByIntData())) && fs.exists(new Path(env.getCfByIntData()))) { LOG.info("term and term id data exist: skipping!"); return 0; } conf.setInt(Constants.CollectionTermCount, (int) env.readCollectionTermCount()); conf.set("mapred.child.java.opts", "-Xmx2048m"); Path tmpPath = new Path(env.getTempDirectory()); fs.delete(tmpPath, true); Job job = new Job(conf, BuildDictionary.class.getSimpleName() + ":" + collectionName); job.setJarByClass(BuildDictionary.class); job.setNumReduceTasks(1); FileInputFormat.setInputPaths(job, new Path(env.getTermDfCfDirectory())); FileOutputFormat.setOutputPath(job, tmpPath); job.setInputFormatClass(SequenceFileInputFormat.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(PairOfIntLong.class); job.setOutputKeyClass(Text.class); job.setSortComparatorClass(DictionaryTransformationStrategy.WritableComparator.class); job.setMapperClass(Mapper.class); job.setReducerClass(MyReducer.class); long startTime = System.currentTimeMillis(); job.waitForCompletion(true); LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds"); fs.delete(tmpPath, true); return 0; } }