Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.mahout.vectorizer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.mahout.common.ClassUtils; import org.apache.mahout.common.lucene.AnalyzerUtils; import org.apache.mahout.math.NamedVector; import org.apache.mahout.math.RandomAccessSparseVector; import org.apache.mahout.math.SequentialAccessSparseVector; import org.apache.mahout.math.Vector; import org.apache.mahout.math.VectorWritable; import org.apache.mahout.vectorizer.encoders.FeatureVectorEncoder; import org.apache.mahout.vectorizer.encoders.LuceneTextValueEncoder; import java.io.IOException; /** * The Mapper that does the work of encoding text */ public class EncodingMapper extends Mapper<Text, Text, Text, VectorWritable> { public static final String USE_NAMED_VECTORS = "namedVectors"; public static final String USE_SEQUENTIAL = "sequential"; public static final String ANALYZER_NAME = "analyzer"; public static final String ENCODER_FIELD_NAME = "encoderFieldName"; public static final String ENCODER_CLASS = "encoderClass"; public static final String CARDINALITY = "cardinality"; private boolean sequentialVectors; private boolean namedVectors; private FeatureVectorEncoder encoder; private int cardinality; @Override protected void setup(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); sequentialVectors = conf.getBoolean(USE_SEQUENTIAL, false); namedVectors = conf.getBoolean(USE_NAMED_VECTORS, false); String analyzerName = conf.get(ANALYZER_NAME, StandardAnalyzer.class.getName()); Analyzer analyzer; try { analyzer = AnalyzerUtils.createAnalyzer(analyzerName); } catch (ClassNotFoundException e) { //TODO: hmmm, don't like this approach throw new IOException("Unable to create Analyzer for name: " + analyzerName, e); } String encoderName = conf.get(ENCODER_FIELD_NAME, "text"); cardinality = conf.getInt(CARDINALITY, 5000); String encClass = conf.get(ENCODER_CLASS); encoder = ClassUtils.instantiateAs(encClass, FeatureVectorEncoder.class, new Class[] { String.class }, new Object[] { encoderName }); if (encoder instanceof LuceneTextValueEncoder) { ((LuceneTextValueEncoder) encoder).setAnalyzer(analyzer); } } @Override protected void map(Text key, Text value, Context context) throws IOException, InterruptedException { Vector vector; if (sequentialVectors) { vector = new SequentialAccessSparseVector(cardinality); } else { vector = new RandomAccessSparseVector(cardinality); } if (namedVectors) { vector = new NamedVector(vector, key.toString()); } encoder.addToVector(value.toString(), vector); context.write(new Text(key.toString()), new VectorWritable(vector)); } }