org.apache.mahout.vectorizer.EncodingMapper.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.mahout.vectorizer.EncodingMapper.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.mahout.vectorizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.mahout.common.ClassUtils;
import org.apache.mahout.common.lucene.AnalyzerUtils;
import org.apache.mahout.math.NamedVector;
import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.SequentialAccessSparseVector;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorWritable;
import org.apache.mahout.vectorizer.encoders.FeatureVectorEncoder;
import org.apache.mahout.vectorizer.encoders.LuceneTextValueEncoder;

import java.io.IOException;

/**
 * The Mapper that does the work of encoding text
 */
public class EncodingMapper extends Mapper<Text, Text, Text, VectorWritable> {

    public static final String USE_NAMED_VECTORS = "namedVectors";
    public static final String USE_SEQUENTIAL = "sequential";
    public static final String ANALYZER_NAME = "analyzer";
    public static final String ENCODER_FIELD_NAME = "encoderFieldName";
    public static final String ENCODER_CLASS = "encoderClass";
    public static final String CARDINALITY = "cardinality";
    private boolean sequentialVectors;
    private boolean namedVectors;
    private FeatureVectorEncoder encoder;
    private int cardinality;

    @Override
    protected void setup(Context context) throws IOException, InterruptedException {
        Configuration conf = context.getConfiguration();
        sequentialVectors = conf.getBoolean(USE_SEQUENTIAL, false);
        namedVectors = conf.getBoolean(USE_NAMED_VECTORS, false);
        String analyzerName = conf.get(ANALYZER_NAME, StandardAnalyzer.class.getName());
        Analyzer analyzer;
        try {
            analyzer = AnalyzerUtils.createAnalyzer(analyzerName);
        } catch (ClassNotFoundException e) {
            //TODO: hmmm, don't like this approach
            throw new IOException("Unable to create Analyzer for name: " + analyzerName, e);
        }

        String encoderName = conf.get(ENCODER_FIELD_NAME, "text");
        cardinality = conf.getInt(CARDINALITY, 5000);
        String encClass = conf.get(ENCODER_CLASS);
        encoder = ClassUtils.instantiateAs(encClass, FeatureVectorEncoder.class, new Class[] { String.class },
                new Object[] { encoderName });
        if (encoder instanceof LuceneTextValueEncoder) {
            ((LuceneTextValueEncoder) encoder).setAnalyzer(analyzer);
        }
    }

    @Override
    protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
        Vector vector;
        if (sequentialVectors) {
            vector = new SequentialAccessSparseVector(cardinality);
        } else {
            vector = new RandomAccessSparseVector(cardinality);
        }
        if (namedVectors) {
            vector = new NamedVector(vector, key.toString());
        }
        encoder.addToVector(value.toString(), vector);
        context.write(new Text(key.toString()), new VectorWritable(vector));
    }
}