org.apache.mahout.classifier.df.tools.UDistrib.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.mahout.classifier.df.tools.UDistrib.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.mahout.classifier.df.tools;

import java.io.File;
import java.io.IOException;
import java.util.Locale;
import java.util.Random;
import java.util.Scanner;

import com.google.common.base.Preconditions;
import com.google.common.io.Closeables;
import org.apache.commons.cli2.CommandLine;
import org.apache.commons.cli2.Group;
import org.apache.commons.cli2.Option;
import org.apache.commons.cli2.OptionException;
import org.apache.commons.cli2.builder.ArgumentBuilder;
import org.apache.commons.cli2.builder.DefaultOptionBuilder;
import org.apache.commons.cli2.builder.GroupBuilder;
import org.apache.commons.cli2.commandline.Parser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.mahout.classifier.df.data.DataConverter;
import org.apache.mahout.classifier.df.data.Dataset;
import org.apache.mahout.classifier.df.data.Instance;
import org.apache.mahout.common.CommandLineUtil;
import org.apache.mahout.common.RandomUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * This tool is used to uniformly distribute the class of all the tuples of the dataset over a given number of
 * partitions.<br>
 * This class can be used when the criterion variable is the categorical attribute.
 */
public final class UDistrib {

    private static final Logger log = LoggerFactory.getLogger(UDistrib.class);

    private UDistrib() {
    }

    /**
     * Launch the uniform distribution tool. Requires the following command line arguments:<br>
     * 
     * data : data path dataset : dataset path numpartitions : num partitions output : output path
     *
     * @throws java.io.IOException
     */
    public static void main(String[] args) throws IOException {

        DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
        ArgumentBuilder abuilder = new ArgumentBuilder();
        GroupBuilder gbuilder = new GroupBuilder();

        Option dataOpt = obuilder.withLongName("data").withShortName("d").withRequired(true)
                .withArgument(abuilder.withName("data").withMinimum(1).withMaximum(1).create())
                .withDescription("Data path").create();

        Option datasetOpt = obuilder.withLongName("dataset").withShortName("ds").withRequired(true)
                .withArgument(abuilder.withName("dataset").withMinimum(1).create()).withDescription("Dataset path")
                .create();

        Option outputOpt = obuilder.withLongName("output").withShortName("o").withRequired(true)
                .withArgument(abuilder.withName("output").withMinimum(1).withMaximum(1).create())
                .withDescription("Path to generated files").create();

        Option partitionsOpt = obuilder.withLongName("numpartitions").withShortName("p").withRequired(true)
                .withArgument(abuilder.withName("numparts").withMinimum(1).withMinimum(1).create())
                .withDescription("Number of partitions to create").create();
        Option helpOpt = obuilder.withLongName("help").withDescription("Print out help").withShortName("h")
                .create();

        Group group = gbuilder.withName("Options").withOption(dataOpt).withOption(outputOpt).withOption(datasetOpt)
                .withOption(partitionsOpt).withOption(helpOpt).create();

        try {
            Parser parser = new Parser();
            parser.setGroup(group);
            CommandLine cmdLine = parser.parse(args);

            if (cmdLine.hasOption(helpOpt)) {
                CommandLineUtil.printHelp(group);
                return;
            }

            String data = cmdLine.getValue(dataOpt).toString();
            String dataset = cmdLine.getValue(datasetOpt).toString();
            int numPartitions = Integer.parseInt(cmdLine.getValue(partitionsOpt).toString());
            String output = cmdLine.getValue(outputOpt).toString();

            runTool(data, dataset, output, numPartitions);
        } catch (OptionException e) {
            log.warn(e.toString(), e);
            CommandLineUtil.printHelp(group);
        }

    }

    private static void runTool(String dataStr, String datasetStr, String output, int numPartitions)
            throws IOException {

        Preconditions.checkArgument(numPartitions > 0, "numPartitions <= 0");

        // make sure the output file does not exist
        Path outputPath = new Path(output);
        Configuration conf = new Configuration();
        FileSystem fs = outputPath.getFileSystem(conf);

        Preconditions.checkArgument(!fs.exists(outputPath), "Output path already exists");

        // create a new file corresponding to each partition
        // Path workingDir = fs.getWorkingDirectory();
        // FileSystem wfs = workingDir.getFileSystem(conf);
        // File parentFile = new File(workingDir.toString());
        // File tempFile = FileUtil.createLocalTempFile(parentFile, "Parts", true);
        // File tempFile = File.createTempFile("df.tools.UDistrib","");
        // tempFile.deleteOnExit();
        File tempFile = FileUtil.createLocalTempFile(new File(""), "df.tools.UDistrib", true);
        Path partsPath = new Path(tempFile.toString());
        FileSystem pfs = partsPath.getFileSystem(conf);

        Path[] partPaths = new Path[numPartitions];
        FSDataOutputStream[] files = new FSDataOutputStream[numPartitions];
        for (int p = 0; p < numPartitions; p++) {
            partPaths[p] = new Path(partsPath, String.format(Locale.ENGLISH, "part.%03d", p));
            files[p] = pfs.create(partPaths[p]);
        }

        Path datasetPath = new Path(datasetStr);
        Dataset dataset = Dataset.load(conf, datasetPath);

        // currents[label] = next partition file where to place the tuple
        int[] currents = new int[dataset.nblabels()];

        // currents is initialized randomly in the range [0, numpartitions[
        Random random = RandomUtils.getRandom();
        for (int c = 0; c < currents.length; c++) {
            currents[c] = random.nextInt(numPartitions);
        }

        // foreach tuple of the data
        Path dataPath = new Path(dataStr);
        FileSystem ifs = dataPath.getFileSystem(conf);
        FSDataInputStream input = ifs.open(dataPath);
        Scanner scanner = new Scanner(input, "UTF-8");
        DataConverter converter = new DataConverter(dataset);

        int id = 0;
        while (scanner.hasNextLine()) {
            if (id % 1000 == 0) {
                log.info("progress : {}", id);
            }

            String line = scanner.nextLine();
            if (line.isEmpty()) {
                continue; // skip empty lines
            }

            // write the tuple in files[tuple.label]
            Instance instance = converter.convert(line);
            int label = (int) dataset.getLabel(instance);
            files[currents[label]].writeBytes(line);
            files[currents[label]].writeChar('\n');

            // update currents
            currents[label]++;
            if (currents[label] == numPartitions) {
                currents[label] = 0;
            }
        }

        // close all the files.
        scanner.close();
        for (FSDataOutputStream file : files) {
            Closeables.close(file, false);
        }

        // merge all output files
        FileUtil.copyMerge(pfs, partsPath, fs, outputPath, true, conf, null);
        /*
         * FSDataOutputStream joined = fs.create(new Path(outputPath, "uniform.data")); for (int p = 0; p <
         * numPartitions; p++) {log.info("Joining part : {}", p); FSDataInputStream partStream =
         * fs.open(partPaths[p]);
         * 
         * IOUtils.copyBytes(partStream, joined, conf, false);
         * 
         * partStream.close(); }
         * 
         * joined.close();
         * 
         * fs.delete(partsPath, true);
         */
    }

}