invertedIndex.startJob.java Source code

Introduction

Here is the source code for invertedIndex.startJob.java

Source

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package invertedIndex;

import hadoopProcesses.map;
import hadoopProcesses.reducer;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.mapred.WordCount;

/**
 *
 * @author madhatter
 */
public class startJob {

    public static void start(String[] args) {
        try {
            JobConf conf = new JobConf(WordCount.class);
            conf.setJobName("wordcount");

            conf.setOutputKeyClass(Text.class);
            conf.setOutputValueClass(IntWritable.class);

            lineIndexMapper Map = new lineIndexMapper();
            conf.setMapperClass(Map.getClass());

            lineIndexReducer Reduce = new lineIndexReducer();
            conf.setCombinerClass(Reduce.getClass());
            conf.setReducerClass(Reduce.getClass());

            conf.setInputFormat(TextInputFormat.class);
            conf.setOutputFormat(TextOutputFormat.class);

            FileInputFormat.setInputPaths(conf, new Path(args[1]));

            Path outputDir = new Path(args[2]);

            outputDir.getFileSystem(conf).delete(outputDir, true);
            FileSystem fs = FileSystem.get(conf);
            fs.delete(outputDir, true);

            FileOutputFormat.setOutputPath(conf, outputDir);

            JobClient.runJob(conf);

        } catch (Exception Exp) {

            Exp.printStackTrace();
        }
    }
}