Example usage for org.apache.hadoop.mapreduce Job waitForCompletion

List of usage examples for org.apache.hadoop.mapreduce Job waitForCompletion

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. waitForCompletion

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job waitForCompletion.

Prototype

public boolean waitForCompletion(boolean verbose)
        throws IOException, InterruptedException, ClassNotFoundException

Source Link

Document

Submit the job to the cluster and wait for it to finish.

Usage

From source file:com.jbw.recommendsystem.add.AddMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("input"));
    Path out = new Path(conf.get("output"));

    Job job = Job.getInstance(conf);
    job.setJobName("5");
    job.setJarByClass(AddMRD.class);

    job.setMapperClass(AddMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);

    job.setReducerClass(AddReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    TextInputFormat.addInputPath(job, in);
    TextOutputFormat.setOutputPath(job, out);

    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.cooc.CoocMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("input"));
    Path out = new Path(conf.get("output"));

    Job surJob = Job.getInstance(conf);
    surJob.setJarByClass(CoocMRD.class);
    surJob.setJobName("Coor");

    surJob.setMapperClass(CoocMapper.class);
    surJob.setReducerClass(IntSumReducer.class);

    surJob.setMapOutputKeyClass(Text.class);
    surJob.setMapOutputValueClass(IntWritable.class);

    surJob.setOutputKeyClass(Text.class);
    surJob.setOutputValueClass(IntWritable.class);

    surJob.setInputFormatClass(TextInputFormat.class);
    surJob.setOutputFormatClass(TextOutputFormat.class);

    TextInputFormat.addInputPath(surJob, in);
    TextOutputFormat.setOutputPath(surJob, out);

    return surJob.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.filter.FilterMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path rPath = new Path(conf.get("rin"));
    Path aPath = new Path(conf.get("ain"));
    Path out = new Path(conf.get("out"));

    Job job = Job.getInstance(conf);
    job.setJobName("666");
    job.setJarByClass(FilterMRD.class);

    MultipleInputs.addInputPath(job, rPath, TextInputFormat.class, RelationMapper.class);
    MultipleInputs.addInputPath(job, aPath, TextInputFormat.class, AddMapper.class);
    job.setMapOutputKeyClass(Text.class);

    job.setReducerClass(FilterReducer.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    TextOutputFormat.setOutputPath(job, out);

    job.setOutputKeyClass(Text.class);

    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.guiyihua.GYHMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("in"));
    Path out = new Path(conf.get("out"));

    Job job = Job.getInstance(conf);
    job.setJarByClass(GYHMRD.class);
    job.setJobName("fdsjh");
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);
    job.setInputFormatClass(TextInputFormat.class);

    job.setPartitionerClass(XXPartition.class);

    job.setOutputFormatClass(TextOutputFormat.class);
    job.setOutputKeyClass(LongWritable.class);
    job.setOutputValueClass(Text.class);

    job.setNumReduceTasks(2);//  ww  w. j  a v  a2  s.c o  m

    TextInputFormat.addInputPath(job, in);
    TextOutputFormat.setOutputPath(job, out);
    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.iteamlist.ItemMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("input"));
    Path out = new Path(conf.get("output"));

    Job surJob = Job.getInstance(conf);
    surJob.setJarByClass(ItemMRD.class);
    surJob.setJobName("item");

    surJob.setMapperClass(ItemMapper.class);
    surJob.setReducerClass(ItemReducer.class);

    surJob.setMapOutputKeyClass(Text.class);
    surJob.setMapOutputValueClass(Text.class);

    surJob.setOutputKeyClass(Text.class);
    surJob.setOutputValueClass(Text.class);

    surJob.setInputFormatClass(TextInputFormat.class);
    surJob.setOutputFormatClass(TextOutputFormat.class);

    TextInputFormat.addInputPath(surJob, in);
    TextOutputFormat.setOutputPath(surJob, out);

    return surJob.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.joinim.JoinMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path itemPath = new Path(conf.get("iteminput"));
    Path matrixPath = new Path(conf.get("matrixinput"));
    Path out = new Path(conf.get("output"));

    Job job = Job.getInstance(conf);
    job.setJobName("jjj");
    job.setJarByClass(JoinMRD.class);

    MultipleInputs.addInputPath(job, itemPath, TextInputFormat.class, IteamMapper.class);
    MultipleInputs.addInputPath(job, matrixPath, TextInputFormat.class, MatrixMapper.class);
    job.setMapOutputKeyClass(Text.class);

    job.setReducerClass(JoinReducer.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    TextOutputFormat.setOutputPath(job, out);

    job.setOutputKeyClass(Text.class);

    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.martrixlist.MartrixListMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("input"));
    Path out = new Path(conf.get("output"));

    Job surJob = Job.getInstance(conf);
    surJob.setJarByClass(MartrixListMRD.class);
    surJob.setJobName("user");

    surJob.setMapperClass(MListMapper.class);
    surJob.setReducerClass(MListReducer.class);

    surJob.setMapOutputKeyClass(Text.class);
    surJob.setMapOutputValueClass(Text.class);

    surJob.setOutputKeyClass(Text.class);
    surJob.setOutputValueClass(Text.class);

    surJob.setInputFormatClass(TextInputFormat.class);
    surJob.setOutputFormatClass(TextOutputFormat.class);

    TextInputFormat.addInputPath(surJob, in);
    TextOutputFormat.setOutputPath(surJob, out);

    return surJob.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.tar.sf.TarDriver.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("input"));
    Path out = new Path(conf.get("output"));

    Job job = Job.getInstance(conf);
    job.setJarByClass(TarDriver.class);
    job.setJobName("test");

    job.setMapperClass(SmallFileMapper.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setInputFormatClass(TextInputFormat.class);
    TextInputFormat.addInputPath(job, in);

    job.setOutputFormatClass(TarOutputFormat.class);
    TarOutputFormat.setOutputPath(job, out);

    // job.setNumReduceTasks(0);

    /*//from w  w w .j a v  a 2s.co  m
        job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TarOutputFormat.class);
    job.setOutputKeyClass(LongWritable.class);
    job.setOutputValueClass(Text.class);
     */
    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.taroutputformat.JobDriver.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("input"));
    Path out = new Path(conf.get("output"));
    Job job = Job.getInstance();
    job.setJobName("test");
    job.setInputFormatClass(FileInputFormat.class);
    job.setOutputFormatClass(TarOutputFormat.class);
    FileInputFormat.addInputPath(job, in);
    TarOutputFormat.setOutputPath(job, out);
    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jeffy.mr.WordCount.java

License:Apache License

/**
 * @param args/*from w  ww  .j av a 2s .co m*/
 */
public static void main(String[] args) {

    String input = "hdfs://master:8020/tmp/jeffy/input/wordcount.txt";
    String output = "hdfs://master:8020/tmp/jeffy/output";
    Configuration config = new Configuration();
    /**
     * Windows???no jobCtrol
     * http://stackoverflow.com/questions/24075669/mapreduce-job-fail-when-submitted-from-windows-machine
     */
    config.set("mapreduce.app-submission.cross-platform", "true");
    config.set("mapred.remote.os", "Linux");
    try {
        Job job = Job.getInstance(config);
        //Windows???
        job.setJarByClass(WordCount.class);
        //?????
        job.setJar("D:\\bigdata\\mapreduce-demo\\src\\main\\java\\WordCount.jar");
        job.setJobName("Wordcount job");
        job.setMapperClass(WordCountMapper.class);
        job.setReducerClass(WordCountReducer.class);
        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        TextInputFormat.setInputPaths(job, new Path(input));
        TextOutputFormat.setOutputPath(job, new Path(output));
        // Submit the job, then poll for progress until the job is complete
        try {
            job.waitForCompletion(true);
        } catch (ClassNotFoundException | InterruptedException e) {
            e.printStackTrace();
        }
    } catch (IOException e) {
        e.printStackTrace();
    }

}