Example usage for org.apache.hadoop.mapreduce Job getInstance

List of usage examples for org.apache.hadoop.mapreduce Job getInstance

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. getInstance

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job getInstance.

Prototype

@Deprecated
public static Job getInstance(Cluster ignored) throws IOException

Source Link

Document

Creates a new Job with no particular Cluster .

Usage

From source file:com.jbw.recommendsystem.add.AddMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("input"));
    Path out = new Path(conf.get("output"));

    Job job = Job.getInstance(conf);
    job.setJobName("5");
    job.setJarByClass(AddMRD.class);

    job.setMapperClass(AddMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);

    job.setReducerClass(AddReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    TextInputFormat.addInputPath(job, in);
    TextOutputFormat.setOutputPath(job, out);

    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.cooc.CoocMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("input"));
    Path out = new Path(conf.get("output"));

    Job surJob = Job.getInstance(conf);
    surJob.setJarByClass(CoocMRD.class);
    surJob.setJobName("Coor");

    surJob.setMapperClass(CoocMapper.class);
    surJob.setReducerClass(IntSumReducer.class);

    surJob.setMapOutputKeyClass(Text.class);
    surJob.setMapOutputValueClass(IntWritable.class);

    surJob.setOutputKeyClass(Text.class);
    surJob.setOutputValueClass(IntWritable.class);

    surJob.setInputFormatClass(TextInputFormat.class);
    surJob.setOutputFormatClass(TextOutputFormat.class);

    TextInputFormat.addInputPath(surJob, in);
    TextOutputFormat.setOutputPath(surJob, out);

    return surJob.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.filter.FilterMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path rPath = new Path(conf.get("rin"));
    Path aPath = new Path(conf.get("ain"));
    Path out = new Path(conf.get("out"));

    Job job = Job.getInstance(conf);
    job.setJobName("666");
    job.setJarByClass(FilterMRD.class);

    MultipleInputs.addInputPath(job, rPath, TextInputFormat.class, RelationMapper.class);
    MultipleInputs.addInputPath(job, aPath, TextInputFormat.class, AddMapper.class);
    job.setMapOutputKeyClass(Text.class);

    job.setReducerClass(FilterReducer.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    TextOutputFormat.setOutputPath(job, out);

    job.setOutputKeyClass(Text.class);

    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.guiyihua.GYHMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("in"));
    Path out = new Path(conf.get("out"));

    Job job = Job.getInstance(conf);
    job.setJarByClass(GYHMRD.class);
    job.setJobName("fdsjh");
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);
    job.setInputFormatClass(TextInputFormat.class);

    job.setPartitionerClass(XXPartition.class);

    job.setOutputFormatClass(TextOutputFormat.class);
    job.setOutputKeyClass(LongWritable.class);
    job.setOutputValueClass(Text.class);

    job.setNumReduceTasks(2);/* w w w .j  a v a2 s  . c  o m*/

    TextInputFormat.addInputPath(job, in);
    TextOutputFormat.setOutputPath(job, out);
    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.iteamlist.ItemMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("input"));
    Path out = new Path(conf.get("output"));

    Job surJob = Job.getInstance(conf);
    surJob.setJarByClass(ItemMRD.class);
    surJob.setJobName("item");

    surJob.setMapperClass(ItemMapper.class);
    surJob.setReducerClass(ItemReducer.class);

    surJob.setMapOutputKeyClass(Text.class);
    surJob.setMapOutputValueClass(Text.class);

    surJob.setOutputKeyClass(Text.class);
    surJob.setOutputValueClass(Text.class);

    surJob.setInputFormatClass(TextInputFormat.class);
    surJob.setOutputFormatClass(TextOutputFormat.class);

    TextInputFormat.addInputPath(surJob, in);
    TextOutputFormat.setOutputPath(surJob, out);

    return surJob.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.joinim.JoinMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path itemPath = new Path(conf.get("iteminput"));
    Path matrixPath = new Path(conf.get("matrixinput"));
    Path out = new Path(conf.get("output"));

    Job job = Job.getInstance(conf);
    job.setJobName("jjj");
    job.setJarByClass(JoinMRD.class);

    MultipleInputs.addInputPath(job, itemPath, TextInputFormat.class, IteamMapper.class);
    MultipleInputs.addInputPath(job, matrixPath, TextInputFormat.class, MatrixMapper.class);
    job.setMapOutputKeyClass(Text.class);

    job.setReducerClass(JoinReducer.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    TextOutputFormat.setOutputPath(job, out);

    job.setOutputKeyClass(Text.class);

    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.martrixlist.MartrixListMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("input"));
    Path out = new Path(conf.get("output"));

    Job surJob = Job.getInstance(conf);
    surJob.setJarByClass(MartrixListMRD.class);
    surJob.setJobName("user");

    surJob.setMapperClass(MListMapper.class);
    surJob.setReducerClass(MListReducer.class);

    surJob.setMapOutputKeyClass(Text.class);
    surJob.setMapOutputValueClass(Text.class);

    surJob.setOutputKeyClass(Text.class);
    surJob.setOutputValueClass(Text.class);

    surJob.setInputFormatClass(TextInputFormat.class);
    surJob.setOutputFormatClass(TextOutputFormat.class);

    TextInputFormat.addInputPath(surJob, in);
    TextOutputFormat.setOutputPath(surJob, out);

    return surJob.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.tar.sf.TarDriver.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("input"));
    Path out = new Path(conf.get("output"));

    Job job = Job.getInstance(conf);
    job.setJarByClass(TarDriver.class);
    job.setJobName("test");

    job.setMapperClass(SmallFileMapper.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setInputFormatClass(TextInputFormat.class);
    TextInputFormat.addInputPath(job, in);

    job.setOutputFormatClass(TarOutputFormat.class);
    TarOutputFormat.setOutputPath(job, out);

    // job.setNumReduceTasks(0);

    /*//from  w  w w. j av a  2s . c  o m
        job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TarOutputFormat.class);
    job.setOutputKeyClass(LongWritable.class);
    job.setOutputValueClass(Text.class);
     */
    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jeffy.mr.WordCount.java

License:Apache License

/**
 * @param args//from w  w w .ja v  a2 s .com
 */
public static void main(String[] args) {

    String input = "hdfs://master:8020/tmp/jeffy/input/wordcount.txt";
    String output = "hdfs://master:8020/tmp/jeffy/output";
    Configuration config = new Configuration();
    /**
     * Windows???no jobCtrol
     * http://stackoverflow.com/questions/24075669/mapreduce-job-fail-when-submitted-from-windows-machine
     */
    config.set("mapreduce.app-submission.cross-platform", "true");
    config.set("mapred.remote.os", "Linux");
    try {
        Job job = Job.getInstance(config);
        //Windows???
        job.setJarByClass(WordCount.class);
        //?????
        job.setJar("D:\\bigdata\\mapreduce-demo\\src\\main\\java\\WordCount.jar");
        job.setJobName("Wordcount job");
        job.setMapperClass(WordCountMapper.class);
        job.setReducerClass(WordCountReducer.class);
        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        TextInputFormat.setInputPaths(job, new Path(input));
        TextOutputFormat.setOutputPath(job, new Path(output));
        // Submit the job, then poll for progress until the job is complete
        try {
            job.waitForCompletion(true);
        } catch (ClassNotFoundException | InterruptedException e) {
            e.printStackTrace();
        }
    } catch (IOException e) {
        e.printStackTrace();
    }

}

From source file:com.knewton.mapreduce.example.SSTableMRExample.java

License:Apache License

private static Job getJobConf(CommandLine cli) throws URISyntaxException, IOException {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf);
    ClassLoader loader = SSTableMRExample.class.getClassLoader();
    URL url = loader.getResource("knewton-site.xml");
    conf.addResource(url);/* w  w  w.j  ava  2  s .c o m*/

    SSTableInputFormat.setPartitionerClass(RandomPartitioner.class.getName(), job);
    SSTableInputFormat.setComparatorClass(LongType.class.getName(), job);
    SSTableInputFormat.setColumnFamilyName("StudentEvents", job);
    SSTableInputFormat.setKeyspaceName("demoKeyspace", job);

    if (cli.hasOption('s')) {
        conf.set(PropertyConstants.START_DATE.txt, cli.getOptionValue('s'));
    }
    if (cli.hasOption('e')) {
        conf.set(PropertyConstants.END_DATE.txt, cli.getOptionValue('e'));
    }
    return job;
}