Example usage for org.apache.hadoop.mapreduce Job setOutputKeyClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job setOutputKeyClass.

Prototype

public void setOutputKeyClass(Class<?> theClass) throws IllegalStateException 

Source Link

Document

Set the key class for the job output data.

Usage

From source file:com.alectenharmsel.hadoop.qa.LineCount.java

License:Apache License

public static void main(String[] args) throws Exception {
    GenericOptionsParser parse = new GenericOptionsParser(new Configuration(), args);
    Configuration conf = parse.getConfiguration();

    String[] remainingArgs = parse.getRemainingArgs();
    if (remainingArgs.length != 2) {
        System.err.println("Usage: LineCount <input> <output>");
        System.exit(-1);//  ww w  . j a v a2 s . c  om
    }

    Job job = Job.getInstance(conf, "LineCount");
    job.setJarByClass(LineCount.class);

    job.setMapperClass(Map.class);
    job.setCombinerClass(Reduce.class);
    job.setReducerClass(Reduce.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    FileInputFormat.addInputPath(job, new Path(remainingArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(remainingArgs[1]));

    boolean success = job.waitForCompletion(true);

    int res = success ? 0 : 1;
    System.exit(res);
}

---

From source file:com.alectenharmsel.research.FileCombine.java

License:Apache License

public int run(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.println("Usage: FileCombine <input> <output>");
        System.exit(-1);/*  ww w.j a  va 2  s.c  o  m*/
    }

    Job job = new Job(getConf(), "FileCombine");
    job.setJarByClass(FileCombine.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(FileCombineMapper.class);
    job.setReducerClass(FileCombineReducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    return job.waitForCompletion(true) ? 0 : 1;
}

---

From source file:com.alectenharmsel.research.hadoop.CodeTokenizer.java

License:Apache License

public int run(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.println("Usage: MoabLicenses <input> <output>");
        System.exit(-1);//from w  ww .  ja v a2  s.c  o m
    }

    Configuration conf = getConf();
    Job job = new Job(conf, "SrcTok");
    job.setJarByClass(CodeTokenizer.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(Map.class);
    job.setReducerClass(Reduce.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(LongWritable.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    boolean success = job.waitForCompletion(true);

    return success ? 0 : 1;
}

---

From source file:com.alectenharmsel.research.hadoop.FileCombine.java

License:Apache License

public int run(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.println("Usage: FileCombine <input> <output>");
        System.exit(-1);//  w w w  . j  ava2s  .c  om
    }

    Job job = new Job(getConf(), "FileCombine");
    job.setJarByClass(FileCombine.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(Map.class);
    job.setReducerClass(Reduce.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    return job.waitForCompletion(true) ? 0 : 1;
}

---

From source file:com.alectenharmsel.research.hadoop.LcCounters.java

License:Apache License

public static void main(String[] args) throws Exception {
    GenericOptionsParser parse = new GenericOptionsParser(new Configuration(), args);
    Configuration conf = parse.getConfiguration();

    String[] remainingArgs = parse.getRemainingArgs();
    if (remainingArgs.length != 2) {
        System.err.println("Usage: LineCount <input> <output>");
        System.exit(-1);/*from w w  w.  java 2  s.  co  m*/
    }

    Job job = Job.getInstance(conf, "LineCount");
    job.setJarByClass(LineCount.class);

    job.setMapperClass(Map.class);
    job.setCombinerClass(Reduce.class);
    job.setReducerClass(Reduce.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    FileInputFormat.addInputPath(job, new Path(remainingArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(remainingArgs[1]));

    boolean success = job.waitForCompletion(true);

    //Get the counter here and print it
    Counters counters = job.getCounters();
    long total = counters.findCounter(LcCounters.NUM_LINES).getValue();
    System.out.println(Long.toString(total));

    int res = success ? 0 : 1;
    System.exit(res);
}

---

From source file:com.alectenharmsel.research.hadoop.MoabLicenseInfo.java

License:Apache License

public static void main(String[] args) throws Exception {
    GenericOptionsParser parser = new GenericOptionsParser(new Configuration(), args);
    Configuration conf = parser.getConfiguration();
    conf.set("mapreduce.output.textoutputformat.separator", ",");

    String[] remainingArgs = parser.getRemainingArgs();
    if (remainingArgs.length != 2) {
        System.err.println("Usage: LineCount <input> <output>");
        System.exit(-1);/*from   w  w  w.j  a  v a 2 s . co m*/
    }

    Job job = Job.getInstance(conf, "MoabLicenseInfo");
    job.setJarByClass(MoabLicenseInfo.class);

    job.setMapperClass(Map.class);
    job.setReducerClass(Reduce.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    FileInputFormat.addInputPath(job, new Path(remainingArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(remainingArgs[1]));

    int res = job.waitForCompletion(true) ? 0 : 1;
    System.exit(res);
}

---

From source file:com.alectenharmsel.research.LcCounters.java

License:Apache License

public int run(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.println("Usage: LineCounter <input> <output>");
        System.exit(-1);/*from w  w w.j a  v  a2 s . com*/
    }

    Job job = new Job(getConf(), "LineCount");
    job.setJarByClass(LineCount.class);

    job.setInputFormatClass(WholeBlockInputFormat.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(LineCountMapper.class);
    job.setReducerClass(LineCountReducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    Configuration check = job.getConfiguration();
    boolean success = job.waitForCompletion(true);

    //Get the counter here, output to a file called total in the dir
    Counters counters = job.getCounters();

    //Throw it in the file
    Path outPath = new Path(args[1]);
    FileSystem fs = outPath.getFileSystem(check);
    OutputStream out = fs.create(new Path(outPath, "total"));
    String total = counters.findCounter(LcCounters.NUM_LINES).getValue() + "\n";
    out.write(total.getBytes());
    out.close();
    return success ? 0 : 1;
}

---

From source file:com.alectenharmsel.research.MoabLicenses.java

License:Apache License

public int run(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.println("Usage: MoabLicenses <input> <output>");
        System.exit(-1);//from ww  w .j  a  v a  2  s  . c o m
    }

    Configuration conf = getConf();
    Job job = new Job(conf, "MoabLicenses");
    job.setJarByClass(MoabLicenses.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(MoabLicensesMapper.class);
    job.setReducerClass(MoabLicensesReducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    Configuration check = job.getConfiguration();
    boolean success = job.waitForCompletion(true);

    return success ? 0 : 1;
}

---

From source file:com.alectenharmsel.research.MoabLogSearch.java

License:Apache License

public int run(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.println("Usage: MoabLogSearch <input> <output>");
        System.exit(-1);/*from   w  w w  .jav  a2  s .c  om*/
    }

    Job job = new Job(getConf(), "MoabLogSearch");
    job.setJarByClass(MoabLogSearch.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(MoabLogSearchMapper.class);
    job.setReducerClass(MoabLogSearchReducer.class);

    job.setOutputKeyClass(LongWritable.class);
    job.setOutputValueClass(Text.class);

    Configuration check = job.getConfiguration();
    boolean success = job.waitForCompletion(true);

    return success ? 0 : 1;
}

---

From source file:com.alectenharmsel.research.SrcTok.java

License:Apache License

public int run(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.println("Usage: MoabLicenses <input> <output>");
        System.exit(-1);//  w w  w .  j  av  a2 s . c om
    }

    Configuration conf = getConf();
    Job job = new Job(conf, "SrcTok");
    job.setJarByClass(SrcTok.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(SrcTokMapper.class);
    job.setReducerClass(SrcTokReducer.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(LongWritable.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    boolean success = job.waitForCompletion(true);

    return success ? 0 : 1;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »