List of usage examples for org.apache.hadoop.mapreduce Job setMapOutputValueClass
public void setMapOutputValueClass(Class<?> theClass) throws IllegalStateException
From source file:com.yourcompany.hadoop.mapreduce.KoreanWordcountDriver.java
License:Apache License
public int run(String[] args) throws Exception { Job job = new Job(); parseArguements(args, job);//from ww w. j a v a 2 s. co m job.setJarByClass(KoreanWordcountDriver.class); job.setMapperClass(KoreanWordcountMapper.class); job.setReducerClass(KoreanWordcountReducer.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); return job.waitForCompletion(true) ? 0 : 1; }
From source file:com.yourcompany.hadoop.mapreduce.lexical.LexicalAnalyzerDriver.java
License:Apache License
public int run(String[] args) throws Exception { Job job = new Job(); parseArguements(args, job);/*from w w w . j a v a 2 s .c o m*/ job.setJarByClass(LexicalAnalyzerDriver.class); // Mapper & Reducer Class job.setMapperClass(LexicalAnalyzerMapper.class); // Mapper Output Key & Value Type after Hadoop 0.20 job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(Text.class); job.setNumReduceTasks(0); // Run a Hadoop Job return job.waitForCompletion(true) ? 0 : 1; }
From source file:com.yunrang.hadoop.app.utils.CustomizedUtil.java
License:Apache License
@SuppressWarnings("rawtypes") public static void initSnapshotMapperJob(String snapshotName, Path tableRootDir, Scan scan, Class<? extends TableMapper> mapper, Class<?> outputKeyClass, Class<?> outputValueClass, Job job, boolean addDependencyJars) throws IOException { SnapshotInputFormat.setInput(job.getConfiguration(), snapshotName, tableRootDir); Configuration conf = job.getConfiguration(); job.setInputFormatClass(SnapshotInputFormat.class); if (outputValueClass != null) { job.setMapOutputValueClass(outputValueClass); }// w ww .j a va 2s . c o m if (outputKeyClass != null) { job.setMapOutputKeyClass(outputKeyClass); } job.setMapperClass(mapper); conf.set(TableInputFormat.SCAN, convertScanToString(scan)); if (addDependencyJars) { TableMapReduceUtil.addDependencyJars(job); } }
From source file:com.zinnia.nectar.util.hadoop.JobCreatorUtil.java
License:Apache License
public Job createJob(Class<? extends Mapper> mapperClass, Class<? extends WritableComparable> mapperOutputKeyClass, Class<? extends WritableComparable> mapperOutputValueClass, Class<? extends InputFormat> inputFormatClass, String[] inputFilePaths, String outputFilePath) throws IOException { Job job = new Job(); job.setMapperClass(mapperClass);/*from w w w . ja va 2s . c om*/ job.setMapOutputKeyClass(mapperOutputKeyClass); job.setMapOutputValueClass(mapperOutputValueClass); for (String inputFilePath : inputFilePaths) { FileInputFormat.addInputPath(job, new Path(inputFilePath)); } FileOutputFormat.setOutputPath(job, new Path(outputFilePath)); job.setInputFormatClass(inputFormatClass); return job; }
From source file:com.zjy.mongo.util.MongoTool.java
License:Apache License
private int runMapReduceJob(final Configuration conf) throws IOException { final Job job = Job.getInstance(conf, getJobName()); /**//from w ww .j a v a2s . c o m * Any arguments specified with -D <property>=<value> * on the CLI will be picked up and set here * They override any XML level values * Note that -D<space> is important - no space will * not work as it gets picked up by Java itself */ // TODO - Do we need to set job name somehow more specifically? // This may or may not be correct/sane job.setJarByClass(getClass()); final Class<? extends Mapper> mapper = MongoConfigUtil.getMapper(conf); if (LOG.isDebugEnabled()) { LOG.debug("Mapper Class: " + mapper); LOG.debug("Input URI: " + conf.get(MongoConfigUtil.INPUT_URI)); } job.setMapperClass(mapper); Class<? extends Reducer> combiner = MongoConfigUtil.getCombiner(conf); if (combiner != null) { job.setCombinerClass(combiner); } job.setReducerClass(MongoConfigUtil.getReducer(conf)); job.setOutputFormatClass(MongoConfigUtil.getOutputFormat(conf)); job.setOutputKeyClass(MongoConfigUtil.getOutputKey(conf)); job.setOutputValueClass(MongoConfigUtil.getOutputValue(conf)); job.setInputFormatClass(MongoConfigUtil.getInputFormat(conf)); Class mapOutputKeyClass = MongoConfigUtil.getMapperOutputKey(conf); Class mapOutputValueClass = MongoConfigUtil.getMapperOutputValue(conf); if (mapOutputKeyClass != null) { job.setMapOutputKeyClass(mapOutputKeyClass); } if (mapOutputValueClass != null) { job.setMapOutputValueClass(mapOutputValueClass); } /** * Determines if the job will run verbosely e.g. print debug output * Only works with foreground jobs */ final boolean verbose = MongoConfigUtil.isJobVerbose(conf); /** * Run job in foreground aka wait for completion or background? */ final boolean background = MongoConfigUtil.isJobBackground(conf); try { if (background) { LOG.info("Setting up and running MapReduce job in background."); job.submit(); return 0; } else { LOG.info("Setting up and running MapReduce job in foreground, will wait for results. {Verbose? " + verbose + "}"); return job.waitForCompletion(true) ? 0 : 1; } } catch (final Exception e) { LOG.error("Exception while executing job... ", e); return 1; } }
From source file:com.zqh.hadoop.mr.Financial.HighLowDayDriver.java
License:Apache License
@Override public int run(String[] args) throws Exception { String input, output;/*from ww w . j a v a 2s . com*/ if (args.length == 2) { input = args[0]; output = args[1]; } else { System.err.println("Incorrect number of arguments. Expected: input output"); return -1; } Job job = new Job(getConf()); job.setJarByClass(HighLowDayDriver.class); job.setJobName("High Low per Day"); FileInputFormat.setInputPaths(job, new Path(input)); FileOutputFormat.setOutputPath(job, new Path(output)); job.setMapperClass(HighLowDayMapper.class); job.setReducerClass(HighLowDayReducer.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(DoubleWritable.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); boolean success = job.waitForCompletion(true); return success ? 0 : 1; }
From source file:com.zqh.hadoop.mr.Financial.HighLowStockDriver.java
License:Apache License
@Override public int run(String[] args) throws Exception { String input, output;// w w w . j ava 2s .c o m if (args.length == 2) { input = args[0]; output = args[1]; } else { System.err.println("Incorrect number of arguments. Expected: input output"); return -1; } Job job = new Job(getConf()); job.setJarByClass(HighLowStockDriver.class); job.setJobName("High Low per Stock"); FileInputFormat.setInputPaths(job, new Path(input)); FileOutputFormat.setOutputPath(job, new Path(output)); job.setMapperClass(HighLowStockMapper.class); job.setReducerClass(HighLowStockReducer.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(DoubleWritable.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); boolean success = job.waitForCompletion(true); return success ? 0 : 1; }
From source file:com.zqh.hadoop.mr.Financial.HighLowWritableDriver.java
License:Apache License
@Override public int run(String[] args) throws Exception { String input, output;/*from w w w . j ava 2s .co m*/ if (args.length == 2) { input = args[0]; output = args[1]; } else { System.err.println("Incorrect number of arguments. Expected: input output"); return -1; } Job job = new Job(getConf()); job.setJarByClass(HighLowWritableDriver.class); job.setJobName("High Low per Stock with day"); FileInputFormat.setInputPaths(job, new Path(input)); FileOutputFormat.setOutputPath(job, new Path(output)); job.setMapperClass(HighLowWritableMapper.class); job.setReducerClass(HighLowWritableReducer.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(StockWritable.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); boolean success = job.waitForCompletion(true); return success ? 0 : 1; }
From source file:corr.job.CIAJob.java
License:Apache License
@Override public int run(String[] args) throws Exception { if (args.length != 1) { System.err.println("usage: CIAJob <input path>"); ToolRunner.printGenericCommandUsage(System.err); return -1; }/* w ww .j a v a2 s . co m*/ String inPath = args[0]; Formatter formatter = new Formatter(); String outPath = "." + args[0] + "_" + formatter.format("%1$tm%1$td%1$tH%1$tM%1$tS", new Date()); Configuration config = getConf(); Job job = new Job(config, "Algorithm A Concordance Index"); job.setJarByClass(CIAJob.class); FileInputFormat.addInputPath(job, new Path(inPath)); FileOutputFormat.setOutputPath(job, new Path(outPath)); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapperClass(CIAMapper.class); job.setReducerClass(CIAReducer.class); job.setMapOutputKeyClass(VariableIndexPairsWritable.class); job.setMapOutputValueClass(VariableValuePairsWritable.class); job.setOutputKeyClass(VariableIndexPairsWritable.class); job.setOutputValueClass(CIComputationWritable.class); return job.waitForCompletion(true) ? 0 : 1; }
From source file:corr.job.CIJob.java
License:Apache License
@Override public int run(String[] args) throws Exception { if (args.length != 1) { System.err.println("usage: CIJob <input path>"); ToolRunner.printGenericCommandUsage(System.err); return -1; }//from ww w . j av a 2 s . c o m String inPath = args[0]; Formatter formatter = new Formatter(); String outPath = "." + args[0] + "_" + formatter.format("%1$tm%1$td%1$tH%1$tM%1$tS", new Date()); Configuration config = getConf(); Job job = new Job(config, "Seqfile Concordance Index"); job.setJarByClass(CIJob.class); FileInputFormat.addInputPath(job, new Path(inPath)); FileOutputFormat.setOutputPath(job, new Path(outPath)); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapperClass(CIMapper.class); job.setReducerClass(CIReducer.class); //job.setMapOutputKeyClass(LongWritable.class); job.setMapOutputValueClass(DoubleWritable.class); //job.setOutputKeyClass(LongWritable.class); job.setOutputValueClass(DoubleWritable.class); return job.waitForCompletion(true) ? 0 : 1; }