List of usage examples for org.apache.hadoop.conf Configuration Configuration
public Configuration()
From source file:Assignment3_P5_Top25Movies.Top25MovieRatingDriver.java
/** * @param args the command line arguments *///from w w w . j ava 2s .c o m public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job1 = Job.getInstance(conf, "Movie Rating Count"); job1.setJarByClass(Top25MovieRatingDriver.class); // the usual - get basic mapred ready job1.setMapperClass(Top25MovieRating_Mapper.class); job1.setCombinerClass(Top25MovieRating_Reducer.class); job1.setReducerClass(Top25MovieRating_Reducer.class); // this will basically out -> movieId, average rating job1.setOutputKeyClass(IntWritable.class); job1.setOutputValueClass(FloatWritable.class); FileInputFormat.addInputPath(job1, new Path(args[0])); FileOutputFormat.setOutputPath(job1, new Path(args[1])); boolean complete = job1.waitForCompletion(true); // here's where we sort Configuration conf2 = new Configuration(); Job job2 = Job.getInstance(conf2, "Movie Rating Count"); if (complete) { job2.setJarByClass(Top25MovieRatingDriver.class); // namesake fellow, take it and go types - mostly useless job2.setMapperClass(Top25MovieRating_Mapper1.class); job2.setMapOutputKeyClass(FloatWritable.class); job2.setMapOutputValueClass(IntWritable.class); // this is where we would ideally sort descendingly job2.setSortComparatorClass(Top25MovieRating_SortComparator.class); // o/p top 25, man job2.setNumReduceTasks(1); job2.setReducerClass(Top25MovieRating_Reducer1.class); job2.setOutputKeyClass(FloatWritable.class); job2.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job2, new Path(args[1])); FileOutputFormat.setOutputPath(job2, new Path(args[2])); System.exit(job2.waitForCompletion(true) ? 0 : 1); } }
From source file:Assignment4_P2_StockAverageWithCombiner.StockAverageDriver.java
/** * @param args the command line arguments */// w w w .ja v a 2s. c om public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Average Stock Price"); job.setJarByClass(StockAverageDriver.class); job.setMapperClass(StockAverage_Mapper.class); job.setCombinerClass(StockAverage_Combiner.class); job.setReducerClass(StockAverage_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(StockAverage_CompositeValueWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment4_P3_InMemoryStdDeviation.MovieRatingStdDevDriver.java
/** * @param args the command line arguments *//* w w w.j a v a 2s . c o m*/ public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Movie Rating Standard Deviation"); job.setJarByClass(MovieRatingStdDevDriver.class); job.setMapperClass(MovieRatingStdDev_Mapper.class); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(FloatWritable.class); job.setReducerClass(MovieRatingStdDev_Reducer.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(Text.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment4_P4_MemoryConscious.MovieRatingMemConsciousDriver.java
/** * @param args the command line arguments *//*from ww w . j a v a 2 s.co m*/ public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Movie Rating Mem Conscious Standard Deviation"); job.setJarByClass(MovieRatingMemConsciousDriver.class); job.setMapperClass(MovieRatingMemConscious_Mapper.class); job.setCombinerClass(MovingRatingMemConscious_Combiner.class); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(SortedMapWritable.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(Text.class); job.setReducerClass(MovieRatingMemConscious_Reducer.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment5_P2_DistinctIPAddress.DistinctIPAddressDriver.java
/** * @param args the command line arguments *//* ww w .jav a2 s . co m*/ public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "IP Address Count"); job.setJarByClass(DistinctIPAddressDriver.class); job.setMapperClass(DistinctIPAddress_Mapper.class); job.setCombinerClass(DistinctIPAddress_Reducer.class); job.setReducerClass(DistinctIPAddress_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(NullWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment5_P3_PartitionPattern.Partition_IPAddress_By_MonthDriver.java
/** * @param args the command line arguments *///from w w w. ja v a 2s.c om public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "IP Address By Date"); job.setJarByClass(Partition_IPAddress_By_MonthDriver.class); job.setMapperClass(Partition_IPAddress_By_Month_Mapper.class); //job.setCombinerClass(Partition_IPAddress_By_Month_Reducer.class); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(Text.class); // partitioner class inclusion job.setPartitionerClass(Partition_IPAddress_By_Month_Partitioner.class); // set num of reduce tasks based on partition we need (here we need 12 cos total no.of months in a year) job.setNumReduceTasks(12); job.setReducerClass(Partition_IPAddress_By_Month_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(NullWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment5_P4_BinningPattern.Binning_IPAddress_By_DayDriver.java
/** * @param args the command line arguments *//* w ww . j a va 2s . c o m*/ public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "IP Address By Hour"); job.setJarByClass(Binning_IPAddress_By_DayDriver.class); job.setMapperClass(Binning_IPAddress_By_Day_Mapper.class); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(Text.class); MultipleOutputs.addNamedOutput(job, "textualBins", TextOutputFormat.class, NullWritable.class, Text.class); MultipleOutputs.addNamedOutput(job, "massaBins", TextOutputFormat.class, NullWritable.class, Text.class); MultipleOutputs.setCountersEnabled(job, true); // set num of reduce tasks to 0 job.setNumReduceTasks(0); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment5_P6_StructureToHierarchyPattern.Structure_HierarchyDriver.java
/** * @param args the command line arguments */// w ww .ja v a2s . c om public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Structure to Hierarchy"); job.setJarByClass(Structure_HierarchyDriver.class); // pass file 1 to this mapper in Text format MultipleInputs.addInputPath(job, new Path(args[0]), TextInputFormat.class, Structure_Hierarchy_Movie_Mapper.class); // pass file 2 to this mapper in Text format MultipleInputs.addInputPath(job, new Path(args[1]), TextInputFormat.class, Structure_Hierarchy_Tag_Mapper.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setReducerClass(Structure_Hierarchy_Reducer.class); FileOutputFormat.setOutputPath(job, new Path(args[2])); System.exit(job.waitForCompletion(true) ? 0 : 2); }
From source file:at.illecker.hadoop.rootbeer.examples.matrixmultiplication.cpu.MatrixMultiplicationCpu.java
License:Apache License
public static Configuration createMatrixMultiplicationCpuConf(Path aPath, Path bPath, Path outPath, int outCardinality, boolean isDebugging) { return createMatrixMultiplicationCpuConf(new Configuration(), aPath, bPath, outPath, outCardinality, isDebugging);/* w w w.ja va 2s. com*/ }
From source file:at.illecker.hadoop.rootbeer.examples.matrixmultiplication.DistributedRowMatrix.java
License:Apache License
/** * This implements matrix this.transpose().times(other) * /*from www. ja v a 2s . co m*/ * @param other a DistributedRowMatrix * @param outPath path to write result to * @return a DistributedRowMatrix containing the product */ public DistributedRowMatrix times(DistributedRowMatrix other, Path outPath) throws IOException { if (numRows != other.numRows()) { throw new CardinalityException(numRows, other.numRows()); } Configuration initialConf = getConf() == null ? new Configuration() : getConf(); Configuration conf = MatrixMultiplicationJob.createMatrixMultiplyJobConf(initialConf, rowPath, other.rowPath, outPath, other.numCols); JobClient.runJob(new JobConf(conf)); DistributedRowMatrix out = new DistributedRowMatrix(outPath, outputTmpPath, numCols, other.numCols()); out.setConf(conf); return out; }