List of usage examples for org.apache.hadoop.mapreduce Job setMapOutputKeyClass
public void setMapOutputKeyClass(Class<?> theClass) throws IllegalStateException
From source file:Analysis.A5_Min_Max_Median_Age_Top_Countries.Min_Max_Age_By_Country_Driver.java
/** * @param args the command line arguments *//*from w w w. ja v a 2 s . c o m*/ public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Min, Max, Average, Median age of users by country"); job.setJarByClass(Min_Max_Age_By_Country_Driver.class); job.setMapperClass(Min_Max_Age_By_Country_Mapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(Min_Max_Age_By_Country_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setNumReduceTasks(1); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Analysis.A6_User_Differentiation_By_Age.Partition_Users_By_Age_Driver.java
/** * @param args the command line arguments *///from ww w. ja va 2 s . com public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Users by Age"); job.setJarByClass(Partition_Users_By_Age_Driver.class); job.setMapperClass(Partition_Users_By_Age_Mapper.class); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(Text.class); // partitioner class inclusion job.setPartitionerClass(Partition_Users_By_Age_Partitioner.class); // set multiple formats for custom naming partitioning MultipleOutputs.addNamedOutput(job, "ageBins", TextOutputFormat.class, Text.class, NullWritable.class); MultipleOutputs.setCountersEnabled(job, true); //11-17, 18-25, 26-35, 36-49,50-65,66-80, 81-99 // set num of reduce tasks based on partition we need (here we need 10 cos total no.of countries) job.setNumReduceTasks(8); job.setReducerClass(Partition_Users_By_Age_Reducer.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Analysis.A8_Top_10_Most_Popular_Tracks.Top_10_Most_Popular_Tracks_Driver.java
/** * @param args the command line arguments *///from w w w.ja va2 s .c o m public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Top 10 most popular tracks "); job.setJarByClass(Top_10_Most_Popular_Tracks_Driver.class); job.setMapperClass(Top_10_Most_Popular_Tracks_Mapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setNumReduceTasks(1); job.setReducerClass(Top_10_Most_Popular_Tracks_Reducer.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Analysis.A9_Max_Activity_By_Time_of_Day.Most_Listens_By_Time_of_Day_Driver.java
/** * @param args the command line arguments *///from w w w. j a va 2 s . c o m public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Most listens by Time of the Day"); job.setJarByClass(Most_Listens_By_Time_of_Day_Driver.class); job.setMapperClass(Most_Listens_By_Time_of_Day_Mapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(NullWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); int code = job.waitForCompletion(true) ? 0 : 1; if (code == 0) { for (Counter counter : job.getCounters() .getGroup(Most_Listens_By_Time_of_Day_Mapper.HOUR_COUNTER_GROUP)) { System.out.println(counter.getDisplayName() + "\t" + counter.getValue()); } } FileSystem.get(conf).delete(new Path(args[1]), true); System.exit(code); }
From source file:arpserver.HadoopTool.java
@Override public int run(String[] strings) throws Exception { Configuration conf = new Configuration(); String in = strings[0];//from w w w . j ava2 s . c o m String out = strings[1]; FileSystem fs = FileSystem.get(conf); if (fs.exists(new Path(out))) { fs.delete(new Path(out), true); fs.delete(new Path(out + "Src"), true); fs.delete(new Path(out + "Mitm"), true); fs.delete(new Path(out + "ArpScn"), true); fs.delete(new Path(out + "s"), true); fs.delete(new Path(out + "d"), true); fs.delete(new Path(out + "t"), true); } Job job = new Job(); Job job2 = new Job(); Job job3 = new Job(); Job job4 = new Job(); Job job5 = new Job(); Job job6 = new Job(); Job job7 = new Job(); job.setJobName("Q"); job2.setJobName("Src"); job3.setJobName("Mitm"); job4.setJobName("ArpScn"); job5.setJobName("s"); job6.setJobName("d"); job7.setJobName("time"); job.setJarByClass(QuickDetect.class); job.setMapperClass(Qmapper.class); job.setReducerClass(Qreducer.class); job2.setMapperClass(Srcmapper.class); job2.setReducerClass(Srcreducer.class); job3.setMapperClass(ArpScanmapper.class); job3.setReducerClass(ArpScanreducer.class); job4.setMapperClass(Mitmmapper.class); job4.setReducerClass(Mitmreducer.class); job5.setMapperClass(Smapper.class); job5.setReducerClass(Sreducer.class); job6.setMapperClass(Dmapper.class); job6.setReducerClass(Dreducer.class); job7.setMapperClass(timemapper.class); job7.setReducerClass(timereducer.class); //testFinal168.txt job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(Text.class); job2.setOutputKeyClass(NullWritable.class); job2.setOutputValueClass(Text.class); job3.setOutputKeyClass(NullWritable.class); job3.setOutputValueClass(IntWritable.class); job4.setOutputKeyClass(NullWritable.class); job4.setOutputValueClass(Text.class); job5.setOutputKeyClass(NullWritable.class); job5.setOutputValueClass(Text.class); job6.setOutputKeyClass(NullWritable.class); job6.setOutputValueClass(Text.class); job7.setOutputKeyClass(NullWritable.class); job7.setOutputValueClass(Text.class); job.setMapOutputKeyClass(QuickDetect.class); job.setMapOutputValueClass(IntWritable.class); //job.setOutputFormatClass(YearMultipleTextOutputFormat.class); job2.setMapOutputKeyClass(DetectSrc.class); job2.setMapOutputValueClass(IntWritable.class); job3.setMapOutputKeyClass(DetectArpScan.class); job3.setMapOutputValueClass(IntWritable.class); job4.setMapOutputKeyClass(DetectMitm.class); job4.setMapOutputValueClass(IntWritable.class); job5.setMapOutputKeyClass(SMac.class); job5.setMapOutputValueClass(IntWritable.class); job6.setMapOutputKeyClass(DMac.class); job6.setMapOutputValueClass(IntWritable.class); job7.setMapOutputKeyClass(timeMac.class); job7.setMapOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(in)); FileOutputFormat.setOutputPath(job, new Path(out)); if (job.waitForCompletion(true)) { FileInputFormat.addInputPath(job2, new Path(in)); FileOutputFormat.setOutputPath(job2, new Path(out + "Src")); if (job2.waitForCompletion(true)) { FileInputFormat.addInputPath(job3, new Path(in)); FileOutputFormat.setOutputPath(job3, new Path(out + "ArpScn")); if (job3.waitForCompletion(true)) { FileInputFormat.addInputPath(job4, new Path(in)); FileOutputFormat.setOutputPath(job4, new Path(out + "Mitm")); if (job4.waitForCompletion(true)) { FileInputFormat.addInputPath(job5, new Path(in)); FileOutputFormat.setOutputPath(job5, new Path(out + "s")); if (job5.waitForCompletion(true)) { FileInputFormat.addInputPath(job6, new Path(in)); FileOutputFormat.setOutputPath(job6, new Path(out + "d")); if (job6.waitForCompletion(true)) { FileInputFormat.addInputPath(job7, new Path(in)); FileOutputFormat.setOutputPath(job7, new Path(out + "t")); job7.waitForCompletion(true); } else { return 1; } } else { return 1; } } else { return 1; } } else { return 1; } } else { return 1; } } else { return 1; } return 0; }
From source file:Assignment3_P4_DateStock.DateStockDriver.java
/** * @param args the command line arguments *//*from w w w . j a va 2 s . c o m*/ public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Average Stock Price"); job.setJarByClass(DateStockDriver.class); job.setMapperClass(DateStock_Mapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(DateStock_CompositeValueWritable.class); job.setReducerClass(DateStock_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment3_P5_Top25Movies.Top25MovieRatingDriver.java
/** * @param args the command line arguments */// ww w.java 2 s . c o m public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job1 = Job.getInstance(conf, "Movie Rating Count"); job1.setJarByClass(Top25MovieRatingDriver.class); // the usual - get basic mapred ready job1.setMapperClass(Top25MovieRating_Mapper.class); job1.setCombinerClass(Top25MovieRating_Reducer.class); job1.setReducerClass(Top25MovieRating_Reducer.class); // this will basically out -> movieId, average rating job1.setOutputKeyClass(IntWritable.class); job1.setOutputValueClass(FloatWritable.class); FileInputFormat.addInputPath(job1, new Path(args[0])); FileOutputFormat.setOutputPath(job1, new Path(args[1])); boolean complete = job1.waitForCompletion(true); // here's where we sort Configuration conf2 = new Configuration(); Job job2 = Job.getInstance(conf2, "Movie Rating Count"); if (complete) { job2.setJarByClass(Top25MovieRatingDriver.class); // namesake fellow, take it and go types - mostly useless job2.setMapperClass(Top25MovieRating_Mapper1.class); job2.setMapOutputKeyClass(FloatWritable.class); job2.setMapOutputValueClass(IntWritable.class); // this is where we would ideally sort descendingly job2.setSortComparatorClass(Top25MovieRating_SortComparator.class); // o/p top 25, man job2.setNumReduceTasks(1); job2.setReducerClass(Top25MovieRating_Reducer1.class); job2.setOutputKeyClass(FloatWritable.class); job2.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job2, new Path(args[1])); FileOutputFormat.setOutputPath(job2, new Path(args[2])); System.exit(job2.waitForCompletion(true) ? 0 : 1); } }
From source file:Assignment4_P3_InMemoryStdDeviation.MovieRatingStdDevDriver.java
/** * @param args the command line arguments */// www.j a v a 2 s .co m public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Movie Rating Standard Deviation"); job.setJarByClass(MovieRatingStdDevDriver.class); job.setMapperClass(MovieRatingStdDev_Mapper.class); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(FloatWritable.class); job.setReducerClass(MovieRatingStdDev_Reducer.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(Text.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment4_P4_MemoryConscious.MovieRatingMemConsciousDriver.java
/** * @param args the command line arguments *///w w w. j a v a 2 s . c o m public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Movie Rating Mem Conscious Standard Deviation"); job.setJarByClass(MovieRatingMemConsciousDriver.class); job.setMapperClass(MovieRatingMemConscious_Mapper.class); job.setCombinerClass(MovingRatingMemConscious_Combiner.class); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(SortedMapWritable.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(Text.class); job.setReducerClass(MovieRatingMemConscious_Reducer.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment5_P3_PartitionPattern.Partition_IPAddress_By_MonthDriver.java
/** * @param args the command line arguments *//*from www. j a v a 2 s . com*/ public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "IP Address By Date"); job.setJarByClass(Partition_IPAddress_By_MonthDriver.class); job.setMapperClass(Partition_IPAddress_By_Month_Mapper.class); //job.setCombinerClass(Partition_IPAddress_By_Month_Reducer.class); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(Text.class); // partitioner class inclusion job.setPartitionerClass(Partition_IPAddress_By_Month_Partitioner.class); // set num of reduce tasks based on partition we need (here we need 12 cos total no.of months in a year) job.setNumReduceTasks(12); job.setReducerClass(Partition_IPAddress_By_Month_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(NullWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }