List of usage examples for org.apache.hadoop.mapreduce Job getInstance
@Deprecated public static Job getInstance(Cluster ignored, Configuration conf) throws IOException
From source file:Analysis.A6_User_Differentiation_By_Age.Partition_Users_By_Age_Driver.java
/** * @param args the command line arguments *///from ww w . jav a 2s . c o m public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Users by Age"); job.setJarByClass(Partition_Users_By_Age_Driver.class); job.setMapperClass(Partition_Users_By_Age_Mapper.class); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(Text.class); // partitioner class inclusion job.setPartitionerClass(Partition_Users_By_Age_Partitioner.class); // set multiple formats for custom naming partitioning MultipleOutputs.addNamedOutput(job, "ageBins", TextOutputFormat.class, Text.class, NullWritable.class); MultipleOutputs.setCountersEnabled(job, true); //11-17, 18-25, 26-35, 36-49,50-65,66-80, 81-99 // set num of reduce tasks based on partition we need (here we need 10 cos total no.of countries) job.setNumReduceTasks(8); job.setReducerClass(Partition_Users_By_Age_Reducer.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Analysis.A7_Total_Signups_By_Year.Total_Signup_by_Year_Driver.java
/** * @param args the command line arguments *//* ww w. j av a 2 s .c o m*/ public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Total Signups by Year"); job.setJarByClass(Total_Signup_by_Year_Driver.class); job.setMapperClass(Total_Signup_by_Year_Mapper.class); job.setCombinerClass(Total_Signup_by_Year_Reducer.class); job.setReducerClass(Total_Signup_by_Year_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Analysis.A8_Top_10_Most_Popular_Tracks.Top_10_Most_Popular_Tracks_Driver.java
/** * @param args the command line arguments *///w w w.j a v a2 s. com public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Top 10 most popular tracks "); job.setJarByClass(Top_10_Most_Popular_Tracks_Driver.class); job.setMapperClass(Top_10_Most_Popular_Tracks_Mapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setNumReduceTasks(1); job.setReducerClass(Top_10_Most_Popular_Tracks_Reducer.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Analysis.A9_Max_Activity_By_Time_of_Day.Most_Listens_By_Time_of_Day_Driver.java
/** * @param args the command line arguments *//*w w w . j ava2 s.c o m*/ public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Most listens by Time of the Day"); job.setJarByClass(Most_Listens_By_Time_of_Day_Driver.class); job.setMapperClass(Most_Listens_By_Time_of_Day_Mapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(NullWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); int code = job.waitForCompletion(true) ? 0 : 1; if (code == 0) { for (Counter counter : job.getCounters() .getGroup(Most_Listens_By_Time_of_Day_Mapper.HOUR_COUNTER_GROUP)) { System.out.println(counter.getDisplayName() + "\t" + counter.getValue()); } } FileSystem.get(conf).delete(new Path(args[1]), true); System.exit(code); }
From source file:Assignment2_P2_StockExchangeCount.StockPriceDriver.java
/** * @param args the command line arguments *///from w w w . j a v a 2 s.co m public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Average Stock Price"); job.setJarByClass(StockPriceDriver.class); job.setMapperClass(StockPrice_Mapper.class); job.setCombinerClass(StockPrice_Reducer.class); job.setReducerClass(StockPrice_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(FloatWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment2_P3_GenderMovieCount.GenderMovieRatingDriver.java
/** * @param args the command line arguments *//*from w w w . j ava2 s .c o m*/ public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Gender Movie Rating Count"); job.setJarByClass(GenderMovieRatingDriver.class); job.setMapperClass(GenderMovieRating_Mapper.class); job.setCombinerClass(GenderMovieRating_Reducer.class); job.setReducerClass(GenderMovieRating_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment2_P4_MovieRatingCount.MovieRatingDriver.java
/** * @param args the command line arguments *//*from w ww .j a v a2s . com*/ public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Movie Rating Count"); job.setJarByClass(MovieRatingDriver.class); job.setMapperClass(MovieRating_Mapper.class); job.setCombinerClass(MovieRating_Reducer.class); job.setReducerClass(MovieRating_Reducer.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment2_P5_IPAddressCount.IPAddressDriver.java
/** * @param args the command line arguments *//* ww w. j a v a 2 s . co m*/ public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "IP Address Count"); job.setJarByClass(IPAddressDriver.class); job.setMapperClass(IPAddress_Mapper.class); job.setCombinerClass(IPAddress_Reducer.class); job.setReducerClass(IPAddress_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment3_P2_MergeStockAverageCount.StockPriceMergeDriver.java
/** * @param args the command line arguments *///from w ww . j a v a2s .co m public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); // local file system handle FileSystem local = FileSystem.getLocal(conf); // hdfs file system handle FileSystem hdfs = FileSystem.get(conf); // local input directory Path inputDir = new Path(args[0]); // hdfs i/p directory Path inputDir1 = new Path(args[1]); // local input files in local dir FileStatus[] inputFiles = local.listStatus(inputDir); // o/p stream FSDataOutputStream out = hdfs.create(inputDir1); // open each file and extract contents of file for (int i = 0; i < inputFiles.length; i++) { System.out.println("File name ----------------------------------------------------------------> " + inputFiles[i].getPath().getName()); FSDataInputStream in = local.open(inputFiles[i].getPath()); byte buffer[] = new byte[256]; int bytesRead = 0; // extract all contents of file while ((bytesRead = in.read(buffer)) > 0) { out.write(buffer, 0, bytesRead); } // close input stream in.close(); } Job job = Job.getInstance(conf, "Average Stock Price"); job.setJarByClass(StockPriceMergeDriver.class); job.setMapperClass(StockPriceMerge_Mapper.class); job.setCombinerClass(StockPriceMerge_Reducer.class); job.setReducerClass(StockPriceMerge_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(FloatWritable.class); FileInputFormat.addInputPath(job, new Path(args[1])); // above programs output will be input for mapper FileOutputFormat.setOutputPath(job, new Path(args[2])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment3_P4_DateStock.DateStockDriver.java
/** * @param args the command line arguments *///from ww w.j ava 2 s . c om public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Average Stock Price"); job.setJarByClass(DateStockDriver.class); job.setMapperClass(DateStock_Mapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(DateStock_CompositeValueWritable.class); job.setReducerClass(DateStock_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }