Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.pig.test.pigmix.mapreduce; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Properties; import java.util.Map; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.DoubleWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.FileOutputFormat; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.KeyValueTextInputFormat; import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hadoop.mapred.jobcontrol.Job; import org.apache.hadoop.mapred.jobcontrol.JobControl; import org.apache.hadoop.mapred.lib.IdentityMapper; import org.apache.pig.test.pigmix.mapreduce.Library; public class L12 { public static class HighestValuePagePerUser extends MapReduceBase implements Mapper<LongWritable, Text, Text, DoubleWritable>, Reducer<Text, DoubleWritable, Text, DoubleWritable> { public void map(LongWritable k, Text val, OutputCollector<Text, DoubleWritable> oc, Reporter reporter) throws IOException { List<Text> fields = Library.splitLine(val, ''); // Filter out null users or query terms. if (fields.get(0).getLength() == 0 || fields.get(3).getLength() == 0) return; try { oc.collect(fields.get(0), new DoubleWritable(Double.valueOf(fields.get(6).toString()))); } catch (NumberFormatException nfe) { oc.collect(fields.get(0), new DoubleWritable(0)); } } public void reduce(Text key, Iterator<DoubleWritable> iter, OutputCollector<Text, DoubleWritable> oc, Reporter reporter) throws IOException { double max = Double.NEGATIVE_INFINITY; while (iter.hasNext()) { double d = iter.next().get(); if (max < d) max = d; } oc.collect(key, new DoubleWritable(max)); } } public static class TotalTimespentPerTerm extends MapReduceBase implements Mapper<LongWritable, Text, Text, LongWritable>, Reducer<Text, LongWritable, Text, LongWritable> { public void map(LongWritable k, Text val, OutputCollector<Text, LongWritable> oc, Reporter reporter) throws IOException { List<Text> fields = Library.splitLine(val, ''); // Filter out non-null users if (fields.get(0).getLength() != 0) return; try { oc.collect(fields.get(3), new LongWritable(Long.valueOf(fields.get(2).toString()))); } catch (NumberFormatException nfe) { oc.collect(fields.get(3), new LongWritable(0)); } } public void reduce(Text key, Iterator<LongWritable> iter, OutputCollector<Text, LongWritable> oc, Reporter reporter) throws IOException { long sum = 0; while (iter.hasNext()) sum += iter.next().get(); oc.collect(key, new LongWritable(sum)); } } public static class QueriesPerAction extends MapReduceBase implements Mapper<LongWritable, Text, Text, LongWritable>, Reducer<Text, LongWritable, Text, LongWritable> { public void map(LongWritable k, Text val, OutputCollector<Text, LongWritable> oc, Reporter reporter) throws IOException { List<Text> fields = Library.splitLine(val, ''); // Filter out non-null users and non-null queries if (fields.get(0).getLength() == 0 || fields.get(3).getLength() != 0) return; oc.collect(fields.get(1), new LongWritable(1)); } public void reduce(Text key, Iterator<LongWritable> iter, OutputCollector<Text, LongWritable> oc, Reporter reporter) throws IOException { long cnt = 0; while (iter.hasNext()) { LongWritable l = iter.next(); cnt += l.get(); } oc.collect(key, new LongWritable(cnt)); } } public static void main(String[] args) throws IOException { if (args.length != 3) { System.out.println("Parameters: inputDir outputDir parallel"); System.exit(1); } String inputDir = args[0]; String outputDir = args[1]; String parallel = args[2]; String user = System.getProperty("user.name"); JobConf lp = new JobConf(L12.class); lp.setJobName("L12 Find Highest Value Page Per User"); lp.setInputFormat(TextInputFormat.class); lp.setOutputKeyClass(Text.class); lp.setOutputValueClass(DoubleWritable.class); lp.setMapperClass(HighestValuePagePerUser.class); lp.setCombinerClass(HighestValuePagePerUser.class); lp.setReducerClass(HighestValuePagePerUser.class); Properties props = System.getProperties(); for (Map.Entry<Object, Object> entry : props.entrySet()) { lp.set((String) entry.getKey(), (String) entry.getValue()); } FileInputFormat.addInputPath(lp, new Path(inputDir + "/page_views")); FileOutputFormat.setOutputPath(lp, new Path(outputDir + "/highest_value_page_per_user")); lp.setNumReduceTasks(Integer.parseInt(parallel)); Job loadPages = new Job(lp); JobConf lu = new JobConf(L12.class); lu.setJobName("L12 Find Total Timespent per Term"); lu.setInputFormat(TextInputFormat.class); lu.setOutputKeyClass(Text.class); lu.setOutputValueClass(LongWritable.class); lu.setMapperClass(TotalTimespentPerTerm.class); lu.setCombinerClass(TotalTimespentPerTerm.class); lu.setReducerClass(TotalTimespentPerTerm.class); props = System.getProperties(); for (Map.Entry<Object, Object> entry : props.entrySet()) { lu.set((String) entry.getKey(), (String) entry.getValue()); } FileInputFormat.addInputPath(lu, new Path(inputDir + "/page_views")); FileOutputFormat.setOutputPath(lu, new Path(outputDir + "/total_timespent_per_term")); lu.setNumReduceTasks(Integer.parseInt(parallel)); Job loadUsers = new Job(lu); JobConf join = new JobConf(L12.class); join.setJobName("L12 Find Queries Per Action"); join.setInputFormat(TextInputFormat.class); join.setOutputKeyClass(Text.class); join.setOutputValueClass(LongWritable.class); join.setMapperClass(QueriesPerAction.class); join.setCombinerClass(QueriesPerAction.class); join.setReducerClass(QueriesPerAction.class); props = System.getProperties(); for (Map.Entry<Object, Object> entry : props.entrySet()) { join.set((String) entry.getKey(), (String) entry.getValue()); } FileInputFormat.addInputPath(join, new Path(inputDir + "/page_views")); FileOutputFormat.setOutputPath(join, new Path(outputDir + "/queries_per_action")); join.setNumReduceTasks(Integer.parseInt(parallel)); Job joinJob = new Job(join); JobControl jc = new JobControl("L12 join"); jc.addJob(loadPages); jc.addJob(loadUsers); jc.addJob(joinJob); new Thread(jc).start(); int i = 0; while (!jc.allFinished()) { ArrayList<Job> failures = jc.getFailedJobs(); if (failures != null && failures.size() > 0) { for (Job failure : failures) { System.err.println(failure.getMessage()); } break; } try { Thread.sleep(5000); } catch (InterruptedException e) { } if (i % 10000 == 0) { System.out.println("Running jobs"); ArrayList<Job> running = jc.getRunningJobs(); if (running != null && running.size() > 0) { for (Job r : running) { System.out.println(r.getJobName()); } } System.out.println("Ready jobs"); ArrayList<Job> ready = jc.getReadyJobs(); if (ready != null && ready.size() > 0) { for (Job r : ready) { System.out.println(r.getJobName()); } } System.out.println("Waiting jobs"); ArrayList<Job> waiting = jc.getWaitingJobs(); if (waiting != null && waiting.size() > 0) { for (Job r : ready) { System.out.println(r.getJobName()); } } System.out.println("Successful jobs"); ArrayList<Job> success = jc.getSuccessfulJobs(); if (success != null && success.size() > 0) { for (Job r : ready) { System.out.println(r.getJobName()); } } } i++; } ArrayList<Job> failures = jc.getFailedJobs(); if (failures != null && failures.size() > 0) { for (Job failure : failures) { System.err.println(failure.getMessage()); } } jc.stop(); } }