Java tutorial
/** * Copyright 2011 Nube Technologies * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR * CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and limitations under the License. */ package co.nubetech.hiho.merge; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.apache.log4j.Logger; import co.nubetech.hiho.common.HIHOConf; import co.nubetech.hiho.common.HIHOException; import co.nubetech.hiho.dedup.DelimitedTextInputFormat; import co.nubetech.hiho.dedup.HihoTuple; public class MergeJob extends Configured implements Tool { final static Logger logger = Logger.getLogger(co.nubetech.hiho.merge.MergeJob.class); private String oldPath = null; private String newPath = null; private String mergeBy = null; private String delimiter = ","; private int column = 1; private String inputFormat = null; private String inputKeyClassName = null; private String inputValueClassName = null; private String outputPath = null; private String outputFormat = null; private long output; private long badRecords; private long totalRecordsNew; private long totalRecordsOld; public long getOutput() { return output; } public void setOutput(long output) { this.output = output; } public long getBadRecords() { return badRecords; } public void setBadRecords(long badRecords) { this.badRecords = badRecords; } public long getTotalRecordsNew() { return totalRecordsNew; } public void setTotalRecordsNew(long totalRecordsNew) { this.totalRecordsNew = totalRecordsNew; } public long getTotalRecordsOld() { return totalRecordsOld; } public void setTotalRecordsOld(long totalRecordsOld) { this.totalRecordsOld = totalRecordsOld; } public void populateConfiguration(String[] args) { for (int i = 0; i < args.length - 1; i++) { if ("-inputFormat".equals(args[i])) { inputFormat = args[++i]; if (inputFormat.equals("org.apache.hadoop.mapreduce.lib.input.TextInputFormat")) { inputKeyClassName = "org.apache.hadoop.io.LongWritable"; inputValueClassName = "org.apache.hadoop.io.Text"; } else if (inputFormat.equals("co.nubetech.hiho.dedup.DelimitedTextInputFormat")) { inputKeyClassName = "org.apache.hadoop.io.Text"; inputValueClassName = "org.apache.hadoop.io.Text"; outputFormat = "co.nubetech.hiho.mapreduce.lib.output.NoKeyOnlyValueOutputFormat"; } else if (inputFormat.equals("org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat")) { outputFormat = "org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat"; } } else if ("-oldPath".equals(args[i])) { oldPath = args[++i]; } else if ("-newPath".equals(args[i])) { newPath = args[++i]; } else if ("-mergeBy".equals(args[i])) { mergeBy = args[++i]; } else if ("-inputKeyClassName".equals(args[i])) { inputKeyClassName = args[++i]; } else if ("-inputValueClassName".equals(args[i])) { inputValueClassName = args[++i]; } else if ("-outputPath".equals(args[i])) { outputPath = args[++i]; } else if ("-delimiter".equals(args[i])) { delimiter = args[++i]; } else if ("-column".equals(args[i])) { column = Integer.parseInt(args[++i]); } else if ("-outputFormat".equals(args[i])) { outputFormat = args[++i]; } } } public void checkMandatoryConfs() throws HIHOException { if (inputFormat == null) { throw new HIHOException("The provided input format is empty, please specify inputFormat"); } if (mergeBy == null) { throw new HIHOException("The provided value of dedupBy is empty, please specify either key or value"); } if ((!mergeBy.equals("key")) && (!mergeBy.equals("value"))) { throw new HIHOException( "The provided value of mergeBy is Incorrect, please specify either key or value"); } if (inputKeyClassName == null) { throw new HIHOException("The provided input key class name is empty, please specify inputKeyClassName"); } if (inputValueClassName == null) { throw new HIHOException( "The provided input value class name is empty, please specify inputValueClassName"); } if (oldPath == null) { throw new HIHOException("The provided old path is empty, please specify oldPath"); } if (newPath == null) { throw new HIHOException("The provided new path is empty, please specify newPath"); } if (outputPath == null) { throw new HIHOException("The provided output path is empty, please specify outputPath"); } if (outputFormat == null) { System.out.println(outputFormat); throw new HIHOException("The provided output format is empty, please specify outputFormat"); } } @Override public int run(String[] args) throws Exception { populateConfiguration(args); try { checkMandatoryConfs(); } catch (HIHOException e1) { e1.printStackTrace(); throw new Exception(e1); } Class inputFormatClass = Class.forName(inputFormat); Class outputFormatClass = Class.forName(outputFormat); Class inputKeyClass = Class.forName(inputKeyClassName); Class inputValueClass = Class.forName(inputValueClassName); Configuration conf = getConf(); conf.set(HIHOConf.MERGE_OLD_PATH, oldPath); conf.set(HIHOConf.MERGE_NEW_PATH, newPath); Job job = new Job(conf); job.setJobName("Merge job"); job.setJarByClass(MergeJob.class); if (mergeBy.equals("key")) { job.setMapperClass(MergeKeyMapper.class); job.setReducerClass(MergeKeyReducer.class); } else if (mergeBy.equals("value")) { job.setMapperClass(MergeValueMapper.class); job.setReducerClass(MergeValueReducer.class); } job.setInputFormatClass(inputFormatClass); DelimitedTextInputFormat.setProperties(job, delimiter, column); job.setMapOutputKeyClass(HihoTuple.class); job.setMapOutputValueClass(HihoValue.class); job.setOutputKeyClass(inputKeyClass); job.setOutputValueClass(inputValueClass); FileInputFormat.setInputPaths(job, oldPath + "," + newPath); job.setOutputFormatClass(outputFormatClass); FileOutputFormat.setOutputPath(job, new Path(outputPath)); try { logger.debug("Output format class is " + job.getOutputFormatClass()); logger.debug("Class is " + ReflectionUtils .newInstance(job.getOutputFormatClass(), job.getConfiguration()).getClass().getName()); job.waitForCompletion(false); if (job.isComplete()) { Counters counters = job.getCounters(); totalRecordsOld = counters.findCounter(MergeRecordCounter.TOTAL_RECORDS_OLD).getValue(); totalRecordsNew = counters.findCounter(MergeRecordCounter.TOTAL_RECORDS_NEW).getValue(); badRecords = counters.findCounter(MergeRecordCounter.BAD_RECORD).getValue(); output = counters.findCounter(MergeRecordCounter.OUTPUT).getValue(); logger.info("Total old records read are: " + totalRecordsOld); logger.info("Total new records read are: " + totalRecordsNew); logger.info("Bad Records are: " + badRecords); logger.info("Output records are: " + output); } } catch (Exception e) { e.printStackTrace(); } return 0; } public static void main(String[] args) throws Exception { MergeJob job = new MergeJob(); int res = ToolRunner.run(new Configuration(), job, args); System.exit(res); } }