Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package hitune.analysis.mapreduce.processor; import hitune.analysis.mapreduce.AnalysisProcessorConfiguration; import hitune.analysis.mapreduce.CSVFileOutputFormat; import hitune.analysis.mapreduce.HiTuneRecord; import hitune.analysis.mapreduce.MultiSequenceFileInputFormat; import hitune.analysis.mapreduce.TextArrayWritable; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord; import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.FileOutputFormat; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.record.Record; import org.apache.log4j.Logger; /** * Filter the metrics data between user defined period of time. * The job only reports out those focusing(chosen by user in the configuration file) metrics, * in case overloading for the results diagnosis. * * */ public class HadoopMetrics extends AnalysisProcessor { static Logger log = Logger.getLogger(HadoopMetrics.class); /** * Filter each record with the time, metrics, and node list * * */ public static class MapClass<K extends Record, V extends Record> extends MapReduceBase implements Mapper<K, V, K, V> { static List<String> nodelist = new ArrayList<String>(); static List<String> metrics = new ArrayList<String>(); static long starttime = 0; static long endtime = 0; JobConf conf = null; @Override public void configure(JobConf jobConf) { super.configure(jobConf); this.conf = jobConf; init(); } private void init() { this.starttime = Long.parseLong(conf.get(AnalysisProcessorConfiguration.starttime)); this.endtime = Long.parseLong(conf.get(AnalysisProcessorConfiguration.endtime)); String nodes = conf.get(AnalysisProcessorConfiguration.nodes); this.nodelist = String2List(nodes, SEPERATOR_COMMA); String metrics = conf.get(AnalysisProcessorConfiguration.metrics); this.metrics = String2List(metrics, SEPERATOR_COMMA); } @Override public void map(K key, V value, OutputCollector<K, V> output, Reporter reporter) throws IOException { // TODO Auto-generated method stub //doing the filter String category = this.conf.get(AnalysisProcessorConfiguration.category); HiTuneRecord valproxy = new HiTuneRecord(value); long timestamp = valproxy.getTime(); String hostname = valproxy.getHost(); if (timestamp >= this.starttime && timestamp <= this.endtime) { if (isMatched(this.nodelist, hostname)) { try { V newvalue = (V) value.getClass().getConstructor().newInstance(); HiTuneRecord newvalproxy = new HiTuneRecord(newvalue); newvalproxy.copyCommonFields(value); for (String metric : this.metrics) { String val = valproxy.getValue(metric); if (val == null) { val = ""; } newvalproxy.add(metric, val); if (category != null) { newvalproxy.add(AnalysisProcessorConfiguration.category, category); } } output.collect(key, (V) newvalproxy.getObject()); } catch (IllegalArgumentException e) { // TODO Auto-generated catch block log.warn(e); e.printStackTrace(); } catch (SecurityException e) { // TODO Auto-generated catch block log.warn(e); e.printStackTrace(); } catch (InstantiationException e) { // TODO Auto-generated catch block log.warn(e); e.printStackTrace(); } catch (IllegalAccessException e) { // TODO Auto-generated catch block log.warn(e); e.printStackTrace(); } catch (InvocationTargetException e) { // TODO Auto-generated catch block log.warn(e); e.printStackTrace(); } catch (NoSuchMethodException e) { // TODO Auto-generated catch block log.warn(e); e.printStackTrace(); } } } } } /** * @param conf */ public HadoopMetrics(Configuration conf) { super(conf); // TODO Auto-generated constructor stub } /* (non-Javadoc) * @see org.apache.hadoop.chukwa.analysis.HiTune.AnalysisProcessor#run() */ @Override public void run() { // TODO Auto-generated method stub long timestamp = System.currentTimeMillis(); JobConf conf = new JobConf(this.conf, HadoopMetrics.class); try { conf.setJobName(this.getClass().getSimpleName() + timestamp); conf.setInputFormat(MultiSequenceFileInputFormat.class); conf.setMapperClass(HadoopMetrics.MapClass.class); conf.setReducerClass(SystemLog.ReduceClass.class); Class<? extends WritableComparable> outputKeyClass = Class .forName(conf.get(AnalysisProcessorConfiguration.mapoutputKeyClass)) .asSubclass(WritableComparable.class); Class<? extends Writable> outputValueClass = Class .forName(conf.get(AnalysisProcessorConfiguration.mapoutputValueClass)) .asSubclass(Writable.class); conf.setMapOutputKeyClass(outputKeyClass); conf.setMapOutputValueClass(outputValueClass); conf.setOutputKeyClass(Text.class); conf.setOutputValueClass(TextArrayWritable.class); conf.setOutputFormat(CSVFileOutputFormat.class); String outputPaths = conf.get(AnalysisProcessorConfiguration.reportfolder) + "/" + conf.get(AnalysisProcessorConfiguration.reportfile); String temp_outputPaths = getTempOutputDir(outputPaths); if (this.inputfiles != null) { log.debug("inputPaths:" + inputfiles); FileInputFormat.setInputPaths(conf, inputfiles); FileOutputFormat.setOutputPath(conf, new Path(temp_outputPaths)); try { JobClient.runJob(conf); moveResults(conf, outputPaths, temp_outputPaths); } catch (IOException e) { // TODO Auto-generated catch block log.warn("For " + getOutputFileName() + " :JOB fails!"); log.warn(e); e.printStackTrace(); this.MOVE_DONE = false; } } else { log.warn("For " + getOutputFileName() + " :No input path!"); } } catch (Exception e) { log.warn("Job preparation failure!"); log.warn(e); e.printStackTrace(); } } /** * @param args */ public static void main(String[] args) { // TODO Auto-generated method stub } }