org.apache.hadoop.mapred.GenReduce.java Source code

Introduction

Here is the source code for org.apache.hadoop.mapred.GenReduce.java

Source

package org.apache.hadoop.mapred;
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.GeneralConstant;

import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.ReflectionUtils;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;

@SuppressWarnings("deprecation")
public class GenReduce extends MapReduceBase implements Reducer<Text, Text, Text, Text>, GeneralConstant {
    private static final Log LOG = LogFactory.getLog(GenReduce.class);
    private Configuration conf;

    public void configure(JobConf configuration) {
        conf = configuration;
    }

    private static final ObjectMapper mapper = new ObjectMapper();

    @Override
    public void reduce(Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter reporter)
            throws IOException {
        FSDataOutputStream out = null;
        try {
            int size = 0;
            FileSystem fs = FileSystem.get(conf);
            double averageIORate = 0;
            List<Float> list = new ArrayList<Float>();

            String output_dir = conf.get(OUTPUT_DIR_KEY);
            Path result_file = new Path(output_dir, "results");
            if (fs.exists(result_file)) {
                fs.delete(result_file);
            }
            out = fs.create(result_file, true);

            long nmaps = conf.getLong(NUMBER_OF_MAPS_KEY, NMAPS);
            long nthreads = conf.getLong(NUMBER_OF_THREADS_KEY, NTHREADS);

            out.writeChars("-----------------------------\n");
            out.writeChars("Number of mapper :\t\t\t" + nmaps + "\n");
            out.writeChars("Number of threads :\t\t\t" + nthreads + "\n");

            float min = Float.MAX_VALUE;
            float max = Float.MIN_VALUE;
            Class<?> clazz = conf.getClass(THREAD_CLASS_KEY, null);
            if (clazz == null) {
                throw new IOException("Class " + conf.get(THREAD_CLASS_KEY) + " not found");
            }
            GenThread t = (GenThread) ReflectionUtils.newInstance(clazz, conf);
            t.reset();
            long total_files = 0;
            long total_processed_size = 0;
            long total_num_errors = 0;
            String total_error = "";
            TypeReference<Map<String, String>> type = new TypeReference<Map<String, String>>() {
            };
            while (values.hasNext()) {
                Map<String, String> stat = mapper.readValue(values.next().toString(), type);
                size++;
                total_files += Long.parseLong(stat.get("files"));
                total_processed_size += Long.parseLong(stat.get("size"));
                total_num_errors += Long.parseLong(stat.get("nerrors"));
                total_error += stat.get("errors");
                double ioRate = Double.parseDouble(stat.get("rate"));
                if (ioRate > max)
                    max = (float) ioRate;
                if (ioRate < min)
                    min = (float) ioRate;
                list.add((float) ioRate);
                averageIORate += ioRate;
                t.analyze(stat);
            }

            out.writeChars("Number of files processed:\t\t" + total_files + "\n");
            out.writeChars("Number of size processed:\t\t" + total_processed_size + "\n");
            out.writeChars("Min IO Rate(MB/sec): \t\t\t" + min + "\n");
            out.writeChars("Max IO Rate(MB/sec): \t\t\t" + max + "\n");
            averageIORate /= size;
            float temp = (float) 0.0;
            for (int i = 0; i < list.size(); i++) {
                temp += Math.pow(list.get(i) - averageIORate, 2);
            }
            out.writeChars("Average(MB/sec): \t\t\t" + averageIORate + "\n");
            float dev = (float) Math.sqrt(temp / size);
            out.writeChars("Std. dev: \t\t\t\t" + dev + "\n");
            out.writeChars("Total number of errors:\t\t\t" + total_num_errors + "\n");
            out.writeChars(total_error);
            t.output(out);
        } catch (IOException e) {
            LOG.error("Error:", e);
            throw e;
        } finally {
            out.close();

        }
    }
}