com.alexholmes.hadooputils.combine.common.mapred.SplitMetricsCombineInputFormat.java Source code

Introduction

Here is the source code for com.alexholmes.hadooputils.combine.common.mapred.SplitMetricsCombineInputFormat.java

Source

/*
 * Copyright 2013 Alex Holmes
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.alexholmes.hadooputils.combine.common.mapred;

import com.alexholmes.hadooputils.combine.common.CombineFileSplitAdapter;
import com.alexholmes.hadooputils.combine.common.LoggerSink;
import com.alexholmes.hadooputils.combine.common.MetricsSink;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
import org.apache.hadoop.mapred.lib.CombineFileSplit;
import org.apache.hadoop.util.ReflectionUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * An input format that writes split details to a sink, the default sink being a logger.
 *
 * @param <K> The key type.
 * @param <V> The value type.
 */
public abstract class SplitMetricsCombineInputFormat<K, V> extends CombineFileInputFormat<K, V> {
    private static final Log LOG = LogFactory.getLog(SplitMetricsCombineInputFormat.class);

    @Override
    public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
        InputSplit[] splits = super.getSplits(job, numSplits);

        if (job.getBoolean("hadooputils.combine.sink.enabled", false)) {
            writeSplitsToSink(job, organizeSplitsByLocation(splits));
        }

        return splits;
    }

    protected void writeSplitsToSink(Configuration conf, Map<String, List<CombineFileSplitAdapter>> splits) {

        Class<? extends MetricsSink> theClass = conf.getClass("hadooputils.combine.sink.class", LoggerSink.class,
                MetricsSink.class);

        MetricsSink sink = ReflectionUtils.newInstance(theClass, conf);

        for (Map.Entry<String, List<CombineFileSplitAdapter>> entry : splits.entrySet()) {
            sink.pushLocation(entry.getKey(), entry.getValue());
        }
    }

    public static Map<String, List<CombineFileSplitAdapter>> organizeSplitsByLocation(final InputSplit[] splits)
            throws IOException {
        Map<String, List<CombineFileSplitAdapter>> locationSplits = new HashMap<String, List<CombineFileSplitAdapter>>();

        for (InputSplit split : splits) {
            CombineFileSplit csplit = (CombineFileSplit) split;

            String location = extractLocation(csplit);

            List<CombineFileSplitAdapter> splitsForLocation = locationSplits.get(location);
            if (splitsForLocation == null) {
                splitsForLocation = new ArrayList<CombineFileSplitAdapter>();
                locationSplits.put(location, splitsForLocation);
            }
            splitsForLocation.add(new CombineFileSplitAdapter(csplit));
        }
        return locationSplits;
    }

    public static String extractLocation(CombineFileSplit split) throws IOException {
        if (split.getLocations() == null || split.getLocations().length == 0) {
            return null;
        }
        return split.getLocations()[0];
    }
}