Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hive.ql.exec.tez; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; import java.util.Comparator; import java.util.List; import com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.split.SplitLocationProvider; import org.apache.hadoop.mapreduce.split.TezMapReduceSplitsGrouper; import org.apache.hadoop.util.ReflectionUtils; import org.apache.tez.common.TezUtils; import org.apache.tez.dag.api.TaskLocationHint; import org.apache.tez.dag.api.VertexLocationHint; import org.apache.tez.dag.api.event.VertexStateUpdate; import org.apache.tez.mapreduce.hadoop.InputSplitInfoMem; import org.apache.tez.mapreduce.hadoop.MRInputHelpers; import org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRInputUserPayloadProto; import org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto; import org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitsProto; import org.apache.tez.runtime.api.Event; import org.apache.tez.runtime.api.InputInitializer; import org.apache.tez.runtime.api.InputInitializerContext; import org.apache.tez.runtime.api.InputSpecUpdate; import org.apache.tez.runtime.api.events.InputConfigureVertexTasksEvent; import org.apache.tez.runtime.api.events.InputDataInformationEvent; import org.apache.tez.runtime.api.events.InputInitializerEvent; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; /** * This class is used to generate splits inside the AM on the cluster. It * optionally groups together splits based on available head room as well as * making sure that splits from different partitions are only grouped if they * are of the same schema, format and serde */ public class HiveSplitGenerator extends InputInitializer { private static final Logger LOG = LoggerFactory.getLogger(HiveSplitGenerator.class); private final DynamicPartitionPruner pruner; private final Configuration conf; private final JobConf jobConf; private final MRInputUserPayloadProto userPayloadProto; private final MapWork work; private final SplitGrouper splitGrouper = new SplitGrouper(); private final SplitLocationProvider splitLocationProvider; public HiveSplitGenerator(Configuration conf, MapWork work) throws IOException { super(null); this.conf = conf; this.work = work; this.jobConf = new JobConf(conf); // Assuming grouping enabled always. userPayloadProto = MRInputUserPayloadProto.newBuilder().setGroupingEnabled(true).build(); this.splitLocationProvider = Utils.getSplitLocationProvider(conf, LOG); LOG.info("SplitLocationProvider: " + splitLocationProvider); // Read all credentials into the credentials instance stored in JobConf. ShimLoader.getHadoopShims().getMergedCredentials(jobConf); // Events can start coming in the moment the InputInitializer is created. The pruner // must be setup and initialized here so that it sets up it's structures to start accepting events. // Setting it up in initialize leads to a window where events may come in before the pruner is // initialized, which may cause it to drop events. // No dynamic partition pruning pruner = null; } public HiveSplitGenerator(InputInitializerContext initializerContext) throws IOException, SerDeException { super(initializerContext); Preconditions.checkNotNull(initializerContext); userPayloadProto = MRInputHelpers.parseMRInputPayload(initializerContext.getInputUserPayload()); this.conf = TezUtils.createConfFromByteString(userPayloadProto.getConfigurationBytes()); this.jobConf = new JobConf(conf); this.splitLocationProvider = Utils.getSplitLocationProvider(conf, LOG); LOG.info("SplitLocationProvider: " + splitLocationProvider); // Read all credentials into the credentials instance stored in JobConf. ShimLoader.getHadoopShims().getMergedCredentials(jobConf); this.work = Utilities.getMapWork(jobConf); // Events can start coming in the moment the InputInitializer is created. The pruner // must be setup and initialized here so that it sets up it's structures to start accepting events. // Setting it up in initialize leads to a window where events may come in before the pruner is // initialized, which may cause it to drop events. pruner = new DynamicPartitionPruner(initializerContext, work, jobConf); } @SuppressWarnings("unchecked") @Override public List<Event> initialize() throws Exception { // Setup the map work for this thread. Pruning modified the work instance to potentially remove // partitions. The same work instance must be used when generating splits. Utilities.setMapWork(jobConf, work); try { boolean sendSerializedEvents = conf .getBoolean("mapreduce.tez.input.initializer.serialize.event.payload", true); // perform dynamic partition pruning if (pruner != null) { pruner.prune(); } InputSplitInfoMem inputSplitInfo = null; boolean generateConsistentSplits = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_TEZ_GENERATE_CONSISTENT_SPLITS); LOG.info("GenerateConsistentSplitsInHive=" + generateConsistentSplits); String realInputFormatName = conf.get("mapred.input.format.class"); boolean groupingEnabled = userPayloadProto.getGroupingEnabled(); if (groupingEnabled) { // Need to instantiate the realInputFormat InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtils .newInstance(JavaUtils.loadClass(realInputFormatName), jobConf); int totalResource = 0; int taskResource = 0; int availableSlots = 0; // FIXME. Do the right thing Luke. if (getContext() == null) { // for now, totalResource = taskResource for llap availableSlots = 1; } if (getContext() != null) { totalResource = getContext().getTotalAvailableResource().getMemory(); taskResource = getContext().getVertexTaskResource().getMemory(); availableSlots = totalResource / taskResource; } if (HiveConf.getLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 1) <= 1) { // broken configuration from mapred-default.xml final long blockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); final long minGrouping = conf.getLong(TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE, TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE_DEFAULT); final long preferredSplitSize = Math.min(blockSize / 2, minGrouping); HiveConf.setLongVar(jobConf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, preferredSplitSize); LOG.info("The preferred split size is " + preferredSplitSize); } // Create the un-grouped splits float waves = conf.getFloat(TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_WAVES, TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_WAVES_DEFAULT); // Raw splits InputSplit[] splits = inputFormat.getSplits(jobConf, (int) (availableSlots * waves)); // Sort the splits, so that subsequent grouping is consistent. Arrays.sort(splits, new InputSplitComparator()); LOG.info("Number of input splits: " + splits.length + ". " + availableSlots + " available slots, " + waves + " waves. Input format is: " + realInputFormatName); if (work.getIncludedBuckets() != null) { splits = pruneBuckets(work, splits); } Multimap<Integer, InputSplit> groupedSplits = splitGrouper.generateGroupedSplits(jobConf, conf, splits, waves, availableSlots, splitLocationProvider); // And finally return them in a flat array InputSplit[] flatSplits = groupedSplits.values().toArray(new InputSplit[0]); LOG.info("Number of split groups: " + flatSplits.length); List<TaskLocationHint> locationHints = splitGrouper.createTaskLocationHints(flatSplits, generateConsistentSplits); inputSplitInfo = new InputSplitInfoMem(flatSplits, locationHints, flatSplits.length, null, jobConf); } else { // no need for grouping and the target #of tasks. // This code path should never be triggered at the moment. If grouping is disabled, // DAGUtils uses MRInputAMSplitGenerator. // If this is used in the future - make sure to disable grouping in the payload, if it isn't already disabled throw new RuntimeException( "HiveInputFormat does not support non-grouped splits, InputFormatName is: " + realInputFormatName); // inputSplitInfo = MRInputHelpers.generateInputSplitsToMem(jobConf, false, 0); } return createEventList(sendSerializedEvents, inputSplitInfo); } finally { Utilities.clearWork(jobConf); } } private InputSplit[] pruneBuckets(MapWork work, InputSplit[] splits) { final BitSet buckets = work.getIncludedBuckets(); final String bucketIn = buckets.toString(); List<InputSplit> filteredSplits = new ArrayList<InputSplit>(splits.length / 2); for (InputSplit split : splits) { final int bucket = Utilities.parseSplitBucket(split); if (bucket < 0 || buckets.get(bucket)) { // match or UNKNOWN filteredSplits.add(split); } else { LOG.info("Pruning with IN ({}) - removing {}", bucketIn, split); } } if (filteredSplits.size() < splits.length) { // reallocate only if any filters pruned splits = filteredSplits.toArray(new InputSplit[filteredSplits.size()]); } return splits; } private List<Event> createEventList(boolean sendSerializedEvents, InputSplitInfoMem inputSplitInfo) { List<Event> events = Lists.newArrayListWithCapacity(inputSplitInfo.getNumTasks() + 1); InputConfigureVertexTasksEvent configureVertexEvent = InputConfigureVertexTasksEvent.create( inputSplitInfo.getNumTasks(), VertexLocationHint.create(inputSplitInfo.getTaskLocationHints()), InputSpecUpdate.getDefaultSinglePhysicalInputSpecUpdate()); events.add(configureVertexEvent); if (sendSerializedEvents) { MRSplitsProto splitsProto = inputSplitInfo.getSplitsProto(); int count = 0; for (MRSplitProto mrSplit : splitsProto.getSplitsList()) { InputDataInformationEvent diEvent = InputDataInformationEvent.createWithSerializedPayload(count++, mrSplit.toByteString().asReadOnlyByteBuffer()); events.add(diEvent); } } else { int count = 0; for (org.apache.hadoop.mapred.InputSplit split : inputSplitInfo.getOldFormatSplits()) { InputDataInformationEvent diEvent = InputDataInformationEvent.createWithObjectPayload(count++, split); events.add(diEvent); } } return events; } @Override public void onVertexStateUpdated(VertexStateUpdate stateUpdate) { pruner.processVertex(stateUpdate.getVertexName()); } @Override public void handleInputInitializerEvent(List<InputInitializerEvent> events) throws Exception { for (InputInitializerEvent e : events) { pruner.addEvent(e); } } // Descending sort based on split size| Followed by file name. Followed by startPosition. static class InputSplitComparator implements Comparator<InputSplit> { @Override public int compare(InputSplit o1, InputSplit o2) { try { long len1 = o1.getLength(); long len2 = o2.getLength(); if (len1 < len2) { return 1; } else if (len1 == len2) { // If the same size. Sort on file name followed by startPosition. if (o1 instanceof FileSplit && o2 instanceof FileSplit) { FileSplit fs1 = (FileSplit) o1; FileSplit fs2 = (FileSplit) o2; if (fs1.getPath() != null && fs2.getPath() != null) { int pathComp = (fs1.getPath().compareTo(fs2.getPath())); if (pathComp == 0) { // Compare start Position long startPos1 = fs1.getStart(); long startPos2 = fs2.getStart(); if (startPos1 > startPos2) { return 1; } else if (startPos1 < startPos2) { return -1; } else { return 0; } } else { return pathComp; } } } // No further checks if not a file split. Return equality. return 0; } else { return -1; } } catch (IOException e) { throw new RuntimeException("Problem getting input split size", e); } } } }