Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hive.ql.io; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Queue; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.parse.SplitSample; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.shims.HadoopShims.CombineFileInputFormatShim; import org.apache.hadoop.hive.shims.HadoopShimsSecure.InputSplitShim; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.compress.CompressionCodecFactory; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hadoop.mapred.lib.CombineFileSplit; /** * CombineHiveInputFormat is a parameterized InputFormat which looks at the path * name and determine the correct InputFormat for that path name from * mapredPlan.pathToPartitionInfo(). It can be used to read files with different * input format in the same map-reduce job. */ public class CombineHiveInputFormat<K extends WritableComparable, V extends Writable> extends HiveInputFormat<K, V> { private static final String CLASS_NAME = CombineHiveInputFormat.class.getName(); public static final Log LOG = LogFactory.getLog(CLASS_NAME); // max number of threads we can use to check non-combinable paths private static final int MAX_CHECK_NONCOMBINABLE_THREAD_NUM = 50; private static final int DEFAULT_NUM_PATH_PER_THREAD = 100; private class CheckNonCombinablePathCallable implements Callable<Set<Integer>> { private final Path[] paths; private final int start; private final int length; private final JobConf conf; public CheckNonCombinablePathCallable(Path[] paths, int start, int length, JobConf conf) { this.paths = paths; this.start = start; this.length = length; this.conf = conf; } @Override public Set<Integer> call() throws Exception { Set<Integer> nonCombinablePathIndices = new HashSet<Integer>(); for (int i = 0; i < length; i++) { PartitionDesc part = HiveFileFormatUtils.getPartitionDescFromPathRecursively(pathToPartitionInfo, paths[i + start], IOPrepareCache.get().allocatePartitionDescMap()); // Use HiveInputFormat if any of the paths is not splittable Class<? extends InputFormat> inputFormatClass = part.getInputFileFormatClass(); InputFormat<WritableComparable, Writable> inputFormat = getInputFormatFromCache(inputFormatClass, conf); if (inputFormat instanceof AvoidSplitCombination && ((AvoidSplitCombination) inputFormat).shouldSkipCombine(paths[i + start], conf)) { if (LOG.isDebugEnabled()) { LOG.debug("The path [" + paths[i + start] + "] is being parked for HiveInputFormat.getSplits"); } nonCombinablePathIndices.add(i); } } return nonCombinablePathIndices; } } /** * CombineHiveInputSplit encapsulates an InputSplit with its corresponding * inputFormatClassName. A CombineHiveInputSplit comprises of multiple chunks * from different files. Since, they belong to a single directory, there is a * single inputformat for all the chunks. */ public static class CombineHiveInputSplit extends InputSplitShim { private String inputFormatClassName; private CombineFileSplit inputSplitShim; private Map<String, PartitionDesc> pathToPartitionInfo; public CombineHiveInputSplit() throws IOException { this(ShimLoader.getHadoopShims().getCombineFileInputFormat().getInputSplitShim()); } public CombineHiveInputSplit(CombineFileSplit inputSplitShim) throws IOException { this(inputSplitShim.getJob(), inputSplitShim); } public CombineHiveInputSplit(JobConf job, CombineFileSplit inputSplitShim) throws IOException { this(job, inputSplitShim, null); } public CombineHiveInputSplit(JobConf job, CombineFileSplit inputSplitShim, Map<String, PartitionDesc> pathToPartitionInfo) throws IOException { this.inputSplitShim = inputSplitShim; this.pathToPartitionInfo = pathToPartitionInfo; if (job != null) { if (this.pathToPartitionInfo == null) { this.pathToPartitionInfo = Utilities.getMapWork(job).getPathToPartitionInfo(); } // extract all the inputFormatClass names for each chunk in the // CombinedSplit. Path[] ipaths = inputSplitShim.getPaths(); if (ipaths.length > 0) { PartitionDesc part = HiveFileFormatUtils.getPartitionDescFromPathRecursively( this.pathToPartitionInfo, ipaths[0], IOPrepareCache.get().getPartitionDescMap()); inputFormatClassName = part.getInputFileFormatClass().getName(); } } } public CombineFileSplit getInputSplitShim() { return inputSplitShim; } /** * Returns the inputFormat class name for the i-th chunk. */ public String inputFormatClassName() { return inputFormatClassName; } public void setInputFormatClassName(String inputFormatClassName) { this.inputFormatClassName = inputFormatClassName; } @Override public JobConf getJob() { return inputSplitShim.getJob(); } @Override public long getLength() { return inputSplitShim.getLength(); } /** Returns an array containing the startoffsets of the files in the split. */ @Override public long[] getStartOffsets() { return inputSplitShim.getStartOffsets(); } /** Returns an array containing the lengths of the files in the split. */ @Override public long[] getLengths() { return inputSplitShim.getLengths(); } /** Returns the start offset of the i<sup>th</sup> Path. */ @Override public long getOffset(int i) { return inputSplitShim.getOffset(i); } /** Returns the length of the i<sup>th</sup> Path. */ @Override public long getLength(int i) { return inputSplitShim.getLength(i); } /** Returns the number of Paths in the split. */ @Override public int getNumPaths() { return inputSplitShim.getNumPaths(); } /** Returns the i<sup>th</sup> Path. */ @Override public Path getPath(int i) { return inputSplitShim.getPath(i); } /** Returns all the Paths in the split. */ @Override public Path[] getPaths() { return inputSplitShim.getPaths(); } /** Returns all the Paths where this input-split resides. */ @Override public String[] getLocations() throws IOException { return inputSplitShim.getLocations(); } /** * Prints this obejct as a string. */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append(inputSplitShim.toString()); sb.append("InputFormatClass: " + inputFormatClassName); sb.append("\n"); return sb.toString(); } /** * Writable interface. */ @Override public void readFields(DataInput in) throws IOException { inputSplitShim.readFields(in); inputFormatClassName = in.readUTF(); } /** * Writable interface. */ @Override public void write(DataOutput out) throws IOException { inputSplitShim.write(out); if (inputFormatClassName == null) { if (pathToPartitionInfo == null) { pathToPartitionInfo = Utilities.getMapWork(getJob()).getPathToPartitionInfo(); } // extract all the inputFormatClass names for each chunk in the // CombinedSplit. PartitionDesc part = HiveFileFormatUtils.getPartitionDescFromPathRecursively(pathToPartitionInfo, inputSplitShim.getPath(0), IOPrepareCache.get().getPartitionDescMap()); // create a new InputFormat instance if this is the first time to see // this class inputFormatClassName = part.getInputFileFormatClass().getName(); } out.writeUTF(inputFormatClassName); } } // Splits are not shared across different partitions with different input formats. // For example, 2 partitions (1 sequencefile and 1 rcfile) will have 2 different splits private static class CombinePathInputFormat { private final List<Operator<? extends OperatorDesc>> opList; private final String inputFormatClassName; private final String deserializerClassName; public CombinePathInputFormat(List<Operator<? extends OperatorDesc>> opList, String inputFormatClassName, String deserializerClassName) { this.opList = opList; this.inputFormatClassName = inputFormatClassName; this.deserializerClassName = deserializerClassName; } @Override public boolean equals(Object o) { if (o instanceof CombinePathInputFormat) { CombinePathInputFormat mObj = (CombinePathInputFormat) o; return (opList.equals(mObj.opList)) && (inputFormatClassName.equals(mObj.inputFormatClassName)) && (deserializerClassName == null ? (mObj.deserializerClassName == null) : deserializerClassName.equals(mObj.deserializerClassName)); } return false; } @Override public int hashCode() { return (opList == null) ? 0 : opList.hashCode(); } } /** * Create Hive splits based on CombineFileSplit. */ private InputSplit[] getCombineSplits(JobConf job, int numSplits, Map<String, PartitionDesc> pathToPartitionInfo) throws IOException { init(job); Map<String, ArrayList<String>> pathToAliases = mrwork.getPathToAliases(); Map<String, Operator<? extends OperatorDesc>> aliasToWork = mrwork.getAliasToWork(); CombineFileInputFormatShim combine = ShimLoader.getHadoopShims().getCombineFileInputFormat(); InputSplit[] splits = null; if (combine == null) { splits = super.getSplits(job, numSplits); return splits; } if (combine.getInputPathsShim(job).length == 0) { throw new IOException("No input paths specified in job"); } ArrayList<InputSplit> result = new ArrayList<InputSplit>(); // combine splits only from same tables and same partitions. Do not combine splits from multiple // tables or multiple partitions. Path[] paths = combine.getInputPathsShim(job); List<Path> inpDirs = new ArrayList<Path>(); List<Path> inpFiles = new ArrayList<Path>(); Map<CombinePathInputFormat, CombineFilter> poolMap = new HashMap<CombinePathInputFormat, CombineFilter>(); Set<Path> poolSet = new HashSet<Path>(); for (Path path : paths) { PartitionDesc part = HiveFileFormatUtils.getPartitionDescFromPathRecursively(pathToPartitionInfo, path, IOPrepareCache.get().allocatePartitionDescMap()); TableDesc tableDesc = part.getTableDesc(); if ((tableDesc != null) && tableDesc.isNonNative()) { return super.getSplits(job, numSplits); } // Use HiveInputFormat if any of the paths is not splittable Class inputFormatClass = part.getInputFileFormatClass(); String inputFormatClassName = inputFormatClass.getName(); InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job); String deserializerClassName = null; try { deserializerClassName = part.getDeserializer(job).getClass().getName(); } catch (Exception e) { // ignore } FileSystem inpFs = path.getFileSystem(job); // Since there is no easy way of knowing whether MAPREDUCE-1597 is present in the tree or not, // we use a configuration variable for the same if (this.mrwork != null && !this.mrwork.getHadoopSupportsSplittable()) { // The following code should be removed, once // https://issues.apache.org/jira/browse/MAPREDUCE-1597 is fixed. // Hadoop does not handle non-splittable files correctly for CombineFileInputFormat, // so don't use CombineFileInputFormat for non-splittable files //ie, dont't combine if inputformat is a TextInputFormat and has compression turned on if (inputFormat instanceof TextInputFormat) { Queue<Path> dirs = new LinkedList<Path>(); FileStatus fStats = inpFs.getFileStatus(path); // If path is a directory if (fStats.isDir()) { dirs.offer(path); } else if ((new CompressionCodecFactory(job)).getCodec(path) != null) { //if compresssion codec is set, use HiveInputFormat.getSplits (don't combine) splits = super.getSplits(job, numSplits); return splits; } while (dirs.peek() != null) { Path tstPath = dirs.remove(); FileStatus[] fStatus = inpFs.listStatus(tstPath, FileUtils.HIDDEN_FILES_PATH_FILTER); for (int idx = 0; idx < fStatus.length; idx++) { if (fStatus[idx].isDir()) { dirs.offer(fStatus[idx].getPath()); } else if ((new CompressionCodecFactory(job)) .getCodec(fStatus[idx].getPath()) != null) { //if compresssion codec is set, use HiveInputFormat.getSplits (don't combine) splits = super.getSplits(job, numSplits); return splits; } } } } } //don't combine if inputformat is a SymlinkTextInputFormat if (inputFormat instanceof SymlinkTextInputFormat) { splits = super.getSplits(job, numSplits); return splits; } Path filterPath = path; // Does a pool exist for this path already CombineFilter f = null; List<Operator<? extends OperatorDesc>> opList = null; if (!mrwork.isMapperCannotSpanPartns()) { //if mapper can span partitions, make sure a splits does not contain multiple // opList + inputFormatClassName + deserializerClassName combination // This is done using the Map of CombinePathInputFormat to PathFilter opList = HiveFileFormatUtils.doGetWorksFromPath(pathToAliases, aliasToWork, filterPath); CombinePathInputFormat combinePathInputFormat = new CombinePathInputFormat(opList, inputFormatClassName, deserializerClassName); f = poolMap.get(combinePathInputFormat); if (f == null) { f = new CombineFilter(filterPath); LOG.info("CombineHiveInputSplit creating pool for " + path + "; using filter path " + filterPath); combine.createPool(job, f); poolMap.put(combinePathInputFormat, f); } else { LOG.info("CombineHiveInputSplit: pool is already created for " + path + "; using filter path " + filterPath); f.addPath(filterPath); } } else { // In the case of tablesample, the input paths are pointing to files rather than directories. // We need to get the parent directory as the filtering path so that all files in the same // parent directory will be grouped into one pool but not files from different parent // directories. This guarantees that a split will combine all files in the same partition // but won't cross multiple partitions if the user has asked so. if (!path.getFileSystem(job).getFileStatus(path).isDir()) { // path is not directory filterPath = path.getParent(); inpFiles.add(path); poolSet.add(filterPath); } else { inpDirs.add(path); } } } // Processing directories List<CombineFileSplit> iss = new ArrayList<CombineFileSplit>(); if (!mrwork.isMapperCannotSpanPartns()) { //mapper can span partitions //combine into as few as one split, subject to the PathFilters set // using combine.createPool. iss = Arrays.asList(combine.getSplits(job, 1)); } else { for (Path path : inpDirs) { processPaths(job, combine, iss, path); } if (inpFiles.size() > 0) { // Processing files for (Path filterPath : poolSet) { combine.createPool(job, new CombineFilter(filterPath)); } processPaths(job, combine, iss, inpFiles.toArray(new Path[0])); } } if (mrwork.getNameToSplitSample() != null && !mrwork.getNameToSplitSample().isEmpty()) { iss = sampleSplits(iss); } for (CombineFileSplit is : iss) { CombineHiveInputSplit csplit = new CombineHiveInputSplit(job, is, pathToPartitionInfo); result.add(csplit); } LOG.info("number of splits " + result.size()); return result.toArray(new CombineHiveInputSplit[result.size()]); } /** * Create Hive splits based on CombineFileSplit. */ @Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { PerfLogger perfLogger = PerfLogger.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.GET_SPLITS); init(job); ArrayList<InputSplit> result = new ArrayList<InputSplit>(); Path[] paths = getInputPaths(job); List<Path> nonCombinablePaths = new ArrayList<Path>(paths.length / 2); List<Path> combinablePaths = new ArrayList<Path>(paths.length / 2); int numThreads = Math.min(MAX_CHECK_NONCOMBINABLE_THREAD_NUM, (int) Math.ceil((double) paths.length / DEFAULT_NUM_PATH_PER_THREAD)); int numPathPerThread = (int) Math.ceil((double) paths.length / numThreads); // This check is necessary because for Spark branch, the result array from // getInputPaths() above could be empty, and therefore numThreads could be 0. // In that case, Executors.newFixedThreadPool will fail. if (numThreads > 0) { LOG.info("Total number of paths: " + paths.length + ", launching " + numThreads + " threads to check non-combinable ones."); ExecutorService executor = Executors.newFixedThreadPool(numThreads); List<Future<Set<Integer>>> futureList = new ArrayList<Future<Set<Integer>>>(numThreads); try { for (int i = 0; i < numThreads; i++) { int start = i * numPathPerThread; int length = i != numThreads - 1 ? numPathPerThread : paths.length - start; futureList.add(executor.submit(new CheckNonCombinablePathCallable(paths, start, length, job))); } Set<Integer> nonCombinablePathIndices = new HashSet<Integer>(); for (Future<Set<Integer>> future : futureList) { nonCombinablePathIndices.addAll(future.get()); } for (int i = 0; i < paths.length; i++) { if (nonCombinablePathIndices.contains(i)) { nonCombinablePaths.add(paths[i]); } else { combinablePaths.add(paths[i]); } } } catch (Exception e) { LOG.error("Error checking non-combinable path", e); perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS); throw new IOException(e); } finally { executor.shutdownNow(); } } // Store the previous value for the path specification String oldPaths = job.get(HiveConf.ConfVars.HADOOPMAPREDINPUTDIR.varname); if (LOG.isDebugEnabled()) { LOG.debug("The received input paths are: [" + oldPaths + "] against the property " + HiveConf.ConfVars.HADOOPMAPREDINPUTDIR.varname); } // Process the normal splits if (nonCombinablePaths.size() > 0) { FileInputFormat.setInputPaths(job, nonCombinablePaths.toArray(new Path[nonCombinablePaths.size()])); InputSplit[] splits = super.getSplits(job, numSplits); for (InputSplit split : splits) { result.add(split); } } // Process the combine splits if (combinablePaths.size() > 0) { FileInputFormat.setInputPaths(job, combinablePaths.toArray(new Path[combinablePaths.size()])); Map<String, PartitionDesc> pathToPartitionInfo = this.pathToPartitionInfo != null ? this.pathToPartitionInfo : Utilities.getMapWork(job).getPathToPartitionInfo(); InputSplit[] splits = getCombineSplits(job, numSplits, pathToPartitionInfo); for (InputSplit split : splits) { result.add(split); } } // Restore the old path information back // This is just to prevent incompatibilities with previous versions Hive // if some application depends on the original value being set. if (oldPaths != null) { job.set(HiveConf.ConfVars.HADOOPMAPREDINPUTDIR.varname, oldPaths); } // clear work from ThreadLocal after splits generated in case of thread is reused in pool. Utilities.clearWorkMapForConf(job); LOG.info("Number of all splits " + result.size()); perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS); return result.toArray(new InputSplit[result.size()]); } private void processPaths(JobConf job, CombineFileInputFormatShim combine, List<CombineFileSplit> iss, Path... path) throws IOException { JobConf currJob = new JobConf(job); FileInputFormat.setInputPaths(currJob, path); iss.addAll(Arrays.asList(combine.getSplits(currJob, 1))); } /** * This function is used to sample inputs for clauses like "TABLESAMPLE(1 PERCENT)" * * First, splits are grouped by alias they are for. If one split serves more than one * alias or not for any sampled alias, we just directly add it to returned list. * Then we find a list of exclusive splits for every alias to be sampled. * For each alias, we start from position of seedNumber%totalNumber, and keep add * splits until the total size hits percentage. * @param splits * @return the sampled splits */ private List<CombineFileSplit> sampleSplits(List<CombineFileSplit> splits) { HashMap<String, SplitSample> nameToSamples = mrwork.getNameToSplitSample(); List<CombineFileSplit> retLists = new ArrayList<CombineFileSplit>(); Map<String, ArrayList<CombineFileSplit>> aliasToSplitList = new HashMap<String, ArrayList<CombineFileSplit>>(); Map<String, ArrayList<String>> pathToAliases = mrwork.getPathToAliases(); Map<String, ArrayList<String>> pathToAliasesNoScheme = removeScheme(pathToAliases); // Populate list of exclusive splits for every sampled alias // for (CombineFileSplit split : splits) { String alias = null; for (Path path : split.getPaths()) { boolean schemeless = path.toUri().getScheme() == null; List<String> l = HiveFileFormatUtils .doGetAliasesFromPath(schemeless ? pathToAliasesNoScheme : pathToAliases, path); // a path for a split unqualified the split from being sampled if: // 1. it serves more than one alias // 2. the alias it serves is not sampled // 3. it serves different alias than another path for the same split if (l.size() != 1 || !nameToSamples.containsKey(l.get(0)) || (alias != null && l.get(0) != alias)) { alias = null; break; } alias = l.get(0); } if (alias != null) { // split exclusively serves alias, which needs to be sampled // add it to the split list of the alias. if (!aliasToSplitList.containsKey(alias)) { aliasToSplitList.put(alias, new ArrayList<CombineFileSplit>()); } aliasToSplitList.get(alias).add(split); } else { // The split doesn't exclusively serve one alias retLists.add(split); } } // for every sampled alias, we figure out splits to be sampled and add // them to return list // for (Map.Entry<String, ArrayList<CombineFileSplit>> entry : aliasToSplitList.entrySet()) { ArrayList<CombineFileSplit> splitList = entry.getValue(); long totalSize = 0; for (CombineFileSplit split : splitList) { totalSize += split.getLength(); } SplitSample splitSample = nameToSamples.get(entry.getKey()); long targetSize = splitSample.getTargetSize(totalSize); int startIndex = splitSample.getSeedNum() % splitList.size(); long size = 0; for (int i = 0; i < splitList.size(); i++) { CombineFileSplit split = splitList.get((startIndex + i) % splitList.size()); retLists.add(split); long splitgLength = split.getLength(); if (size + splitgLength >= targetSize) { LOG.info("Sample alias " + entry.getValue() + " using " + (i + 1) + "splits"); if (size + splitgLength > targetSize) { ((InputSplitShim) split).shrinkSplit(targetSize - size); } break; } size += splitgLength; } } return retLists; } Map<String, ArrayList<String>> removeScheme(Map<String, ArrayList<String>> pathToAliases) { Map<String, ArrayList<String>> result = new HashMap<String, ArrayList<String>>(); for (Map.Entry<String, ArrayList<String>> entry : pathToAliases.entrySet()) { String newKey = new Path(entry.getKey()).toUri().getPath(); result.put(newKey, entry.getValue()); } return result; } /** * Create a generic Hive RecordReader than can iterate over all chunks in a * CombinedFileSplit. */ @Override public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { if (!(split instanceof CombineHiveInputSplit)) { return super.getRecordReader(split, job, reporter); } CombineHiveInputSplit hsplit = (CombineHiveInputSplit) split; String inputFormatClassName = null; Class inputFormatClass = null; try { inputFormatClassName = hsplit.inputFormatClassName(); inputFormatClass = job.getClassByName(inputFormatClassName); } catch (Exception e) { throw new IOException("cannot find class " + inputFormatClassName); } pushProjectionsAndFilters(job, inputFormatClass, hsplit.getPath(0).toString(), hsplit.getPath(0).toUri().getPath()); return ShimLoader.getHadoopShims().getCombineFileInputFormat().getRecordReader(job, (CombineFileSplit) split, reporter, CombineHiveRecordReader.class); } static class CombineFilter implements PathFilter { private final Set<String> pStrings = new HashSet<String>(); // store a path prefix in this TestFilter // PRECONDITION: p should always be a directory public CombineFilter(Path p) { // we need to keep the path part only because the Hadoop CombineFileInputFormat will // pass the path part only to accept(). // Trailing the path with a separator to prevent partial matching. addPath(p); } public void addPath(Path p) { String pString = p.toUri().getPath(); pStrings.add(pString); } // returns true if the specified path matches the prefix stored // in this TestFilter. @Override public boolean accept(Path path) { boolean find = false; while (path != null && !find) { if (pStrings.contains(path.toUri().getPath())) { find = true; break; } path = path.getParent(); } return find; } @Override public String toString() { StringBuilder s = new StringBuilder(); s.append("PathFilter: "); for (String pString : pStrings) { s.append(pString + " "); } return s.toString(); } } /** * This is a marker interface that is used to identify the formats where * combine split generation is not applicable */ public interface AvoidSplitCombination { boolean shouldSkipCombine(Path path, Configuration conf) throws IOException; } }