Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapred; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; /** * Utility class for skip bad records functionality. It contains various * settings related to skipping of bad records. * * <p>Hadoop provides an optional mode of execution in which the bad records * are detected and skipped in further attempts. * * <p>This feature can be used when map/reduce tasks crashes deterministically on * certain input. This happens due to bugs in the map/reduce function. The usual * course would be to fix these bugs. But sometimes this is not possible; * perhaps the bug is in third party libraries for which the source code is * not available. Due to this, the task never reaches to completion even with * multiple attempts and complete data for that task is lost.</p> * * <p>With this feature, only a small portion of data is lost surrounding * the bad record, which may be acceptable for some user applications. * see {@link SkipBadRecords#setMapperMaxSkipRecords(Configuration, long)}</p> * * <p>The skipping mode gets kicked off after certain no of failures * see {@link SkipBadRecords#setAttemptsToStartSkipping(Configuration, int)}</p> * * <p>In the skipping mode, the map/reduce task maintains the record range which * is getting processed at all times. Before giving the input to the * map/reduce function, it sends this record range to the Task tracker. * If task crashes, the Task tracker knows which one was the last reported * range. On further attempts that range get skipped.</p> */ @InterfaceAudience.Public @InterfaceStability.Stable public class SkipBadRecords { /** * Special counters which are written by the application and are * used by the framework for detecting bad records. For detecting bad records * these counters must be incremented by the application. */ public static final String COUNTER_GROUP = "SkippingTaskCounters"; /** * Number of processed map records. * @see SkipBadRecords#getAutoIncrMapperProcCount(Configuration) */ public static final String COUNTER_MAP_PROCESSED_RECORDS = "MapProcessedRecords"; /** * Number of processed reduce groups. * @see SkipBadRecords#getAutoIncrReducerProcCount(Configuration) */ public static final String COUNTER_REDUCE_PROCESSED_GROUPS = "ReduceProcessedGroups"; private static final String ATTEMPTS_TO_START_SKIPPING = JobContext.SKIP_START_ATTEMPTS; private static final String AUTO_INCR_MAP_PROC_COUNT = JobContext.MAP_SKIP_INCR_PROC_COUNT; private static final String AUTO_INCR_REDUCE_PROC_COUNT = JobContext.REDUCE_SKIP_INCR_PROC_COUNT; private static final String OUT_PATH = JobContext.SKIP_OUTDIR; private static final String MAPPER_MAX_SKIP_RECORDS = JobContext.MAP_SKIP_MAX_RECORDS; private static final String REDUCER_MAX_SKIP_GROUPS = JobContext.REDUCE_SKIP_MAXGROUPS; /** * Get the number of Task attempts AFTER which skip mode * will be kicked off. When skip mode is kicked off, the * tasks reports the range of records which it will process * next to the TaskTracker. So that on failures, TT knows which * ones are possibly the bad records. On further executions, * those are skipped. * Default value is 2. * * @param conf the configuration * @return attemptsToStartSkipping no of task attempts */ public static int getAttemptsToStartSkipping(Configuration conf) { return conf.getInt(ATTEMPTS_TO_START_SKIPPING, 2); } /** * Set the number of Task attempts AFTER which skip mode * will be kicked off. When skip mode is kicked off, the * tasks reports the range of records which it will process * next to the TaskTracker. So that on failures, TT knows which * ones are possibly the bad records. On further executions, * those are skipped. * Default value is 2. * * @param conf the configuration * @param attemptsToStartSkipping no of task attempts */ public static void setAttemptsToStartSkipping(Configuration conf, int attemptsToStartSkipping) { conf.setInt(ATTEMPTS_TO_START_SKIPPING, attemptsToStartSkipping); } /** * Get the flag which if set to true, * {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS} is incremented * by MapRunner after invoking the map function. This value must be set to * false for applications which process the records asynchronously * or buffer the input records. For example streaming. * In such cases applications should increment this counter on their own. * Default value is true. * * @param conf the configuration * @return <code>true</code> if auto increment * {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS}. * <code>false</code> otherwise. */ public static boolean getAutoIncrMapperProcCount(Configuration conf) { return conf.getBoolean(AUTO_INCR_MAP_PROC_COUNT, true); } /** * Set the flag which if set to true, * {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS} is incremented * by MapRunner after invoking the map function. This value must be set to * false for applications which process the records asynchronously * or buffer the input records. For example streaming. * In such cases applications should increment this counter on their own. * Default value is true. * * @param conf the configuration * @param autoIncr whether to auto increment * {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS}. */ public static void setAutoIncrMapperProcCount(Configuration conf, boolean autoIncr) { conf.setBoolean(AUTO_INCR_MAP_PROC_COUNT, autoIncr); } /** * Get the flag which if set to true, * {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS} is incremented * by framework after invoking the reduce function. This value must be set to * false for applications which process the records asynchronously * or buffer the input records. For example streaming. * In such cases applications should increment this counter on their own. * Default value is true. * * @param conf the configuration * @return <code>true</code> if auto increment * {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS}. * <code>false</code> otherwise. */ public static boolean getAutoIncrReducerProcCount(Configuration conf) { return conf.getBoolean(AUTO_INCR_REDUCE_PROC_COUNT, true); } /** * Set the flag which if set to true, * {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS} is incremented * by framework after invoking the reduce function. This value must be set to * false for applications which process the records asynchronously * or buffer the input records. For example streaming. * In such cases applications should increment this counter on their own. * Default value is true. * * @param conf the configuration * @param autoIncr whether to auto increment * {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS}. */ public static void setAutoIncrReducerProcCount(Configuration conf, boolean autoIncr) { conf.setBoolean(AUTO_INCR_REDUCE_PROC_COUNT, autoIncr); } /** * Get the directory to which skipped records are written. By default it is * the sub directory of the output _logs directory. * User can stop writing skipped records by setting the value null. * * @param conf the configuration. * @return path skip output directory. Null is returned if this is not set * and output directory is also not set. */ public static Path getSkipOutputPath(Configuration conf) { String name = conf.get(OUT_PATH); if (name != null) { if ("none".equals(name)) { return null; } return new Path(name); } Path outPath = FileOutputFormat.getOutputPath(new JobConf(conf)); return outPath == null ? null : new Path(outPath, "_logs" + Path.SEPARATOR + "skip"); } /** * Set the directory to which skipped records are written. By default it is * the sub directory of the output _logs directory. * User can stop writing skipped records by setting the value null. * * @param conf the configuration. * @param path skip output directory path */ public static void setSkipOutputPath(JobConf conf, Path path) { String pathStr = null; if (path == null) { pathStr = "none"; } else { pathStr = path.toString(); } conf.set(OUT_PATH, pathStr); } /** * Get the number of acceptable skip records surrounding the bad record PER * bad record in mapper. The number includes the bad record as well. * To turn the feature of detection/skipping of bad records off, set the * value to 0. * The framework tries to narrow down the skipped range by retrying * until this threshold is met OR all attempts get exhausted for this task. * Set the value to Long.MAX_VALUE to indicate that framework need not try to * narrow down. Whatever records(depends on application) get skipped are * acceptable. * Default value is 0. * * @param conf the configuration * @return maxSkipRecs acceptable skip records. */ public static long getMapperMaxSkipRecords(Configuration conf) { return conf.getLong(MAPPER_MAX_SKIP_RECORDS, 0); } /** * Set the number of acceptable skip records surrounding the bad record PER * bad record in mapper. The number includes the bad record as well. * To turn the feature of detection/skipping of bad records off, set the * value to 0. * The framework tries to narrow down the skipped range by retrying * until this threshold is met OR all attempts get exhausted for this task. * Set the value to Long.MAX_VALUE to indicate that framework need not try to * narrow down. Whatever records(depends on application) get skipped are * acceptable. * Default value is 0. * * @param conf the configuration * @param maxSkipRecs acceptable skip records. */ public static void setMapperMaxSkipRecords(Configuration conf, long maxSkipRecs) { conf.setLong(MAPPER_MAX_SKIP_RECORDS, maxSkipRecs); } /** * Get the number of acceptable skip groups surrounding the bad group PER * bad group in reducer. The number includes the bad group as well. * To turn the feature of detection/skipping of bad groups off, set the * value to 0. * The framework tries to narrow down the skipped range by retrying * until this threshold is met OR all attempts get exhausted for this task. * Set the value to Long.MAX_VALUE to indicate that framework need not try to * narrow down. Whatever groups(depends on application) get skipped are * acceptable. * Default value is 0. * * @param conf the configuration * @return maxSkipGrps acceptable skip groups. */ public static long getReducerMaxSkipGroups(Configuration conf) { return conf.getLong(REDUCER_MAX_SKIP_GROUPS, 0); } /** * Set the number of acceptable skip groups surrounding the bad group PER * bad group in reducer. The number includes the bad group as well. * To turn the feature of detection/skipping of bad groups off, set the * value to 0. * The framework tries to narrow down the skipped range by retrying * until this threshold is met OR all attempts get exhausted for this task. * Set the value to Long.MAX_VALUE to indicate that framework need not try to * narrow down. Whatever groups(depends on application) get skipped are * acceptable. * Default value is 0. * * @param conf the configuration * @param maxSkipGrps acceptable skip groups. */ public static void setReducerMaxSkipGroups(Configuration conf, long maxSkipGrps) { conf.setLong(REDUCER_MAX_SKIP_GROUPS, maxSkipGrps); } }