Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapred; import java.io.DataInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.Vector; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.corona.SessionPriority; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.TaskStatus; import org.apache.hadoop.mapred.JobHistory.Values; import org.apache.hadoop.mapred.JobInProgress.Counter; import org.apache.hadoop.mapred.TaskStatus.Phase; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; import org.apache.hadoop.net.TopologyCache; import org.apache.hadoop.util.StringUtils; public class CoronaJobInProgress extends JobInProgressTraits { static final Log LOG = LogFactory.getLog(CoronaJobInProgress.class); Clock clock; @SuppressWarnings("deprecation") JobID jobId; SessionPriority priority; JobProfile profile; JobStatus status; private final JobStats jobStats = new JobStats(); long startTime; long launchTime; long finishTime; long deadline; String user; @SuppressWarnings("deprecation") JobConf jobConf; Path jobFile; // non-local Path localJobFile; // XXX: Do not limit number of tasks ourselves. Let Cluster Manager handle it. int maxTasks = 0; int numMapTasks; int numReduceTasks; long memoryPerMap; long memoryPerReduce; volatile int numSlotsPerMap = 1; volatile int numSlotsPerReduce = 1; List<TaskCompletionEvent> taskCompletionEvents; private int taskCompletionEventCounter = 0; static final float DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART = 0.05f; public static final String SPECULATIVE_MAP_UNFINISHED_THRESHOLD_KEY = "mapred.map.tasks.speculation.unfinished.threshold"; public static final String SPECULATIVE_REDUCE_UNFINISHED_THRESHOLD_KEY = "mapred.reduce.tasks.speculation.unfinished.threshold"; static final int NUM_SLOTS_PER_MAP = 1; static final int NUM_SLOTS_PER_REDUCE = 1; private final boolean jobSetupCleanupNeeded; private final boolean jobFinishWhenReducesDone; private final boolean taskCleanupNeeded; private volatile boolean launchedSetup = false; private volatile boolean tasksInited = false; private final AtomicBoolean reduceResourcesRequested = new AtomicBoolean(false); private volatile boolean launchedCleanup = false; private volatile boolean jobKilled = false; private volatile boolean jobFailed = false; String[][] mapLocations; // Has an array of locations for each map. List<TaskInProgress> nonRunningMaps; List<TaskInProgress> nonRunningReduces = new LinkedList<TaskInProgress>(); int runningMapTasks = 0; int runningReduceTasks = 0; int failedMapTasks = 0; // includes killed int failedReduceTasks = 0; // includes killed int killedMapTasks = 0; int killedReduceTasks = 0; int failedMapTIPs = 0; int failedReduceTIPs = 0; int finishedMapTasks = 0; int finishedReduceTasks = 0; int completedMapsForReduceSlowstart = 0; int rushReduceReduces = 5; int rushReduceMaps = 5; float speculativeMapUnfininshedThreshold = 0.001F; float speculativeReduceUnfininshedThreshold = 0.001F; float speculativeMapLogRateThreshold = 0.001F; int speculativeMapLogNumThreshold = 3; float speculativeReduceLogRateThreshold = 0.001F; int speculativeReduceLogNumThreshold = 3; Set<TaskInProgress> runningMaps = new LinkedHashSet<TaskInProgress>(); Set<TaskInProgress> runningReduces = new LinkedHashSet<TaskInProgress>(); @SuppressWarnings("deprecation") List<TaskAttemptID> mapCleanupTasks = new LinkedList<TaskAttemptID>(); @SuppressWarnings("deprecation") List<TaskAttemptID> reduceCleanupTasks = new LinkedList<TaskAttemptID>(); int maxLevel; int anyCacheLevel; private final LocalityStats localityStats; private final Thread localityStatsThread; // runningMapTasks include speculative tasks, so we need to capture // speculative tasks separately int speculativeMapTasks = 0; int speculativeReduceTasks = 0; int mapFailuresPercent = 0; int reduceFailuresPercent = 0; @SuppressWarnings("deprecation") Counters jobCounters = new Counters(); TaskErrorCollector taskErrorCollector; // Maximum no. of fetch-failure notifications after which // the map task is killed private static final int MAX_FETCH_FAILURES_NOTIFICATIONS = 3; private static final int MAX_FETCH_FAILURES_PER_MAP_DEFAULT = 50; private static final String MAX_FETCH_FAILURES_PER_MAP_KEY = "mapred.job.per.map.maxfetchfailures"; private int maxFetchFailuresPerMapper; // The key for the property holding the job deadline value private static final String JOB_DEADLINE_KEY = "mapred.job.deadline"; // The default value of the job deadline property private static final long JOB_DEADLINE_DEFAULT_VALUE = 0L; // The key for the property holding the job priority private static final String SESSION_PRIORITY_KEY = "mapred.job.priority"; // The default value of the job priority private static final String SESSION_PRIORITY_DEFAULT = "NORMAL"; // The maximum percentage of fetch failures allowed for a map private static final double MAX_ALLOWED_FETCH_FAILURES_PERCENT = 0.5; // Map of mapTaskId -> no. of fetch failures private final Map<TaskAttemptID, Integer> mapTaskIdToFetchFailuresMap = new TreeMap<TaskAttemptID, Integer>(); // Don't lower speculativeCap below one TT's worth (for small clusters) private static final int MIN_SPEC_CAP = 10; public static final String SPECULATIVE_SLOWTASK_THRESHOLD = "mapreduce.job.speculative.slowtaskthreshold"; public static final String RUSH_REDUCER_MAP_THRESHOLD = "mapred.job.rushreduce.map.threshold"; public static final String RUSH_REDUCER_REDUCE_THRESHOLD = "mapred.job.rushreduce.reduce.threshold"; public static final String SPECULATIVECAP = "mapreduce.job.speculative.speculativecap"; public static final String SPECULATIVE_SLOWNODE_THRESHOLD = "mapreduce.job.speculative.slownodethreshold"; public static final String SPECULATIVE_REFRESH_TIMEOUT = "mapreduce.job.speculative.refresh.timeout"; public static final String LOG_CANNOTSPECULATIVE_INTERVAL = "mapreduce.job.log.cannotspeculative.interval"; public static final String SPECULATIVE_STDDEVMEANRATIO_MAX = "mapreduce.job.speculative.stddevmeanratio.max"; // thresholds for speculative execution float slowTaskThreshold; float speculativeCap; float slowNodeThreshold; private long lastSpeculativeMapRefresh, lastSpeculativeReduceRefresh; private long lastTimeCannotspeculativeMapLog, lastTimeCannotspeculativeReduceLog; private final long speculativeRefreshTimeout; private final long logCannotspeculativeInterval; private final float speculativeStddevMeanRatioMax; volatile private boolean hasSpeculativeMaps; volatile private boolean hasSpeculativeReduces; private List<TaskInProgress> candidateSpeculativeMaps, candidateSpeculativeReduces; //Statistics are maintained for a couple of things //mapTaskStats is used for maintaining statistics about //the completion time of map tasks on the trackers. On a per //tracker basis, the mean time for task completion is maintained private final DataStatistics mapTaskStats = new DataStatistics(); //reduceTaskStats is used for maintaining statistics about //the completion time of reduce tasks on the trackers. On a per //tracker basis, the mean time for task completion is maintained private final DataStatistics reduceTaskStats = new DataStatistics(); //trackerMapStats used to maintain a mapping from the tracker to the //the statistics about completion time of map tasks private final Map<String, DataStatistics> trackerMapStats = new HashMap<String, DataStatistics>(); //trackerReduceStats used to maintain a mapping from the tracker to the //the statistics about completion time of reduce tasks private final Map<String, DataStatistics> trackerReduceStats = new HashMap<String, DataStatistics>(); //runningMapStats used to maintain the RUNNING map tasks' statistics private final DataStatistics runningMapTaskStats = new DataStatistics(); //runningReduceStats used to maintain the RUNNING reduce tasks' statistics private final DataStatistics runningReduceTaskStats = new DataStatistics(); private static final String JOB_KILLED_REASON = "Job killed"; private static final String EMPTY_TRACKER_NAME = "tracker_:localhost.localdomain/127.0.0.1:"; protected CoronaJobTracker.TaskLookupTable taskLookupTable; private final TaskStateChangeListener taskStateChangeListener; private final Object lockObject; private final CoronaJobHistory jobHistory; private String jobTrackerId; private int terminated = -1; @SuppressWarnings("deprecation") public CoronaJobInProgress(Object lockObject, JobID jobId, Path systemDir, JobConf jobConf, CoronaJobTracker.TaskLookupTable taskLookupTable, TaskStateChangeListener taskStateChangeListener, TopologyCache topologyCache, CoronaJobHistory jobHistory, String url, String jobTrackerId) throws IOException { this.lockObject = lockObject; this.clock = JobTracker.getClock(); this.jobId = jobId; this.jobConf = jobConf; this.taskLookupTable = taskLookupTable; this.taskStateChangeListener = taskStateChangeListener; this.jobHistory = jobHistory; this.jobTrackerId = jobTrackerId; // Status. this.startTime = clock.getTime(); this.status = new JobStatus(jobId, 0.0f, 0.0f, JobStatus.PREP); status.setStartTime(startTime); // Job file. this.jobFile = getJobFile(systemDir, jobId); this.user = jobConf.getUser(); this.profile = new JobProfile(user, jobId, jobFile.toString(), url, jobConf.getJobName(), jobConf.getQueueName()); this.numMapTasks = jobConf.getNumMapTasks(); this.numReduceTasks = jobConf.getNumReduceTasks(); this.memoryPerMap = jobConf.getMemoryForMapTask(); this.memoryPerReduce = jobConf.getMemoryForReduceTask(); this.taskCompletionEvents = new ArrayList<TaskCompletionEvent>(numMapTasks + numReduceTasks + 10); this.jobSetupCleanupNeeded = jobConf.getJobSetupCleanupNeeded(); this.jobFinishWhenReducesDone = jobConf.getJobFinishWhenReducesDone(); this.taskCleanupNeeded = jobConf.getTaskCleanupNeeded(); this.mapFailuresPercent = jobConf.getMaxMapTaskFailuresPercent(); this.reduceFailuresPercent = jobConf.getMaxReduceTaskFailuresPercent(); this.maxLevel = jobConf.getInt("mapred.task.cache.levels", NetworkTopology.DEFAULT_HOST_LEVEL); this.anyCacheLevel = this.maxLevel + 1; this.localityStats = new LocalityStats(jobConf, maxLevel, jobCounters, jobStats, topologyCache); localityStatsThread = new Thread(localityStats); localityStatsThread.setName("Locality Stats"); localityStatsThread.setDaemon(true); localityStatsThread.start(); this.taskErrorCollector = new TaskErrorCollector(jobConf, Integer.MAX_VALUE, 1); this.slowTaskThreshold = Math.max(0.0f, jobConf.getFloat(CoronaJobInProgress.SPECULATIVE_SLOWTASK_THRESHOLD, 1.0f)); this.speculativeCap = jobConf.getFloat(CoronaJobInProgress.SPECULATIVECAP, 0.1f); this.slowNodeThreshold = jobConf.getFloat(CoronaJobInProgress.SPECULATIVE_SLOWNODE_THRESHOLD, 1.0f); this.speculativeRefreshTimeout = jobConf.getLong(CoronaJobInProgress.SPECULATIVE_REFRESH_TIMEOUT, 5000L); this.logCannotspeculativeInterval = jobConf.getLong(CoronaJobInProgress.LOG_CANNOTSPECULATIVE_INTERVAL, 150000L); this.speculativeStddevMeanRatioMax = jobConf.getFloat(CoronaJobInProgress.SPECULATIVE_STDDEVMEANRATIO_MAX, 0.33f); this.speculativeMapUnfininshedThreshold = jobConf.getFloat( CoronaJobInProgress.SPECULATIVE_MAP_UNFINISHED_THRESHOLD_KEY, speculativeMapUnfininshedThreshold); this.speculativeReduceUnfininshedThreshold = jobConf.getFloat( CoronaJobInProgress.SPECULATIVE_REDUCE_UNFINISHED_THRESHOLD_KEY, speculativeReduceUnfininshedThreshold); this.deadline = jobConf.getLong(JOB_DEADLINE_KEY, JOB_DEADLINE_DEFAULT_VALUE); this.priority = SessionPriority.valueOf(jobConf.get(SESSION_PRIORITY_KEY, SESSION_PRIORITY_DEFAULT)); hasSpeculativeMaps = jobConf.getMapSpeculativeExecution(); hasSpeculativeReduces = jobConf.getReduceSpeculativeExecution(); LOG.info(jobId + ": hasSpeculativeMaps = " + hasSpeculativeMaps + ", hasSpeculativeReduces = " + hasSpeculativeReduces); } public JobStats getJobStats() { return jobStats; } @Override public String getUser() { return user; } public JobProfile getProfile() { return profile; } @Override public JobStatus getStatus() { return status; } public boolean isSetupCleanupRequired() { return jobSetupCleanupNeeded; } public SessionPriority getPriority() { return this.priority; } public void setPriority(SessionPriority priority) { this.priority = priority; } @Override DataStatistics getRunningTaskStatistics(boolean isMap) { if (isMap) { return runningMapTaskStats; } else { return runningReduceTaskStats; } } @Override public float getSlowTaskThreshold() { return slowTaskThreshold; } @Override public float getStddevMeanRatioMax() { return speculativeStddevMeanRatioMax; } @Override public int getNumRestarts() { return 0; } public long getLaunchTime() { return launchTime; } public long getStartTime() { return startTime; } public long getFinishTime() { return finishTime; } public long getJobDeadline() { return deadline; } public int getNumMapTasks() { return numMapTasks; } public int getNumReduceTasks() { return numReduceTasks; } @SuppressWarnings("deprecation") public Counters getJobCounters() { return jobCounters; } /** * Returns map phase counters by summing over all map tasks in progress. */ @SuppressWarnings("deprecation") public Counters getMapCounters() { synchronized (lockObject) { return incrementTaskCountersUnprotected(new Counters(), maps); } } /** * Returns map phase counters by summing over all map tasks in progress. */ @SuppressWarnings("deprecation") public Counters getReduceCounters() { synchronized (lockObject) { return incrementTaskCountersUnprotected(new Counters(), reduces); } } /** * Get all the tasks of the desired type in this job. */ TaskInProgress[] getTasks(TaskType type) { TaskInProgress[] tasks = null; switch (type) { case MAP: tasks = maps; break; case REDUCE: tasks = reduces; break; case JOB_SETUP: tasks = setup; break; case JOB_CLEANUP: tasks = cleanup; break; default: tasks = new TaskInProgress[0]; break; } return tasks; } public TaskCompletionEvent[] getTaskCompletionEvents(int fromEventId, int maxEvents) { TaskCompletionEvent[] events = TaskCompletionEvent.EMPTY_ARRAY; synchronized (lockObject) { if (!tasksInited) { return events; } if (taskCompletionEvents.size() > fromEventId) { int actualMax = Math.min(maxEvents, (taskCompletionEvents.size() - fromEventId)); events = taskCompletionEvents.subList(fromEventId, actualMax + fromEventId).toArray(events); } return events; } } public int getTaskCompletionEventsSize() { synchronized (lockObject) { return taskCompletionEvents.size(); } } boolean fetchFailureNotification(TaskAttemptID reportingAttempt, TaskInProgress tip, TaskAttemptID mapAttemptId, String trackerName) { jobStats.incNumMapFetchFailures(); synchronized (lockObject) { Integer fetchFailures = mapTaskIdToFetchFailuresMap.get(mapAttemptId); fetchFailures = (fetchFailures == null) ? 1 : (fetchFailures + 1); mapTaskIdToFetchFailuresMap.put(mapAttemptId, fetchFailures); LOG.info("Failed fetch notification #" + fetchFailures + " by " + reportingAttempt + " for task " + mapAttemptId + " tracker " + trackerName); float failureRate = (float) fetchFailures / runningReduceTasks; // declare faulty if fetch-failures >= max-allowed-failures final boolean isMapFaulty = (failureRate >= MAX_ALLOWED_FETCH_FAILURES_PERCENT) || fetchFailures > maxFetchFailuresPerMapper; if (fetchFailures >= MAX_FETCH_FAILURES_NOTIFICATIONS && isMapFaulty) { String reason = "Too many fetch-failures (" + fetchFailures + ") at " + new Date(); LOG.info(reason + " for " + mapAttemptId + " ... killing it"); final boolean isFailed = true; TaskTrackerInfo ttStatus = null; jobStats.incNumMapTasksFailedByFetchFailures(); failedTask(tip, mapAttemptId, reason, (tip.isMapTask() ? TaskStatus.Phase.MAP : TaskStatus.Phase.REDUCE), isFailed, trackerName, ttStatus); mapTaskIdToFetchFailuresMap.remove(mapAttemptId); return true; } } return false; } @SuppressWarnings("deprecation") public static Path getJobFile(Path systemDir, JobID jobId) { Path systemDirForJob = new Path(systemDir, jobId.toString()); return new Path(systemDirForJob, "job.xml"); } public int getMaxTasksPerJob() { return 0; // No maximum. } public void close() { localityStats.stop(); localityStatsThread.interrupt(); // The thread may be stuck in DNS lookups. This thread is a daemon, // so it will not prevent process exit. try { localityStatsThread.join(1000); } catch (InterruptedException e) { LOG.warn("localityStatsThread.join interrupted"); } } /** * Read input splits and create a map per split. */ public void initTasks() throws IOException { // log job info jobHistory.logSubmitted(jobFile.toString(), this.startTime, this.jobTrackerId); // log the job priority JobClient.RawSplit[] splits = null; splits = JobClient.getAndRemoveCachedSplits(jobId); if (splits == null) { FileSystem fs = jobFile.getFileSystem(jobConf); Path splitFile = new Path(jobFile.getParent(), "job.split"); LOG.info("Reading splits from " + splitFile); DataInputStream splitFileIn = fs.open(splitFile); try { splits = JobClient.readSplitFile(splitFileIn); } finally { splitFileIn.close(); } } initTasksFromSplits(splits); jobHistory.logInited(this.launchTime, numMapTasks, numReduceTasks); } /** * Used by test code. */ void initTasksFromSplits(JobClient.RawSplit[] splits) throws IOException { synchronized (lockObject) { initTasksFromSplitsUnprotected(splits); } } private void initTasksFromSplitsUnprotected(JobClient.RawSplit[] splits) throws IOException { String jobFile = profile.getJobFile(); numMapTasks = splits.length; if (maxTasks > 0 && numMapTasks + numReduceTasks > maxTasks) { throw new IOException("The number of tasks for this job " + (numMapTasks + numReduceTasks) + " exceeds the configured limit " + maxTasks); } long inputLength = 0; maps = new TaskInProgress[numMapTasks]; mapLocations = new String[numMapTasks][]; nonRunningMaps = new ArrayList<TaskInProgress>(numMapTasks); for (int i = 0; i < numMapTasks; i++) { inputLength += splits[i].getDataLength(); maps[i] = new TaskInProgress(jobId, jobFile, splits[i], jobConf, this, i, 1); // numSlotsPerMap = 1 nonRunningMaps.add(maps[i]); mapLocations[i] = splits[i].getLocations(); } LOG.info("Input size for job " + jobId + " = " + inputLength + ". Number of splits = " + splits.length); this.launchTime = clock.getTime(); LOG.info("Number of splits for job " + jobId + " = " + splits.length); // Create reduce tasks this.reduces = new TaskInProgress[numReduceTasks]; for (int i = 0; i < numReduceTasks; i++) { reduces[i] = new TaskInProgress(jobId, jobFile, numMapTasks, i, jobConf, this, 1); // numSlotsPerReduce = 1 nonRunningReduces.add(reduces[i]); } LOG.info("Number of reduces for job " + jobId + " = " + reduces.length); // Calculate the minimum number of maps to be complete before // we should start scheduling reduces completedMapsForReduceSlowstart = (int) Math .ceil((jobConf.getFloat("mapred.reduce.slowstart.completed.maps", DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART) * numMapTasks)); // The thresholds of total maps and reduces for scheduling reducers // immediately. rushReduceMaps = jobConf.getInt(RUSH_REDUCER_MAP_THRESHOLD, rushReduceMaps); rushReduceReduces = jobConf.getInt(RUSH_REDUCER_REDUCE_THRESHOLD, rushReduceReduces); maxFetchFailuresPerMapper = jobConf.getInt(MAX_FETCH_FAILURES_PER_MAP_KEY, MAX_FETCH_FAILURES_PER_MAP_DEFAULT); // Proceed to Setup/Cleanup. if (jobSetupCleanupNeeded) { // create cleanup two cleanup tips, one map and one reduce. cleanup = new TaskInProgress[2]; // cleanup map tip. This map doesn't use any splits. Just assign an empty // split. JobClient.RawSplit emptySplit = new JobClient.RawSplit(); cleanup[0] = new TaskInProgress(jobId, jobFile, emptySplit, jobConf, this, numMapTasks, 1); cleanup[0].setJobCleanupTask(); // cleanup reduce tip. cleanup[1] = new TaskInProgress(jobId, jobFile, numMapTasks, numReduceTasks, jobConf, this, 1); cleanup[1].setJobCleanupTask(); // create two setup tips, one map and one reduce. setup = new TaskInProgress[2]; // setup map tip. This map doesn't use any split. Just assign an empty // split. setup[0] = new TaskInProgress(jobId, jobFile, emptySplit, jobConf, this, numMapTasks + 1, 1); setup[0].setJobSetupTask(); // setup reduce tip. setup[1] = new TaskInProgress(jobId, jobFile, numMapTasks, numReduceTasks + 1, jobConf, this, 1); setup[1].setJobSetupTask(); } tasksInited = true; } boolean scheduleReducesUnprotected() { // Start scheduling reducers if we have enough maps finished or // if the job has very few mappers or reducers. return numMapTasks <= rushReduceMaps || numReduceTasks <= rushReduceReduces || finishedMapTasks >= completedMapsForReduceSlowstart; } /** * Signals that the reduce resources are being requested * as soon as one process starts this nobody else should be * trying to request reduce resources, so get current and set to true * * @return false if the resources have not been requested yet, * true if they have */ boolean initializeReducers() { return this.reduceResourcesRequested.getAndSet(true); } boolean areReducersInitialized() { return this.reduceResourcesRequested.get(); } public Task obtainNewMapTaskForTip(String taskTrackerName, String hostName, TaskInProgress intendedTip) { synchronized (lockObject) { Task result = obtainTaskCleanupTask(taskTrackerName, intendedTip); if (result != null) { return result; } if (status.getRunState() != JobStatus.RUNNING) { return null; } TaskInProgress tip = removeMatchingTipUnprotected(nonRunningMaps, hostName, intendedTip); if (tip != null) { LOG.info("Running task " + tip.getTIPId() + " on " + taskTrackerName + "(" + hostName + ")"); } if (tip == null && hasSpeculativeMaps) { SpeculationStatus speculationStatus = confirmSpeculativeTaskUnprotected(candidateSpeculativeMaps, intendedTip, taskTrackerName, hostName, TaskType.MAP); if (speculationStatus == SpeculationStatus.CAN_BE_SPECULATED) { tip = intendedTip; LOG.info( "Speculating task " + tip.getTIPId() + " on " + taskTrackerName + "(" + hostName + ")"); } else { LOG.warn("Cant speculate for given resource " + taskTrackerName + " because " + speculationStatus); } } if (tip == null) { return null; } else if (tip != intendedTip) { throw new RuntimeException( "Logic error:" + tip.getTIPId() + " was chosen instead of " + intendedTip.getTIPId()); } scheduleMapUnprotected(tip); result = tip.getTaskToRun(taskTrackerName); if (result != null) { addRunningTaskToTIPUnprotected(tip, result.getTaskID(), taskTrackerName, hostName, true); } return result; } } /** * Registers new task attempt for given task * @param taskTrackerName name of destination tracker * @param hostName hostname of tracker * @param forcedTip task in progress to run * @return task to run */ public Task forceNewMapTaskForTip(String taskTrackerName, String hostName, TaskInProgress forcedTip) { synchronized (lockObject) { Task result = obtainTaskCleanupTask(taskTrackerName, forcedTip); if (result != null) { return result; } removeMatchingTipUnprotectedUnconditional(nonRunningMaps, forcedTip); LOG.info("Running task " + forcedTip.getTIPId() + " on " + taskTrackerName + "(" + hostName + ")"); scheduleMapUnprotected(forcedTip); result = forcedTip.getTaskToRun(taskTrackerName); if (result != null) { addRunningTaskToTIPUnprotected(forcedTip, result.getTaskID(), taskTrackerName, hostName, true); // Handle cleanup task setJobCleanupTaskState(result); } return result; } } @Override public boolean hasSpeculativeMaps() { return hasSpeculativeMaps; } @Override public boolean hasSpeculativeReduces() { return hasSpeculativeReduces; } private void refreshCandidateSpeculativeMapsUnprotected() { long now = clock.getTime(); if ((now - lastSpeculativeMapRefresh) > speculativeRefreshTimeout) { // update the progress rates of all the candidate tips .. for (TaskInProgress tip : runningMaps) { tip.updateProgressRate(now); } candidateSpeculativeMaps = findSpeculativeTaskCandidatesUnprotected(runningMaps); int cap = getSpeculativeCap(TaskType.MAP); cap = Math.max(0, cap - speculativeMapTasks); cap = Math.min(candidateSpeculativeMaps.size(), cap); candidateSpeculativeMaps = candidateSpeculativeMaps.subList(0, cap); lastSpeculativeMapRefresh = now; } } public int getSpeculativeCap(TaskType type) { int cap = 0; synchronized (lockObject) { int numRunningTasks = (type == TaskType.MAP) ? (runningMapTasks - speculativeMapTasks) : (runningReduceTasks - speculativeReduceTasks); cap = (int) Math.max(MIN_SPEC_CAP, speculativeCap * numRunningTasks); } return cap; } public List<TaskInProgress> getSpeculativeCandidates(TaskType type) { synchronized (lockObject) { if (TaskType.MAP == type) { return candidateSpeculativeMaps; } else { return candidateSpeculativeReduces; } } } public void updateSpeculationCandidates() { synchronized (lockObject) { if (hasSpeculativeMaps()) { refreshCandidateSpeculativeMapsUnprotected(); } if (hasSpeculativeReduces()) { refreshCandidateSpeculativeReducesUnprotected(); } } } /** * Given a candidate set of tasks, find and order the ones that * can be speculated and return the same. */ protected List<TaskInProgress> findSpeculativeTaskCandidatesUnprotected(Collection<TaskInProgress> list) { ArrayList<TaskInProgress> candidates = new ArrayList<TaskInProgress>(); long now = clock.getTime(); Iterator<TaskInProgress> iter = list.iterator(); while (iter.hasNext()) { TaskInProgress tip = iter.next(); if (tip.canBeSpeculated(now)) { candidates.add(tip); } } if (candidates.size() > 0) { Comparator<TaskInProgress> LateComparator = new JobInProgress.EstimatedTimeLeftComparator(now); Collections.sort(candidates, LateComparator); } return candidates; } private enum SpeculationStatus { CAN_BE_SPECULATED, CAN_NO_LONGER_BE_SPECULATED, HAS_RUN_ON_MACHINE, MACHINE_IS_SLOW }; public boolean confirmSpeculativeTask(TaskInProgress tip, String taskTrackerName, String taskTrackerHost) { synchronized (lockObject) { if (tip.isMapTask()) { return confirmSpeculativeTaskUnprotected(candidateSpeculativeMaps, tip, taskTrackerName, taskTrackerHost, TaskType.MAP) == SpeculationStatus.CAN_BE_SPECULATED; } else { return confirmSpeculativeTaskUnprotected(candidateSpeculativeReduces, tip, taskTrackerName, taskTrackerHost, TaskType.REDUCE) == SpeculationStatus.CAN_BE_SPECULATED; } } } public boolean isBadSpeculativeResource(TaskInProgress tip, String taskTrackerName, String taskTrackerHost) { synchronized (lockObject) { SpeculationStatus status = null; if (tip.isMapTask()) { status = confirmSpeculativeTaskUnprotected(candidateSpeculativeMaps, tip, taskTrackerName, taskTrackerHost, TaskType.MAP); } else { status = confirmSpeculativeTaskUnprotected(candidateSpeculativeReduces, tip, taskTrackerName, taskTrackerHost, TaskType.REDUCE); } return status == SpeculationStatus.HAS_RUN_ON_MACHINE || status == SpeculationStatus.MACHINE_IS_SLOW; } } protected SpeculationStatus confirmSpeculativeTaskUnprotected(List<TaskInProgress> candidates, TaskInProgress intendedTip, String taskTrackerName, String taskTrackerHost, TaskType taskType) { if ((candidates == null) || candidates.isEmpty()) { return null; } if (isSlowTrackerUnprotected(taskTrackerName)) { // TODO: request another resource if this happens. return SpeculationStatus.MACHINE_IS_SLOW; } if (!candidates.contains(intendedTip)) { return SpeculationStatus.CAN_NO_LONGER_BE_SPECULATED; } if (intendedTip.hasRunOnMachine(taskTrackerHost, taskTrackerName)) { return SpeculationStatus.HAS_RUN_ON_MACHINE; } long now = clock.getTime(); if (intendedTip.canBeSpeculated(now)) { return SpeculationStatus.CAN_BE_SPECULATED; } else { // if it can't be speculated, then: // A. it has completed/failed etc. - in which case makes sense to never // speculate again // B. it's relative progress does not allow speculation. in this case // it's fair to treat it as if it was never eligible for speculation // to begin with. return SpeculationStatus.CAN_NO_LONGER_BE_SPECULATED; } } /** * Compares the ave progressRate of tasks that have finished on this * taskTracker to the ave of all succesfull tasks thus far to see if this * TT one is too slow for speculating. * slowNodeThreshold is used to determine the number of standard deviations * @param taskTracker the name of the TaskTracker we are checking * @return is this TaskTracker slow */ protected boolean isSlowTrackerUnprotected(String taskTracker) { // TODO - use statistics here. return false; } private TaskInProgress removeMatchingTipUnprotected(List<TaskInProgress> taskList, String hostName, TaskInProgress intendedTip) { for (Iterator<TaskInProgress> iter = taskList.iterator(); iter.hasNext();) { TaskInProgress t = iter.next(); if (t == intendedTip && t.isRunnable() && !t.isRunning() && !t.hasFailedOnMachine(hostName)) { iter.remove(); return t; } } return null; } /** * Removes matching TIP without checking any conditions * @param taskList list of tasks to remove from * @param intendedTip tip to remove * @return removed tip */ private TaskInProgress removeMatchingTipUnprotectedUnconditional(List<TaskInProgress> taskList, TaskInProgress intendedTip) { for (Iterator<TaskInProgress> iter = taskList.iterator(); iter.hasNext();) { TaskInProgress t = iter.next(); if (t.getTIPId().equals(intendedTip.getTIPId())) { iter.remove(); return t; } } return null; } /** * Find a non-running task in the passed list of TIPs * @param tips a collection of TIPs * @param hostName the host name of tracker that has requested a task to run * @param removeFailedTip whether to remove the failed tips */ private static TaskInProgress findTaskFromList(Collection<TaskInProgress> tips, String hostName, boolean removeFailedTip) { Iterator<TaskInProgress> iter = tips.iterator(); while (iter.hasNext()) { TaskInProgress tip = iter.next(); // Select a tip if // 1. runnable : still needs to be run and is not completed // 2. ~running : no other node is running it // 3. earlier attempt failed : has not failed on this host // and has failed on all the other hosts // A TIP is removed from the list if // (1) this tip is scheduled // (2) if the passed list is a level 0 (host) cache // (3) when the TIP is non-schedulable (running, killed, complete) if (tip.isRunnable() && !tip.isRunning()) { // check if the tip has failed on this host if (!tip.hasFailedOnMachine(hostName)) { // TODO: check if the tip has failed on all the nodes iter.remove(); return tip; } else if (removeFailedTip) { // the case where we want to remove a failed tip from the host cache // point#3 in the TIP removal logic above iter.remove(); } } else { // see point#3 in the comment above for TIP removal logic iter.remove(); } } return null; } public Task obtainNewReduceTaskForTip(String taskTrackerName, String hostName, TaskInProgress intendedTip) { synchronized (lockObject) { // TODO: check for resource constraints. Task result = obtainTaskCleanupTask(taskTrackerName, intendedTip); if (result != null) { return result; } TaskInProgress tip = removeMatchingTipUnprotected(nonRunningReduces, hostName, intendedTip); if (tip != null) { LOG.info("Running task " + tip.getTIPId() + " on " + taskTrackerName + "(" + hostName + ")"); } // 2. check for a reduce tip to be speculated if (tip == null && hasSpeculativeReduces) { SpeculationStatus speculationStatus = confirmSpeculativeTaskUnprotected(candidateSpeculativeReduces, intendedTip, taskTrackerName, hostName, TaskType.REDUCE); if (speculationStatus == SpeculationStatus.CAN_BE_SPECULATED) { tip = intendedTip; LOG.info( "Speculating task " + tip.getTIPId() + " on " + taskTrackerName + "(" + hostName + ")"); } else { LOG.warn("Cant speculate for given resource " + taskTrackerName + " because " + speculationStatus); } } if (tip == null) { return null; } else if (tip != intendedTip) { throw new RuntimeException( "Logic error:" + tip.getTIPId() + " was chosen instead of " + intendedTip.getTIPId()); } scheduleReduceUnprotected(tip); result = tip.getTaskToRun(taskTrackerName); if (result != null) { addRunningTaskToTIPUnprotected(tip, result.getTaskID(), taskTrackerName, hostName, true); } return result; } } /** * Registers new task attempt for given task * @param taskTrackerName name of destination tracker * @param hostName hostname of tracker * @param forcedTip task in progress to run * @return task to run */ public Task forceNewReduceTaskForTip(String taskTrackerName, String hostName, TaskInProgress forcedTip) { synchronized (lockObject) { Task result = obtainTaskCleanupTask(taskTrackerName, forcedTip); if (result != null) { return result; } removeMatchingTipUnprotectedUnconditional(nonRunningMaps, forcedTip); LOG.info("Running task " + forcedTip.getTIPId() + " on " + taskTrackerName + "(" + hostName + ")"); scheduleReduceUnprotected(forcedTip); result = forcedTip.getTaskToRun(taskTrackerName); if (result != null) { addRunningTaskToTIPUnprotected(forcedTip, result.getTaskID(), taskTrackerName, hostName, true); // Handle cleanup task setJobCleanupTaskState(result); } return result; } } private void refreshCandidateSpeculativeReducesUnprotected() { long now = clock.getTime(); if ((now - lastSpeculativeReduceRefresh) > speculativeRefreshTimeout) { // update the progress rates of all the candidate tips .. for (TaskInProgress tip : runningReduces) { tip.updateProgressRate(now); } candidateSpeculativeReduces = findSpeculativeTaskCandidatesUnprotected(runningReduces); int cap = getSpeculativeCap(TaskType.REDUCE); cap = Math.max(0, cap - speculativeReduceTasks); cap = Math.min(candidateSpeculativeReduces.size(), cap); candidateSpeculativeReduces = candidateSpeculativeReduces.subList(0, cap); lastSpeculativeReduceRefresh = now; } } /** * Can a tracker be used for a TIP? */ public boolean canTrackerBeUsed(String taskTracker, String trackerHost, TaskInProgress tip) { synchronized (lockObject) { return !tip.hasFailedOnMachine(trackerHost); } } /** * Return a CleanupTask, if appropriate, to run on the given tasktracker */ public Task obtainJobCleanupTask(String taskTrackerName, String hostName, boolean isMapSlot) { synchronized (lockObject) { if (!tasksInited || !jobSetupCleanupNeeded) { return null; } if (!canLaunchJobCleanupTaskUnprotected()) { return null; } List<TaskInProgress> cleanupTaskList = new ArrayList<TaskInProgress>(); if (isMapSlot) { cleanupTaskList.add(cleanup[0]); } else { cleanupTaskList.add(cleanup[1]); } TaskInProgress tip = findTaskFromList(cleanupTaskList, hostName, false); if (tip == null) { return null; } // Now launch the cleanupTask Task result = tip.getTaskToRun(taskTrackerName); if (result != null) { addRunningTaskToTIPUnprotected(tip, result.getTaskID(), taskTrackerName, hostName, true); // Handle cleanup task setJobCleanupTaskState(result); } return result; } } /** * Sets task state according to job state if given task is cleanup one * @param task task to handle */ private void setJobCleanupTaskState(Task task) { if (task.isJobCleanupTask()) { if (jobFailed) { task.setJobCleanupTaskState(org.apache.hadoop.mapreduce.JobStatus.State.FAILED); } else if (jobKilled) { task.setJobCleanupTaskState(org.apache.hadoop.mapreduce.JobStatus.State.KILLED); } else { task.setJobCleanupTaskState(org.apache.hadoop.mapreduce.JobStatus.State.SUCCEEDED); } } } @SuppressWarnings("deprecation") public boolean needsTaskCleanup(TaskInProgress tip) { synchronized (lockObject) { Iterator<TaskAttemptID> cleanupCandidates; if (tip.isMapTask()) { cleanupCandidates = mapCleanupTasks.iterator(); } else { cleanupCandidates = reduceCleanupTasks.iterator(); } while (cleanupCandidates.hasNext()) { if (cleanupCandidates.next().getTaskID().equals(tip.getTIPId())) { return true; } } } return false; } @SuppressWarnings("deprecation") public Task obtainTaskCleanupTask(String taskTracker, TaskInProgress tip) { synchronized (lockObject) { if (!tasksInited) { return null; } if (this.status.getRunState() != JobStatus.RUNNING || jobFailed || jobKilled) { return null; } if (tip.isMapTask()) { if (mapCleanupTasks.isEmpty()) return null; } else { if (reduceCleanupTasks.isEmpty()) return null; } if (this.status.getRunState() != JobStatus.RUNNING || jobFailed || jobKilled) { return null; } TaskAttemptID taskid = null; Iterator<TaskAttemptID> cleanupCandidates = null; boolean foundCleanup = false; if (tip.isMapTask()) { if (!mapCleanupTasks.isEmpty()) { cleanupCandidates = mapCleanupTasks.iterator(); } } else { if (!reduceCleanupTasks.isEmpty()) { cleanupCandidates = reduceCleanupTasks.iterator(); } } while (cleanupCandidates.hasNext()) { taskid = cleanupCandidates.next(); if (taskid.getTaskID().equals(tip.getTIPId())) { // The task requires a cleanup so we are going to do that right now cleanupCandidates.remove(); foundCleanup = true; break; } } if (foundCleanup) { return tip.addRunningTask(taskid, taskTracker, true); } return null; } } /** * Return a SetupTask, if appropriate, to run on the given tasktracker */ public Task obtainJobSetupTask(String taskTrackerName, String hostName, boolean isMapSlot) { synchronized (lockObject) { if (!tasksInited || !jobSetupCleanupNeeded) { return null; } if (!(tasksInited && status.getRunState() == JobStatus.PREP && !launchedSetup && !jobKilled && !jobFailed)) { return null; } List<TaskInProgress> setupTaskList = new ArrayList<TaskInProgress>(); if (isMapSlot) { setupTaskList.add(setup[0]); } else { setupTaskList.add(setup[1]); } TaskInProgress tip = findTaskFromList(setupTaskList, hostName, false); if (tip == null) { return null; } // Now launch the setupTask Task result = tip.getTaskToRun(taskTrackerName); if (result != null) { addRunningTaskToTIPUnprotected(tip, result.getTaskID(), taskTrackerName, hostName, true); } return result; } } public void updateTaskStatus(TaskInProgress tip, TaskStatus status, TaskTrackerInfo ttStatus) { synchronized (lockObject) { updateTaskStatusUnprotected(tip, status, ttStatus); } } private boolean isTaskKilledHighMemory(TaskStatus status, String keyword) { String diagnosticInfo = status.getDiagnosticInfo(); if (diagnosticInfo == null) { return false; } String[] splitdiagnosticInfo = diagnosticInfo.split("\\s+"); for (String info : splitdiagnosticInfo) { if (keyword.equals(info)) { return true; } } return false; } private boolean isTaskKilledWithHighMemory(TaskStatus status) { return isTaskKilledHighMemory(status, TaskMemoryManagerThread.HIGH_MEMORY_KEYWORD); } private boolean isTaskKilledWithCGroupMemory(TaskStatus status) { return isTaskKilledHighMemory(status, CGroupMemoryWatcher.CGROUPHIGH_MEMORY_KEYWORD); } private void updateCGResourceCounters(TaskStatus status, boolean isMap) { Counters taskCounters = status.getCounters(); long maxMem = taskCounters.getCounter(Task.Counter.MAX_MEMORY_BYTES); long rssMem = taskCounters.getCounter(Task.Counter.MAX_RSS_MEMORY_BYTES); long instMem = taskCounters.getCounter(Task.Counter.INST_MEMORY_BYTES); if (isMap) { if (jobCounters.getCounter(Counter.MAX_MAP_MEM_BYTES) < maxMem) { jobCounters.findCounter(Counter.MAX_MAP_MEM_BYTES).setValue(maxMem); } if (jobCounters.getCounter(Counter.MAX_MAP_RSS_MEM_BYTES) < rssMem) { jobCounters.findCounter(Counter.MAX_MAP_RSS_MEM_BYTES).setValue(rssMem); } if (jobCounters.getCounter(Counter.MAX_MAP_INST_MEM_BYTES) < instMem) { jobCounters.findCounter(Counter.MAX_MAP_INST_MEM_BYTES).setValue(instMem); } } else { if (jobCounters.getCounter(Counter.MAX_REDUCE_MEM_BYTES) < maxMem) { jobCounters.findCounter(Counter.MAX_REDUCE_MEM_BYTES).setValue(maxMem); } if (jobCounters.getCounter(Counter.MAX_REDUCE_RSS_MEM_BYTES) < rssMem) { jobCounters.findCounter(Counter.MAX_REDUCE_RSS_MEM_BYTES).setValue(rssMem); } if (jobCounters.getCounter(Counter.MAX_REDUCE_INST_MEM_BYTES) < instMem) { jobCounters.findCounter(Counter.MAX_REDUCE_INST_MEM_BYTES).setValue(instMem); } } } @SuppressWarnings("deprecation") private void updateTaskStatusUnprotected(TaskInProgress tip, TaskStatus status, TaskTrackerInfo ttStatus) { double oldProgress = tip.getProgress(); // save old progress boolean wasRunning = tip.isRunning(); boolean wasComplete = tip.isComplete(); boolean wasPending = tip.isOnlyCommitPending(); TaskAttemptID taskid = status.getTaskID(); boolean wasAttemptRunning = tip.isAttemptRunning(taskid); // If the TIP is already completed and the task reports as SUCCEEDED then // mark the task as KILLED. // In case of task with no promotion the task tracker will mark the task // as SUCCEEDED. // User has requested to kill the task, but TT reported SUCCEEDED, // mark the task KILLED. if ((wasComplete || tip.wasKilled(taskid)) && (status.getRunState() == TaskStatus.State.SUCCEEDED)) { status.setRunState(TaskStatus.State.KILLED); } // When a task has just reported its state as FAILED_UNCLEAN/KILLED_UNCLEAN, // if the job is complete or cleanup task is switched off, // make the task's state FAILED/KILLED without launching cleanup attempt. // Note that if task is already a cleanup attempt, // we don't change the state to make sure the task gets a killTaskAction if ((this.status.isJobComplete() || jobFailed || jobKilled || !taskCleanupNeeded) && !tip.isCleanupAttempt(taskid)) { if (status.getRunState() == TaskStatus.State.FAILED_UNCLEAN) { status.setRunState(TaskStatus.State.FAILED); } else if (status.getRunState() == TaskStatus.State.KILLED_UNCLEAN) { status.setRunState(TaskStatus.State.KILLED); } } // aggregate the task tracker reported cgroup resource counters in job updateCGResourceCounters(status, tip.isMapTask()); boolean change = tip.updateStatus(status); if (change) { TaskStatus.State state = status.getRunState(); String httpTaskLogLocation = null; // TODO fix this if (ttStatus != null) { String host; if (NetUtils.getStaticResolution(ttStatus.getHost()) != null) { host = NetUtils.getStaticResolution(ttStatus.getHost()); } else { host = ttStatus.getHost(); } httpTaskLogLocation = "http://" + host + ":" + ttStatus.getHttpPort(); } TaskCompletionEvent taskEvent = null; if (state == TaskStatus.State.SUCCEEDED) { taskEvent = new TaskCompletionEvent(taskCompletionEventCounter, taskid, tip.idWithinJob(), status.getIsMap() && !tip.isJobCleanupTask() && !tip.isJobSetupTask(), TaskCompletionEvent.Status.SUCCEEDED, httpTaskLogLocation); taskEvent.setTaskRunTime((int) (status.getFinishTime() - status.getStartTime())); tip.setSuccessEventNumber(taskCompletionEventCounter); } else if (state == TaskStatus.State.COMMIT_PENDING) { // If it is the first attempt reporting COMMIT_PENDING // ask the task to commit. if (!wasComplete && !wasPending) { tip.doCommit(taskid); } return; } else if (state == TaskStatus.State.FAILED_UNCLEAN || state == TaskStatus.State.KILLED_UNCLEAN) { tip.incompleteSubTask(taskid, this.status); // add this task, to be rescheduled as cleanup attempt if (tip.isMapTask()) { mapCleanupTasks.add(taskid); } else { reduceCleanupTasks.add(taskid); } if (isTaskKilledWithCGroupMemory(status)) { //Increment the High Memory killed count for Reduce and Map Tasks if (status.getIsMap()) { jobCounters.incrCounter(Counter.TOTAL_CGROUP_MEMORY_MAP_TASK_KILLED, 1); } else { jobCounters.incrCounter(Counter.TOTAL_CGROUP_MEMORY_REDUCE_TASK_KILLED, 1); } } } //For a failed task update the JT datastructures. else if (state == TaskStatus.State.FAILED || state == TaskStatus.State.KILLED) { if (isTaskKilledWithHighMemory(status)) { //Increment the High Memory killed count for Reduce and Map Tasks if (status.getIsMap()) { jobCounters.incrCounter(Counter.TOTAL_HIGH_MEMORY_MAP_TASK_KILLED, 1); } else { jobCounters.incrCounter(Counter.TOTAL_HIGH_MEMORY_REDUCE_TASK_KILLED, 1); } } if (isTaskKilledWithCGroupMemory(status)) { //Increment the High Memory killed count for Reduce and Map Tasks if (status.getIsMap()) { jobCounters.incrCounter(Counter.TOTAL_CGROUP_MEMORY_MAP_TASK_KILLED, 1); } else { jobCounters.incrCounter(Counter.TOTAL_CGROUP_MEMORY_REDUCE_TASK_KILLED, 1); } } // Get the event number for the (possibly) previously successful // task. If there exists one, then set that status to OBSOLETE int eventNumber; if ((eventNumber = tip.getSuccessEventNumber()) != -1) { TaskCompletionEvent t = this.taskCompletionEvents.get(eventNumber); if (t.getTaskAttemptId().equals(taskid)) t.setTaskStatus(TaskCompletionEvent.Status.OBSOLETE); } // Tell the job to fail the relevant task failedTask(tip, taskid, status, ttStatus, wasRunning, wasComplete, wasAttemptRunning); // Did the task failure lead to tip failure? TaskCompletionEvent.Status taskCompletionStatus = (state == TaskStatus.State.FAILED) ? TaskCompletionEvent.Status.FAILED : TaskCompletionEvent.Status.KILLED; if (tip.isFailed()) { taskCompletionStatus = TaskCompletionEvent.Status.TIPFAILED; } taskEvent = new TaskCompletionEvent(taskCompletionEventCounter, taskid, tip.idWithinJob(), status.getIsMap() && !tip.isJobCleanupTask() && !tip.isJobSetupTask(), taskCompletionStatus, httpTaskLogLocation); } // Add the 'complete' task i.e. successful/failed // It _is_ safe to add the TaskCompletionEvent.Status.SUCCEEDED // *before* calling TIP.completedTask since: // a. One and only one task of a TIP is declared as a SUCCESS, the // other (speculative tasks) are marked KILLED by the TaskCommitThread // b. TIP.completedTask *does not* throw _any_ exception at all. if (taskEvent != null) { this.taskCompletionEvents.add(taskEvent); taskCompletionEventCounter++; if (state == TaskStatus.State.SUCCEEDED) { completedTask(tip, status, ttStatus); } } taskStateChangeListener.taskStateChange(state, tip, taskid, (ttStatus == null ? "null" : ttStatus.getHost())); } // // Update CoronaJobInProgress status // if (LOG.isDebugEnabled()) { LOG.debug( "Taking progress for " + tip.getTIPId() + " from " + oldProgress + " to " + tip.getProgress()); } if (!tip.isJobCleanupTask() && !tip.isJobSetupTask()) { double progressDelta = tip.getProgress() - oldProgress; if (tip.isMapTask()) { this.status.setMapProgress((float) (this.status.mapProgress() + progressDelta / maps.length)); } else { this.status.setReduceProgress( (float) (this.status.reduceProgress() + (progressDelta / reduces.length))); } } } /** * Should we reuse the resource of this succeeded task attempt * return true if we should reuse */ boolean shouldReuseTaskResource(TaskInProgress tip) { synchronized (lockObject) { return tip.isJobSetupTask() || canLaunchJobCleanupTaskUnprotected() || tip.isJobCleanupTask(); } // TIP is a job setup/cleanup task or job is ready for cleanup. // // Since job setup/cleanup does not get an explicit resource, reuse // the resource. For Map/Reduce tasks, we would want normally want // to release the resource. But if the job is ready for cleanup, // releasing the resource could mean that the job cleanup task can't // run, so reuse the resource. } public boolean completedTask(TaskInProgress tip, TaskStatus status, TaskTrackerInfo ttStatus) { synchronized (lockObject) { return completedTaskUnprotected(tip, status, ttStatus); } } /** * A taskId assigned to this CoronaJobInProgress has reported in successfully. */ @SuppressWarnings("deprecation") private boolean completedTaskUnprotected(TaskInProgress tip, TaskStatus status, TaskTrackerInfo ttStatus) { int oldNumAttempts = tip.getActiveTasks().size(); // Metering meterTaskAttemptUnprotected(tip, status); // It _is_ safe to not decrement running{Map|Reduce}Tasks and // finished{Map|Reduce}Tasks variables here because one and only // one task-attempt of a TIP gets to completedTask. This is because // the TaskCommitThread in the JobTracker marks other, completed, // speculative tasks as _complete_. TaskAttemptID taskId = status.getTaskID(); if (tip.isComplete()) { // Mark this task as KILLED tip.alreadyCompletedTask(taskId); return false; } LOG.info("Task '" + taskId + "' has completed " + tip.getTIPId() + " successfully."); // Mark the TIP as complete tip.completed(taskId); // Update jobhistory String taskType = getTaskType(tip); if (status.getIsMap()) { jobHistory.logMapTaskStarted(status.getTaskID(), status.getStartTime(), status.getTaskTracker(), ttStatus.getHttpPort(), taskType); jobHistory.logMapTaskFinished(status.getTaskID(), status.getFinishTime(), ttStatus.getHost(), taskType, status.getStateString(), status.getCounters()); } else { jobHistory.logReduceTaskStarted(status.getTaskID(), status.getStartTime(), status.getTaskTracker(), ttStatus.getHttpPort(), taskType); jobHistory.logReduceTaskFinished(status.getTaskID(), status.getShuffleFinishTime(), status.getSortFinishTime(), status.getFinishTime(), ttStatus.getHost(), taskType, status.getStateString(), status.getCounters()); } jobHistory.logTaskFinished(tip.getTIPId(), taskType, tip.getExecFinishTime(), status.getCounters()); int newNumAttempts = tip.getActiveTasks().size(); if (tip.isJobSetupTask()) { // setup task has finished. kill the extra setup tip killSetupTipUnprotected(!tip.isMapTask()); setupCompleteUnprotected(); } else if (tip.isJobCleanupTask()) { // cleanup task has finished. Kill the extra cleanup tip if (tip.isMapTask()) { // kill the reduce tip cleanup[1].kill(); } else { cleanup[0].kill(); } if (jobFailed) { terminateJob(JobStatus.FAILED); } else if (jobKilled) { terminateJob(JobStatus.KILLED); } else { jobCompleteUnprotected(); } } else if (tip.isMapTask()) { runningMapTasks--; // Update locality counters. long inputBytes = tip.getCounters().getGroup("org.apache.hadoop.mapred.Task$Counter") .getCounter("Map input bytes"); jobStats.incTotalMapInputBytes(inputBytes); localityStats.record(tip, ttStatus.getHost(), inputBytes); // check if this was a speculative task. if (oldNumAttempts > 1) { speculativeMapTasks -= (oldNumAttempts - newNumAttempts); jobStats.incNumSpeculativeSucceededMaps(); } finishedMapTasks += 1; jobStats.incNumMapTasksCompleted(); if (!tip.isJobSetupTask() && hasSpeculativeMaps) { updateTaskTrackerStats(tip, ttStatus, trackerMapStats, mapTaskStats); } // remove the completed map from the resp running caches retireMapUnprotected(tip); if ((finishedMapTasks + failedMapTIPs) == (numMapTasks)) { this.status.setMapProgress(1.0f); } } else { runningReduceTasks -= 1; if (oldNumAttempts > 1) { speculativeReduceTasks -= (oldNumAttempts - newNumAttempts); jobStats.incNumSpeculativeSucceededReduces(); } finishedReduceTasks += 1; jobStats.incNumReduceTasksCompleted(); if (!tip.isJobSetupTask() && hasSpeculativeReduces) { updateTaskTrackerStats(tip, ttStatus, trackerReduceStats, reduceTaskStats); } // remove the completed reduces from the running reducers set retireReduceUnprotected(tip); if ((finishedReduceTasks + failedReduceTIPs) == (numReduceTasks)) { this.status.setReduceProgress(1.0f); } } // is job complete? if (!jobSetupCleanupNeeded && canLaunchJobCleanupTaskUnprotected()) { jobCompleteUnprotected(); } return true; } /** * Fail a task with a given reason, but without a status object. */ @SuppressWarnings("deprecation") public void failedTask(TaskInProgress tip, TaskAttemptID taskid, String reason, TaskStatus.Phase phase, boolean isFailed, String trackerName, TaskTrackerInfo ttStatus) { TaskStatus.State state = isFailed ? TaskStatus.State.FAILED : TaskStatus.State.KILLED; TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskid, 0.0f, 1, state, reason, reason, trackerName, phase, new Counters()); synchronized (lockObject) { // update the actual start-time of the attempt TaskStatus oldStatus = tip.getTaskStatus(taskid); long startTime = oldStatus == null ? JobTracker.getClock().getTime() : oldStatus.getStartTime(); if (startTime < 0) { startTime = JobTracker.getClock().getTime(); } status.setStartTime(startTime); status.setFinishTime(JobTracker.getClock().getTime()); boolean wasComplete = tip.isComplete(); updateTaskStatus(tip, status, ttStatus); boolean isComplete = tip.isComplete(); if (wasComplete && !isComplete) { // mark a successful tip as failed String taskType = getTaskType(tip); JobHistory.Task.logFailed(tip.getTIPId(), taskType, tip.getExecFinishTime(), reason, taskid); } } } @SuppressWarnings("deprecation") public void failedTask(TaskInProgress tip, TaskAttemptID taskid, TaskStatus status, TaskTrackerInfo taskTrackerStatus, boolean wasRunning, boolean wasComplete, boolean wasAttemptRunning) { synchronized (lockObject) { failedTaskUnprotected(tip, taskid, status, taskTrackerStatus, wasRunning, wasComplete, wasAttemptRunning); } } /** * A task assigned to this CoronaJobInProgress has reported in as failed. * Most of the time, we'll just reschedule execution. However, after * many repeated failures we may instead decide to allow the entire * job to fail or succeed if the user doesn't care about a few tasks failing. * * Even if a task has reported as completed in the past, it might later * be reported as failed. That's because the TaskTracker that hosts a map * task might die before the entire job can complete. If that happens, * we need to schedule reexecution so that downstream reduce tasks can * obtain the map task's output. */ @SuppressWarnings("deprecation") private void failedTaskUnprotected(TaskInProgress tip, TaskAttemptID taskid, TaskStatus status, TaskTrackerInfo taskTrackerStatus, boolean wasRunning, boolean wasComplete, boolean wasAttemptRunning) { taskErrorCollector.collect(tip, taskid, clock.getTime()); // check if the TIP is already failed boolean wasFailed = tip.isFailed(); // Mark the taskid as FAILED or KILLED tip.incompleteSubTask(taskid, this.status); boolean isRunning = tip.isRunning(); boolean isComplete = tip.isComplete(); if (wasAttemptRunning) { if (!tip.isJobCleanupTask() && !tip.isJobSetupTask()) { long timeSpent = clock.getTime() - status.getStartTime(); boolean isSpeculative = tip.isSpeculativeAttempt(taskid); if (tip.isMapTask()) { runningMapTasks -= 1; if (wasFailed) { jobStats.incNumMapTasksFailed(); jobStats.incFailedMapTime(timeSpent); } else { jobStats.incNumMapTasksKilled(); jobStats.incKilledMapTime(timeSpent); if (isSpeculative) { jobStats.incNumSpeculativeWasteMaps(); jobStats.incSpeculativeMapTimeWaste(timeSpent); } } } else { runningReduceTasks -= 1; if (wasFailed) { jobStats.incNumReduceTasksFailed(); jobStats.incFailedReduceTime(timeSpent); } else { jobStats.incNumReduceTasksKilled(); jobStats.incKilledReduceTime(timeSpent); if (isSpeculative) { jobStats.incNumSpeculativeWasteReduces(); jobStats.incSpeculativeReduceTimeWaste(timeSpent); } } } } // Metering meterTaskAttemptUnprotected(tip, status); } //update running count on task failure. if (wasRunning && !isRunning) { if (tip.isJobCleanupTask()) { launchedCleanup = false; } else if (tip.isJobSetupTask()) { launchedSetup = false; } else if (tip.isMapTask()) { // remove from the running queue and put it in the non-running cache // if the tip is not complete i.e if the tip still needs to be run if (!isComplete) { failMapUnprotected(tip); } } else { // remove from the running queue and put in the failed queue if the tip // is not complete if (!isComplete) { failReduceUnprotected(tip); } } } // The case when the map was complete but the task tracker went down. // However, we don't need to do any metering here... if (wasComplete && !isComplete) { if (tip.isMapTask()) { // Put the task back in the cache. This will help locality for cases // where we have a different TaskTracker from the same rack/switch // asking for a task. // We bother about only those TIPs that were successful // earlier (wasComplete and !isComplete) // (since they might have been removed from the cache of other // racks/switches, if the input split blocks were present there too) failMapUnprotected(tip); finishedMapTasks -= 1; } } // update job history // get taskStatus from tip TaskStatus taskStatus = tip.getTaskStatus(taskid); String taskTrackerName = taskStatus.getTaskTracker(); String taskTrackerHostName = convertTrackerNameToHostName(taskTrackerName); int taskTrackerPort = -1; if (taskTrackerStatus != null) { taskTrackerPort = taskTrackerStatus.getHttpPort(); } long startTime = taskStatus.getStartTime(); long finishTime = taskStatus.getFinishTime(); List<String> taskDiagnosticInfo = tip.getDiagnosticInfo(taskid); String diagInfo = taskDiagnosticInfo == null ? "" : StringUtils.arrayToString(taskDiagnosticInfo.toArray(new String[0])); String taskType = getTaskType(tip); if (taskStatus.getIsMap()) { jobHistory.logMapTaskStarted(taskid, startTime, taskTrackerName, taskTrackerPort, taskType); if (taskStatus.getRunState() == TaskStatus.State.FAILED) { jobHistory.logMapTaskFailed(taskid, finishTime, taskTrackerHostName, diagInfo, taskType); } else { jobHistory.logMapTaskKilled(taskid, finishTime, taskTrackerHostName, diagInfo, taskType); } } else { jobHistory.logReduceTaskStarted(taskid, startTime, taskTrackerName, taskTrackerPort, taskType); if (taskStatus.getRunState() == TaskStatus.State.FAILED) { jobHistory.logReduceTaskFailed(taskid, finishTime, taskTrackerHostName, diagInfo, taskType); } else { jobHistory.logReduceTaskKilled(taskid, finishTime, taskTrackerHostName, diagInfo, taskType); } } // After this, try to assign tasks with the one after this, so that // the failed task goes to the end of the list. if (!tip.isJobCleanupTask() && !tip.isJobSetupTask()) { if (tip.isMapTask()) { failedMapTasks++; if (taskStatus.getRunState() != TaskStatus.State.FAILED) { killedMapTasks++; } } else { failedReduceTasks++; if (taskStatus.getRunState() != TaskStatus.State.FAILED) { killedReduceTasks++; } } } // // Check if we need to kill the job because of too many failures or // if the job is complete since all component tasks have completed // We do it once per TIP and that too for the task that fails the TIP if (!wasFailed && tip.isFailed()) { // // Allow upto 'mapFailuresPercent' of map tasks to fail or // 'reduceFailuresPercent' of reduce tasks to fail // boolean killJob = tip.isJobCleanupTask() || tip.isJobSetupTask() ? true : tip.isMapTask() ? ((++failedMapTIPs * 100) > (mapFailuresPercent * numMapTasks)) : ((++failedReduceTIPs * 100) > (reduceFailuresPercent * numReduceTasks)); if (killJob) { LOG.info("Aborting job " + profile.getJobID()); jobHistory.logTaskFailed(tip.getTIPId(), taskType, finishTime, diagInfo); if (tip.isJobCleanupTask()) { // kill the other tip if (tip.isMapTask()) { cleanup[1].kill(); } else { cleanup[0].kill(); } terminateJob(JobStatus.FAILED); } else { if (tip.isJobSetupTask()) { // kill the other tip killSetupTipUnprotected(!tip.isMapTask()); } fail(); } } // // Update the counters // if (!tip.isJobCleanupTask() && !tip.isJobSetupTask()) { if (tip.isMapTask()) { jobCounters.incrCounter(Counter.NUM_FAILED_MAPS, 1); } else { jobCounters.incrCounter(Counter.NUM_FAILED_REDUCES, 1); } } } } /** * Get the task type for logging it to {@link JobHistory}. */ private String getTaskType(TaskInProgress tip) { if (tip.isJobCleanupTask()) { return Values.CLEANUP.name(); } else if (tip.isJobSetupTask()) { return Values.SETUP.name(); } else if (tip.isMapTask()) { return Values.MAP.name(); } else { return Values.REDUCE.name(); } } /** * Adds the failed TIP in the front of the list for non-running maps * @param tip the tip that needs to be failed */ private void failMapUnprotected(TaskInProgress tip) { if (nonRunningMaps == null) { LOG.warn("Non-running cache for maps missing!! " + "Job details are missing."); return; } nonRunningMaps.add(tip); } /** * Adds a failed TIP in the front of the list for non-running reduces * @param tip the tip that needs to be failed */ private void failReduceUnprotected(TaskInProgress tip) { if (nonRunningReduces == null) { LOG.warn("Failed cache for reducers missing!! " + "Job details are missing."); return; } nonRunningReduces.add(0, tip); } @SuppressWarnings("deprecation") private void clearUncleanTasksUnprotected() { TaskAttemptID taskid = null; TaskInProgress tip = null; while (!mapCleanupTasks.isEmpty()) { taskid = mapCleanupTasks.remove(0); tip = maps[taskid.getTaskID().getId()]; updateTaskStatus(tip, tip.getTaskStatus(taskid), null); } while (!reduceCleanupTasks.isEmpty()) { taskid = reduceCleanupTasks.remove(0); tip = reduces[taskid.getTaskID().getId()]; updateTaskStatus(tip, tip.getTaskStatus(taskid), null); } } private void killSetupTipUnprotected(boolean isMap) { if (isMap) { setup[0].kill(); } else { setup[1].kill(); } } private void setupCompleteUnprotected() { status.setSetupProgress(1.0f); if (this.status.getRunState() == JobStatus.PREP) { changeStateTo(JobStatus.RUNNING); jobHistory.logStarted(); } } private void jobComplete() { synchronized (lockObject) { jobCompleteUnprotected(); } } /** * The job is done since all it's component tasks are either * successful or have failed. */ @SuppressWarnings("deprecation") private void jobCompleteUnprotected() { // // All tasks are complete, then the job is done! if (this.terminated == JobStatus.FAILED || this.terminated == JobStatus.KILLED) { terminate(this.terminated); } if (this.status.getRunState() == JobStatus.RUNNING || this.status.getRunState() == JobStatus.PREP) { changeStateTo(JobStatus.SUCCEEDED); this.status.setCleanupProgress(1.0f); if (maps.length == 0) { this.status.setMapProgress(1.0f); } if (reduces.length == 0) { this.status.setReduceProgress(1.0f); } this.finishTime = clock.getTime(); LOG.info("Job " + this.status.getJobID() + " has completed successfully."); // Log the job summary (this should be done prior to logging to // job-history to ensure job-counters are in-sync JobSummary.logJobSummary(this); Counters counters = getCounters(); // Log job-history jobHistory.logFinished(finishTime, this.finishedMapTasks, this.finishedReduceTasks, failedMapTasks, failedReduceTasks, killedMapTasks, killedReduceTasks, getMapCounters(), getReduceCounters(), counters); } } /** * Job state change must happen thru this call */ private void changeStateTo(int newState) { synchronized (lockObject) { int oldState = this.status.getRunState(); if (oldState == newState) { return; //old and new states are same } this.status.setRunState(newState); } } /** * Metering: Occupied Slots * (Finish - Start) * @param tip {@link TaskInProgress} to be metered which just completed, * cannot be <code>null</code> * @param status {@link TaskStatus} of the completed task, cannot be * <code>null</code> */ @SuppressWarnings("deprecation") private void meterTaskAttemptUnprotected(TaskInProgress tip, TaskStatus status) { Counter slotCounter = (tip.isMapTask()) ? Counter.SLOTS_MILLIS_MAPS : Counter.SLOTS_MILLIS_REDUCES; jobCounters.incrCounter(slotCounter, tip.getNumSlotsRequired() * (status.getFinishTime() - status.getStartTime())); if (!tip.isMapTask()) { jobCounters.incrCounter(Counter.SLOTS_MILLIS_REDUCES_COPY, tip.getNumSlotsRequired() * (status.getShuffleFinishTime() - status.getStartTime())); jobCounters.incrCounter(Counter.SLOTS_MILLIS_REDUCES_SORT, tip.getNumSlotsRequired() * (status.getSortFinishTime() - status.getShuffleFinishTime())); jobCounters.incrCounter(Counter.SLOTS_MILLIS_REDUCES_REDUCE, tip.getNumSlotsRequired() * (status.getFinishTime() - status.getSortFinishTime())); } } /** * Populate the data structures as a task is scheduled. * * Assuming {@link JobTracker} is locked on entry. * * @param tip The tip for which the task is added * @param id The attempt-id for the task * @param taskTracker task tracker name * @param hostName host name for the task tracker * @param isScheduled Whether this task is scheduled from the JT or has * joined back upon restart */ @SuppressWarnings("deprecation") private void addRunningTaskToTIPUnprotected(TaskInProgress tip, TaskAttemptID id, String taskTracker, String hostName, boolean isScheduled) { // Make an entry in the tip if the attempt is not scheduled i.e externally // added if (!isScheduled) { tip.addRunningTask(id, taskTracker); } // keeping the earlier ordering intact String name; String splits = ""; Enum<Counter> counter = null; if (tip.isJobSetupTask()) { launchedSetup = true; name = Values.SETUP.name(); } else if (tip.isJobCleanupTask()) { launchedCleanup = true; name = Values.CLEANUP.name(); } else if (tip.isMapTask()) { ++runningMapTasks; name = Values.MAP.name(); counter = Counter.TOTAL_LAUNCHED_MAPS; splits = tip.getSplitNodes(); if (tip.getActiveTasks().size() > 1) { speculativeMapTasks++; jobStats.incNumSpeculativeMaps(); } jobStats.incNumMapTasksLaunched(); } else { ++runningReduceTasks; name = Values.REDUCE.name(); counter = Counter.TOTAL_LAUNCHED_REDUCES; if (tip.getActiveTasks().size() > 1) { speculativeReduceTasks++; jobStats.incNumSpeculativeReduces(); } jobStats.incNumReduceTasksLaunched(); } // Note that the logs are for the scheduled tasks only. Tasks that join on // restart has already their logs in place. if (tip.isFirstAttempt(id)) { jobHistory.logTaskStarted(tip.getTIPId(), name, tip.getExecStartTime(), splits); } if (!tip.isJobSetupTask() && !tip.isJobCleanupTask()) { jobCounters.incrCounter(counter, 1); } if (tip.isMapTask() && !tip.isJobSetupTask() && !tip.isJobCleanupTask()) { localityStats.record(tip, hostName, -1); } } /** * Kill the job and all its component tasks. */ public void kill() { terminate(JobStatus.KILLED); } void fail() { terminate(JobStatus.FAILED); } private void terminate(int jobTerminationState) { this.terminated = jobTerminationState; synchronized (lockObject) { terminateUnprotected(jobTerminationState); } } /** * Terminate the job and all its component tasks. * Calling this will lead to marking the job as failed/killed. Cleanup * tip will be launched. If the job has not inited, it will directly call * terminateJob as there is no need to launch cleanup tip. * This method is reentrant. * @param jobTerminationState job termination state */ private void terminateUnprotected(int jobTerminationState) { this.terminated = jobTerminationState; if (!tasksInited) { //init could not be done, we just terminate directly. terminateJob(jobTerminationState); return; } if ((status.getRunState() == JobStatus.RUNNING) || (status.getRunState() == JobStatus.PREP)) { LOG.info("Killing job '" + this.status.getJobID() + "'"); if (jobTerminationState == JobStatus.FAILED) { if (jobFailed) {//reentrant return; } jobFailed = true; } else if (jobTerminationState == JobStatus.KILLED) { if (jobKilled) {//reentrant return; } jobKilled = true; } // clear all unclean tasks clearUncleanTasksUnprotected(); // // kill all TIPs. // for (int i = 0; i < setup.length; i++) { setup[i].kill(); } for (int i = 0; i < maps.length; i++) { maps[i].kill(); TreeMap<TaskAttemptID, String> activeTasks = maps[i].getActiveTasksCopy(); for (TaskAttemptID attempt : activeTasks.keySet()) { TaskStatus status = maps[i].getTaskStatus(attempt); if (status != null) { failedTask(maps[i], attempt, JOB_KILLED_REASON, status.getPhase(), false, status.getTaskTracker(), null); } else { failedTask(maps[i], attempt, JOB_KILLED_REASON, Phase.MAP, false, EMPTY_TRACKER_NAME, null); } } } for (int i = 0; i < reduces.length; i++) { reduces[i].kill(); TreeMap<TaskAttemptID, String> activeTasks = reduces[i].getActiveTasksCopy(); for (TaskAttemptID attempt : activeTasks.keySet()) { TaskStatus status = reduces[i].getTaskStatus(attempt); if (status != null) { failedTask(reduces[i], attempt, JOB_KILLED_REASON, status.getPhase(), false, status.getTaskTracker(), null); } else { failedTask(reduces[i], attempt, JOB_KILLED_REASON, Phase.REDUCE, false, EMPTY_TRACKER_NAME, null); } } } // Moved job to a terminal state if no job cleanup is needed. In case the // job is killed, we do not perform cleanup. This is because cleanup // cannot be guaranteed - the process running Corona JT could just be killed. if (!jobSetupCleanupNeeded || jobTerminationState == JobStatus.KILLED) { terminateJobUnprotected(jobTerminationState); } } } private void terminateJob(int jobTerminationState) { synchronized (lockObject) { terminateJobUnprotected(jobTerminationState); } } @SuppressWarnings("deprecation") private void terminateJobUnprotected(int jobTerminationState) { if ((status.getRunState() == JobStatus.RUNNING) || (status.getRunState() == JobStatus.PREP)) { this.finishTime = clock.getTime(); this.status.setMapProgress(1.0f); this.status.setReduceProgress(1.0f); this.status.setCleanupProgress(1.0f); Counters counters = getCounters(); setExtendedMetricsCountersUnprotected(counters); if (jobTerminationState == JobStatus.FAILED) { changeStateTo(JobStatus.FAILED); // Log the job summary JobSummary.logJobSummary(this); // Log to job-history jobHistory.logFailed(finishTime, this.finishedMapTasks, this.finishedReduceTasks, counters); } else { changeStateTo(JobStatus.KILLED); // Log the job summary JobSummary.logJobSummary(this); // Log to job-history jobHistory.logKilled(finishTime, this.finishedMapTasks, this.finishedReduceTasks, counters); } } } @SuppressWarnings("deprecation") private void setExtendedMetricsCountersUnprotected(Counters counters) { counters.incrCounter("extMet", "submit_time", getLaunchTime() - getStartTime()); for (int i = 0; i < setup.length; i++) { if (setup[i].isComplete()) { counters.incrCounter("extMet", "setup_time", setup[i].getExecFinishTime() - setup[i].getStartTime()); break; } } for (int i = cleanup.length - 1; i >= 0; i--) { if (cleanup[i].isComplete()) { counters.incrCounter("extMet", "cleanup_time", cleanup[i].getExecFinishTime() - cleanup[i].getStartTime()); break; } } long totalMapWaitTime = 0; long maxMapWaitTime = 0; long totalMaps = 0; for (int i = 0; i < maps.length; i++) { if (maps[i].isComplete()) { long waitTime = maps[i].getExecStartTime() - getLaunchTime(); if (waitTime > maxMapWaitTime) maxMapWaitTime = waitTime; totalMapWaitTime += waitTime; ++totalMaps; } } counters.incrCounter("extMet", "avg_map_wait_time", totalMaps > 0 ? (totalMapWaitTime / totalMaps) : 0); counters.incrCounter("extMet", "max_map_wait_time", maxMapWaitTime); } /** * Remove a map TIP from the lists for running maps. * Called when a map fails/completes (note if a map is killed, * it won't be present in the list since it was completed earlier) * @param tip the tip that needs to be retired */ private void retireMapUnprotected(TaskInProgress tip) { if (runningMaps == null) { LOG.warn("Running cache for maps missing!! " + "Job details are missing."); return; } runningMaps.remove(tip); } /** * Remove a reduce TIP from the list for running-reduces * Called when a reduce fails/completes * @param tip the tip that needs to be retired */ private void retireReduceUnprotected(TaskInProgress tip) { if (runningReduces == null) { LOG.warn("Running list for reducers missing!! " + "Job details are missing."); return; } runningReduces.remove(tip); } protected void scheduleMapUnprotected(TaskInProgress tip) { runningMapTaskStats.add(0.0f); runningMaps.add(tip); } protected void scheduleReduceUnprotected(TaskInProgress tip) { runningReduceTaskStats.add(0.0f); runningReduces.add(tip); } /** * Check if the job needs the JobCleanup task launched. * @return true if the job needs a cleanup task launched, false otherwise */ public boolean canLaunchJobCleanupTask() { synchronized (lockObject) { return canLaunchJobCleanupTaskUnprotected(); } } /** * Check whether cleanup task can be launched for the job. * * Cleanup task can be launched if it is not already launched * or job is Killed * or all maps and reduces are complete * @return true/false */ private boolean canLaunchJobCleanupTaskUnprotected() { // check if the job is running if (status.getRunState() != JobStatus.RUNNING && status.getRunState() != JobStatus.PREP) { return false; } // check if cleanup task has been launched already or if setup isn't // launched already. The later check is useful when number of maps is // zero. if (launchedCleanup || !isSetupFinishedUnprotected()) { return false; } // check if job has failed or killed if (jobKilled || jobFailed) { return true; } boolean mapsDone = ((finishedMapTasks + failedMapTIPs) == (numMapTasks)); boolean reducesDone = ((finishedReduceTasks + failedReduceTIPs) == numReduceTasks); boolean mapOnlyJob = (numReduceTasks == 0); if (mapOnlyJob) { return mapsDone; } if (jobFinishWhenReducesDone) { return reducesDone; } return mapsDone && reducesDone; } boolean isSetupFinishedUnprotected() { // if there is no setup to be launched, consider setup is finished. if ((tasksInited && setup.length == 0) || setup[0].isComplete() || setup[0].isFailed() || setup[1].isComplete() || setup[1].isFailed()) { return true; } return false; } /** * Returns the total job counters, by adding together the job, * the map and the reduce counters. */ @SuppressWarnings("deprecation") public Counters getCounters() { synchronized (lockObject) { Counters result = new Counters(); result.incrAllCounters(getJobCounters()); incrementTaskCountersUnprotected(result, maps); return incrementTaskCountersUnprotected(result, reduces); } } public Counters getErrorCounters() { return taskErrorCollector.getErrorCountsCounters(); } public Object getSchedulingInfo() { return null; // TODO } boolean isJobEmpty() { return maps.length == 0 && reduces.length == 0 && !jobSetupCleanupNeeded; } void completeEmptyJob() { jobComplete(); } void completeSetup() { synchronized (lockObject) { setupCompleteUnprotected(); } } /** * Increments the counters with the counters from each task. * @param counters the counters to increment * @param tips the tasks to add in to counters * @return counters the same object passed in as counters */ @SuppressWarnings("deprecation") private Counters incrementTaskCountersUnprotected(Counters counters, TaskInProgress[] tips) { for (TaskInProgress tip : tips) { counters.incrAllCounters(tip.getCounters()); } return counters; } static class JobSummary { static final Log LOG = LogFactory.getLog(JobSummary.class); // Escape sequences static final char EQUALS = '='; static final char[] charsToEscape = { StringUtils.COMMA, EQUALS, StringUtils.ESCAPE_CHAR }; /** * Log a summary of the job's runtime. * * @param job {@link JobInProgress} whose summary is to be logged, cannot * be <code>null</code>. */ @SuppressWarnings("deprecation") public static void logJobSummary(CoronaJobInProgress job) { JobStatus status = job.getStatus(); JobProfile profile = job.getProfile(); String user = StringUtils.escapeString(profile.getUser(), StringUtils.ESCAPE_CHAR, charsToEscape); String queue = StringUtils.escapeString(profile.getQueueName(), StringUtils.ESCAPE_CHAR, charsToEscape); Counters jobCounters = job.getJobCounters(); long mapSlotSeconds = (jobCounters.getCounter(Counter.SLOTS_MILLIS_MAPS) + jobCounters.getCounter(Counter.FALLOW_SLOTS_MILLIS_MAPS)) / 1000; long reduceSlotSeconds = (jobCounters.getCounter(Counter.SLOTS_MILLIS_REDUCES) + jobCounters.getCounter(Counter.FALLOW_SLOTS_MILLIS_REDUCES)) / 1000; LOG.info("jobId=" + profile.getJobID() + StringUtils.COMMA + "submitTime" + EQUALS + job.getStartTime() + StringUtils.COMMA + "launchTime" + EQUALS + job.getLaunchTime() + StringUtils.COMMA + "finishTime" + EQUALS + job.getFinishTime() + StringUtils.COMMA + "numMaps" + EQUALS + job.getTasks(TaskType.MAP).length + StringUtils.COMMA + "numSlotsPerMap" + EQUALS + NUM_SLOTS_PER_MAP + StringUtils.COMMA + "numReduces" + EQUALS + job.getTasks(TaskType.REDUCE).length + StringUtils.COMMA + "numSlotsPerReduce" + EQUALS + NUM_SLOTS_PER_REDUCE + StringUtils.COMMA + "user" + EQUALS + user + StringUtils.COMMA + "queue" + EQUALS + queue + StringUtils.COMMA + "status" + EQUALS + JobStatus.getJobRunState(status.getRunState()) + StringUtils.COMMA + "mapSlotSeconds" + EQUALS + mapSlotSeconds + StringUtils.COMMA + "reduceSlotsSeconds" + EQUALS + reduceSlotSeconds + StringUtils.COMMA); } } @Override public boolean inited() { return tasksInited; } private void updateTaskTrackerStats(TaskInProgress tip, TaskTrackerInfo ttStatus, Map<String, DataStatistics> trackerStats, DataStatistics overallStats) { synchronized (lockObject) { float tipDuration = tip.getExecFinishTime() - tip.getDispatchTime(tip.getSuccessfulTaskid()); DataStatistics ttStats = trackerStats.get(ttStatus.getTrackerName()); double oldMean = 0.0d; //We maintain the mean of TaskTrackers' means. That way, we get a single //data-point for every tracker (used in the evaluation in isSlowTracker) if (ttStats != null) { oldMean = ttStats.mean(); ttStats.add(tipDuration); overallStats.updateStatistics(oldMean, ttStats.mean()); } else { trackerStats.put(ttStatus.getTrackerName(), (ttStats = new DataStatistics(tipDuration))); overallStats.add(tipDuration); } if (LOG.isDebugEnabled()) { LOG.debug("Added mean of " + ttStats.mean() + " to trackerStats of type " + (tip.isMapTask() ? "Map" : "Reduce") + " on " + ttStatus.getTrackerName() + ". DataStatistics is now: " + trackerStats.get(ttStatus.getTrackerName())); } } } @Override public Vector<TaskInProgress> reportTasksInProgress(boolean shouldBeMap, boolean shouldBeComplete) { synchronized (lockObject) { return super.reportTasksInProgress(shouldBeMap, shouldBeComplete); } } @Override public Vector<TaskInProgress> reportCleanupTIPs(boolean shouldBeComplete) { synchronized (lockObject) { return super.reportCleanupTIPs(shouldBeComplete); } } @Override public Vector<TaskInProgress> reportSetupTIPs(boolean shouldBeComplete) { synchronized (lockObject) { return super.reportSetupTIPs(shouldBeComplete); } } @Override public TaskInProgress getTaskInProgress(TaskID tipid) { synchronized (lockObject) { return super.getTaskInProgress(tipid); } } public Configuration getConf() { return jobConf; } @Override public boolean shouldLogCannotspeculativeMaps() { long now = clock.getTime(); if ((now - lastTimeCannotspeculativeMapLog) <= logCannotspeculativeInterval) return false; int unfinished = numMapTasks - finishedMapTasks; if (unfinished <= numMapTasks * speculativeMapLogRateThreshold || unfinished <= speculativeMapLogNumThreshold) { lastTimeCannotspeculativeMapLog = now; return true; } return false; } @Override public boolean shouldLogCannotspeculativeReduces() { long now = clock.getTime(); if ((now - lastTimeCannotspeculativeReduceLog) <= logCannotspeculativeInterval) return false; int unfinished = numReduceTasks - finishedReduceTasks; if (unfinished <= numReduceTasks * speculativeReduceLogRateThreshold || unfinished <= speculativeReduceLogNumThreshold) { lastTimeCannotspeculativeReduceLog = now; return true; } return false; } @Override public boolean shouldSpeculateAllRemainingMaps() { int unfinished = numMapTasks - finishedMapTasks; if (unfinished < numMapTasks * speculativeMapUnfininshedThreshold || unfinished == 1) { return true; } return false; } @Override public boolean shouldSpeculateAllRemainingReduces() { int unfinished = numReduceTasks - finishedReduceTasks; if (unfinished < numReduceTasks * speculativeReduceUnfininshedThreshold || unfinished == 1) { return true; } return false; } @Override DataStatistics getRunningTaskStatistics(Phase phase) { throw new RuntimeException("Not yet implemented."); } public static void uploadCachedSplits(JobID jobId, JobConf jobConf, String systemDir) throws IOException { Path jobDir = new Path(systemDir, jobId.toString()); Path splitFile = new Path(jobDir, "job.split"); LOG.info("Uploading splits file for " + jobId + " to " + splitFile); List<JobClient.RawSplit> splits = Arrays.asList(JobClient.getAndRemoveCachedSplits(jobId)); JobClient.writeComputedSplits(jobConf, splits, splitFile); } }