Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapred; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapred.JobQueueJobInProgressListener.JobSchedulingInfo; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.TreeMap; /** * */ class JobQueue extends AbstractQueue { static final Log LOG = LogFactory.getLog(JobQueue.class); public JobQueue(AbstractQueue parent, QueueSchedulingContext qsc) { super(parent, qsc); if (qsc.supportsPriorities()) { // use the default priority-aware comparator comparator = JobQueueJobInProgressListener.FIFO_JOB_QUEUE_COMPARATOR; } else { comparator = STARTTIME_JOB_COMPARATOR; } waitingJobs = new TreeMap<JobSchedulingInfo, JobInProgress>(comparator); runningJobs = new TreeMap<JobSchedulingInfo, JobInProgress>(comparator); } /* * If a queue supports priorities, jobs must be * sorted on priorities, and then on their start times (technically, * their insertion time. * If a queue doesn't support priorities, jobs are * sorted based on their start time. */ static final Comparator<JobSchedulingInfo> STARTTIME_JOB_COMPARATOR; static { STARTTIME_JOB_COMPARATOR = new Comparator<JobSchedulingInfo>() { // comparator for jobs in queues that don't support priorities public int compare(JobSchedulingInfo o1, JobSchedulingInfo o2) { // the job that started earlier wins if (o1.getStartTime() < o2.getStartTime()) { return -1; } else { return (o1.getStartTime() == o2.getStartTime() ? o1.getJobID().compareTo(o2.getJobID()) : 1); } } }; } /** * This involves updating each qC structure. * * @param mapClusterCapacity * @param reduceClusterCapacity */ @Override public void update(int mapClusterCapacity, int reduceClusterCapacity) { super.update(mapClusterCapacity, reduceClusterCapacity); for (JobInProgress j : this.getRunningJobs()) { updateStatsOnRunningJob(qsc, j); } } private void updateStatsOnRunningJob(QueueSchedulingContext qC, JobInProgress j) { if (j.getStatus().getRunState() != JobStatus.RUNNING) { return; } TaskSchedulingContext mapTSI = qC.getMapTSC(); TaskSchedulingContext reduceTSI = qC.getReduceTSC(); int numMapsRunningForThisJob = j.runningMaps(); int numReducesRunningForThisJob = j.runningReduces(); TaskDataView mapScheduler = TaskDataView.getTaskDataView(TaskType.MAP); TaskDataView reduceScheduler = TaskDataView.getTaskDataView(TaskType.REDUCE); int numRunningMapSlots = numMapsRunningForThisJob * mapScheduler.getSlotsPerTask(j); int numRunningReduceSlots = numReducesRunningForThisJob * reduceScheduler.getSlotsPerTask(j); int numMapSlotsForThisJob = mapScheduler.getSlotsOccupied(j); int numReduceSlotsForThisJob = reduceScheduler.getSlotsOccupied(j); int numReservedMapSlotsForThisJob = (mapScheduler.getNumReservedTaskTrackers(j) * mapScheduler.getSlotsPerTask(j)); int numReservedReduceSlotsForThisJob = (reduceScheduler.getNumReservedTaskTrackers(j) * reduceScheduler.getSlotsPerTask(j)); j.setSchedulingInfo( getJobQueueSchedInfo(numMapsRunningForThisJob, numRunningMapSlots, numReservedMapSlotsForThisJob, numReducesRunningForThisJob, numRunningReduceSlots, numReservedReduceSlotsForThisJob)); mapTSI.setNumRunningTasks(mapTSI.getNumRunningTasks() + numMapsRunningForThisJob); reduceTSI.setNumRunningTasks(reduceTSI.getNumRunningTasks() + numReducesRunningForThisJob); mapTSI.setNumSlotsOccupied(mapTSI.getNumSlotsOccupied() + numMapSlotsForThisJob); reduceTSI.setNumSlotsOccupied(reduceTSI.getNumSlotsOccupied() + numReduceSlotsForThisJob); Integer i = mapTSI.getNumSlotsOccupiedByUser().get(j.getProfile().getUser()); mapTSI.getNumSlotsOccupiedByUser().put(j.getProfile().getUser(), i.intValue() + numMapSlotsForThisJob); i = reduceTSI.getNumSlotsOccupiedByUser().get(j.getProfile().getUser()); reduceTSI.getNumSlotsOccupiedByUser().put(j.getProfile().getUser(), i.intValue() + numReduceSlotsForThisJob); if (LOG.isDebugEnabled()) { synchronized (j) { LOG.debug(String.format( "updateQSI: job %s: run(m)=%d, " + "occupied(m)=%d, run(r)=%d, occupied(r)=%d, finished(m)=%d," + " finished(r)=%d, failed(m)=%d, failed(r)=%d, " + "spec(m)=%d, spec(r)=%d, total(m)=%d, total(r)=%d", j.getJobID().toString(), numMapsRunningForThisJob, numMapSlotsForThisJob, numReducesRunningForThisJob, numReduceSlotsForThisJob, j.finishedMaps(), j.finishedReduces(), j.failedMapTasks, j.failedReduceTasks, j.speculativeMapTasks, j.speculativeReduceTasks, j.numMapTasks, j.numReduceTasks)); } } /* * it's fine walking down the entire list of running jobs - there * probably will not be many, plus, we may need to go through the * list to compute numSlotsOccupiedByUser. If this is expensive, we * can keep a list of running jobs per user. Then we only need to * consider the first few jobs per user. */ } private static final int JOBQUEUE_SCHEDULINGINFO_INITIAL_LENGTH = 175; static String getJobQueueSchedInfo(int numMapsRunningForThisJob, int numRunningMapSlots, int numReservedMapSlotsForThisJob, int numReducesRunningForThisJob, int numRunningReduceSlots, int numReservedReduceSlotsForThisJob) { StringBuilder sb = new StringBuilder(JOBQUEUE_SCHEDULINGINFO_INITIAL_LENGTH); sb.append(numMapsRunningForThisJob).append(" running map tasks using ").append(numRunningMapSlots) .append(" map slots. ").append(numReservedMapSlotsForThisJob).append(" additional slots reserved. ") .append(numReducesRunningForThisJob).append(" running reduce tasks using ") .append(numRunningReduceSlots).append(" reduce slots. ").append(numReservedReduceSlotsForThisJob) .append(" additional slots reserved."); return sb.toString(); } Map<JobSchedulingInfo, JobInProgress> waitingJobs; // for waiting jobs Map<JobSchedulingInfo, JobInProgress> runningJobs; // for running jobs public Comparator<JobSchedulingInfo> comparator; Collection<JobInProgress> getWaitingJobs() { synchronized (waitingJobs) { return Collections.unmodifiableCollection(new LinkedList<JobInProgress>(waitingJobs.values())); } } Collection<JobInProgress> getRunningJobs() { synchronized (runningJobs) { return Collections.unmodifiableCollection(new LinkedList<JobInProgress>(runningJobs.values())); } } private void addRunningJob(JobInProgress job) { synchronized (runningJobs) { runningJobs.put(new JobSchedulingInfo(job), job); } } private JobInProgress removeRunningJob(JobSchedulingInfo jobInfo) { synchronized (runningJobs) { return runningJobs.remove(jobInfo); } } JobInProgress removeWaitingJob(JobSchedulingInfo schedInfo) { synchronized (waitingJobs) { JobInProgress jip = waitingJobs.remove(schedInfo); this.qsc.setNumOfWaitingJobs(waitingJobs.size()); return jip; } } private void addWaitingJob(JobInProgress job) { synchronized (waitingJobs) { waitingJobs.put(new JobSchedulingInfo(job), job); this.qsc.setNumOfWaitingJobs(waitingJobs.size()); } } int getWaitingJobCount() { synchronized (waitingJobs) { return waitingJobs.size(); } } // called when a job is added synchronized void jobAdded(JobInProgress job) throws IOException { // add job to waiting queue. It will end up in the right place, // based on priority. addWaitingJob(job); // update user-specific info Integer i = qsc.getNumJobsByUser().get(job.getProfile().getUser()); if (null == i) { i = 1; // set the count for running tasks to 0 qsc.getMapTSC().getNumSlotsOccupiedByUser().put(job.getProfile().getUser(), 0); qsc.getReduceTSC().getNumSlotsOccupiedByUser().put(job.getProfile().getUser(), 0); } else { i++; } qsc.getNumJobsByUser().put(job.getProfile().getUser(), i); // setup scheduler specific job information preInitializeJob(job); if (LOG.isDebugEnabled()) { LOG.debug("Job " + job.getJobID().toString() + " is added under user " + job.getProfile().getUser() + ", user now has " + i + " jobs"); } } /** * Setup {@link CapacityTaskScheduler} specific information prior to * job initialization. * <p/> * TO DO: Currently this method uses , CapacityTaskScheduler based variables * need to shift those. */ void preInitializeJob(JobInProgress job) { JobConf jobConf = job.getJobConf(); // Compute number of slots required to run a single map/reduce task int slotsPerMap = 1; int slotsPerReduce = 1; if (MemoryMatcher.isSchedulingBasedOnMemEnabled()) { slotsPerMap = jobConf.computeNumSlotsPerMap(MemoryMatcher.getMemSizeForMapSlot()); slotsPerReduce = jobConf.computeNumSlotsPerReduce(MemoryMatcher.getMemSizeForReduceSlot()); } job.setNumSlotsPerMap(slotsPerMap); job.setNumSlotsPerReduce(slotsPerReduce); } // called when a job completes synchronized void jobCompleted(JobInProgress job) { if (LOG.isDebugEnabled()) { LOG.debug("Job to be removed for user " + job.getProfile().getUser()); } Integer i = qsc.getNumJobsByUser().get(job.getProfile().getUser()); i--; if (0 == i.intValue()) { qsc.getNumJobsByUser().remove(job.getProfile().getUser()); // remove job footprint from our TSIs qsc.getMapTSC().getNumSlotsOccupiedByUser().remove(job.getProfile().getUser()); qsc.getReduceTSC().getNumSlotsOccupiedByUser().remove(job.getProfile().getUser()); if (LOG.isDebugEnabled()) { LOG.debug("No more jobs for user, number of users = " + qsc.getNumJobsByUser().size()); } } else { qsc.getNumJobsByUser().put(job.getProfile().getUser(), i); if (LOG.isDebugEnabled()) { LOG.debug("User still has " + i + " jobs, number of users = " + qsc.getNumJobsByUser().size()); } } } // This is used to reposition a job in the queue. A job can get repositioned // because of the change in the job priority or job start-time. private void reorderJobs(JobInProgress job, JobSchedulingInfo oldInfo) { if (removeWaitingJob(oldInfo) != null) { addWaitingJob(job); } if (removeRunningJob(oldInfo) != null) { addRunningJob(job); } } /** * @return */ @Override List<AbstractQueue> getDescendentJobQueues() { List<AbstractQueue> l = new ArrayList<AbstractQueue>(); l.add(this); return l; } @Override List<AbstractQueue> getDescendantContainerQueues() { return new ArrayList<AbstractQueue>(); } public void jobUpdated(JobChangeEvent event) { // Check if this is the status change if (event instanceof JobStatusChangeEvent) { jobStateChanged((JobStatusChangeEvent) event); } } /** * @return */ @Override List<AbstractQueue> getChildren() { return null; } /** * Dont do anything in sort , this is leaf level queue. * * @param queueComparator */ @Override public void sort(Comparator queueComparator) { return; } // Update the scheduler as job's state has changed private void jobStateChanged(JobStatusChangeEvent event) { JobInProgress job = event.getJobInProgress(); JobSchedulingInfo oldJobStateInfo = new JobSchedulingInfo(event.getOldStatus()); // Check if the ordering of the job has changed // For now priority and start-time can change the job ordering if (event.getEventType() == JobStatusChangeEvent.EventType.PRIORITY_CHANGED || event.getEventType() == JobStatusChangeEvent.EventType.START_TIME_CHANGED) { // Make a priority change reorderJobs(job, oldJobStateInfo); } else if (event.getEventType() == JobStatusChangeEvent.EventType.RUN_STATE_CHANGED) { // Check if the job is complete int runState = job.getStatus().getRunState(); if (runState == JobStatus.SUCCEEDED || runState == JobStatus.FAILED || runState == JobStatus.KILLED) { jobCompleted(job, oldJobStateInfo); } else if (runState == JobStatus.RUNNING) { // Removing of the job from job list is responsibility of the //initialization poller. // Add the job to the running queue addRunningJob(job); } } } /* * Method removes the jobs from both running and waiting job queue in * job queue manager. */ private void jobCompleted(JobInProgress job, JobSchedulingInfo oldInfo) { LOG.info("Job " + job.getJobID().toString() + " submitted to queue " + job.getProfile().getQueueName() + " has completed"); //remove jobs from both queue's a job can be in //running and waiting queue at the same time. removeRunningJob(oldInfo); removeWaitingJob(oldInfo); // let scheduler know jobCompleted(job); } @Override public void addChild(AbstractQueue queue) { throw new UnsupportedOperationException("addChildren is not allowed for " + "" + getName()); } @Override void distributeUnConfiguredCapacity() { return; } }