Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import com.google.common.collect.ImmutableList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; @Private @Unstable public class FSParentQueue extends FSQueue { private static final Log LOG = LogFactory.getLog(FSParentQueue.class.getName()); private final List<FSQueue> childQueues = new ArrayList<>(); private Resource demand = Resources.createResource(0); private int runnableApps; private ReadWriteLock rwLock = new ReentrantReadWriteLock(); private Lock readLock = rwLock.readLock(); private Lock writeLock = rwLock.writeLock(); public FSParentQueue(String name, FairScheduler scheduler, FSParentQueue parent) { super(name, scheduler, parent); } public void addChildQueue(FSQueue child) { writeLock.lock(); try { childQueues.add(child); } finally { writeLock.unlock(); } } public void removeChildQueue(FSQueue child) { writeLock.lock(); try { childQueues.remove(child); } finally { writeLock.unlock(); } } @Override public void recomputeShares() { readLock.lock(); try { policy.computeShares(childQueues, getFairShare()); for (FSQueue childQueue : childQueues) { childQueue.getMetrics().setFairShare(childQueue.getFairShare()); childQueue.recomputeShares(); } } finally { readLock.unlock(); } } public void recomputeSteadyShares() { readLock.lock(); try { policy.computeSteadyShares(childQueues, getSteadyFairShare()); for (FSQueue childQueue : childQueues) { childQueue.getMetrics().setSteadyFairShare(childQueue.getSteadyFairShare()); if (childQueue instanceof FSParentQueue) { ((FSParentQueue) childQueue).recomputeSteadyShares(); } } } finally { readLock.unlock(); } } @Override public void updatePreemptionVariables() { super.updatePreemptionVariables(); // For child queues readLock.lock(); try { for (FSQueue childQueue : childQueues) { childQueue.updatePreemptionVariables(); } } finally { readLock.unlock(); } } @Override public Resource getDemand() { readLock.lock(); try { return Resource.newInstance(demand.getMemorySize(), demand.getVirtualCores()); } finally { readLock.unlock(); } } @Override public Resource getResourceUsage() { Resource usage = Resources.createResource(0); readLock.lock(); try { for (FSQueue child : childQueues) { Resources.addTo(usage, child.getResourceUsage()); } } finally { readLock.unlock(); } return usage; } @Override public void updateDemand() { // Compute demand by iterating through apps in the queue // Limit demand to maxResources Resource maxRes = scheduler.getAllocationConfiguration().getMaxResources(getName()); writeLock.lock(); try { demand = Resources.createResource(0); for (FSQueue childQueue : childQueues) { childQueue.updateDemand(); Resource toAdd = childQueue.getDemand(); if (LOG.isDebugEnabled()) { LOG.debug("Counting resource from " + childQueue.getName() + " " + toAdd + "; Total resource consumption for " + getName() + " now " + demand); } demand = Resources.add(demand, toAdd); demand = Resources.componentwiseMin(demand, maxRes); if (Resources.equals(demand, maxRes)) { break; } } } finally { writeLock.unlock(); } if (LOG.isDebugEnabled()) { LOG.debug("The updated demand for " + getName() + " is " + demand + "; the max is " + maxRes); } } private QueueUserACLInfo getUserAclInfo(UserGroupInformation user) { List<QueueACL> operations = new ArrayList<>(); for (QueueACL operation : QueueACL.values()) { if (hasAccess(operation, user)) { operations.add(operation); } } return QueueUserACLInfo.newInstance(getQueueName(), operations); } @Override public List<QueueUserACLInfo> getQueueUserAclInfo(UserGroupInformation user) { List<QueueUserACLInfo> userAcls = new ArrayList<QueueUserACLInfo>(); // Add queue acls userAcls.add(getUserAclInfo(user)); // Add children queue acls readLock.lock(); try { for (FSQueue child : childQueues) { userAcls.addAll(child.getQueueUserAclInfo(user)); } } finally { readLock.unlock(); } return userAcls; } @Override public Resource assignContainer(FSSchedulerNode node) { Resource assigned = Resources.none(); // If this queue is over its limit, reject if (!assignContainerPreCheck(node)) { return assigned; } // Hold the write lock when sorting childQueues writeLock.lock(); try { Collections.sort(childQueues, policy.getComparator()); } finally { writeLock.unlock(); } /* * We are releasing the lock between the sort and iteration of the * "sorted" list. There could be changes to the list here: * 1. Add a child queue to the end of the list, this doesn't affect * container assignment. * 2. Remove a child queue, this is probably good to take care of so we * don't assign to a queue that is going to be removed shortly. */ readLock.lock(); try { for (FSQueue child : childQueues) { assigned = child.assignContainer(node); if (!Resources.equals(assigned, Resources.none())) { break; } } } finally { readLock.unlock(); } return assigned; } @Override public RMContainer preemptContainer() { RMContainer toBePreempted = null; // Find the childQueue which is most over fair share FSQueue candidateQueue = null; Comparator<Schedulable> comparator = policy.getComparator(); readLock.lock(); try { for (FSQueue queue : childQueues) { // Skip selection for non-preemptable queue if (!queue.canBePreempted()) { if (LOG.isDebugEnabled()) { LOG.debug("skipping from queue=" + getName() + " because it's a non-preemptable queue or there is no" + " sub-queues whose resource usage exceeds fair share."); } continue; } if (candidateQueue == null || comparator.compare(queue, candidateQueue) > 0) { candidateQueue = queue; } } } finally { readLock.unlock(); } // Let the selected queue choose which of its container to preempt if (candidateQueue != null) { toBePreempted = candidateQueue.preemptContainer(); } return toBePreempted; } @Override public List<FSQueue> getChildQueues() { readLock.lock(); try { return ImmutableList.copyOf(childQueues); } finally { readLock.unlock(); } } @Override public void setPolicy(SchedulingPolicy policy) throws AllocationConfigurationException { boolean allowed = SchedulingPolicy.isApplicableTo(policy, (parent == null) ? SchedulingPolicy.DEPTH_ROOT : SchedulingPolicy.DEPTH_INTERMEDIATE); if (!allowed) { throwPolicyDoesnotApplyException(policy); } super.policy = policy; } public void incrementRunnableApps() { writeLock.lock(); try { runnableApps++; } finally { writeLock.unlock(); } } public void decrementRunnableApps() { writeLock.lock(); try { runnableApps--; } finally { writeLock.unlock(); } } @Override public int getNumRunnableApps() { readLock.lock(); try { return runnableApps; } finally { readLock.unlock(); } } @Override public void collectSchedulerApplications(Collection<ApplicationAttemptId> apps) { readLock.lock(); try { for (FSQueue childQueue : childQueues) { childQueue.collectSchedulerApplications(apps); } } finally { readLock.unlock(); } } @Override public ActiveUsersManager getActiveUsersManager() { // Should never be called since all applications are submitted to LeafQueues return null; } @Override public void recoverContainer(Resource clusterResource, SchedulerApplicationAttempt schedulerAttempt, RMContainer rmContainer) { // TODO Auto-generated method stub } }