Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.mapred; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.PriorityQueue; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.metrics.MetricsRecord; /** * Helper utilies to calculate the fairness of pools. */ public class PoolFairnessCalculator { /** Class logger */ private static final Log LOG = LogFactory.getLog(PoolFairnessCalculator.class); /** * This is a utility class, do not construct. */ private PoolFairnessCalculator() { } /** Prefix for metrics of fairness resource difference */ private static final String FAIRNESS_DIFFERENCE_COUNT_PREFIX = "fairness_difference_count_"; /** Prefix for metrics of average difference per pool */ private static final String FAIRNESS_DIFFERENCE_PER_POOL_PREFIX = "fairness_difference_per_pool_"; /** Prefix for metrics of unfairness percentage */ private static final String FAIRNESS_PERCENT_UNFAIR_PREFIX = "fairness_percent_unfair_"; /** Prefix of metrics for standard deviation of unfairness */ private static final String FAIRNESS_UNFAIR_STD_DEV_PERFIX = "fairness_unfair_std_dev_"; /** Prefix of total resources available for printing */ private static final String TOTAL_RESOURCES_PREFIX = "total_resources_"; /** * Helper object to keep track of the metadata for all resources of a * particular type. */ private static class TotalResourceMetadata { /** Total resources that were allocated by the calculator */ private int totalAllocated = 0; /** Total resources that were allocated by the scheduler */ private int totalAvailable = 0; /** * Total difference between the expected and actual resources * allocated per pool */ private int totalFairnessDifference = 0; /** * Differences between expected and actual resources squared for all pools */ private int totalFairnessDifferenceSquared = 0; /** % of resources unfairly allocated for this resource type [0,100] */ private float percentUnfair; /** Standard deviation of unfairly allocated resources */ private float stdDevUnfair; /** Average resource unfairness per pool */ private float averageUnfairPerPool; /** * Number of resources this type represents (usually one, except for * when this represents all the resource types) */ private int resourceTypeCount = 1; } /** * A comparator for ResourceMetadata that sorts by the largest * min(min guaranteed, desired). */ private static class GuaranteedDesiredComparator implements Comparator<ResourceMetadata> { @Override public int compare(ResourceMetadata left, ResourceMetadata right) { return right.getGuaranteedUsedAndDesired() - left.getGuaranteedUsedAndDesired(); } } /** * A comparator for ResourceMetadata that sorts by the smallest expected * used. */ private static class ExpectedUsedComparator implements Comparator<ResourceMetadata> { @Override public int compare(ResourceMetadata left, ResourceMetadata right) { return left.getExpectedUsed() - right.getExpectedUsed(); } } /** * This method takes a list of {@link PoolMetadata} objects and calculates * fairness metrics of how well scheduling is doing. * * The goals of the fair scheduling are to insure that every pool is getting * an equal share. The expected share of resources for each pool is * complicated by the pools not requiring an equal share * or pools that have a minimum or maximum allocation of resources. * * @param poolMetadataList List of all pool metadata * @param metricsRecord Where to write the metrics */ public static void calculateFairness(final List<PoolMetadata> poolMetadataList, final MetricsRecord metricsRecord) { if (poolMetadataList == null || poolMetadataList.isEmpty()) { return; } // Find the total available usage and guaranteed resources by resource // type. Add the resource metadata to the sorted set to schedule if // there is something to schedule (desiredAfterConstraints > 0) long startTime = System.currentTimeMillis(); Map<String, TotalResourceMetadata> resourceTotalMap = new HashMap<String, TotalResourceMetadata>(); Map<String, Set<ResourceMetadata>> resourceSchedulablePoolMap = new HashMap<String, Set<ResourceMetadata>>(); for (PoolMetadata poolMetadata : poolMetadataList) { for (String resourceName : poolMetadata.getResourceMetadataKeys()) { ResourceMetadata resourceMetadata = poolMetadata.getResourceMetadata(resourceName); TotalResourceMetadata totalResourceMetadata = resourceTotalMap.get(resourceName); if (totalResourceMetadata == null) { totalResourceMetadata = new TotalResourceMetadata(); resourceTotalMap.put(resourceName, totalResourceMetadata); } totalResourceMetadata.totalAvailable += resourceMetadata.getCurrentlyUsed(); Set<ResourceMetadata> schedulablePoolSet = resourceSchedulablePoolMap.get(resourceName); if (schedulablePoolSet == null) { schedulablePoolSet = new HashSet<ResourceMetadata>(); resourceSchedulablePoolMap.put(resourceName, schedulablePoolSet); } if (resourceMetadata.getDesiredAfterConstraints() > 0) { if (!schedulablePoolSet.add(resourceMetadata)) { throw new RuntimeException( "Duplicate resource metadata " + resourceMetadata + " in " + schedulablePoolSet); } } } } // First, allocate resources for all the min guaranteed resources // for the pools. Ordering is done by the largest // min(min guaranteed, desired). GuaranteedDesiredComparator guarantedDesiredComparator = new GuaranteedDesiredComparator(); List<ResourceMetadata> removePoolList = new ArrayList<ResourceMetadata>(); for (Map.Entry<String, TotalResourceMetadata> entry : resourceTotalMap.entrySet()) { List<ResourceMetadata> resourceMetadataList = new ArrayList<ResourceMetadata>( resourceSchedulablePoolMap.get(entry.getKey())); TotalResourceMetadata totalResourceMetadata = entry.getValue(); Collections.sort(resourceMetadataList, guarantedDesiredComparator); while ((totalResourceMetadata.totalAllocated < totalResourceMetadata.totalAvailable) && !resourceMetadataList.isEmpty()) { removePoolList.clear(); for (ResourceMetadata resourceMetadata : resourceMetadataList) { if (resourceMetadata.getExpectedUsed() == resourceMetadata.getGuaranteedUsedAndDesired()) { removePoolList.add(resourceMetadata); continue; } resourceMetadata.incrExpectedUsed(); ++totalResourceMetadata.totalAllocated; } resourceMetadataList.removeAll(removePoolList); } LOG.info("After allocating min guaranteed and desired - " + "Resource type " + entry.getKey() + " totalAvailable=" + totalResourceMetadata.totalAvailable + ", totalAllocated=" + totalResourceMetadata.totalAllocated); } // At this point, all pools have been allocated their guaranteed used and // desired resources. If there are any more resources to allocate, give // resources to lowest allocated pool that hasn't reached desired // until all the resources are gone ExpectedUsedComparator expectedUsedComparator = new ExpectedUsedComparator(); PriorityQueue<ResourceMetadata> minHeap = new PriorityQueue<ResourceMetadata>(100, expectedUsedComparator); for (Map.Entry<String, TotalResourceMetadata> entry : resourceTotalMap.entrySet()) { minHeap.addAll(resourceSchedulablePoolMap.get(entry.getKey())); TotalResourceMetadata totalResourceMetadata = entry.getValue(); while ((totalResourceMetadata.totalAllocated < totalResourceMetadata.totalAvailable) && !minHeap.isEmpty()) { ResourceMetadata resourceMetadata = minHeap.remove(); if (resourceMetadata.getExpectedUsed() == resourceMetadata.getDesiredAfterConstraints()) { continue; } resourceMetadata.incrExpectedUsed(); ++totalResourceMetadata.totalAllocated; minHeap.add(resourceMetadata); } minHeap.clear(); } // Now calculate the difference of the expected allocation and the // actual allocation to get the following metrics. When calculating // the percent bad allocated divide by 2 because the difference double // counts a bad allocation // 1) total tasks difference between expected and actual allocation // 0 is totally fair, higher is less fair // 2) % of tasks incorrectly allocated // 0 is totally fair, higher is less fair // 3) average difference per pool // 0 is totally fair, higher is less fair // 4) standard deviation per pool // 0 is totally fair, higher is less fair for (PoolMetadata poolMetadata : poolMetadataList) { for (String resourceName : poolMetadata.getResourceMetadataKeys()) { ResourceMetadata resourceMetadata = poolMetadata.getResourceMetadata(resourceName); int diff = Math.abs(resourceMetadata.getExpectedUsed() - resourceMetadata.getCurrentlyUsed()); LOG.info("Pool " + poolMetadata.getPoolName() + ", resourceName=" + resourceName + ", expectedUsed=" + resourceMetadata.getExpectedUsed() + ", currentUsed=" + resourceMetadata.getCurrentlyUsed() + ", maxAllowed=" + resourceMetadata.getMaxAllowed() + ", desiredAfterConstraints=" + resourceMetadata.getDesiredAfterConstraints() + ", guaranteedUsedAndDesired=" + resourceMetadata.getGuaranteedUsedAndDesired() + ", diff=" + diff); resourceTotalMap.get(resourceName).totalFairnessDifference += diff; resourceTotalMap.get(resourceName).totalFairnessDifferenceSquared += diff * diff; } } TotalResourceMetadata allResourceMetadata = new TotalResourceMetadata(); allResourceMetadata.resourceTypeCount = resourceTotalMap.size(); for (TotalResourceMetadata totalResourceMetadata : resourceTotalMap.values()) { allResourceMetadata.totalAvailable += totalResourceMetadata.totalAvailable; allResourceMetadata.totalFairnessDifference += totalResourceMetadata.totalFairnessDifference; allResourceMetadata.totalFairnessDifferenceSquared += totalResourceMetadata.totalFairnessDifferenceSquared; } resourceTotalMap.put("all", allResourceMetadata); StringBuilder metricsBuilder = new StringBuilder(); for (Map.Entry<String, TotalResourceMetadata> entry : resourceTotalMap.entrySet()) { TotalResourceMetadata totalResourceMetadata = entry.getValue(); totalResourceMetadata.percentUnfair = (totalResourceMetadata.totalAvailable == 0) ? 0 : totalResourceMetadata.totalFairnessDifference * 100f / 2 / totalResourceMetadata.totalAvailable; totalResourceMetadata.stdDevUnfair = (float) Math .sqrt((double) totalResourceMetadata.totalFairnessDifferenceSquared / poolMetadataList.size() / totalResourceMetadata.resourceTypeCount); totalResourceMetadata.averageUnfairPerPool = (float) totalResourceMetadata.totalFairnessDifference / poolMetadataList.size() / totalResourceMetadata.resourceTypeCount; metricsRecord.setMetric(FAIRNESS_DIFFERENCE_COUNT_PREFIX + entry.getKey(), totalResourceMetadata.totalFairnessDifference); metricsBuilder.append(FAIRNESS_DIFFERENCE_COUNT_PREFIX + entry.getKey() + "=" + totalResourceMetadata.totalFairnessDifference + "\n"); metricsRecord.setMetric(FAIRNESS_PERCENT_UNFAIR_PREFIX + entry.getKey(), totalResourceMetadata.percentUnfair); metricsBuilder.append(FAIRNESS_PERCENT_UNFAIR_PREFIX + entry.getKey() + "=" + totalResourceMetadata.percentUnfair + "\n"); metricsRecord.setMetric(FAIRNESS_DIFFERENCE_PER_POOL_PREFIX + entry.getKey(), totalResourceMetadata.averageUnfairPerPool); metricsBuilder.append(FAIRNESS_DIFFERENCE_PER_POOL_PREFIX + entry.getKey() + "=" + totalResourceMetadata.averageUnfairPerPool + "\n"); metricsRecord.setMetric(FAIRNESS_UNFAIR_STD_DEV_PERFIX + entry.getKey(), totalResourceMetadata.stdDevUnfair); metricsBuilder.append(FAIRNESS_UNFAIR_STD_DEV_PERFIX + entry.getKey() + "=" + totalResourceMetadata.stdDevUnfair + "\n"); metricsBuilder.append( TOTAL_RESOURCES_PREFIX + entry.getKey() + "=" + totalResourceMetadata.totalAvailable + "\n"); } if (LOG.isInfoEnabled()) { LOG.info("calculateFairness took " + (System.currentTimeMillis() - startTime) + " millisecond(s)."); } if (LOG.isDebugEnabled()) { LOG.debug("\n" + metricsBuilder.toString()); } } }