Java tutorial
/** * Licensed to Ravel, Inc. under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. Ravel, Inc. licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package org.goldenorb.util; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.mapred.InvalidJobConfException; import org.goldenorb.conf.OrbConfiguration; import org.goldenorb.jet.OrbTrackerMember; public class ResourceAllocator<M extends OrbTrackerMember> { final Logger logger = LoggerFactory.getLogger(this.getClass()); public static final int TRACKER_AVAILABLE = 0; public static final int TRACKER_RESERVED = 1; private OrbConfiguration conf; private Collection<M> orbTrackers; private List<M> trackersByAvailable; private List<M> trackersByReserved; private int totalAvailable = 0; private int totalReserved = 0; private Map<M, Integer[]> orbTrackerAssignments = new HashMap<M, Integer[]>(); private Map<M, Integer[]> orbTrackerAvailability = new HashMap<M, Integer[]>(); /** * Constructor * * @param conf * - OrbConfiguration * @param orbTrackers */ public ResourceAllocator(OrbConfiguration conf, Collection<M> orbTrackers) { this.setConf(conf); this.setOrbTrackers(orbTrackers); } /** * Assigns resources. * * @returns Map<M,Integer[]> */ public Map<M, Integer[]> assignResources(OrbConfiguration jobConf) throws InvalidJobConfException { logger.info("ResourceAllocator: assignResources()"); int requestedPartitions = jobConf.getOrbRequestedPartitions(); // if no partitions are requested, the job is invalid if (requestedPartitions <= 0) { logger.error("missing number of requested partitions for job"); throw new InvalidJobConfException("missing number of requested partitions for job"); } int reservedPartitions = jobConf.getOrbReservedPartitions(); // if this is zero, it screws up the resource allocation so we set it to Int max to move through // the allocation process int partitionsPerMachine = (conf.getNumberOfPartitionsPerMachine() == 0 ? Integer.MAX_VALUE : conf.getNumberOfPartitionsPerMachine()); logger.debug("requestedPartitions: {}", requestedPartitions); logger.debug("reservedPartitions: {}", reservedPartitions); logger.debug("partitionsPerMachine: {}", partitionsPerMachine); // some setup, organize the trackers and capture availability/reserved info sortTrackers(); updateOrbTrackerAvailability(); // if you request more than what's available, you get nothing (caller should wait and try again) if (requestedPartitions > totalAvailable || reservedPartitions > totalReserved) { return null; } // requestedPartitions functions as the countdown mechanism while (requestedPartitions > 0) { // until all partitions are assigned, iterate through trackers for (Iterator<M> iter = trackersByAvailable.iterator(); iter.hasNext() && requestedPartitions > 0;) { M tracker = iter.next(); // tracker has > partitionsPerMachine available if (partitionsPerMachine <= orbTrackerAvailability.get(tracker)[TRACKER_AVAILABLE] && requestedPartitions >= partitionsPerMachine) { assignAvailable(tracker, partitionsPerMachine, TRACKER_AVAILABLE); requestedPartitions -= partitionsPerMachine; decrementAvailable(tracker, partitionsPerMachine, TRACKER_AVAILABLE); } // tracker has at least one available else if (orbTrackerAvailability.get(tracker)[TRACKER_AVAILABLE] > 0) { assignAvailable(tracker, 1, TRACKER_AVAILABLE); requestedPartitions -= 1; decrementAvailable(tracker, 1, TRACKER_AVAILABLE); } // tracker is full, skip it else { continue; } } } // requestedPartitions functions as the countdown mechanism while (reservedPartitions > 0) { // until all partitions are assigned, iterate through trackers for (Iterator<M> iter = trackersByReserved.iterator(); iter.hasNext() && reservedPartitions > 0;) { M tracker = iter.next(); // tracker has at least one available if (orbTrackerAvailability.get(tracker)[TRACKER_RESERVED] > 0) { assignAvailable(tracker, 1, TRACKER_RESERVED); reservedPartitions -= 1; decrementAvailable(tracker, 1, TRACKER_RESERVED); } // tracker is full, skip it else { continue; } } } return orbTrackerAssignments; } /** * * @param tracker * - M * @param count * - int * @param field * - int */ private void assignAvailable(M tracker, int count, int field) { Integer[] v = orbTrackerAssignments.get(tracker); v[field] += count; orbTrackerAssignments.put(tracker, v); } /** * * @param tracker * - M * @param count * - int * @param field * - int */ private void decrementAvailable(M tracker, int count, int field) { Integer[] v = orbTrackerAvailability.get(tracker); v[field] -= count; orbTrackerAvailability.put(tracker, v); } /** * Sorts available and reserved trackers. */ private void sortTrackers() { trackersByAvailable = new ArrayList<M>(getOrbTrackers()); Collections.sort(trackersByAvailable, new Comparator<M>() { @Override public int compare(M o1, M o2) { Integer a1 = o1.getAvailablePartitions(); Integer a2 = o2.getAvailablePartitions(); return a1.compareTo(a2); } }); trackersByReserved = new ArrayList<M>(getOrbTrackers()); Collections.sort(trackersByReserved, new Comparator<M>() { @Override public int compare(M o1, M o2) { Integer a1 = o1.getReservedPartitions(); Integer a2 = o2.getReservedPartitions(); return a1.compareTo(a2); } }); } /** * Updates OrbTracker availability. */ private void updateOrbTrackerAvailability() { totalAvailable = 0; totalReserved = 0; for (M tracker : getOrbTrackers()) { int reserved = tracker.getReservedPartitions(); int available = tracker.getAvailablePartitions(); Integer[] v = new Integer[2]; v[TRACKER_AVAILABLE] = available; v[TRACKER_RESERVED] = reserved; orbTrackerAvailability.put(tracker, v); Integer[] empty = { 0, 0 }; orbTrackerAssignments.put(tracker, empty); totalReserved += reserved; totalAvailable += available; } } /** * Set the conf * * @param conf * - OrbConfiguration */ public void setConf(OrbConfiguration conf) { this.conf = conf; } /** * Return the conf */ public OrbConfiguration getConf() { return conf; } /** * Return the orbTrackers */ public Collection<M> getOrbTrackers() { return orbTrackers; } /** * Set the orbTrackers * * @param orbTrackers */ public void setOrbTrackers(Collection<M> orbTrackers) { this.orbTrackers = orbTrackers; } }