org.apache.hadoop.mapred.YTPoolManager.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.mapred.YTPoolManager.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.mapred;

import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.HashSet;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.Yunti3Scheduler.JobInfo;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.dom.Text;
import org.xml.sax.SAXException;

/**
 * Maintains a hierarchy of pools.
 */
public class YTPoolManager {
    public static final Log LOG = LogFactory.getLog("org.apache.hadoop.mapred.PoolManager");

    /** Time to wait between checks of the allocation file */
    public static final long ALLOC_RELOAD_INTERVAL = 10 * 1000;

    /**
     * Time to wait after the allocation has been modified before reloading it
     * (this is done to prevent loading a file that hasn't been fully written).
     */
    public static final long ALLOC_RELOAD_WAIT = 5 * 1000;

    // Map and reduce minimum allocations for each pool
    private Map<String, Integer> mapAllocs = new HashMap<String, Integer>();
    private Map<String, Integer> reduceAllocs = new HashMap<String, Integer>();

    // Max concurrent running jobs for each pool and for each user; in addition,
    // for users that have no max specified, we use the userMaxJobsDefault.
    private Map<String, Integer> poolMaxJobs = new HashMap<String, Integer>();
    private Map<String, Integer> userMaxJobs = new HashMap<String, Integer>();

    // Sharing weights for each pool 
    private Map<String, Integer> poolWeights = new HashMap<String, Integer>();

    // whether use FIFO
    private Map<String, Boolean> poolUseFIFO = new HashMap<String, Boolean>();

    // Map: pool name -> ReduceWatcher object.
    private Map<String, ReduceWatcher> poolReduceWatcher = new HashMap<String, ReduceWatcher>();

    private int poolWeightDefault = 1;

    // A list of ordered pools to be scheduled. A pool with 2x weight has two slots. 
    private List<YTPool> orderedPools = new ArrayList<YTPool>();

    // current scheduled pool's index in orderedPools
    private int poolIndexLastAccessed;

    //too big will cause out of memory when doing: List<JobInProgress> nextRunnableJobs = new ArrayList<JobInProgress>(cntPoolMaxJobs);
    private final int POOL_MAX_JOBS_DEFAULT = 32;
    private int userMaxJobsDefault = 32;

    private String allocFile; // Path to XML file containing allocations
    private String poolNameProperty; // Jobconf property to use for determining a
                                     // job's pool name (default: mapred.job.queue.name)

    private Map<String, YTPool> pools = new HashMap<String, YTPool>();

    private long lastReloadAttempt; // Last time we tried to reload the pools file
    private long lastSuccessfulReload; // Last time we successfully reloaded pools
    private boolean lastReloadAttemptFailed = false;

    private URI siteConf;
    private long lastSuccessfulReloadSiteConf;

    static class ReduceWatcher {
        private Map<Integer, Double> levelPercent = new HashMap<Integer, Double>();
        private double defaultPercent = 0.1;

        public ReduceWatcher() {

        }

        // Example: 0-1:1;2-3:0.5;default:0.1
        public ReduceWatcher(String rawText) {
            try {
                String[] items = rawText.split(";");
                for (String item : items) {
                    String[] pair = item.split(":", 2);
                    double percent = Double.parseDouble(pair[1]);
                    if ("default".equals(pair[0])) {
                        defaultPercent = percent;
                    } else {
                        String[] range = pair[0].split("-", 2);
                        int start = Integer.parseInt(range[0]);
                        int end = Integer.parseInt(range[1]);
                        for (int i = start; i <= end; i++) {
                            levelPercent.put(i, percent);
                        }
                    }
                }
            } catch (Exception e) {
                levelPercent.clear();
                LOG.warn("Catch Exception when parse '" + rawText + "', use default instead. " + e.getMessage());
            }
        }

        public boolean shouldRunReduces(int finishedMaps, int totalMaps, int jobLevel) {
            Double percent = levelPercent.get(jobLevel);
            if (percent == null) {
                percent = defaultPercent;
            }
            return finishedMaps >= Math.max(1, totalMaps * percent);
        }

        @Override
        public String toString() {
            StringBuilder sb = new StringBuilder();
            sb.append("default: " + defaultPercent);
            for (Map.Entry<Integer, Double> entry : levelPercent.entrySet()) {
                sb.append(" Level: " + entry.getKey() + "->" + entry.getValue());
            }
            return sb.toString();
        }
    }

    public YTPoolManager(Configuration conf)
            throws IOException, SAXException, YTAllocationConfigurationException, ParserConfigurationException {
        this.poolNameProperty = conf.get("mapred.yunti3scheduler.poolnameproperty", "mapred.job.queue.name");
        this.allocFile = conf.get("mapred.yunti3scheduler.allocation.file");
        if (allocFile == null) {
            LOG.warn("No mapred.yunti3scheduler.allocation.file given in jobconf - "
                    + "the fair scheduler will not use any queues.");
        }
        reloadAllocs();
        lastSuccessfulReload = System.currentTimeMillis();
        lastReloadAttempt = System.currentTimeMillis();
        lastSuccessfulReloadSiteConf = System.currentTimeMillis();

        try {
            siteConf = conf.getResource("hadoop-site.xml").toURI();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }

    /**
     * Get a pool by name, creating it if necessary
     */
    public synchronized YTPool getPool(String name) {
        YTPool pool = pools.get(name);
        if (pool == null) {
            pool = new YTPool(name);
            pools.put(name, pool);
        }
        return pool;
    }

    /**
     * Remove a pool by name
     */
    public synchronized void removePool(String name) {
        pools.remove(name);
    }

    /**
     * Reload allocations file if it hasn't been loaded in a while
     */
    public void reloadAllocsIfNecessary(TaskTrackerManager taskTrackerManager) {
        long time = System.currentTimeMillis();
        if (time > lastReloadAttempt + ALLOC_RELOAD_INTERVAL) {
            lastReloadAttempt = time;
            try {
                File file = new File(allocFile);
                long lastModified = file.lastModified();
                if (lastModified > lastSuccessfulReload && time > lastModified + ALLOC_RELOAD_WAIT) {
                    reloadAllocs();

                    lastSuccessfulReload = time;
                    lastReloadAttemptFailed = false;
                }
            } catch (Exception e) {
                // Throwing the error further out here won't help - the RPC thread
                // will catch it and report it in a loop. Instead, just log it and
                // hope somebody will notice from the log.
                // We log the error only on the first failure so we don't fill up the
                // JobTracker's log with these messages.
                if (!lastReloadAttemptFailed) {
                    LOG.error("Failed to reload allocations file - " + "will use existing allocations.", e);
                }
                lastReloadAttemptFailed = true;
            }

            // reload queue info from hadoop-site.xml
            // added by megatron
            try {
                File file = new File(siteConf);
                long lastModified = file.lastModified();
                if (lastModified > lastSuccessfulReloadSiteConf && time > lastModified + ALLOC_RELOAD_WAIT) {
                    QueueManager queueManager = taskTrackerManager.getQueueManager();
                    queueManager.refresh(new Configuration());

                    // reset schedulerInfo of queueManager added by liangly
                    if (taskTrackerManager instanceof JobTracker) {
                        TaskScheduler scheduler = ((JobTracker) taskTrackerManager).getTaskScheduler();
                        if (scheduler instanceof Yunti3Scheduler) {
                            ((Yunti3Scheduler) scheduler).setSchedulerInfo();
                        }
                    }

                    lastSuccessfulReloadSiteConf = time;
                }
            } catch (Exception e) {
                LOG.error("queueManager.refresh error", e);
            }
        }
    }

    /**
     * Updates the allocation list from the allocation config file. This file is
     * expected to be in the following whitespace-separated format:
     * 
     * <code>
     * poolName1 mapAlloc reduceAlloc
     * poolName2 mapAlloc reduceAlloc
     * ...
     * </code>
     * 
     * Blank lines and lines starting with # are ignored.
     *  
     * @throws IOException if the config file cannot be read.
     * @throws YTAllocationConfigurationException if allocations are invalid.
     * @throws ParserConfigurationException if XML parser is misconfigured.
     * @throws SAXException if config file is malformed.
     */
    public void reloadAllocs()
            throws IOException, ParserConfigurationException, SAXException, YTAllocationConfigurationException {
        if (allocFile == null)
            return;
        // Create some temporary hashmaps to hold the new allocs, and we only save
        // them in our fields if we have parsed the entire allocs file successfully.
        Map<String, Integer> mapAllocs = new HashMap<String, Integer>();
        Map<String, Integer> reduceAllocs = new HashMap<String, Integer>();
        Map<String, Integer> poolMaxJobs = new HashMap<String, Integer>();
        Map<String, Integer> userMaxJobs = new HashMap<String, Integer>();
        Map<String, Integer> poolWeights = new HashMap<String, Integer>();
        Map<String, Boolean> poolUseFIFO = new HashMap<String, Boolean>();
        Map<String, ReduceWatcher> poolReduceWatcher = new HashMap<String, ReduceWatcher>();
        int userMaxJobsDefault = POOL_MAX_JOBS_DEFAULT;

        // Remember all pool names so we can display them on web UI, etc.
        List<String> poolNamesInAllocFile = new ArrayList<String>();

        // Read and parse the allocations file.
        DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
        docBuilderFactory.setIgnoringComments(true);
        DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
        Document doc = builder.parse(new File(allocFile));
        Element root = doc.getDocumentElement();
        if (!"allocations".equals(root.getTagName()))
            throw new YTAllocationConfigurationException(
                    "Bad allocations file: " + "top-level element not <allocations>");
        NodeList elements = root.getChildNodes();
        for (int i = 0; i < elements.getLength(); i++) {
            Node node = elements.item(i);
            if (!(node instanceof Element))
                continue;
            Element element = (Element) node;
            if ("pool".equals(element.getTagName())) {
                String poolName = element.getAttribute("name");
                poolNamesInAllocFile.add(poolName);
                NodeList fields = element.getChildNodes();
                for (int j = 0; j < fields.getLength(); j++) {
                    Node fieldNode = fields.item(j);
                    if (!(fieldNode instanceof Element))
                        continue;
                    Element field = (Element) fieldNode;
                    if ("minMaps".equals(field.getTagName())) {
                        String text = ((Text) field.getFirstChild()).getData().trim();
                        int val = Integer.parseInt(text);
                        mapAllocs.put(poolName, val);
                    } else if ("minReduces".equals(field.getTagName())) {
                        String text = ((Text) field.getFirstChild()).getData().trim();
                        int val = Integer.parseInt(text);
                        reduceAllocs.put(poolName, val);
                    } else if ("maxRunningJobs".equals(field.getTagName())) {
                        String text = ((Text) field.getFirstChild()).getData().trim();
                        int val = Integer.parseInt(text);
                        poolMaxJobs.put(poolName, val);
                    } else if ("weight".equals(field.getTagName())) {
                        String text = ((Text) field.getFirstChild()).getData().trim();
                        int val = Integer.parseInt(text);
                        poolWeights.put(poolName, val);
                    } else if ("useFIFO".equals(field.getTagName())) {
                        String text = ((Text) field.getFirstChild()).getData().trim();
                        boolean val = Boolean.parseBoolean(text);
                        poolUseFIFO.put(poolName, val);
                    } else if ("lazyReduce".equals(field.getTagName())) {
                        String text = ((Text) field.getFirstChild()).getData().trim();
                        ReduceWatcher watcher = new ReduceWatcher(text);
                        poolReduceWatcher.put(poolName, watcher);
                    }
                }
            } else if ("user".equals(element.getTagName())) {
                String userName = element.getAttribute("name");
                NodeList fields = element.getChildNodes();
                for (int j = 0; j < fields.getLength(); j++) {
                    Node fieldNode = fields.item(j);
                    if (!(fieldNode instanceof Element))
                        continue;
                    Element field = (Element) fieldNode;
                    if ("maxRunningJobs".equals(field.getTagName())) {
                        String text = ((Text) field.getFirstChild()).getData().trim();
                        int val = Integer.parseInt(text);
                        userMaxJobs.put(userName, val);
                    }
                }
            } else if ("userMaxJobsDefault".equals(element.getTagName())) {
                String text = ((Text) element.getFirstChild()).getData().trim();
                int val = Integer.parseInt(text);
                userMaxJobsDefault = val;
            } else {
                LOG.warn("Bad element in allocations file: " + element.getTagName());
            }
        }

        // Commit the reload; also create any pool defined in the alloc file
        // if it does not already exist, so it can be displayed on the web UI.
        synchronized (this) {
            this.mapAllocs = mapAllocs;
            this.reduceAllocs = reduceAllocs;
            this.poolMaxJobs = poolMaxJobs;
            this.userMaxJobs = userMaxJobs;
            this.userMaxJobsDefault = userMaxJobsDefault;
            this.poolWeights = poolWeights;
            this.poolUseFIFO = poolUseFIFO;
            this.poolReduceWatcher = poolReduceWatcher;

            // remove deleted queue
            Set<String> uncontainedPoolNameSet = new HashSet<String>();
            for (String key : pools.keySet())
                uncontainedPoolNameSet.add(key);

            for (String name : poolNamesInAllocFile.toArray(new String[0])) {
                if (uncontainedPoolNameSet.contains(name)) {
                    uncontainedPoolNameSet.remove(name);
                }
            }
            for (String name : uncontainedPoolNameSet) {
                if (getPool(name).getJobs().size() == 0)
                    pools.remove(name);
            }
            uncontainedPoolNameSet = null;

            for (String name : poolNamesInAllocFile) {
                getPool(name);
            }

            // create a list of ordered pools to be scheduled. 
            this.orderedPools = orderPools();
            this.poolIndexLastAccessed = 0;
        }
    }

    /**
     * Get the allocation for a particular pool
     */
    public int getAllocation(String pool, YTTaskType taskType) {
        Map<String, Integer> allocationMap = (taskType == YTTaskType.MAP ? mapAllocs : reduceAllocs);
        Integer alloc = allocationMap.get(pool);
        return (alloc == null ? 0 : alloc);
    }

    /**
     * Add a job in the appropriate pool
     */
    public synchronized void addJob(JobInProgress job) {
        getPool(getPoolName(job)).addJob(job);
    }

    /**
     * Add a job in the appropriate pool
     */
    public synchronized void addJob(JobInProgress job, JobInfo info) {
        getPool(getPoolName(job)).addJob(job, info);
    }

    /**
     * Remove a job
     */
    public synchronized void removeJob(JobInProgress job) {
        getPool(getPoolName(job)).removeJob(job);
    }

    /**
     * Get a collection of all pools
     */
    public synchronized Collection<YTPool> getPools() {
        return pools.values();
    }

    /**
     * Get the pool name for a JobInProgress from its configuration. This uses
     * the "project" property in the jobconf by default, or the property set with
     * "mapred.yunti3scheduler.poolnameproperty".
     */
    public String getPoolName(JobInProgress job) {
        JobConf conf = job.getJobConf();
        return conf.get(poolNameProperty, YTPool.DEFAULT_POOL_NAME).trim();
    }

    /**
     * Get all pool names that have been seen either in the allocation file or in
     * a MapReduce job.
     */
    public synchronized Collection<String> getPoolNames() {
        List<String> list = new ArrayList<String>();
        for (YTPool pool : getPools()) {
            list.add(pool.getName());
        }
        Collections.sort(list);
        return list;
    }

    public int getUserMaxJobs(String user) {
        if (userMaxJobs.containsKey(user)) {
            return userMaxJobs.get(user);
        } else {
            return userMaxJobsDefault;
        }
    }

    public int getPoolMaxJobs(String pool) {
        if (poolMaxJobs.containsKey(pool)) {
            return poolMaxJobs.get(pool);
        } else {
            return POOL_MAX_JOBS_DEFAULT;
        }
    }

    public int getPoolWeight(String pool) {
        if (poolWeights.containsKey(pool)) {
            return poolWeights.get(pool);
        } else {
            return poolWeightDefault;
        }
    }

    public boolean getPoolUseFIFO(String pool) {
        if (poolUseFIFO.containsKey(pool)) {
            return poolUseFIFO.get(pool);
        }
        return false;
    }

    public ReduceWatcher getReduceWatcher(String pool) {
        return poolReduceWatcher.get(pool);
    }

    /**
     * Update the lass scheduled pool index to next slot. This should be called at the
     * end of a heartbeat's tasks assign.
     */
    public synchronized void updateAccessedPoolIndex() {
        if (!orderedPools.isEmpty()) {
            poolIndexLastAccessed = (poolIndexLastAccessed + 1) % orderedPools.size();
        }
    }

    /**
     * Get the pool which will be scheduled in the coming heartbeat.
     * @return a pool if exits, else null
     */
    synchronized YTPool getCurrentScheduledPool() {
        if (!orderedPools.isEmpty()) {
            return orderedPools.get(poolIndexLastAccessed);
        } else {
            return null;
        }
    }

    /**
     * Create a list of ordered pools to be scheduled. According to pool's weight.
     * A pool with 2x weight has two slots. 
     * @return a list of ordered pools to be scheduled
     */
    private synchronized List<YTPool> orderPools() {
        List<YTPool> pools = new ArrayList<YTPool>();
        for (YTPool pool : getPools()) {
            Integer weight = poolWeights.get(pool.getName());
            if (weight == null)
                weight = poolWeightDefault;

            for (int i = 0; i < weight; i++) {
                pools.add(pool);
            }
        }
        return pools;
    }

}