List of usage examples for org.w3c.dom Element getFirstChild
public Node getFirstChild();
From source file:org.apache.hadoop.hbase.mapred.IndexConfiguration.java
private void propertyFromXML(Element prop, Properties properties) { NodeList fields = prop.getChildNodes(); String attr = null;// w ww . j av a2 s . com String value = null; for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) { continue; } Element field = (Element) fieldNode; if ("name".equals(field.getTagName())) { attr = ((Text) field.getFirstChild()).getData(); } if ("value".equals(field.getTagName()) && field.hasChildNodes()) { value = ((Text) field.getFirstChild()).getData(); } } if (attr != null && value != null) { if (properties == null) { set(attr, value); } else { properties.setProperty(attr, value); } } }
From source file:org.apache.hadoop.hdfs.server.hightidenode.ConfigManager.java
/** * Updates the in-memory data structures from the config file. This file is * expected to be in the following whitespace-separated format: * Blank lines and lines starting with # are ignored. * /*from w w w . j a v a2s. c o m*/ * @throws IOException if the config file cannot be read. * @throws HighTideConfigurationException if configuration entries are invalid. * @throws ClassNotFoundException if user-defined policy classes cannot be loaded * @throws ParserConfigurationException if XML parser is misconfigured. * @throws SAXException if config file is malformed. * @returns A new set of policy categories. */ void reloadConfigs() throws IOException, ParserConfigurationException, SAXException, ClassNotFoundException, HighTideConfigurationException { if (configFileName == null) { return; } File file = new File(configFileName); if (!file.exists()) { throw new HighTideConfigurationException("Configuration file " + configFileName + " does not exist."); } // Read and parse the configuration file. // allow include files in configuration file DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); docBuilderFactory.setNamespaceAware(true); try { docBuilderFactory.setXIncludeAware(true); } catch (UnsupportedOperationException e) { LOG.error("Failed to set setXIncludeAware(true) for raid parser " + docBuilderFactory + ":" + e, e); } LOG.error("Reloading config file " + file); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = builder.parse(file); Element root = doc.getDocumentElement(); if (!"configuration".equalsIgnoreCase(root.getTagName())) throw new HighTideConfigurationException( "Bad configuration file: " + "top-level element not <configuration>"); NodeList elements = root.getChildNodes(); Set<PolicyInfo> existingPolicies = new HashSet<PolicyInfo>(); // loop through all the configured source paths. for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (!(node instanceof Element)) { continue; } Element element = (Element) node; String elementTagName = element.getTagName(); String policyName = null; if ("srcPath".equalsIgnoreCase(elementTagName)) { String srcPathPrefix = element.getAttribute("name"); if (srcPathPrefix == null || srcPathPrefix.length() == 0) { throw new HighTideConfigurationException( "Bad configuration file: " + "srcPath node does not have a path."); } PolicyInfo policyInfo = new PolicyInfo(srcPathPrefix, conf); policyName = srcPathPrefix; Properties policyProperties; // loop through all elements of this policy NodeList policies = element.getChildNodes(); for (int j = 0; j < policies.getLength(); j++) { Node node1 = policies.item(j); if (!(node1 instanceof Element)) { continue; } Element policy = (Element) node1; if ((!"property".equalsIgnoreCase(policy.getTagName())) && (!"destPath".equalsIgnoreCase(policy.getTagName()))) { throw new HighTideConfigurationException( "Bad configuration file: " + "Expecting <property> or <destPath> for srcPath " + srcPathPrefix + " but found " + policy.getTagName()); } // parse the <destPath> items if ("destPath".equalsIgnoreCase(policy.getTagName())) { String destPath = policy.getAttribute("name"); if (destPath == null) { throw new HighTideConfigurationException("Bad configuration file: " + "<destPath> tag should have an attribute named 'name'."); } NodeList properties = policy.getChildNodes(); Properties destProperties = new Properties(); for (int k = 0; k < properties.getLength(); k++) { Node node2 = properties.item(k); if (!(node2 instanceof Element)) { continue; } Element property = (Element) node2; String propertyName = property.getTagName(); if (!("property".equalsIgnoreCase(propertyName))) { throw new HighTideConfigurationException( "Bad configuration file: " + "<destPath> can have only <property> children." + " but found " + propertyName); } NodeList nl = property.getChildNodes(); String pname = null, pvalue = null; for (int l = 0; l < nl.getLength(); l++) { Node node3 = nl.item(l); if (!(node3 instanceof Element)) { continue; } Element item = (Element) node3; String itemName = item.getTagName(); if ("name".equalsIgnoreCase(itemName)) { pname = ((Text) item.getFirstChild()).getData().trim(); } else if ("value".equalsIgnoreCase(itemName)) { pvalue = ((Text) item.getFirstChild()).getData().trim(); } } if (pname == null || pvalue == null) { throw new HighTideConfigurationException("Bad configuration file: " + "All property for destPath " + destPath + " must have name and value "); } LOG.info(policyName + "." + pname + " = " + pvalue); destProperties.setProperty(pname, pvalue); } policyInfo.addDestPath(destPath, destProperties); } else if ("property".equalsIgnoreCase(policy.getTagName())) { Element property = (Element) node1; NodeList nl = property.getChildNodes(); String pname = null, pvalue = null; for (int l = 0; l < nl.getLength(); l++) { Node node3 = nl.item(l); if (!(node3 instanceof Element)) { continue; } Element item = (Element) node3; String itemName = item.getTagName(); if ("name".equalsIgnoreCase(itemName)) { pname = ((Text) item.getFirstChild()).getData().trim(); } else if ("value".equalsIgnoreCase(itemName)) { pvalue = ((Text) item.getFirstChild()).getData().trim(); } } if (pname == null || pvalue == null) { throw new HighTideConfigurationException("Bad configuration file: " + "All property for srcPath " + srcPathPrefix + " must have name and value "); } LOG.info(policyName + "." + pname + " = " + pvalue); policyInfo.setProperty(pname, pvalue); } } existingPolicies.add(policyInfo); } else { throw new HighTideConfigurationException("Bad configuration file: " + "The top level item must be srcPath but found " + elementTagName); } } validateAllPolicies(existingPolicies); setAllPolicies(existingPolicies); return; }
From source file:org.apache.hadoop.mapred.PoolManager.java
/** * Updates the allocation list from the allocation config file. This file is * expected to be in the following whitespace-separated format: * //from ww w . jav a 2 s . c o m * <code> * poolName1 mapAlloc reduceAlloc * poolName2 mapAlloc reduceAlloc * ... * </code> * * Blank lines and lines starting with # are ignored. * * @throws IOException if the config file cannot be read. * @throws AllocationConfigurationException if allocations are invalid. * @throws ParserConfigurationException if XML parser is misconfigured. * @throws SAXException if config file is malformed. */ public void reloadAllocs() throws IOException, ParserConfigurationException, SAXException, AllocationConfigurationException { if (allocFile == null) return; // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. Map<String, Integer> mapAllocs = new HashMap<String, Integer>(); Map<String, Integer> reduceAllocs = new HashMap<String, Integer>(); Map<String, Integer> poolMaxJobs = new HashMap<String, Integer>(); Map<String, Integer> userMaxJobs = new HashMap<String, Integer>(); Map<String, Integer> poolMaxMaps = new HashMap<String, Integer>(); Map<String, Integer> poolMaxReduces = new HashMap<String, Integer>(); Map<String, Double> poolWeights = new HashMap<String, Double>(); Map<String, SchedulingMode> poolModes = new HashMap<String, SchedulingMode>(); Map<String, Long> minSharePreemptionTimeouts = new HashMap<String, Long>(); int userMaxJobsDefault = Integer.MAX_VALUE; int poolMaxJobsDefault = Integer.MAX_VALUE; long fairSharePreemptionTimeout = Long.MAX_VALUE; long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; SchedulingMode defaultSchedulingMode = SchedulingMode.FAIR; // Remember all pool names so we can display them on web UI, etc. List<String> poolNamesInAllocFile = new ArrayList<String>(); // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc; if (allocFile instanceof String) { doc = builder.parse(new File((String) allocFile)); } else { doc = builder.parse(allocFile.toString()); } Element root = doc.getDocumentElement(); if (!"allocations".equals(root.getTagName())) throw new AllocationConfigurationException( "Bad fair scheduler config " + "file: top-level element not <allocations>"); NodeList elements = root.getChildNodes(); for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (!(node instanceof Element)) continue; Element element = (Element) node; if ("pool".equals(element.getTagName())) { String poolName = element.getAttribute("name"); poolNamesInAllocFile.add(poolName); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("minMaps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); mapAllocs.put(poolName, val); } else if ("minReduces".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); reduceAllocs.put(poolName, val); } else if ("maxMaps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxMaps.put(poolName, val); } else if ("maxReduces".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxReduces.put(poolName, val); } else if ("maxRunningJobs".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxJobs.put(poolName, val); } else if ("weight".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); double val = Double.parseDouble(text); poolWeights.put(poolName, val); } else if ("minSharePreemptionTimeout".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; minSharePreemptionTimeouts.put(poolName, val); } else if ("schedulingMode".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); poolModes.put(poolName, parseSchedulingMode(text)); } } if (poolMaxMaps.containsKey(poolName) && mapAllocs.containsKey(poolName) && poolMaxMaps.get(poolName) < mapAllocs.get(poolName)) { LOG.warn(String.format("Pool %s has max maps %d less than min maps %d", poolName, poolMaxMaps.get(poolName), mapAllocs.get(poolName))); } if (poolMaxReduces.containsKey(poolName) && reduceAllocs.containsKey(poolName) && poolMaxReduces.get(poolName) < reduceAllocs.get(poolName)) { LOG.warn(String.format("Pool %s has max reduces %d less than min reduces %d", poolName, poolMaxReduces.get(poolName), reduceAllocs.get(poolName))); } } else if ("user".equals(element.getTagName())) { String userName = element.getAttribute("name"); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("maxRunningJobs".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxJobs.put(userName, val); } } } else if ("userMaxJobsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxJobsDefault = val; } else if ("poolMaxJobsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxJobsDefault = val; } else if ("fairSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; fairSharePreemptionTimeout = val; } else if ("defaultMinSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; defaultMinSharePreemptionTimeout = val; } else if ("defaultPoolSchedulingMode".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); defaultSchedulingMode = parseSchedulingMode(text); } else { LOG.warn("Bad element in allocations file: " + element.getTagName()); } } // Commit the reload; also create any pool defined in the alloc file // if it does not already exist, so it can be displayed on the web UI. synchronized (this) { this.mapAllocs = mapAllocs; this.reduceAllocs = reduceAllocs; this.poolMaxMaps = poolMaxMaps; this.poolMaxReduces = poolMaxReduces; this.poolMaxJobs = poolMaxJobs; this.userMaxJobs = userMaxJobs; this.poolWeights = poolWeights; this.minSharePreemptionTimeouts = minSharePreemptionTimeouts; this.userMaxJobsDefault = userMaxJobsDefault; this.poolMaxJobsDefault = poolMaxJobsDefault; this.fairSharePreemptionTimeout = fairSharePreemptionTimeout; this.defaultMinSharePreemptionTimeout = defaultMinSharePreemptionTimeout; this.defaultSchedulingMode = defaultSchedulingMode; for (String name : poolNamesInAllocFile) { Pool pool = getPool(name); if (poolModes.containsKey(name)) { pool.setSchedulingMode(poolModes.get(name)); } else { pool.setSchedulingMode(defaultSchedulingMode); } } } }
From source file:org.apache.hadoop.mapred.TaskErrorCollector.java
private static String getText(Element element) { return ((Text) element.getFirstChild()).getData().trim(); }
From source file:org.apache.hadoop.mapred.YTPoolManager.java
/** * Updates the allocation list from the allocation config file. This file is * expected to be in the following whitespace-separated format: * /*from w w w.j a v a2 s. co m*/ * <code> * poolName1 mapAlloc reduceAlloc * poolName2 mapAlloc reduceAlloc * ... * </code> * * Blank lines and lines starting with # are ignored. * * @throws IOException if the config file cannot be read. * @throws YTAllocationConfigurationException if allocations are invalid. * @throws ParserConfigurationException if XML parser is misconfigured. * @throws SAXException if config file is malformed. */ public void reloadAllocs() throws IOException, ParserConfigurationException, SAXException, YTAllocationConfigurationException { if (allocFile == null) return; // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. Map<String, Integer> mapAllocs = new HashMap<String, Integer>(); Map<String, Integer> reduceAllocs = new HashMap<String, Integer>(); Map<String, Integer> poolMaxJobs = new HashMap<String, Integer>(); Map<String, Integer> userMaxJobs = new HashMap<String, Integer>(); Map<String, Integer> poolWeights = new HashMap<String, Integer>(); Map<String, Boolean> poolUseFIFO = new HashMap<String, Boolean>(); Map<String, ReduceWatcher> poolReduceWatcher = new HashMap<String, ReduceWatcher>(); int userMaxJobsDefault = POOL_MAX_JOBS_DEFAULT; // Remember all pool names so we can display them on web UI, etc. List<String> poolNamesInAllocFile = new ArrayList<String>(); // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = builder.parse(new File(allocFile)); Element root = doc.getDocumentElement(); if (!"allocations".equals(root.getTagName())) throw new YTAllocationConfigurationException( "Bad allocations file: " + "top-level element not <allocations>"); NodeList elements = root.getChildNodes(); for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (!(node instanceof Element)) continue; Element element = (Element) node; if ("pool".equals(element.getTagName())) { String poolName = element.getAttribute("name"); poolNamesInAllocFile.add(poolName); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("minMaps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); mapAllocs.put(poolName, val); } else if ("minReduces".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); reduceAllocs.put(poolName, val); } else if ("maxRunningJobs".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxJobs.put(poolName, val); } else if ("weight".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolWeights.put(poolName, val); } else if ("useFIFO".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); boolean val = Boolean.parseBoolean(text); poolUseFIFO.put(poolName, val); } else if ("lazyReduce".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); ReduceWatcher watcher = new ReduceWatcher(text); poolReduceWatcher.put(poolName, watcher); } } } else if ("user".equals(element.getTagName())) { String userName = element.getAttribute("name"); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("maxRunningJobs".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxJobs.put(userName, val); } } } else if ("userMaxJobsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxJobsDefault = val; } else { LOG.warn("Bad element in allocations file: " + element.getTagName()); } } // Commit the reload; also create any pool defined in the alloc file // if it does not already exist, so it can be displayed on the web UI. synchronized (this) { this.mapAllocs = mapAllocs; this.reduceAllocs = reduceAllocs; this.poolMaxJobs = poolMaxJobs; this.userMaxJobs = userMaxJobs; this.userMaxJobsDefault = userMaxJobsDefault; this.poolWeights = poolWeights; this.poolUseFIFO = poolUseFIFO; this.poolReduceWatcher = poolReduceWatcher; // remove deleted queue Set<String> uncontainedPoolNameSet = new HashSet<String>(); for (String key : pools.keySet()) uncontainedPoolNameSet.add(key); for (String name : poolNamesInAllocFile.toArray(new String[0])) { if (uncontainedPoolNameSet.contains(name)) { uncontainedPoolNameSet.remove(name); } } for (String name : uncontainedPoolNameSet) { if (getPool(name).getJobs().size() == 0) pools.remove(name); } uncontainedPoolNameSet = null; for (String name : poolNamesInAllocFile) { getPool(name); } // create a list of ordered pools to be scheduled. this.orderedPools = orderPools(); this.poolIndexLastAccessed = 0; } }
From source file:org.apache.hadoop.raid.ConfigManager.java
void reloadXmlConfigs() throws IOException, ParserConfigurationException, SAXException, ClassNotFoundException, RaidConfigurationException {/*from w w w . j a va 2 s. c o m*/ if (configFileName == null) { return; } File file = new File(configFileName); if (!file.exists()) { throw new RaidConfigurationException("Configuration file " + configFileName + " does not exist."); } // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. List<PolicyInfo> all = new ArrayList<PolicyInfo>(); long periodicityValue = periodicity; // Read and parse the configuration file. // allow include files in configuration file DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); docBuilderFactory.setNamespaceAware(true); try { docBuilderFactory.setXIncludeAware(true); } catch (UnsupportedOperationException e) { LOG.error("Failed to set setXIncludeAware(true) for raid parser " + docBuilderFactory + ":" + e, e); } LOG.info("Reloading config file " + file); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = builder.parse(file); Element root = doc.getDocumentElement(); if (!"configuration".equalsIgnoreCase(root.getTagName())) throw new RaidConfigurationException( "Bad configuration file: " + "top-level element not <configuration>"); NodeList policies = root.getChildNodes(); Map<String, PolicyInfo> existingPolicies = new HashMap<String, PolicyInfo>(); // loop through all the configured policies. for (int i = 0; i < policies.getLength(); i++) { Node node = policies.item(i); if (!(node instanceof Element)) { continue; } Element policy = (Element) node; if ("policy".equalsIgnoreCase(policy.getTagName())) { String policyName = policy.getAttribute("name"); PolicyInfo curr = new PolicyInfo(policyName, conf); PolicyInfo parent = null; NodeList policyElements = policy.getChildNodes(); for (int j = 0; j < policyElements.getLength(); j++) { Node node1 = policyElements.item(j); if (!(node1 instanceof Element)) { continue; } Element property = (Element) node1; String propertyName = property.getTagName(); if ("srcPath".equalsIgnoreCase(propertyName)) { String srcPathPrefix = property.getAttribute("prefix"); if (srcPathPrefix != null && srcPathPrefix.length() > 0) { curr.setSrcPath(srcPathPrefix); } } else if ("fileList".equalsIgnoreCase(propertyName)) { String text = ((Text) property.getFirstChild()).getData().trim(); LOG.info(policyName + ".fileList = " + text); curr.setFileListPath(new Path(text)); } else if ("codecId".equalsIgnoreCase(propertyName)) { String text = ((Text) property.getFirstChild()).getData().trim(); LOG.info(policyName + ".codecId = " + text); curr.setCodecId(text); } else if ("shouldRaid".equalsIgnoreCase(propertyName)) { String text = ((Text) property.getFirstChild()).getData().trim(); curr.setShouldRaid(Boolean.parseBoolean(text)); } else if ("description".equalsIgnoreCase(propertyName)) { String text = ((Text) property.getFirstChild()).getData().trim(); curr.setDescription(text); } else if ("parentPolicy".equalsIgnoreCase(propertyName)) { String text = ((Text) property.getFirstChild()).getData().trim(); parent = existingPolicies.get(text); } else if ("property".equalsIgnoreCase(propertyName)) { NodeList nl = property.getChildNodes(); String pname = null, pvalue = null; for (int l = 0; l < nl.getLength(); l++) { Node node3 = nl.item(l); if (!(node3 instanceof Element)) { continue; } Element item = (Element) node3; String itemName = item.getTagName(); if ("name".equalsIgnoreCase(itemName)) { pname = ((Text) item.getFirstChild()).getData().trim(); } else if ("value".equalsIgnoreCase(itemName)) { pvalue = ((Text) item.getFirstChild()).getData().trim(); } } if (pname != null && pvalue != null) { LOG.info(policyName + "." + pname + " = " + pvalue); curr.setProperty(pname, pvalue); } } else { LOG.info("Found bad property " + propertyName + " policy name " + policyName + ". Ignoring."); } } // done with all properties of this policy PolicyInfo pinfo; if (parent != null) { pinfo = new PolicyInfo(policyName, conf); pinfo.copyFrom(parent); pinfo.copyFrom(curr); } else { pinfo = curr; } if (pinfo.getSrcPath() != null || pinfo.getFileListPath() != null) { all.add(pinfo); } existingPolicies.put(policyName, pinfo); } } setAllPolicies(all); periodicity = periodicityValue; return; }
From source file:org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService.java
/** * Updates the allocation list from the allocation config file. This file is * expected to be in the XML format specified in the design doc. * * @throws IOException if the config file cannot be read. * @throws AllocationConfigurationException if allocations are invalid. * @throws ParserConfigurationException if XML parser is misconfigured. * @throws SAXException if config file is malformed. */// ww w . ja va2 s.co m public synchronized void reloadAllocations() throws IOException, ParserConfigurationException, SAXException, AllocationConfigurationException { if (allocFile == null) { return; } LOG.info("Loading allocation file " + allocFile); // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. Map<String, Resource> minQueueResources = new HashMap<String, Resource>(); Map<String, Resource> maxQueueResources = new HashMap<String, Resource>(); Map<String, Integer> queueMaxApps = new HashMap<String, Integer>(); Map<String, Integer> userMaxApps = new HashMap<String, Integer>(); Map<String, Float> queueMaxAMShares = new HashMap<String, Float>(); Map<String, ResourceWeights> queueWeights = new HashMap<String, ResourceWeights>(); Map<String, SchedulingPolicy> queuePolicies = new HashMap<String, SchedulingPolicy>(); Map<String, Long> minSharePreemptionTimeouts = new HashMap<String, Long>(); Map<String, Long> fairSharePreemptionTimeouts = new HashMap<String, Long>(); Map<String, Float> fairSharePreemptionThresholds = new HashMap<String, Float>(); Map<String, Map<QueueACL, AccessControlList>> queueAcls = new HashMap<String, Map<QueueACL, AccessControlList>>(); Map<String, Map<ReservationACL, AccessControlList>> reservationAcls = new HashMap<String, Map<ReservationACL, AccessControlList>>(); Set<String> reservableQueues = new HashSet<String>(); int userMaxAppsDefault = Integer.MAX_VALUE; int queueMaxAppsDefault = Integer.MAX_VALUE; Resource queueMaxResourcesDefault = Resources.unbounded(); float queueMaxAMShareDefault = 0.5f; long defaultFairSharePreemptionTimeout = Long.MAX_VALUE; long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; float defaultFairSharePreemptionThreshold = 0.5f; SchedulingPolicy defaultSchedPolicy = SchedulingPolicy.DEFAULT_POLICY; // Reservation global configuration knobs String planner = null; String reservationAgent = null; String reservationAdmissionPolicy = null; QueuePlacementPolicy newPlacementPolicy = null; // Remember all queue names so we can display them on web UI, etc. // configuredQueues is segregated based on whether it is a leaf queue // or a parent queue. This information is used for creating queues // and also for making queue placement decisions(QueuePlacementRule.java). Map<FSQueueType, Set<String>> configuredQueues = new HashMap<FSQueueType, Set<String>>(); for (FSQueueType queueType : FSQueueType.values()) { configuredQueues.put(queueType, new HashSet<String>()); } // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = builder.parse(allocFile); Element root = doc.getDocumentElement(); if (!"allocations".equals(root.getTagName())) throw new AllocationConfigurationException( "Bad fair scheduler config " + "file: top-level element not <allocations>"); NodeList elements = root.getChildNodes(); List<Element> queueElements = new ArrayList<Element>(); Element placementPolicyElement = null; for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (node instanceof Element) { Element element = (Element) node; if ("queue".equals(element.getTagName()) || "pool".equals(element.getTagName())) { queueElements.add(element); } else if ("user".equals(element.getTagName())) { String userName = element.getAttribute("name"); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("maxRunningApps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxApps.put(userName, val); } } } else if ("queueMaxResourcesDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text); queueMaxResourcesDefault = val; } else if ("userMaxAppsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxAppsDefault = val; } else if ("defaultFairSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; defaultFairSharePreemptionTimeout = val; } else if ("fairSharePreemptionTimeout".equals(element.getTagName())) { if (defaultFairSharePreemptionTimeout == Long.MAX_VALUE) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; defaultFairSharePreemptionTimeout = val; } } else if ("defaultMinSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; defaultMinSharePreemptionTimeout = val; } else if ("defaultFairSharePreemptionThreshold".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); float val = Float.parseFloat(text); val = Math.max(Math.min(val, 1.0f), 0.0f); defaultFairSharePreemptionThreshold = val; } else if ("queueMaxAppsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); queueMaxAppsDefault = val; } else if ("queueMaxAMShareDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); float val = Float.parseFloat(text); val = Math.min(val, 1.0f); queueMaxAMShareDefault = val; } else if ("defaultQueueSchedulingPolicy".equals(element.getTagName()) || "defaultQueueSchedulingMode".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); defaultSchedPolicy = SchedulingPolicy.parse(text); } else if ("queuePlacementPolicy".equals(element.getTagName())) { placementPolicyElement = element; } else if ("reservation-planner".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); planner = text; } else if ("reservation-agent".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); reservationAgent = text; } else if ("reservation-policy".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); reservationAdmissionPolicy = text; } else { LOG.warn("Bad element in allocations file: " + element.getTagName()); } } } // Load queue elements. A root queue can either be included or omitted. If // it's included, all other queues must be inside it. for (Element element : queueElements) { String parent = "root"; if (element.getAttribute("name").equalsIgnoreCase("root")) { if (queueElements.size() > 1) { throw new AllocationConfigurationException( "If configuring root queue," + " no other queues can be placed alongside it."); } parent = null; } loadQueue(parent, element, minQueueResources, maxQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, reservationAcls, configuredQueues, reservableQueues); } // Load placement policy and pass it configured queues Configuration conf = getConfig(); if (placementPolicyElement != null) { newPlacementPolicy = QueuePlacementPolicy.fromXml(placementPolicyElement, configuredQueues, conf); } else { newPlacementPolicy = QueuePlacementPolicy.fromConfiguration(conf, configuredQueues); } // Set the min/fair share preemption timeout for the root queue if (!minSharePreemptionTimeouts.containsKey(QueueManager.ROOT_QUEUE)) { minSharePreemptionTimeouts.put(QueueManager.ROOT_QUEUE, defaultMinSharePreemptionTimeout); } if (!fairSharePreemptionTimeouts.containsKey(QueueManager.ROOT_QUEUE)) { fairSharePreemptionTimeouts.put(QueueManager.ROOT_QUEUE, defaultFairSharePreemptionTimeout); } // Set the fair share preemption threshold for the root queue if (!fairSharePreemptionThresholds.containsKey(QueueManager.ROOT_QUEUE)) { fairSharePreemptionThresholds.put(QueueManager.ROOT_QUEUE, defaultFairSharePreemptionThreshold); } ReservationQueueConfiguration globalReservationQueueConfig = new ReservationQueueConfiguration(); if (planner != null) { globalReservationQueueConfig.setPlanner(planner); } if (reservationAdmissionPolicy != null) { globalReservationQueueConfig.setReservationAdmissionPolicy(reservationAdmissionPolicy); } if (reservationAgent != null) { globalReservationQueueConfig.setReservationAgent(reservationAgent); } AllocationConfiguration info = new AllocationConfiguration(minQueueResources, maxQueueResources, queueMaxApps, userMaxApps, queueWeights, queueMaxAMShares, userMaxAppsDefault, queueMaxAppsDefault, queueMaxResourcesDefault, queueMaxAMShareDefault, queuePolicies, defaultSchedPolicy, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, reservationAcls, newPlacementPolicy, configuredQueues, globalReservationQueueConfig, reservableQueues); lastSuccessfulReload = clock.getTime(); lastReloadAttemptFailed = false; reloadListener.onReload(info); }
From source file:org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService.java
/** * Loads a queue from a queue element in the configuration file *//* ww w. jav a 2 s. co m*/ private void loadQueue(String parentName, Element element, Map<String, Resource> minQueueResources, Map<String, Resource> maxQueueResources, Map<String, Integer> queueMaxApps, Map<String, Integer> userMaxApps, Map<String, Float> queueMaxAMShares, Map<String, ResourceWeights> queueWeights, Map<String, SchedulingPolicy> queuePolicies, Map<String, Long> minSharePreemptionTimeouts, Map<String, Long> fairSharePreemptionTimeouts, Map<String, Float> fairSharePreemptionThresholds, Map<String, Map<QueueACL, AccessControlList>> queueAcls, Map<String, Map<ReservationACL, AccessControlList>> resAcls, Map<FSQueueType, Set<String>> configuredQueues, Set<String> reservableQueues) throws AllocationConfigurationException { String queueName = element.getAttribute("name").trim(); if (queueName.contains(".")) { throw new AllocationConfigurationException("Bad fair scheduler config " + "file: queue name (" + queueName + ") shouldn't contain period."); } if (queueName.isEmpty()) { throw new AllocationConfigurationException("Bad fair scheduler config " + "file: queue name shouldn't be empty or " + "consist only of whitespace."); } if (parentName != null) { queueName = parentName + "." + queueName; } Map<QueueACL, AccessControlList> acls = new HashMap<QueueACL, AccessControlList>(); Map<ReservationACL, AccessControlList> racls = new HashMap<>(); NodeList fields = element.getChildNodes(); boolean isLeaf = true; for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("minResources".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text); minQueueResources.put(queueName, val); } else if ("maxResources".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text); maxQueueResources.put(queueName, val); } else if ("maxRunningApps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); queueMaxApps.put(queueName, val); } else if ("maxAMShare".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); float val = Float.parseFloat(text); val = Math.min(val, 1.0f); queueMaxAMShares.put(queueName, val); } else if ("weight".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); double val = Double.parseDouble(text); queueWeights.put(queueName, new ResourceWeights((float) val)); } else if ("minSharePreemptionTimeout".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; minSharePreemptionTimeouts.put(queueName, val); } else if ("fairSharePreemptionTimeout".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; fairSharePreemptionTimeouts.put(queueName, val); } else if ("fairSharePreemptionThreshold".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); float val = Float.parseFloat(text); val = Math.max(Math.min(val, 1.0f), 0.0f); fairSharePreemptionThresholds.put(queueName, val); } else if ("schedulingPolicy".equals(field.getTagName()) || "schedulingMode".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); SchedulingPolicy policy = SchedulingPolicy.parse(text); queuePolicies.put(queueName, policy); } else if ("aclSubmitApps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData(); acls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(text)); } else if ("aclAdministerApps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData(); acls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(text)); } else if ("aclAdministerReservations".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData(); racls.put(ReservationACL.ADMINISTER_RESERVATIONS, new AccessControlList(text)); } else if ("aclListReservations".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData(); racls.put(ReservationACL.LIST_RESERVATIONS, new AccessControlList(text)); } else if ("aclSubmitReservations".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData(); racls.put(ReservationACL.SUBMIT_RESERVATIONS, new AccessControlList(text)); } else if ("reservation".equals(field.getTagName())) { isLeaf = false; reservableQueues.add(queueName); configuredQueues.get(FSQueueType.PARENT).add(queueName); } else if ("queue".endsWith(field.getTagName()) || "pool".equals(field.getTagName())) { loadQueue(queueName, field, minQueueResources, maxQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, resAcls, configuredQueues, reservableQueues); isLeaf = false; } } if (isLeaf) { // if a leaf in the alloc file is marked as type='parent' // then store it under 'parent' if ("parent".equals(element.getAttribute("type"))) { configuredQueues.get(FSQueueType.PARENT).add(queueName); } else { configuredQueues.get(FSQueueType.LEAF).add(queueName); } } else { if ("parent".equals(element.getAttribute("type"))) { throw new AllocationConfigurationException("Both <reservation> and " + "type=\"parent\" found for queue " + queueName + " which is " + "unsupported"); } configuredQueues.get(FSQueueType.PARENT).add(queueName); } queueAcls.put(queueName, acls); resAcls.put(queueName, racls); if (maxQueueResources.containsKey(queueName) && minQueueResources.containsKey(queueName) && !Resources.fitsIn(minQueueResources.get(queueName), maxQueueResources.get(queueName))) { LOG.warn(String.format("Queue %s has max resources %s less than min resources %s", queueName, maxQueueResources.get(queueName), minQueueResources.get(queueName))); } }
From source file:org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.PoolManager.java
/** * Updates the allocation list from the allocation config file. This file is * expected to be in the XML format specified in the design doc. * /*from w w w . j a va 2s . co m*/ * @throws IOException if the config file cannot be read. * @throws AllocationConfigurationException if allocations are invalid. * @throws ParserConfigurationException if XML parser is misconfigured. * @throws SAXException if config file is malformed. */ public void reloadAllocs() throws IOException, ParserConfigurationException, SAXException, AllocationConfigurationException { if (allocFile == null) return; // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. Map<String, Resource> minPoolResources = new HashMap<String, Resource>(); Map<String, Resource> maxPoolResources = new HashMap<String, Resource>(); Map<String, Integer> poolMaxApps = new HashMap<String, Integer>(); Map<String, Integer> userMaxApps = new HashMap<String, Integer>(); Map<String, Double> poolWeights = new HashMap<String, Double>(); Map<String, SchedulingMode> poolModes = new HashMap<String, SchedulingMode>(); Map<String, Long> minSharePreemptionTimeouts = new HashMap<String, Long>(); int userMaxAppsDefault = Integer.MAX_VALUE; int poolMaxAppsDefault = Integer.MAX_VALUE; long fairSharePreemptionTimeout = Long.MAX_VALUE; long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; SchedulingMode defaultSchedulingMode = SchedulingMode.FAIR; // Remember all pool names so we can display them on web UI, etc. List<String> poolNamesInAllocFile = new ArrayList<String>(); // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc; if (allocFile instanceof String) { doc = builder.parse(new File((String) allocFile)); } else { doc = builder.parse(allocFile.toString()); } Element root = doc.getDocumentElement(); if (!"allocations".equals(root.getTagName())) throw new AllocationConfigurationException( "Bad fair scheduler config " + "file: top-level element not <allocations>"); NodeList elements = root.getChildNodes(); for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (!(node instanceof Element)) continue; Element element = (Element) node; if ("pool".equals(element.getTagName())) { String poolName = element.getAttribute("name"); poolNamesInAllocFile.add(poolName); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("minResources".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); minPoolResources.put(poolName, Resources.createResource(val)); } else if ("maxResources".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); maxPoolResources.put(poolName, Resources.createResource(val)); } else if ("maxRunningApps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxApps.put(poolName, val); } else if ("weight".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); double val = Double.parseDouble(text); poolWeights.put(poolName, val); } else if ("minSharePreemptionTimeout".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; minSharePreemptionTimeouts.put(poolName, val); } else if ("schedulingMode".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); poolModes.put(poolName, parseSchedulingMode(text)); } } if (maxPoolResources.containsKey(poolName) && minPoolResources.containsKey(poolName) && Resources.lessThan(maxPoolResources.get(poolName), minPoolResources.get(poolName))) { LOG.warn(String.format("Pool %s has max resources %d less than min resources %d", poolName, maxPoolResources.get(poolName), minPoolResources.get(poolName))); } } else if ("user".equals(element.getTagName())) { String userName = element.getAttribute("name"); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("maxRunningApps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxApps.put(userName, val); } } } else if ("userMaxAppsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxAppsDefault = val; } else if ("poolMaxAppsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxAppsDefault = val; } else if ("fairSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; fairSharePreemptionTimeout = val; } else if ("defaultMinSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; defaultMinSharePreemptionTimeout = val; } else if ("defaultPoolSchedulingMode".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); defaultSchedulingMode = parseSchedulingMode(text); } else { LOG.warn("Bad element in allocations file: " + element.getTagName()); } } // Commit the reload; also create any pool defined in the alloc file // if it does not already exist, so it can be displayed on the web UI. synchronized (this) { this.minPoolResources = minPoolResources; this.maxPoolResources = maxPoolResources; this.poolMaxApps = poolMaxApps; this.userMaxApps = userMaxApps; this.poolWeights = poolWeights; this.minSharePreemptionTimeouts = minSharePreemptionTimeouts; this.userMaxAppsDefault = userMaxAppsDefault; this.poolMaxAppsDefault = poolMaxAppsDefault; this.fairSharePreemptionTimeout = fairSharePreemptionTimeout; this.defaultMinSharePreemptionTimeout = defaultMinSharePreemptionTimeout; this.defaultSchedulingMode = defaultSchedulingMode; for (String name : poolNamesInAllocFile) { Pool pool = getPool(name); if (poolModes.containsKey(name)) { pool.setSchedulingMode(poolModes.get(name)); } else { pool.setSchedulingMode(defaultSchedulingMode); } } } }
From source file:org.apache.impala.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService.java
/** * Updates the allocation list from the allocation config file. This file is * expected to be in the XML format specified in the design doc. * * @throws IOException if the config file cannot be read. * @throws AllocationConfigurationException if allocations are invalid. * @throws ParserConfigurationException if XML parser is misconfigured. * @throws SAXException if config file is malformed. *//*from www . j a v a2s .c om*/ public synchronized void reloadAllocations() throws IOException, ParserConfigurationException, SAXException, AllocationConfigurationException { if (allocFile == null) { return; } LOG.info("Loading allocation file " + allocFile); // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. Map<String, Resource> minQueueResources = new HashMap<>(); Map<String, Resource> maxQueueResources = new HashMap<>(); Map<String, Resource> maxChildQueueResources = new HashMap<>(); Map<String, Integer> queueMaxApps = new HashMap<>(); Map<String, Integer> userMaxApps = new HashMap<>(); Map<String, Float> queueMaxAMShares = new HashMap<>(); Map<String, ResourceWeights> queueWeights = new HashMap<>(); Map<String, SchedulingPolicy> queuePolicies = new HashMap<>(); Map<String, Long> minSharePreemptionTimeouts = new HashMap<>(); Map<String, Long> fairSharePreemptionTimeouts = new HashMap<>(); Map<String, Float> fairSharePreemptionThresholds = new HashMap<>(); Map<String, Map<QueueACL, AccessControlList>> queueAcls = new HashMap<>(); Set<String> nonPreemptableQueues = new HashSet<>(); int userMaxAppsDefault = Integer.MAX_VALUE; int queueMaxAppsDefault = Integer.MAX_VALUE; Resource queueMaxResourcesDefault = Resources.unbounded(); float queueMaxAMShareDefault = 0.5f; long defaultFairSharePreemptionTimeout = Long.MAX_VALUE; long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; float defaultFairSharePreemptionThreshold = 0.5f; SchedulingPolicy defaultSchedPolicy = SchedulingPolicy.DEFAULT_POLICY; QueuePlacementPolicy newPlacementPolicy = null; // Remember all queue names so we can display them on web UI, etc. // configuredQueues is segregated based on whether it is a leaf queue // or a parent queue. This information is used for creating queues // and also for making queue placement decisions(QueuePlacementRule.java). Map<FSQueueType, Set<String>> configuredQueues = new HashMap<>(); for (FSQueueType queueType : FSQueueType.values()) { configuredQueues.put(queueType, new HashSet<String>()); } // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = builder.parse(allocFile); Element root = doc.getDocumentElement(); if (!"allocations".equals(root.getTagName())) throw new AllocationConfigurationException( "Bad fair scheduler config " + "file: top-level element not <allocations>"); NodeList elements = root.getChildNodes(); List<Element> queueElements = new ArrayList<Element>(); Element placementPolicyElement = null; for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (node instanceof Element) { Element element = (Element) node; if ("queue".equals(element.getTagName()) || "pool".equals(element.getTagName())) { queueElements.add(element); } else if ("user".equals(element.getTagName())) { String userName = element.getAttribute("name"); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("maxRunningApps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxApps.put(userName, val); } } } else if ("queueMaxResourcesDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text); queueMaxResourcesDefault = val; } else if ("userMaxAppsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxAppsDefault = val; } else if ("defaultFairSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; defaultFairSharePreemptionTimeout = val; } else if ("fairSharePreemptionTimeout".equals(element.getTagName())) { if (defaultFairSharePreemptionTimeout == Long.MAX_VALUE) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; defaultFairSharePreemptionTimeout = val; } } else if ("defaultMinSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; defaultMinSharePreemptionTimeout = val; } else if ("defaultFairSharePreemptionThreshold".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); float val = Float.parseFloat(text); val = Math.max(Math.min(val, 1.0f), 0.0f); defaultFairSharePreemptionThreshold = val; } else if ("queueMaxAppsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); queueMaxAppsDefault = val; } else if ("queueMaxAMShareDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); float val = Float.parseFloat(text); val = Math.min(val, 1.0f); queueMaxAMShareDefault = val; } else if ("defaultQueueSchedulingPolicy".equals(element.getTagName()) || "defaultQueueSchedulingMode".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); if (text.equalsIgnoreCase("FIFO")) { throw new AllocationConfigurationException( "Bad fair scheduler " + "config file: defaultQueueSchedulingPolicy or " + "defaultQueueSchedulingMode can't be FIFO."); } defaultSchedPolicy = SchedulingPolicy.parse(text); } else if ("queuePlacementPolicy".equals(element.getTagName())) { placementPolicyElement = element; } else { LOG.warn("Bad element in allocations file: " + element.getTagName()); } } } // Load queue elements. A root queue can either be included or omitted. If // it's included, all other queues must be inside it. for (Element element : queueElements) { String parent = "root"; if (element.getAttribute("name").equalsIgnoreCase("root")) { if (queueElements.size() > 1) { throw new AllocationConfigurationException( "If configuring root queue," + " no other queues can be placed alongside it."); } parent = null; } loadQueue(parent, element, minQueueResources, maxQueueResources, maxChildQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, configuredQueues, nonPreemptableQueues); } // Load placement policy and pass it configured queues Configuration conf = getConfig(); if (placementPolicyElement != null) { newPlacementPolicy = QueuePlacementPolicy.fromXml(placementPolicyElement, configuredQueues, conf); } else { newPlacementPolicy = QueuePlacementPolicy.fromConfiguration(conf, configuredQueues); } // Set the min/fair share preemption timeout for the root queue if (!minSharePreemptionTimeouts.containsKey("root")) { minSharePreemptionTimeouts.put("root", defaultMinSharePreemptionTimeout); } if (!fairSharePreemptionTimeouts.containsKey("root")) { fairSharePreemptionTimeouts.put("root", defaultFairSharePreemptionTimeout); } // Set the fair share preemption threshold for the root queue if (!fairSharePreemptionThresholds.containsKey("root")) { fairSharePreemptionThresholds.put("root", defaultFairSharePreemptionThreshold); } AllocationConfiguration info = new AllocationConfiguration(minQueueResources, maxQueueResources, maxChildQueueResources, queueMaxApps, userMaxApps, queueWeights, queueMaxAMShares, userMaxAppsDefault, queueMaxAppsDefault, queueMaxResourcesDefault, queueMaxAMShareDefault, queuePolicies, defaultSchedPolicy, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, newPlacementPolicy, configuredQueues, nonPreemptableQueues); lastSuccessfulReload = clock.getTime(); lastReloadAttemptFailed = false; reloadListener.onReload(info); }