Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.corona; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.util.Collection; import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.Set; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.w3c.dom.Text; import org.xml.sax.SAXException; /** * Reloads corona scheduling parameters periodically. Generates the pools * config if the {@link PoolsConfigDocumentGenerator} class is set. * Needs to be thread safe * * The following is a corona.xml example: * * <?xml version="1.0"?> * <configuration> * <defaultSchedulingMode>FAIR</defaultSchedulingMode> * <nodeLocalityWaitMAP>0</nodeLocalityWaitMAP> * <rackLocalityWaitMAP>5000</rackLocalityWaitMAP> * <preemptedTaskMaxRunningTime>60000</preemptedTaskMaxRunningTime> * <shareStarvingRatio>0.9</shareStarvingRatio> * <starvingTimeForShare>60000</starvingTimeForShare> * <starvingTimeForMinimum>30000</starvingTimeForMinimum> * <group name="group_a"> * <minMAP>200</minMAP> * <minMAP>100</minMAP> * <minREDUCE>100</minREDUCE> * <maxMAP>200</maxMAP> * <maxREDUCE>200</maxREDUCE> * <pool name="pool_sla"> * <minMAP>100</minMAP> * <minREDUCE>100</minREDUCE> * <maxMAP>200</maxMAP> * <maxREDUCE>200</maxREDUCE> * <weight>2.0</weight> * <schedulingMode>FIFO</schedulingMode> * </pool> * <pool name="pool_nonsla"> * </pool> * </group> * <group name ="group_b"> * <maxMAP>200</maxMAP> * <maxREDUCE>200</maxREDUCE> * <weight>3.0</weight> * </group> * </configuration> * * Note that the type strings "MAP" and "REDUCE" must be * defined in {@link CoronaConf} */ public class ConfigManager { /** Configuration xml tag name */ public static final String CONFIGURATION_TAG_NAME = "configuration"; /** Redirect xml tag name */ public static final String REDIRECT_TAG_NAME = "redirect"; /** Group xml tag name */ public static final String GROUP_TAG_NAME = "group"; /** Pool xml tag name */ public static final String POOL_TAG_NAME = "pool"; /** Scheduling mode xml tag name */ public static final String SCHEDULING_MODE_TAG_NAME = "schedulingMode"; /** Preemptability xml tag name */ public static final String PREEMPTABILITY_MODE_TAG_NAME = "preemptable"; /** Request maximum xml tag name */ public static final String REQUEST_MAX_MODE_TAG_NAME = "requestMax"; /** Weight xml tag name */ public static final String WEIGHT_TAG_NAME = "weight"; /** Priority xml tag name */ public static final String PRIORITY_TAG_NAME = "priority"; /** Whitelist xml tag name */ public static final String WHITELIST_TAG_NAME = "whitelist"; /** Min xml tag name prefix */ public static final String MIN_TAG_NAME_PREFIX = "min"; /** Max xml tag name prefix */ public static final String MAX_TAG_NAME_PREFIX = "max"; public static final String REDIRECT_JOB_WITH_LIMIT = "redirectJobWithLimit"; /** Name xml attribute */ public static final String NAME_ATTRIBUTE = "name"; /** Source xml attribute (for redirect) */ public static final String SOURCE_ATTRIBUTE = "source"; /** Destination xml attribute (for redirect) */ public static final String DESTINATION_ATTRIBUTE = "destination"; public static final String JOB_INPUT_SIZE_LIMIT_ATTRIBUTE = "inputSizeLimit"; /** Logger */ private static final Log LOG = LogFactory.getLog(ConfigManager.class); /** The default behavior to schedule from nodes to sessions. */ private static final boolean DEFAULT_SCHEDULE_FROM_NODE_TO_SESSION = false; /** The default max running time to consider for preemption */ private static final long DEFAULT_PREEMPT_TASK_MAX_RUNNING_TIME = 5 * 60 * 1000L; /** The default number of attempts to preempt */ private static final int DEFAULT_PREEMPTION_ROUNDS = 10; /** The default ratio to consider starvation */ private static final double DEFAULT_SHARE_STARVING_RATIO = 0.7; /** The minimum amount of time to wait between preemptions */ public static final long DEFAULT_MIN_PREEMPT_PERIOD = 60 * 1000L; /** * The default number of grants to give out on each iteration of * the Scheduler. */ public static final int DEFAULT_GRANTS_PER_ITERATION = 5000; /** The default allowed starvation time for a share */ public static final long DEFAULT_STARVING_TIME_FOR_SHARE = 5 * 60 * 1000L; /** The default allowed starvation time for a minimum allocation */ public static final long DEFAULT_STARVING_TIME_FOR_MINIMUM = 3 * 60 * 1000L; /** The comparator to use by default within any pool group */ private static final ScheduleComparator DEFAULT_POOL_GROUP_COMPARATOR = ScheduleComparator.PRIORITY; /** The comparator to use by default within any pool */ private static final ScheduleComparator DEFAULT_POOL_COMPARATOR = ScheduleComparator.FIFO; /** If defined, generate the pools config document periodically */ private PoolsConfigDocumentGenerator poolsConfigDocumentGenerator; /** The types this configuration is initialized for */ private final Collection<ResourceType> TYPES; /** The classloader to use when looking for resource in the classpath */ private final ClassLoader classLoader; /** Set of configured pool group names */ private Set<String> poolGroupNameSet; /** Set of configured pool info names */ private Set<PoolInfo> poolInfoSet; /** The Map of max allocations for a given type and group */ private TypePoolGroupNameMap<Integer> typePoolGroupNameToMax = new TypePoolGroupNameMap<Integer>(); /** The Map of min allocations for a given type and group */ private TypePoolGroupNameMap<Integer> typePoolGroupNameToMin = new TypePoolGroupNameMap<Integer>(); /** The Map of max allocations for a given type, group, and pool */ private TypePoolInfoMap<Integer> typePoolInfoToMax = new TypePoolInfoMap<Integer>(); /** The Map of min allocations for a given type, group, and pool */ private TypePoolInfoMap<Integer> typePoolInfoToMin = new TypePoolInfoMap<Integer>(); /** The set of pools that can't be preempted */ private Set<PoolInfo> nonPreemptablePools = new HashSet<PoolInfo>(); /** * The set of pools that limit the number of resource requests by the * pool maximum. */ private Set<PoolInfo> requestMaxPools = new HashSet<PoolInfo>(); /** The Map of node locality wait times for different resource types */ private Map<ResourceType, Long> typeToNodeWait; /** The Map of rack locality wait times for different resource types */ private Map<ResourceType, Long> typeToRackWait; /** The Map of comparators configurations for the pools */ private Map<PoolInfo, ScheduleComparator> poolInfoToComparator; /** The Map of the weights configuration for the pools */ private Map<PoolInfo, Double> poolInfoToWeight; /** The Map of redirections (source -> target) for PoolInfo objects */ private Map<PoolInfo, PoolInfo> poolInfoToRedirect; /** The Map of priorities for the pools */ private Map<PoolInfo, Integer> poolInfoToPriority; /** The Map of whitelist for the pools */ private Map<PoolInfo, String> poolInfoToWhitelist; /** The default comparator for the schedulables within the pool */ private ScheduleComparator defaultPoolComparator; /** The ratio of the share to consider starvation */ private double shareStarvingRatio; /** The allowed starvation time for the share */ private long starvingTimeForShare; /** The minimum period between preemptions. */ private long minPreemptPeriod; /** The maximum Job size for a pool */ private Map<PoolInfo, Long> poolJobSizeLimit; /** The Map of redirections when job exceeds limit for Pool */ private Map<PoolInfo, PoolInfo> jobExceedsLimitPoolRedirect; /** The default FIFO pool info for large query to redirect to */ private PoolInfo defaultFifoPoolInfo; /** Tasks to schedule in one iteration of the scheduler */ private volatile int grantsPerIteration = DEFAULT_GRANTS_PER_ITERATION; /** The allowed starvation time for the min allocation */ private long starvingTimeForMinimum; /** The max time of the task to consider for preemption */ private long preemptedTaskMaxRunningTime; /** The number of times to iterate over pools trying to preempt */ private int preemptionRounds; /** Match nodes to session */ private boolean scheduleFromNodeToSession; /** The flag for the reload thread */ private volatile boolean running = true; /** The thread that monitors and reloads the config file */ private ReloadThread reloadThread; /** The last timestamp when the config was successfully loaded */ private long lastSuccessfulReload = -1L; /** Corona conf used to get static config */ private final CoronaConf conf; /** Pools reload period ms */ private final long poolsReloadPeriodMs; /** Config reload period ms */ private final long configReloadPeriodMs; /** The name of the general config file to use */ private volatile String configFileName; /** The name of the pools config file to use */ private volatile String poolsConfigFileName; /** * The main constructor for the config manager given the types and the name * of the config file to use * @param types the types to initialize the configuration for */ public ConfigManager(Collection<ResourceType> types, CoronaConf conf) { this.TYPES = types; this.conf = conf; Class<?> poolsConfigDocumentGeneratorClass = conf.getPoolsConfigDocumentGeneratorClass(); if (poolsConfigDocumentGeneratorClass != null) { try { this.poolsConfigDocumentGenerator = (PoolsConfigDocumentGenerator) poolsConfigDocumentGeneratorClass .newInstance(); poolsConfigDocumentGenerator.initialize(conf); } catch (InstantiationException e) { LOG.warn("Failed to instantiate " + poolsConfigDocumentGeneratorClass, e); } catch (IllegalAccessException e) { LOG.warn("Failed to instantiate " + poolsConfigDocumentGeneratorClass, e); } } else { poolsConfigDocumentGenerator = null; } poolsReloadPeriodMs = conf.getPoolsReloadPeriodMs(); configReloadPeriodMs = conf.getConfigReloadPeriodMs(); LOG.info("ConfigManager: PoolsConfigDocumentGenerator class = " + poolsConfigDocumentGeneratorClass + ", poolsReloadPeriodMs = " + poolsReloadPeriodMs + ", configReloadPeriodMs = " + configReloadPeriodMs); typeToNodeWait = new EnumMap<ResourceType, Long>(ResourceType.class); typeToRackWait = new EnumMap<ResourceType, Long>(ResourceType.class); defaultPoolComparator = DEFAULT_POOL_COMPARATOR; shareStarvingRatio = DEFAULT_SHARE_STARVING_RATIO; minPreemptPeriod = DEFAULT_MIN_PREEMPT_PERIOD; grantsPerIteration = DEFAULT_GRANTS_PER_ITERATION; starvingTimeForMinimum = DEFAULT_STARVING_TIME_FOR_MINIMUM; starvingTimeForShare = DEFAULT_STARVING_TIME_FOR_SHARE; preemptedTaskMaxRunningTime = DEFAULT_PREEMPT_TASK_MAX_RUNNING_TIME; preemptionRounds = DEFAULT_PREEMPTION_ROUNDS; scheduleFromNodeToSession = DEFAULT_SCHEDULE_FROM_NODE_TO_SESSION; reloadThread = new ReloadThread(); reloadThread.setName("Config reload thread"); reloadThread.setDaemon(true); for (ResourceType type : TYPES) { typeToNodeWait.put(type, 0L); typeToRackWait.put(type, 0L); } ClassLoader cl = Thread.currentThread().getContextClassLoader(); if (cl == null) { cl = ConfigManager.class.getClassLoader(); } classLoader = cl; if (poolsConfigDocumentGenerator != null) { if (generatePoolsConfigIfClassSet() == null) { throw new IllegalStateException("Failed to generate the pools " + "config. Must succeed on initialization of ConfigManager."); } } try { findConfigFiles(); reloadAllConfig(true); } catch (IOException e) { LOG.error("Failed to load " + configFileName, e); } catch (SAXException e) { LOG.error("Failed to load " + configFileName, e); } catch (ParserConfigurationException e) { LOG.error("Failed to load " + configFileName, e); } catch (JSONException e) { LOG.error("Failed to load " + configFileName, e); } } /** * Used for unit tests */ public ConfigManager() { TYPES = null; conf = new CoronaConf(new Configuration()); poolsReloadPeriodMs = conf.getPoolsReloadPeriodMs(); configReloadPeriodMs = conf.getConfigReloadPeriodMs(); classLoader = ConfigManager.class.getClassLoader(); } /** * Find the configuration files as set file names or in the classpath. */ private void findConfigFiles() { // Find the materialized_JSON configuration file. if (configFileName == null) { String jsonConfigFileString = conf.getConfigFile().replace(CoronaConf.DEFAULT_CONFIG_FILE, Configuration.MATERIALIZEDJSON); File jsonConfigFile = new File(jsonConfigFileString); String jsonConfigFileName = null; if (jsonConfigFile.exists()) { jsonConfigFileName = jsonConfigFileString; } else { URL u = classLoader.getResource(jsonConfigFileString); jsonConfigFileName = (u != null) ? u.getPath() : null; } // Check that the materialized_JSON contains the resources // of corona.xml if (jsonConfigFileName != null) { try { jsonConfigFile = new File(jsonConfigFileName); InputStream in = new BufferedInputStream(new FileInputStream(jsonConfigFile)); JSONObject json = conf.instantiateJsonObject(in); if (json.has(conf.xmlToThrift(CoronaConf.DEFAULT_CONFIG_FILE))) { configFileName = jsonConfigFileName; LOG.info("Attempt to find config file " + jsonConfigFileString + " as a file and in class loader returned " + configFileName); } } catch (IOException e) { LOG.warn("IOException: " + "while parsing corona JSON configuration"); } catch (JSONException e) { LOG.warn("JSONException: " + "while parsing corona JSON configuration"); } } } if (configFileName == null) { // Parsing the JSON configuration failed. Look for // the xml configuration. String configFileString = conf.getConfigFile(); File configFile = new File(configFileString); if (configFile.exists()) { configFileName = configFileString; } else { URL u = classLoader.getResource(configFileString); configFileName = (u != null) ? u.getPath() : null; } LOG.info("Attempt to find config file " + configFileString + " as a file and in class loader returned " + configFileName); } if (poolsConfigFileName == null) { String poolsConfigFileString = conf.getPoolsConfigFile(); File poolsConfigFile = new File(poolsConfigFileString); if (poolsConfigFile.exists()) { poolsConfigFileName = poolsConfigFileString; } else { URL u = classLoader.getResource(poolsConfigFileString); poolsConfigFileName = (u != null) ? u.getPath() : null; } LOG.info("Attempt to find pools config file " + poolsConfigFileString + " as a file and in class loader returned " + poolsConfigFileName); } } /** * Start monitoring the configuration for the updates */ public void start() { reloadThread.start(); } /** * Stop monitoring and reloading of the configuration */ public void close() { running = false; reloadThread.interrupt(); } /** * Is this a configured pool group? * @param poolGroup Name of the pool group to check * @return True if configured, false otherwise */ public synchronized boolean isConfiguredPoolGroup(String poolGroup) { if (poolGroupNameSet == null) { return false; } return poolGroupNameSet.contains(poolGroup); } /** * Is this a configured pool info? * @param poolInfo Pool info to check * @return True if configured, false otherwise */ public synchronized boolean isConfiguredPoolInfo(PoolInfo poolInfo) { if (poolInfoSet == null) { return false; } return poolInfoSet.contains(poolInfo); } /** * Get the configured pool infos so that the PoolGroupManager can make sure * they are created for stats and cm.jsp. */ public synchronized Collection<PoolInfo> getConfiguredPoolInfos() { return poolInfoSet; } /** * Get the configured maximum allocation for a given {@link ResourceType} * in a given pool group * @param poolGroupName the name of the pool group * @param type the type of the resource * @return the maximum allocation for the resource in a pool group */ public synchronized int getPoolGroupMaximum(String poolGroupName, ResourceType type) { Integer max = (typePoolGroupNameToMax == null) ? null : typePoolGroupNameToMax.get(type, poolGroupName); return max == null ? Integer.MAX_VALUE : max; } /** * Get the configured minimum allocation for a given {@link ResourceType} * in a given pool group * @param poolGroupName the name of the pool group * @param type the type of the resource * @return the minimum allocation for the resource in a pool group */ public synchronized int getPoolGroupMinimum(String poolGroupName, ResourceType type) { Integer min = (typePoolGroupNameToMin == null) ? null : typePoolGroupNameToMin.get(type, poolGroupName); return min == null ? 0 : min; } /** * Get the configured ScheduleComparator. * in a given pool group * @param poolGroupName the name of the pool group * @return the configured comparator. (returns DEFAULT_POOL_GROUP_COMPARATOR * by default. */ public ScheduleComparator getPoolGroupComparator(String poolGroupName) { // TODO: If it uses a shared data structure, this needs to be thread safe!! // TODO: Currently unconfigurable. return DEFAULT_POOL_GROUP_COMPARATOR; } /** * Get the configured maximum allocation for a given {@link ResourceType} * in a given pool * @param poolInfo Pool info to check * @param type the type of the resource * @return the maximum allocation for the resource in a pool */ public synchronized int getPoolMaximum(PoolInfo poolInfo, ResourceType type) { Integer max = (typePoolInfoToMax == null) ? null : typePoolInfoToMax.get(type, poolInfo); return max == null ? Integer.MAX_VALUE : max; } /** * Get the configured minimum allocation for a given {@link ResourceType} * in a given pool * @param poolInfo Pool info to check * @param type the type of the resource * @return the minimum allocation for the resource in a pool */ public synchronized int getPoolMinimum(PoolInfo poolInfo, ResourceType type) { Integer min = (typePoolInfoToMin == null) ? null : typePoolInfoToMin.get(type, poolInfo); return min == null ? 0 : min; } public synchronized boolean isPoolPreemptable(PoolInfo poolInfo) { return !nonPreemptablePools.contains(poolInfo); } /** * Should this pool use the request max to limit job submission? * @param poolInfo Pool info to check * @return True if using the request max to limit jobs */ public synchronized boolean useRequestMax(PoolInfo poolInfo) { return requestMaxPools.contains(poolInfo); } /** * Get the weight for the pool * @param poolInfo Pool info to check * @return the weight for the pool */ public synchronized double getWeight(PoolInfo poolInfo) { Double weight = (poolInfoToWeight == null) ? null : poolInfoToWeight.get(poolInfo); return weight == null ? 1.0 : weight; } /** * Get the priority for the pool * @param poolInfo Pool info to check * @return the priority for the pool */ public synchronized int getPriority(PoolInfo poolInfo) { Integer priority = (poolInfoToPriority == null) ? null : poolInfoToPriority.get(poolInfo); return priority == null ? 0 : priority; } /** * Get a redirected PoolInfo (destination) from the source. Only supports * one level of redirection. * @param poolInfo Pool info to check for a destination * @return Destination pool info if one exists, else return the input */ public synchronized PoolInfo getRedirect(PoolInfo poolInfo) { PoolInfo destination = (poolInfoToRedirect == null) ? poolInfo : poolInfoToRedirect.get(poolInfo); if (destination == null) { return poolInfo; } return destination; } private synchronized boolean jobExceedsSizeInfoLimit(Long jobSizeInfo, Long sizeInfoLimit) { return jobSizeInfo >= sizeInfoLimit; } public synchronized PoolInfo getRedirect(PoolInfo poolInfo, Long jobSizeInfo) { // Check if the pool specified has a redirect, use result to determine // if we should redirect based on size info limit PoolInfo actualPoolInfo = getRedirect(poolInfo); Long sizeInfoLimit = poolJobSizeLimit.get(actualPoolInfo); if (sizeInfoLimit != null && jobExceedsSizeInfoLimit(jobSizeInfo, sizeInfoLimit)) { return jobExceedsLimitPoolRedirect.get(actualPoolInfo); } return actualPoolInfo; } public synchronized String getWhitelist(PoolInfo poolInfo) { return poolInfoToWhitelist.get(poolInfo); } /** * Get a copy of the map of redirects (used for cm.jsp) * * @return Map of redirects otherwise null if none exists */ public synchronized Map<PoolInfo, PoolInfo> getRedirects() { return (poolInfoToRedirect == null) ? null : new HashMap<PoolInfo, PoolInfo>(poolInfoToRedirect); } /** * Get the comparator to use for scheduling sessions within a pool * @param poolInfo Pool info to check * @return the scheduling comparator to use for the pool */ public synchronized ScheduleComparator getPoolComparator(PoolInfo poolInfo) { ScheduleComparator comparator = (poolInfoToComparator == null) ? null : poolInfoToComparator.get(poolInfo); return comparator == null ? defaultPoolComparator : comparator; } public synchronized long getPreemptedTaskMaxRunningTime() { return preemptedTaskMaxRunningTime; } public synchronized boolean getScheduleFromNodeToSession() { return scheduleFromNodeToSession; } public synchronized int getPreemptionRounds() { return preemptionRounds; } public synchronized double getShareStarvingRatio() { return shareStarvingRatio; } public synchronized long getStarvingTimeForShare() { return starvingTimeForShare; } public synchronized long getStarvingTimeForMinimum() { return starvingTimeForMinimum; } /** * Get the locality wait to be used by the scheduler for a given * ResourceType on a given LocalityLevel * @param type the type of the resource * @param level the locality level * @return the time in milliseconds to use as a locality wait for a resource * of a given type on a given level */ public synchronized long getLocalityWait(ResourceType type, LocalityLevel level) { if (level == LocalityLevel.ANY) { return 0L; } Long wait = level == LocalityLevel.NODE ? typeToNodeWait.get(type) : typeToRackWait.get(type); if (wait == null) { throw new IllegalArgumentException("Unknown type:" + type); } return wait; } public long getMinPreemptPeriod() { return minPreemptPeriod; } public int getGrantsPerIteration() { return grantsPerIteration; } /** * The thread that monitors the config file and reloads the configuration * when the file is updated */ private class ReloadThread extends Thread { @Override public void run() { long lastReloadAttempt = -1L; long lastGenerationAttempt = -1L; while (running) { try { boolean reloadAllConfig = false; long now = ClusterManager.clock.getTime(); if ((poolsConfigDocumentGenerator != null) && (now - lastGenerationAttempt > poolsReloadPeriodMs)) { lastGenerationAttempt = now; generatePoolsConfigIfClassSet(); reloadAllConfig = true; } if (now - lastReloadAttempt > configReloadPeriodMs) { lastReloadAttempt = now; reloadAllConfig = true; } if (reloadAllConfig) { findConfigFiles(); try { reloadAllConfig(false); } catch (IOException e) { LOG.error("Failed to load " + configFileName, e); } catch (SAXException e) { LOG.error("Failed to load " + configFileName, e); } catch (ParserConfigurationException e) { LOG.error("Failed to load " + configFileName, e); } catch (JSONException e) { LOG.error("Failed to load " + configFileName, e); } } } catch (Exception e) { LOG.error("Failed to reload config because of " + "an unknown exception", e); } try { Thread.sleep(Math.min(poolsReloadPeriodMs, configReloadPeriodMs) / 10); } catch (InterruptedException e) { LOG.warn("run: Interrupted", e); } } } } /** * Generate the new pools configuration using the configuration generator. * The generated configuration is written to a temporary file and then * atomically renamed to the specified destination file. * This function may be called concurrently and it is safe to do so because * of the atomic rename to the destination file. * * @return Md5 of the generated file or null if generation failed. */ public String generatePoolsConfigIfClassSet() { if (poolsConfigDocumentGenerator == null) { return null; } Document document = poolsConfigDocumentGenerator.generatePoolsDocument(); if (document == null) { LOG.warn("generatePoolsConfig: Did not generate a valid pools xml file"); return null; } // Write the content into a temporary xml file and rename to the // expected file. File tempXmlFile; try { TransformerFactory transformerFactory = TransformerFactory.newInstance(); transformerFactory.setAttribute("indent-number", new Integer(2)); Transformer transformer = transformerFactory.newTransformer(); transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); transformer.setOutputProperty(OutputKeys.INDENT, "yes"); DOMSource source = new DOMSource(document); tempXmlFile = File.createTempFile("tmpPoolsConfig", "xml"); if (LOG.isDebugEnabled()) { StreamResult stdoutResult = new StreamResult(System.out); transformer.transform(source, stdoutResult); } StreamResult result = new StreamResult(tempXmlFile); transformer.transform(source, result); String md5 = org.apache.commons.codec.digest.DigestUtils.md5Hex(new FileInputStream(tempXmlFile)); File destXmlFile = new File(conf.getPoolsConfigFile()); boolean success = tempXmlFile.renameTo(destXmlFile); LOG.info("generatePoolConfig: Renamed generated file " + tempXmlFile.getAbsolutePath() + " to " + destXmlFile.getAbsolutePath() + " returned " + success + " with md5sum " + md5); return md5; } catch (TransformerConfigurationException e) { LOG.warn("generatePoolConfig: Failed to write file", e); } catch (IOException e) { LOG.warn("generatePoolConfig: Failed to write file", e); } catch (TransformerException e) { LOG.warn("generatePoolConfig: Failed to write file", e); } return null; } /** * Helper function to reloadJsonConfig(). Parses the JSON Object * corresponding to the key "TYPES". * @throws JSONException */ private void loadLocalityWaits(JSONObject json, Map<ResourceType, Long> newTypeToNodeWait, Map<ResourceType, Long> newTypeToRackWait) throws JSONException { Iterator<String> keys = json.keys(); while (keys.hasNext()) { String key = keys.next(); if (!json.isNull(key)) { for (ResourceType type : TYPES) { if (key.equals("nodeLocalityWait" + type)) { long val = Long.parseLong(json.getString(key)); newTypeToNodeWait.put(type, val); } if (key.equals("rackLocalityWait" + type)) { long val = Long.parseLong(json.getString(key)); newTypeToRackWait.put(type, val); } } } } } private boolean loadPoolInfoToRedirect(String source, String destination, Map<PoolInfo, PoolInfo> newPoolInfoToRedirect) { PoolInfo sourcePoolInfo = PoolInfo.createPoolInfo(source); PoolInfo destPoolInfo = PoolInfo.createPoolInfo(destination); if (sourcePoolInfo == null || destPoolInfo == null) { LOG.error("Illegal redirect source " + sourcePoolInfo + " or destination " + destPoolInfo); return false; } else { newPoolInfoToRedirect.put(sourcePoolInfo, destPoolInfo); return true; } } /** * Helper function to reloadJsonConfig(). Parses the JSON Array * corresponding to the key REDIRECT_TAG_NAME. * @throws JSONException */ private void loadPoolInfoToRedirect(JSONArray json, Map<PoolInfo, PoolInfo> newPoolInfoToRedirect) throws JSONException { for (int i = 0; i < json.length(); i++) { JSONObject jsonObj = json.getJSONObject(i); loadPoolInfoToRedirect(jsonObj.getString(SOURCE_ATTRIBUTE), jsonObj.getString(DESTINATION_ATTRIBUTE), newPoolInfoToRedirect); } } private void loadPoolInfoToRedirect(String poolGroupName, String source, String destination, String specifiedJobInputSizeLimit, Map<PoolInfo, PoolInfo> newPoolInfoToRedirectWithLimit, Map<PoolInfo, Long> newPoolInfoJobSizeLimit) { String validSource = PoolInfo.createValidString(poolGroupName, source); String validDestination = PoolInfo.createValidString(poolGroupName, destination); boolean loadedRedirect = loadPoolInfoToRedirect(validSource, validDestination, newPoolInfoToRedirectWithLimit); if (!loadedRedirect) { return; } long jobInputSizeLimit = Long.parseLong(specifiedJobInputSizeLimit); if (jobInputSizeLimit <= 0) { LOG.error("Illegal redirect limit " + jobInputSizeLimit + ". Limit must be > 0."); } else { PoolInfo sourcePoolInfo = PoolInfo.createPoolInfo(validSource); newPoolInfoJobSizeLimit.put(sourcePoolInfo, jobInputSizeLimit); } } /** * Reload the general configuration and update all in-memory values. Should * be invoked under synchronization. * * @throws IOException * @throws SAXException * @throws ParserConfigurationException * @throws JSONException */ private void reloadJsonConfig() throws IOException, SAXException, ParserConfigurationException, JSONException { Map<ResourceType, Long> newTypeToNodeWait; Map<ResourceType, Long> newTypeToRackWait; ScheduleComparator newDefaultPoolComparator = DEFAULT_POOL_COMPARATOR; double newShareStarvingRatio = DEFAULT_SHARE_STARVING_RATIO; long newMinPreemptPeriod = DEFAULT_MIN_PREEMPT_PERIOD; int newGrantsPerIteration = DEFAULT_GRANTS_PER_ITERATION; long newStarvingTimeForMinimum = DEFAULT_STARVING_TIME_FOR_MINIMUM; long newStarvingTimeForShare = DEFAULT_STARVING_TIME_FOR_SHARE; long newPreemptedTaskMaxRunningTime = DEFAULT_PREEMPT_TASK_MAX_RUNNING_TIME; int newPreemptionRounds = DEFAULT_PREEMPTION_ROUNDS; boolean newScheduleFromNodeToSession = DEFAULT_SCHEDULE_FROM_NODE_TO_SESSION; newTypeToNodeWait = new EnumMap<ResourceType, Long>(ResourceType.class); newTypeToRackWait = new EnumMap<ResourceType, Long>(ResourceType.class); Map<PoolInfo, PoolInfo> newPoolInfoToRedirect = new HashMap<PoolInfo, PoolInfo>(); for (ResourceType type : TYPES) { newTypeToNodeWait.put(type, 0L); newTypeToRackWait.put(type, 0L); } // All the configuration files for a cluster are placed in one large // json object. This large json object has keys that map to smaller // json objects which hold the same resources as xml configuration // files. Here, we try to parse the json object that corresponds to // corona.xml File jsonConfigFile = new File(configFileName); InputStream in = new BufferedInputStream(new FileInputStream(jsonConfigFile)); JSONObject json = conf.instantiateJsonObject(in); json = json.getJSONObject(conf.xmlToThrift(CoronaConf.DEFAULT_CONFIG_FILE)); Iterator<String> keys = json.keys(); while (keys.hasNext()) { String key = keys.next(); if (!json.isNull(key)) { if (key.equals("localityWaits")) { JSONObject jsonTypes = json.getJSONObject(key); loadLocalityWaits(jsonTypes, newTypeToNodeWait, newTypeToRackWait); } if (key.equals("defaultSchedulingMode")) { newDefaultPoolComparator = ScheduleComparator.valueOf(json.getString(key)); } if (key.equals("shareStarvingRatio")) { newShareStarvingRatio = json.getDouble(key); if (newShareStarvingRatio < 0 || newShareStarvingRatio > 1.0) { LOG.error("Illegal shareStarvingRatio:" + newShareStarvingRatio); newShareStarvingRatio = DEFAULT_SHARE_STARVING_RATIO; } } if (key.equals("grantsPerIteration")) { newGrantsPerIteration = json.getInt(key); if (newMinPreemptPeriod < 0) { LOG.error("Illegal grantsPerIteration: " + newGrantsPerIteration); newGrantsPerIteration = DEFAULT_GRANTS_PER_ITERATION; } } if (key.equals("minPreemptPeriod")) { newMinPreemptPeriod = json.getLong(key); if (newMinPreemptPeriod < 0) { LOG.error("Illegal minPreemptPeriod: " + newMinPreemptPeriod); newMinPreemptPeriod = DEFAULT_MIN_PREEMPT_PERIOD; } } if (key.equals("starvingTimeForShare")) { newStarvingTimeForShare = json.getLong(key); if (newStarvingTimeForShare < 0) { LOG.error("Illegal starvingTimeForShare:" + newStarvingTimeForShare); newStarvingTimeForShare = DEFAULT_STARVING_TIME_FOR_SHARE; } } if (key.equals("starvingTimeForMinimum")) { newStarvingTimeForMinimum = json.getLong(key); if (newStarvingTimeForMinimum < 0) { LOG.error("Illegal starvingTimeForMinimum:" + newStarvingTimeForMinimum); newStarvingTimeForMinimum = DEFAULT_STARVING_TIME_FOR_MINIMUM; } } if (key.equals("preemptedTaskMaxRunningTime")) { newPreemptedTaskMaxRunningTime = json.getLong(key); if (newPreemptedTaskMaxRunningTime < 0) { LOG.error("Illegal preemptedTaskMaxRunningTime:" + newPreemptedTaskMaxRunningTime); newPreemptedTaskMaxRunningTime = DEFAULT_PREEMPT_TASK_MAX_RUNNING_TIME; } } if (key.equals("preemptionRounds")) { newPreemptionRounds = json.getInt(key); if (newPreemptionRounds < 0) { LOG.error("Illegal preemptedTaskMaxRunningTime:" + newPreemptionRounds); newPreemptionRounds = DEFAULT_PREEMPTION_ROUNDS; } } if (key.equals("scheduleFromNodeToSession")) { newScheduleFromNodeToSession = json.getBoolean(key); } if (key.equals(REDIRECT_TAG_NAME)) { JSONArray jsonPoolInfoToRedirect = json.getJSONArray(key); loadPoolInfoToRedirect(jsonPoolInfoToRedirect, newPoolInfoToRedirect); } } } synchronized (this) { this.typeToNodeWait = newTypeToNodeWait; this.typeToRackWait = newTypeToRackWait; this.defaultPoolComparator = newDefaultPoolComparator; this.shareStarvingRatio = newShareStarvingRatio; this.minPreemptPeriod = newMinPreemptPeriod; this.grantsPerIteration = newGrantsPerIteration; this.starvingTimeForMinimum = newStarvingTimeForMinimum; this.starvingTimeForShare = newStarvingTimeForShare; this.preemptedTaskMaxRunningTime = newPreemptedTaskMaxRunningTime; this.preemptionRounds = newPreemptionRounds; this.scheduleFromNodeToSession = newScheduleFromNodeToSession; this.poolInfoToRedirect = newPoolInfoToRedirect; } } /** * Reload the general configuration and update all in-memory values. Should * be invoked under synchronization. * * @throws IOException * @throws SAXException * @throws ParserConfigurationException */ private void reloadConfig() throws IOException, SAXException, ParserConfigurationException, JSONException { // Loading corona configuration as JSON. if (configFileName != null && configFileName.endsWith(Configuration.MATERIALIZEDJSON)) { reloadJsonConfig(); return; } // Loading corona configuration as XML. XML configurations are // deprecated. We intend to remove all XML configurations and // transition entirely into JSON. Map<ResourceType, Long> newTypeToNodeWait; Map<ResourceType, Long> newTypeToRackWait; ScheduleComparator newDefaultPoolComparator = DEFAULT_POOL_COMPARATOR; double newShareStarvingRatio = DEFAULT_SHARE_STARVING_RATIO; long newMinPreemptPeriod = DEFAULT_MIN_PREEMPT_PERIOD; int newGrantsPerIteration = DEFAULT_GRANTS_PER_ITERATION; long newStarvingTimeForMinimum = DEFAULT_STARVING_TIME_FOR_MINIMUM; long newStarvingTimeForShare = DEFAULT_STARVING_TIME_FOR_SHARE; long newPreemptedTaskMaxRunningTime = DEFAULT_PREEMPT_TASK_MAX_RUNNING_TIME; int newPreemptionRounds = DEFAULT_PREEMPTION_ROUNDS; boolean newScheduleFromNodeToSession = DEFAULT_SCHEDULE_FROM_NODE_TO_SESSION; newTypeToNodeWait = new EnumMap<ResourceType, Long>(ResourceType.class); newTypeToRackWait = new EnumMap<ResourceType, Long>(ResourceType.class); Map<PoolInfo, PoolInfo> newPoolInfoToRedirect = new HashMap<PoolInfo, PoolInfo>(); for (ResourceType type : TYPES) { newTypeToNodeWait.put(type, 0L); newTypeToRackWait.put(type, 0L); } Element root = getRootElement(configFileName); NodeList elements = root.getChildNodes(); for (int i = 0; i < elements.getLength(); ++i) { Node node = elements.item(i); if (!(node instanceof Element)) { continue; } Element element = (Element) node; for (ResourceType type : TYPES) { if (matched(element, "nodeLocalityWait" + type)) { long val = Long.parseLong(getText(element)); newTypeToNodeWait.put(type, val); } if (matched(element, "rackLocalityWait" + type)) { long val = Long.parseLong(getText(element)); newTypeToRackWait.put(type, val); } } if (matched(element, "defaultSchedulingMode")) { newDefaultPoolComparator = ScheduleComparator.valueOf(getText(element)); } if (matched(element, "shareStarvingRatio")) { newShareStarvingRatio = Double.parseDouble(getText(element)); if (newShareStarvingRatio < 0 || newShareStarvingRatio > 1.0) { LOG.error("Illegal shareStarvingRatio:" + newShareStarvingRatio); newShareStarvingRatio = DEFAULT_SHARE_STARVING_RATIO; } } if (matched(element, "grantsPerIteration")) { newGrantsPerIteration = Integer.parseInt(getText(element)); if (newMinPreemptPeriod < 0) { LOG.error("Illegal grantsPerIteration: " + newGrantsPerIteration); newGrantsPerIteration = DEFAULT_GRANTS_PER_ITERATION; } } if (matched(element, "minPreemptPeriod")) { newMinPreemptPeriod = Long.parseLong(getText(element)); if (newMinPreemptPeriod < 0) { LOG.error("Illegal minPreemptPeriod: " + newMinPreemptPeriod); newMinPreemptPeriod = DEFAULT_MIN_PREEMPT_PERIOD; } } if (matched(element, "starvingTimeForShare")) { newStarvingTimeForShare = Long.parseLong(getText(element)); if (newStarvingTimeForShare < 0) { LOG.error("Illegal starvingTimeForShare:" + newStarvingTimeForShare); newStarvingTimeForShare = DEFAULT_STARVING_TIME_FOR_SHARE; } } if (matched(element, "starvingTimeForMinimum")) { newStarvingTimeForMinimum = Long.parseLong(getText(element)); if (newStarvingTimeForMinimum < 0) { LOG.error("Illegal starvingTimeForMinimum:" + newStarvingTimeForMinimum); newStarvingTimeForMinimum = DEFAULT_STARVING_TIME_FOR_MINIMUM; } } if (matched(element, "preemptedTaskMaxRunningTime")) { newPreemptedTaskMaxRunningTime = Long.parseLong(getText(element)); if (newPreemptedTaskMaxRunningTime < 0) { LOG.error("Illegal preemptedTaskMaxRunningTime:" + newPreemptedTaskMaxRunningTime); newPreemptedTaskMaxRunningTime = DEFAULT_PREEMPT_TASK_MAX_RUNNING_TIME; } } if (matched(element, "preemptionRounds")) { newPreemptionRounds = Integer.parseInt(getText(element)); if (newPreemptionRounds < 0) { LOG.error("Illegal preemptedTaskMaxRunningTime:" + newPreemptionRounds); newPreemptionRounds = DEFAULT_PREEMPTION_ROUNDS; } } if (matched(element, "scheduleFromNodeToSession")) { newScheduleFromNodeToSession = Boolean.parseBoolean(getText(element)); } if (matched(element, REDIRECT_TAG_NAME)) { loadPoolInfoToRedirect(element.getAttribute(SOURCE_ATTRIBUTE), element.getAttribute(DESTINATION_ATTRIBUTE), newPoolInfoToRedirect); } } synchronized (this) { this.typeToNodeWait = newTypeToNodeWait; this.typeToRackWait = newTypeToRackWait; this.defaultPoolComparator = newDefaultPoolComparator; this.shareStarvingRatio = newShareStarvingRatio; this.minPreemptPeriod = newMinPreemptPeriod; this.grantsPerIteration = newGrantsPerIteration; this.starvingTimeForMinimum = newStarvingTimeForMinimum; this.starvingTimeForShare = newStarvingTimeForShare; this.preemptedTaskMaxRunningTime = newPreemptedTaskMaxRunningTime; this.preemptionRounds = newPreemptionRounds; this.scheduleFromNodeToSession = newScheduleFromNodeToSession; this.poolInfoToRedirect = newPoolInfoToRedirect; } } /** * Reload the pools config and update all in-memory values * @throws ParserConfigurationException * @throws SAXException * @throws IOException */ private void reloadPoolsConfig() throws IOException, SAXException, ParserConfigurationException { if (poolsConfigFileName == null) { return; } Set<String> newPoolGroupNameSet = new HashSet<String>(); Set<PoolInfo> newPoolInfoSet = new HashSet<PoolInfo>(); TypePoolGroupNameMap<Integer> newTypePoolNameGroupToMax = new TypePoolGroupNameMap<Integer>(); TypePoolGroupNameMap<Integer> newTypePoolNameGroupToMin = new TypePoolGroupNameMap<Integer>(); TypePoolInfoMap<Integer> newTypePoolInfoToMax = new TypePoolInfoMap<Integer>(); TypePoolInfoMap<Integer> newTypePoolInfoToMin = new TypePoolInfoMap<Integer>(); Map<PoolInfo, ScheduleComparator> newPoolInfoToComparator = new HashMap<PoolInfo, ScheduleComparator>(); Map<PoolInfo, Double> newPoolInfoToWeight = new HashMap<PoolInfo, Double>(); Map<PoolInfo, Integer> newPoolInfoToPriority = new HashMap<PoolInfo, Integer>(); Map<PoolInfo, String> newPoolInfoToWhitelist = new HashMap<PoolInfo, String>(); Map<PoolInfo, PoolInfo> newJobExceedsLimitPoolRedirect = new HashMap<PoolInfo, PoolInfo>(); Map<PoolInfo, Long> newPoolJobSizeLimit = new HashMap<PoolInfo, Long>(); Element root = getRootElement(poolsConfigFileName); NodeList elements = root.getChildNodes(); for (int i = 0; i < elements.getLength(); ++i) { Node node = elements.item(i); if (!(node instanceof Element)) { continue; } Element element = (Element) node; if (matched(element, GROUP_TAG_NAME)) { String groupName = element.getAttribute(NAME_ATTRIBUTE); if (!newPoolGroupNameSet.add(groupName)) { LOG.debug("Already added group " + groupName); } NodeList groupFields = element.getChildNodes(); for (int j = 0; j < groupFields.getLength(); ++j) { Node groupNode = groupFields.item(j); if (!(groupNode instanceof Element)) { continue; } Element field = (Element) groupNode; for (ResourceType type : TYPES) { if (matched(field, MIN_TAG_NAME_PREFIX + type)) { int val = Integer.parseInt(getText(field)); newTypePoolNameGroupToMin.put(type, groupName, val); } if (matched(field, MAX_TAG_NAME_PREFIX + type)) { int val = Integer.parseInt(getText(field)); newTypePoolNameGroupToMax.put(type, groupName, val); } } if (matched(field, REDIRECT_JOB_WITH_LIMIT)) { loadPoolInfoToRedirect(groupName, field.getAttribute(SOURCE_ATTRIBUTE), field.getAttribute(DESTINATION_ATTRIBUTE), field.getAttribute(JOB_INPUT_SIZE_LIMIT_ATTRIBUTE), newJobExceedsLimitPoolRedirect, newPoolJobSizeLimit); } if (matched(field, POOL_TAG_NAME)) { PoolInfo poolInfo = new PoolInfo(groupName, field.getAttribute("name")); if (!newPoolInfoSet.add(poolInfo)) { LOG.warn("Already added pool info " + poolInfo); } NodeList poolFields = field.getChildNodes(); for (int k = 0; k < poolFields.getLength(); ++k) { Node poolNode = poolFields.item(k); if (!(poolNode instanceof Element)) { continue; } Element poolField = (Element) poolNode; for (ResourceType type : TYPES) { if (matched(poolField, MIN_TAG_NAME_PREFIX + type)) { int val = Integer.parseInt(getText(poolField)); newTypePoolInfoToMin.put(type, poolInfo, val); } if (matched(poolField, MAX_TAG_NAME_PREFIX + type)) { int val = Integer.parseInt(getText(poolField)); newTypePoolInfoToMax.put(type, poolInfo, val); } } if (matched(poolField, PREEMPTABILITY_MODE_TAG_NAME)) { boolean val = Boolean.parseBoolean(getText(poolField)); if (!val) { nonPreemptablePools.add(poolInfo); } } if (matched(poolField, REQUEST_MAX_MODE_TAG_NAME)) { boolean val = Boolean.parseBoolean(getText(poolField)); if (val) { requestMaxPools.add(poolInfo); } } if (matched(poolField, SCHEDULING_MODE_TAG_NAME)) { ScheduleComparator val = ScheduleComparator.valueOf(getText(poolField)); newPoolInfoToComparator.put(poolInfo, val); } if (matched(poolField, WEIGHT_TAG_NAME)) { double val = Double.parseDouble(getText(poolField)); newPoolInfoToWeight.put(poolInfo, val); } if (matched(poolField, PRIORITY_TAG_NAME)) { int val = Integer.parseInt(getText(poolField)); newPoolInfoToPriority.put(poolInfo, val); } if (matched(poolField, WHITELIST_TAG_NAME)) { newPoolInfoToWhitelist.put(poolInfo, getText(poolField)); } } } } } } synchronized (this) { this.poolGroupNameSet = newPoolGroupNameSet; this.poolInfoSet = Collections.unmodifiableSet(newPoolInfoSet); this.typePoolGroupNameToMax = newTypePoolNameGroupToMax; this.typePoolGroupNameToMin = newTypePoolNameGroupToMin; this.typePoolInfoToMax = newTypePoolInfoToMax; this.typePoolInfoToMin = newTypePoolInfoToMin; this.poolInfoToComparator = newPoolInfoToComparator; this.poolInfoToWeight = newPoolInfoToWeight; this.poolInfoToPriority = newPoolInfoToPriority; this.poolInfoToWhitelist = newPoolInfoToWhitelist; this.jobExceedsLimitPoolRedirect = newJobExceedsLimitPoolRedirect; this.poolJobSizeLimit = newPoolJobSizeLimit; } } /** * Reload all the configuration files if the config changed and * set the last successful reload time. Synchronized due to potential * conflict from a fetch pools config http request. * * @return true if the config was reloaded, false otherwise * @throws IOException * @throws SAXException * @throws ParserConfigurationException * @param init true when the config manager is being initialized. * false on reloads */ public synchronized boolean reloadAllConfig(boolean init) throws IOException, SAXException, ParserConfigurationException, JSONException { if (!isConfigChanged(init)) { return false; } reloadConfig(); reloadPoolsConfig(); this.lastSuccessfulReload = ClusterManager.clock.getTime(); return true; } /** * Check if the config files have changed since they were last read * @return true if the modification time of the file is greater * than that of the last successful reload, false otherwise * @param init true when the config manager is being initialized. * false on reloads */ private boolean isConfigChanged(boolean init) throws IOException { if (init && (configFileName == null || (poolsConfigFileName == null && conf.onlyAllowConfiguredPools()))) { throw new IOException("ClusterManager needs a config and a " + "pools file to start"); } if (configFileName == null && poolsConfigFileName == null) { return false; } boolean configChanged = false; if (configFileName != null) { File file = new File(configFileName); configChanged |= (file.lastModified() == 0 || file.lastModified() > lastSuccessfulReload); } if (poolsConfigFileName != null) { File file = new File(poolsConfigFileName); configChanged |= (file.lastModified() == 0 || file.lastModified() > lastSuccessfulReload); } return configChanged; } /** * Get the root element of the XML document * @return the root element of the XML document * @throws IOException * @throws SAXException * @throws ParserConfigurationException */ private Element getRootElement(String fileName) throws IOException, SAXException, ParserConfigurationException { DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = builder.parse(new File(fileName)); Element root = doc.getDocumentElement(); if (!matched(root, CONFIGURATION_TAG_NAME)) { throw new IOException("Bad " + fileName); } return root; } /** * Check if the element name matches the tagName provided * @param element the xml element * @param tagName the name to check against * @return true if the name of the element matches tagName, false otherwise */ private static boolean matched(Element element, String tagName) { return tagName.equals(element.getTagName()); } /** * Get the text inside of the Xml element * @param element xml element * @return the text inside of the xml element */ private static String getText(Element element) { if (element.getFirstChild() == null) { return ""; } return ((Text) element.getFirstChild()).getData().trim(); } }