Java tutorial
/******************************************************************************* * =========================================================== * Ankush : Big Data Cluster Management Solution * =========================================================== * * (C) Copyright 2014, by Impetus Technologies * * This is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License (LGPL v3) as * published by the Free Software Foundation; * * This software is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See the GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this software; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ******************************************************************************/ package com.impetus.ankush2.hadoop.utils; import java.io.File; import java.io.FileInputStream; import java.io.InputStream; import java.text.DateFormat; import java.text.DecimalFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TimeZone; import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import net.neoremind.sshxcute.core.Result; import net.neoremind.sshxcute.core.SSHExec; import net.neoremind.sshxcute.task.CustomTask; import net.neoremind.sshxcute.task.impl.ExecCommand; import org.apache.commons.io.FilenameUtils; import org.jdom.Element; import org.jdom.input.SAXBuilder; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import com.impetus.ankush.AppStore; import com.impetus.ankush.AppStoreWrapper; import com.impetus.ankush.common.exception.AnkushException; import com.impetus.ankush.common.exception.RegisterClusterException; import com.impetus.ankush.common.scripting.AnkushTask; import com.impetus.ankush.common.service.ConfigurationManager; import com.impetus.ankush.common.utils.AnkushRestClient; import com.impetus.ankush.common.utils.JsonMapperUtil; import com.impetus.ankush.common.utils.ReflectionUtil; import com.impetus.ankush.common.utils.XmlUtil; import com.impetus.ankush2.common.scripting.impl.AddConfProperty; import com.impetus.ankush2.constant.Constant; import com.impetus.ankush2.framework.config.ClusterConfig; import com.impetus.ankush2.framework.config.ComponentConfig; import com.impetus.ankush2.framework.config.NodeConfig; import com.impetus.ankush2.hadoop.config.CmpConfigMapping; import com.impetus.ankush2.hadoop.config.CmpConfigMappingSet; import com.impetus.ankush2.hadoop.config.ComponentConfigContext; import com.impetus.ankush2.hadoop.deployer.configurator.Hadoop1Configurator; import com.impetus.ankush2.hadoop.deployer.configurator.Hadoop2Configurator; import com.impetus.ankush2.hadoop.deployer.configurator.HadoopConfigurator; import com.impetus.ankush2.hadoop.deployer.installer.HadoopInstaller; import com.impetus.ankush2.hadoop.deployer.servicemanager.HadoopServiceManager; import com.impetus.ankush2.hadoop.monitor.Hadoop2Monitor; import com.impetus.ankush2.hadoop.monitor.HadoopDFSManager; import com.impetus.ankush2.hadoop.monitor.HadoopMonitor; import com.impetus.ankush2.hadoop.monitor.commandsmanager.HadoopCommandsManager; import com.impetus.ankush2.hadoop.utils.HadoopConstants.ConfigXmlKeys.Attributes; import com.impetus.ankush2.logger.AnkushLogger; // TODO: Auto-generated Javadoc /** * The Class HadoopUtils. * * @author Akhil */ public class HadoopUtils { /** The log. */ private static AnkushLogger LOG = new AnkushLogger(HadoopUtils.class); /** The conf manager. */ private final static ConfigurationManager confManager = new ConfigurationManager(); /** The Constant TOPOLOGY_DATA_FILE. */ public static final String TOPOLOGY_DATA_FILE = AppStoreWrapper.getAnkushConfReader() .getStringValue("hadoop.conf.topology.data"); /** The Constant JMX_BEAN_NAME_JAVA_RUNTIME. */ public static final String JMX_BEAN_NAME_JAVA_RUNTIME = "java.lang:type=Runtime"; /** The Constant KEY_HADOOP_HOME. */ public static final String KEY_HADOOP_HOME = "HADOOP_HOME"; /** The Constant KEY_HADOOP_PREFIX. */ public static final String KEY_HADOOP_PREFIX = "HADOOP_PREFIX"; /** The Constant KEY_HADOOP_OPTS. */ public static final String KEY_HADOOP_OPTS = "HADOOP_OPTS"; /** The Constant KEY_HADOOP_CONF_DIR. */ public static final String KEY_HADOOP_CONF_DIR = "HADOOP_CONF_DIR"; /** The Constant KEY_PATH_VARIABLE. */ public static final String KEY_PATH_VARIABLE = "PATH"; /** * Adds the and log error. * * @param logger * the logger * @param clusterConfig * the cluster config * @param errMsg * the err msg * @param componentName * the component name * @param host * the host */ public static void addAndLogError(AnkushLogger logger, ClusterConfig clusterConfig, String errMsg, String componentName, String host) { logger.error(errMsg, componentName, host); clusterConfig.addError(host, componentName, errMsg); } /** * Adds the and log error. * * @param logger * the logger * @param clusterConfig * the cluster config * @param errMsg * the err msg * @param componentName * the component name * @param host * the host * @param t * the t */ public static void addAndLogError(AnkushLogger logger, ClusterConfig clusterConfig, String errMsg, String componentName, String host, Throwable t) { logger.error(errMsg, componentName, host, t); clusterConfig.addError(host, componentName, errMsg); } /** * Adds the and log error. * * @param logger * the logger * @param clusterConfig * the cluster config * @param errMsg * the err msg * @param componentName * the component name */ public static void addAndLogError(AnkushLogger logger, ClusterConfig clusterConfig, String errMsg, String componentName) { logger.error(errMsg, componentName); clusterConfig.addError(componentName, errMsg); } /** * Adds the and log error. * * @param logger * the logger * @param clusterConfig * the cluster config * @param errMsg * the err msg * @param componentName * the component name * @param t * the t */ public static void addAndLogError(AnkushLogger logger, ClusterConfig clusterConfig, String errMsg, String componentName, Throwable t) { logger.error(errMsg, componentName, t); clusterConfig.addError(componentName, errMsg); } public static void addAndLogWarning(AnkushLogger logger, ClusterConfig clusterConfig, String warnMsg, String componentName, Throwable t) { logger.warn(warnMsg, componentName, t); clusterConfig.addError(componentName, warnMsg); } /** The Constant TOPOLOGY_SCRIPT_FILE. */ private static final String TOPOLOGY_SCRIPT_FILE = AppStoreWrapper.getAnkushConfReader() .getStringValue("hadoop.conf.topology.script"); /** The Constant TOPOLOGY_SCRIPT_FILE_RELPATH_RESOURCES. */ private static final String TOPOLOGY_SCRIPT_FILE_RELPATH_RESOURCES = AppStoreWrapper.getAnkushConfReader() .getStringValue("hadoop.conf.topology.script.relpath"); /** The Constant VERSION_CDH3. */ public static final String VERSION_CDH3 = "0.20.2-cdh3u1"; /** The Constant VERSION_CDH4.4.0. */ public static final String VERSION_CDH4_4_0 = "2.0.0-cdh4.4.0"; /** The Constant DEFAULT_NODE_IP. */ public static final String DEFAULT_NODE_IP = "0.0.0.0"; /** * Adds the property to xml file. * * @param logger * the logger * @param clusterConfig * the cluster config * @param nodeConfig * the node config * @param filePath * the file path * @param propertyName * the property name * @param propertyValue * the property value * @return true, if successful */ public static boolean addPropertyToXmlFile(AnkushLogger logger, ClusterConfig clusterConfig, NodeConfig nodeConfig, String filePath, String propertyName, String propertyValue) { try { // Configuration manager to save the property file change records. ConfigurationManager confManager = new ConfigurationManager(); Result res = null; AnkushTask addProperty = new AddConfProperty(propertyName, propertyValue, filePath, Constant.File_Extension.XML, clusterConfig.getAgentInstallDir()); res = nodeConfig.getConnection().exec(addProperty); if (!res.isSuccess) { String errMsg = "Could not add " + propertyName + " to " + filePath; HadoopUtils.addAndLogError(logger, clusterConfig, errMsg, Constant.Component.Name.HADOOP, nodeConfig.getHost()); return false; } // saving the Add Event for Audit Trail confManager.saveConfiguration(clusterConfig.getClusterId(), clusterConfig.getCreatedBy(), FilenameUtils.getName(filePath), nodeConfig.getHost(), propertyName, propertyValue); } catch (Exception e) { String errMsg = "Could not add " + propertyName + " to " + filePath; LOG.error(errMsg, Constant.Component.Name.HADOOP, nodeConfig.getHost(), e); HadoopUtils.addAndLogError(logger, clusterConfig, errMsg, Constant.Component.Name.HADOOP, nodeConfig.getHost(), e); return false; } return true; } /** * Gets the gmt from time in millis. * * @param val * the val * @return the gmt from time in millis */ public static String getGmtFromTimeInMillis(String val) { long time = Long.parseLong(val); DateFormat gmtFormat = new SimpleDateFormat("E, dd-MMM-yyyy hh:mm:ss"); TimeZone gmtTime = TimeZone.getTimeZone("GMT"); gmtFormat.setTimeZone(gmtTime); return (gmtFormat.format(new Date(time)) + " GMT"); } /** * Gets the gmt from time in millis. * * @param time * the time * @return the gmt from time in millis */ public static String getGmtFromTimeInMillis(long time) { DateFormat gmtFormat = new SimpleDateFormat("E, dd-MMM-yyyy hh:mm:ss"); TimeZone gmtTime = TimeZone.getTimeZone("GMT"); gmtFormat.setTimeZone(gmtTime); return (gmtFormat.format(new Date(time)) + " GMT"); } /** * Convert millis to time. * * @param val * the val * @return the string */ public static String convertMillisToTime(String val) { long milliseconds = Long.parseLong(val); long seconds, minutes, hours; seconds = milliseconds / 1000; minutes = seconds / 60; seconds = seconds % 60; hours = minutes / 60; minutes = minutes % 60; if (hours > 0) { return (hours + "hrs, " + minutes + "mins, " + seconds + "secs"); } else if (minutes > 0) { return (minutes + "mins, " + seconds + "secs"); } else { return (seconds + "secs"); } } /** * Gets the job id from app id. * * @param appId * the app id * @return the job id from app id */ public static String getJobIdFromAppId(String appId) { return appId.replaceAll("application", "job"); } /** * Gets the hadoop config. * * @param clusterConfig * the cluster config * @return the hadoop config */ public static ComponentConfig getHadoopConfig(ClusterConfig clusterConfig) { ComponentConfig hadoopConfig = clusterConfig.getComponents().get(Constant.Component.Name.HADOOP); return hadoopConfig; } public static boolean isMonitoredByAnkush(ComponentConfig hadoopConfig) { if (hadoopConfig.isRegister()) { HadoopConstants.RegisterLevel registerLevel = HadoopConstants.RegisterLevel.valueOf( (String) hadoopConfig.getAdvanceConf().get(HadoopConstants.AdvanceConfKeys.REGISTER_LEVEL)); if (registerLevel != null) { if (registerLevel.equals(HadoopConstants.RegisterLevel.LEVEL3) || registerLevel.equals(HadoopConstants.RegisterLevel.LEVEL2)) { return true; } else { return false; } } } return true; } public static boolean isManagedByAnkush(ComponentConfig hadoopConfig) { if (hadoopConfig.isRegister()) { HadoopConstants.RegisterLevel registerLevel = HadoopConstants.RegisterLevel.valueOf( (String) hadoopConfig.getAdvanceConf().get(HadoopConstants.AdvanceConfKeys.REGISTER_LEVEL)); if (registerLevel != null) { if (registerLevel.equals(HadoopConstants.RegisterLevel.LEVEL3)) { return true; } else { return false; } } } return true; } /** * Setup passwordless ssh. * * @param logger * the logger * @param clusterConfig * the cluster config * @param sourceHost * the source host * @param destinationHosts * the destination hosts * @return true, if successful */ public static boolean setupPasswordlessSSH(AnkushLogger logger, ClusterConfig clusterConfig, String sourceHost, Set<String> destinationHosts) { try { NodeConfig sourceNodeConfig = clusterConfig.getNodes().get(sourceHost); String sourceHostPublicKey = com.impetus.ankush2.utils.SSHUtils.getFileContents("~/.ssh/id_rsa.pub", sourceNodeConfig); if (sourceHostPublicKey == null) { HadoopUtils.addAndLogError(logger, clusterConfig, "Could not read ~/.ssh/id_rsa.pub file", Constant.Component.Name.HADOOP, sourceHost); return false; } // add host entry in know host. for (String destinationHost : destinationHosts) { String cmdAddToKnowHost = "ssh -o ConnectTimeout=15 -o ConnectionAttempts=5 -o StrictHostKeyChecking=no " + destinationHost + " ls"; CustomTask task = new ExecCommand(cmdAddToKnowHost); sourceNodeConfig.getConnection().exec(task); StringBuilder appendKeysCmd = new StringBuilder(); appendKeysCmd.append("echo \"").append(sourceHostPublicKey).append("\" >> ~/.ssh/authorized_keys"); String changePermissionCmd = "chmod 0600 ~/.ssh/authorized_keys"; task = new ExecCommand(appendKeysCmd.toString(), changePermissionCmd); if (clusterConfig.getNodes().get(destinationHost).getConnection().exec(task).rc != 0) { HadoopUtils.addAndLogError(logger, clusterConfig, "Could not update ~/.ssh/authorized_keys file for source " + sourceHost, Constant.Component.Name.HADOOP, destinationHost); return false; } } } catch (Exception e) { HadoopUtils.addAndLogError(logger, clusterConfig, "Could not configure passwordless SSH " + sourceHost, Constant.Component.Name.HADOOP, sourceHost, e); return false; } return true; } public static boolean setupPasswordlessSSH(AnkushLogger logger, ClusterConfig clusterConfig, String sourceHost, Set<String> destinationHosts, SSHExec sourceConnection) { try { String sourceHostPublicKey = com.impetus.ankush2.utils.SSHUtils.getFileContents("~/.ssh/id_rsa.pub", sourceConnection); if (sourceHostPublicKey == null) { HadoopUtils.addAndLogError(logger, clusterConfig, "Could not read ~/.ssh/id_rsa.pub file", Constant.Component.Name.HADOOP, sourceHost); return false; } // add host entry in know host. for (String destinationHost : destinationHosts) { String cmdAddToKnowHost = "ssh -o ConnectTimeout=15 -o ConnectionAttempts=5 -o StrictHostKeyChecking=no " + destinationHost + " ls"; CustomTask task = new ExecCommand(cmdAddToKnowHost); sourceConnection.exec(task); task = new ExecCommand("echo \"" + sourceHostPublicKey + "\" >> ~/.ssh/authorized_keys", "chmod 0600 ~/.ssh/authorized_keys"); if (clusterConfig.getNodes().get(destinationHost).getConnection().exec(task).rc != 0) { HadoopUtils.addAndLogError(logger, clusterConfig, "Could not update ~/.ssh/authorized_keys file for source " + sourceHost, Constant.Component.Name.HADOOP, destinationHost); return false; } } } catch (Exception e) { HadoopUtils.addAndLogError(logger, clusterConfig, "Could not configure passwordless SSH " + sourceHost, Constant.Component.Name.HADOOP, sourceHost, e); return false; } return true; } public static String getHaNameNodeId1(ComponentConfig hadoopConfig) { return (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_NAMENODEID1); } public static String getHaNameNodeId2(ComponentConfig hadoopConfig) { return (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_NAMENODEID2); } /** * Gets the name node host. * * @param hadoopConfig * the hadoop config * @return the name node host */ public static String getNameNodeHost(ComponentConfig hadoopConfig) { String namenodeHost = (String) hadoopConfig .getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.NAMENODE); if (HadoopUtils.isHadoop2Config(hadoopConfig)) { boolean isHaEnabled = (Boolean) hadoopConfig .getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_ENABLED); if (isHaEnabled) { namenodeHost = (String) hadoopConfig .getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_ACTIVE_NAMENODE); } } return namenodeHost; } public static String getHdfsUri(ComponentConfig hadoopConfig) { StringBuilder hdfsUri = new StringBuilder(HadoopConfigurator.HADOOP_URI_PREFIX); if (HadoopUtils.isHdfsHaEnabled(hadoopConfig)) { hdfsUri.append( (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_NAMESERVICEID)); return hdfsUri.toString(); } hdfsUri.append(HadoopUtils.getNameNodeHost(hadoopConfig)).append(":") .append(HadoopUtils.getNameNodeRpcPort(hadoopConfig)); return hdfsUri.toString(); } public static boolean isHdfsHaEnabled(ComponentConfig hadoopConfig) { if (HadoopUtils.isHadoop2Config(hadoopConfig)) { return (Boolean) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_ENABLED); } else { return false; } } /** * Gets the ha name node hosts. * * @param hadoopConfig * the hadoop config * @return the ha name node hosts */ public static Set<String> getHaNameNodeHosts(ComponentConfig hadoopConfig) { if (hadoopConfig.getAdvanceConf().containsKey(HadoopConstants.AdvanceConfKeys.HA_NAMENODE_HOSTS)) { return (Set<String>) hadoopConfig .getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_NAMENODE_HOSTS); } else { Set<String> haNameNodeHosts = new HashSet<String>(); haNameNodeHosts.add((String) hadoopConfig .getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_ACTIVE_NAMENODE)); haNameNodeHosts.add((String) hadoopConfig .getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_STANDBY_NAMENODE)); hadoopConfig.addAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_NAMENODE_HOSTS, haNameNodeHosts); return haNameNodeHosts; } } /** * Gets the slave hosts. * * @param hadoopConfig * the hadoop config * @return the slave hosts */ public static Set<String> getSlaveHosts(ComponentConfig hadoopConfig) { return (Set<String>) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.SLAVES); } /** * Gets the journal node hosts. * * @param hadoopConfig * the hadoop config * @return the journal node hosts */ public static Set<String> getJournalNodeHosts(ComponentConfig hadoopConfig) { return (Set<String>) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_JOURNALNODES); } /** * Gets the secondary name node host. * * @param hadoopConfig * the hadoop config * @return the secondary name node host */ public static String getSecondaryNameNodeHost(ComponentConfig hadoopConfig) { return (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.SECONDARY_NAMENODE); } /** * Gets the job tracker host. * * @param hadoopConfig * the hadoop config * @return the job tracker host */ public static String getJobTrackerHost(ComponentConfig hadoopConfig) { return (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.JOBTRACKER); } /** * Gets the resource manager host. * * @param hadoopConfig * the hadoop config * @return the resource manager host */ public static String getResourceManagerHost(ComponentConfig hadoopConfig) { return (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.RESOURCE_MANAGER); } /** * Gets the resource manager http port. * * @param hadoopConfig * the hadoop config * @return the resource manager http port */ public static String getResourceManagerHttpPort(ComponentConfig hadoopConfig) { String rmHttpPort = hadoopConfig .getAdvanceConfStringProperty(HadoopConstants.AdvanceConfKeys.HttpPort.RESOURCEMANAGER); if (rmHttpPort == null) { return Hadoop2Configurator.DEFAULT_PORT_HTTP_RESOURCEMANAGER; } return rmHttpPort; } /** * Gets the name node http port. * * @param hadoopConfig * the hadoop config * @return the name node http port */ public static String getNameNodeHttpPort(ComponentConfig hadoopConfig) { String namenodeHttpPort = hadoopConfig .getAdvanceConfStringProperty(HadoopConstants.AdvanceConfKeys.HttpPort.NAMENODE); if (namenodeHttpPort == null) { return Hadoop1Configurator.DEFAULT_PORT_HTTP_NAMENODE; } return namenodeHttpPort; } /** * Gets the name node rpc port. * * @param hadoopConfig * the hadoop config * @return the name node rpc port */ public static String getNameNodeRpcPort(ComponentConfig hadoopConfig) { String namenodeHttpPort = hadoopConfig .getAdvanceConfStringProperty(HadoopConstants.AdvanceConfKeys.RpcPort.NAMENODE); if (namenodeHttpPort == null) { if (HadoopUtils.isHadoop2Config(hadoopConfig)) { return Hadoop2Configurator.DEFAULT_PORT_RPC_NAMENODE; } else { return Hadoop1Configurator.DEFAULT_PORT_RPC_NAMENODE; } } return namenodeHttpPort; } /** * Gets the job tracker http port. * * @param hadoopConfig * the hadoop config * @return the job tracker http port */ public static String getJobTrackerHttpPort(ComponentConfig hadoopConfig) { String jobtrackerHttpPort = hadoopConfig .getAdvanceConfStringProperty(HadoopConstants.AdvanceConfKeys.HttpPort.JOBTRACKER); if (jobtrackerHttpPort == null) { return Hadoop1Configurator.DEFAULT_PORT_HTTP_JOBTRACKER; } return jobtrackerHttpPort; } /** * Gets the job tracker rpc port. * * @param hadoopConfig * the hadoop config * @return the job tracker rpc port */ public static String getJobTrackerRpcPort(ComponentConfig hadoopConfig) { String jobtrackerRpcPort = hadoopConfig .getAdvanceConfStringProperty(HadoopConstants.AdvanceConfKeys.RpcPort.JOBTRACKER); if (jobtrackerRpcPort == null) { return Hadoop1Configurator.DEFAULT_PORT_RPC_JOBTRACKER; } return jobtrackerRpcPort; } /** * Gets the job history server host. * * @param hadoopConfig * the hadoop config * @return the job history server host */ public static String getJobHistoryServerHost(ComponentConfig hadoopConfig) { return (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.JOB_HISTORY_SERVER); } /** * Gets the web app proxy server host. * * @param hadoopConfig * the hadoop config * @return the web app proxy server host */ public static String getWebAppProxyServerHost(ComponentConfig hadoopConfig) { return (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.WEB_APP_PROXY_SERVER); } /** * Gets the active name node host. * * @param hadoopConfig * the hadoop config * @return the active name node host */ public static String getActiveNameNodeHost(ComponentConfig hadoopConfig) { return (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_ACTIVE_NAMENODE); } /** * Gets the stand by name node host. * * @param hadoopConfig * the hadoop config * @return the stand by name node host */ public static String getStandByNameNodeHost(ComponentConfig hadoopConfig) { return (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_STANDBY_NAMENODE); } /** * Gets the rpc port name node. * * @param hadoopConfig * the hadoop config * @return the rpc port name node */ public static String getRpcPortNameNode(ComponentConfig hadoopConfig) { return (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.RpcPort.NAMENODE); } /** * Gets the rpc port resource manager. * * @param hadoopConfig * the hadoop config * @return the rpc port resource manager */ public static String getRpcPortResourceManager(ComponentConfig hadoopConfig) { return (String) hadoopConfig .getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.RpcPort.RESOURCEMANAGER); } /** * Gets the rpc port job tracker. * * @param hadoopConfig * the hadoop config * @return the rpc port job tracker */ public static String getRpcPortJobTracker(ComponentConfig hadoopConfig) { return (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.RpcPort.JOBTRACKER); } /** * Gets the jmx bean url. * * @param nodeIp * the node ip * @param clientPort * the client port * @param beanName * the bean name * @return the jmx bean url */ public static String getJmxBeanUrl(String nodeIp, String clientPort, String beanName) { return ("http://" + nodeIp + ":" + clientPort + "/jmx?qry=" + beanName); } /** * Gets the bean json for jmx data. * * @param urlDfsJmxBean * the url dfs jmx bean * @return the bean json for jmx data */ public JSONObject getBeanJsonForJmxData(String urlDfsJmxBean) { String errMsg = "Exception: Unable to get data using url : " + urlDfsJmxBean; JSONObject json = null; try { AnkushRestClient restClient = new AnkushRestClient(); String data = restClient.getRequest(urlDfsJmxBean); if (data == null) { LOG.error(errMsg); } else { json = (JSONObject) new JSONParser().parse(data); } } catch (Exception e) { LOG.error(errMsg); LOG.error(e.getMessage()); } LOG.info("json " + json.toJSONString()); return json; } /** * Gets the jmx bean using callable. * * @param host * the host * @param clientPort * the client port * @param beanName * the bean name * @return the jmx bean using callable * @throws AnkushException * the ankush exception */ public static Map<String, Object> getJmxBeanUsingCallable(String host, String clientPort, String beanName) throws AnkushException { String errMsg = "Could not get JMX bean data for host-" + host + ", port-" + clientPort + "."; try { long waitTime = AppStoreWrapper.getAnkushConfReader().getLongValue("hadoop.jmxmonitoring.wait.time"); CallableJmxBeanData callableJmxBeanData = new CallableJmxBeanData(host, clientPort, beanName); FutureTask<Map<String, Object>> futureTaskJmxBeanData = new FutureTask<Map<String, Object>>( callableJmxBeanData); AppStoreWrapper.getExecutor().execute(futureTaskJmxBeanData); Map<String, Object> beanObject = futureTaskJmxBeanData.get(waitTime, TimeUnit.MILLISECONDS); if (beanObject == null) { throw new AnkushException(errMsg); } return beanObject; } catch (AnkushException e) { throw e; } catch (Exception e) { e.printStackTrace(); LOG.error(errMsg, Constant.Component.Name.HADOOP, host, e); throw new AnkushException(errMsg); } } /** * Gets the json object using callable. * * @param url * the url * @return the json object using callable * @throws AnkushException * the ankush exception */ public static JSONObject getJsonObjectUsingCallable(String url) throws AnkushException { String errMsg = "Could not get JSON object for URL-" + url + "."; try { long waitTime = AppStoreWrapper.getAnkushConfReader().getLongValue("hadoop.jmxmonitoring.wait.time"); CallableRestJsonData callableRestJsonData = new CallableRestJsonData(url); FutureTask<JSONObject> futureTaskJmxBeanData = new FutureTask<JSONObject>(callableRestJsonData); AppStoreWrapper.getExecutor().execute(futureTaskJmxBeanData); JSONObject beanObject = futureTaskJmxBeanData.get(waitTime, TimeUnit.MILLISECONDS); if (beanObject == null) { throw new AnkushException(errMsg); } return beanObject; } catch (AnkushException e) { throw e; } catch (Exception e) { LOG.error(errMsg, Constant.Component.Name.HADOOP, e); throw new AnkushException(errMsg); } } /** * Gets the jmx bean data. * * @param host * the node ip * @param port * the client port * @param beanName * the bean name * @return the jmx bean data * @throws AnkushException * the ankush exception */ public Map<String, Object> getJmxBeanData(String host, String port, String beanName) throws AnkushException { String urlJmxBean = ""; if (beanName != null && (!beanName.isEmpty())) { urlJmxBean = HadoopUtils.getJmxBeanUrl(host, port, beanName); } else { throw (new AnkushException("Could not fetch JMX data: Invalid Bean Name")); } try { JSONObject jmxDataJson = this.getBeanJsonForJmxData(urlJmxBean); Map<String, Object> beanObject = new HashMap<String, Object>(); if (jmxDataJson != null) { List<JSONObject> beanObjList = (List<JSONObject>) jmxDataJson .get(HadoopConstants.Hadoop.Keys.JMX_DATA_KEY_BEANS); if (beanObjList.size() == 1) { beanObject = (Map<String, Object>) beanObjList.get(0); } else { throw (new AnkushException("Could not fetch JMX data from URL: " + urlJmxBean)); } } return beanObject; } catch (AnkushException e) { throw e; } catch (Exception e) { String errMsg = "Could not fetch JMX data from URL: " + urlJmxBean; LOG.error(errMsg, Constant.Component.Name.HADOOP, host, e); throw (new AnkushException(errMsg)); } } /** * Gets the process start time. * * @param host * the host * @param port * the port * @return the process start time * @throws Exception * the exception */ public static String getProcessStartTime(String host, String port) throws Exception { Map<String, Object> beanJavaRuntime = new HashMap<String, Object>(); beanJavaRuntime = HadoopUtils.getJmxBeanUsingCallable(host, port, HadoopMonitor.JMX_BEAN_NAME_JAVA_RUNTIME); long startTime = ((Number) beanJavaRuntime.get(HadoopConstants.Hadoop.Keys.NameNodeJmxInfo.STARTTIME)) .longValue(); return HadoopUtils.getGmtFromTimeInMillis(startTime); } /** * Gets the process thread count. * * @param host * the host * @param port * the port * @return the process thread count * @throws Exception * the exception */ public static String getProcessThreadCount(String host, String port) throws Exception { Map<String, Object> beanJavaRuntime = new HashMap<String, Object>(); String beanName = HadoopMonitor.JMX_BEAN_NAME_JAVA_THREADING; beanJavaRuntime = HadoopUtils.getJmxBeanUsingCallable(host, port, beanName); long threadCount = ((Number) beanJavaRuntime.get(HadoopConstants.Hadoop.Keys.JMX_DATA_KEY_THREAD_COUNT)) .longValue(); return String.valueOf(threadCount); } /** * Gets the memory used from jmx. * * @param nodeIp * the node ip * @param clientPort * the client port * @param memoryType * the memory type * @return the memory used from jmx * @throws Exception * the exception */ public String getMemoryUsedFromJmx(String nodeIp, String clientPort, ProcessMemoryType memoryType) throws Exception { Map<String, Object> beanJavaMemory = new HashMap<String, Object>(); String beanName = HadoopMonitor.JMX_BEAN_NAME_JAVA_MEMORY; beanJavaMemory = this.getJmxBeanData(nodeIp, clientPort, beanName); String memoryKeyName = ""; switch (memoryType) { case HEAP: memoryKeyName = HadoopConstants.Hadoop.Keys.JMX_DATA_KEY_HEAP_MEMORY_USAGE; break; case NONHEAP: memoryKeyName = HadoopConstants.Hadoop.Keys.JMX_DATA_KEY_NON_HEAP_MEMORY_USAGE; break; } JSONObject memoryUsageJson = (JSONObject) beanJavaMemory.get(memoryKeyName); long heapUsed = ((Number) memoryUsageJson.get(HadoopConstants.Hadoop.Keys.JMX_DATA_KEY_MEMORY_USAGE_USED)) .longValue(); return HadoopDFSManager.convertBytes(heapUsed); } /** * Gets the memory info from jmx bean. * * @param beanJavaMemory * the bean java memory * @param memoryType * the memory type * @param usageType * the usage type * @return the memory info from jmx bean * @throws Exception * the exception */ public static String getMemoryInfoFromJmxBean(Map<String, Object> beanJavaMemory, ProcessMemoryType memoryType, ProcessMemoryUsageType usageType) throws Exception { String memoryTypeKey = new String(); String usageTypeKey = new String(); switch (memoryType) { case HEAP: memoryTypeKey = HadoopConstants.Hadoop.Keys.JMX_DATA_KEY_HEAP_MEMORY_USAGE; break; case NONHEAP: memoryTypeKey = HadoopConstants.Hadoop.Keys.JMX_DATA_KEY_NON_HEAP_MEMORY_USAGE; break; } switch (usageType) { case INIT: usageTypeKey = HadoopConstants.Hadoop.Keys.JMX_DATA_KEY_MEMORY_USAGE_INIT; break; case USED: usageTypeKey = HadoopConstants.Hadoop.Keys.JMX_DATA_KEY_MEMORY_USAGE_USED; break; case MAX: usageTypeKey = HadoopConstants.Hadoop.Keys.JMX_DATA_KEY_MEMORY_USAGE_MAX; break; case COMMITTED: usageTypeKey = HadoopConstants.Hadoop.Keys.JMX_DATA_KEY_MEMORY_USAGE_COMMITTED; break; } JSONObject memoryUsageJson = (JSONObject) beanJavaMemory.get(memoryTypeKey); long heapUsed = ((Number) memoryUsageJson.get(usageTypeKey)).longValue(); return HadoopDFSManager.convertBytes(heapUsed); } /** * Gets the display name for start time. * * @param role * the role * @return the display name for start time */ public static String getDisplayNameForStartTime(String role) { return role + " Started"; } /** * Gets the json from rm rest api. * * @param rmHost * the rm host * @param rmHttpPort * the rm http port * @param relativeUrl * the relative url * @return the json from rm rest api * @throws AnkushException * the ankush exception */ public static JSONObject getJsonFromRmRestApi(String rmHost, String rmHttpPort, String relativeUrl) throws AnkushException { String url = "http://" + rmHost + ":" + rmHttpPort + Hadoop2Monitor.RELATIVE_URL_RM_REST_API + relativeUrl; LOG.info("Connecting to RM REST API - " + url, Constant.Component.Name.HADOOP); try { return HadoopUtils.getJsonObjectUsingCallable(url); } catch (AnkushException e) { throw e; } catch (Exception e) { String errMsg = "Could not fetch data from RM REST API - " + url; LOG.error(errMsg, Constant.Component.Name.HADOOP, rmHost, e); throw new AnkushException(errMsg); } } /** * Gets the json object from app proxy api. * * @param webAppProxyHost * the web app proxy host * @param port * the port * @param relativeUrl * the relative url * @param appId * the app id * @return the json object from app proxy api * @throws AnkushException * the ankush exception */ public static JSONObject getJsonObjectFromAppProxyApi(String webAppProxyHost, String port, String relativeUrl, String appId) throws AnkushException { String url = "http://" + webAppProxyHost + ":" + port + Hadoop2Monitor.RELATIVE_URL_APPPROXY_REST_API .replace(Hadoop2Monitor.APPPROXY_REST_API_TEXT_APPID, appId) + relativeUrl; LOG.info("Connecting to Web Application ProxyServer REST API - " + url, Constant.Component.Name.HADOOP); try { return HadoopUtils.getJsonObjectUsingCallable(url); } catch (AnkushException e) { throw e; } catch (Exception e) { String errMsg = "Could not fetch data from Web Application ProxyServer REST API - " + url; LOG.error(errMsg, Constant.Component.Name.HADOOP, webAppProxyHost, e); throw new AnkushException(errMsg); } } /** * Gets the json object from jhs api. * * @param jhsHost * the jhs host * @param jhsHttpPort * the jhs http port * @param relativeUrl * the relative url * @return the json object from jhs api * @throws AnkushException * the ankush exception */ public static JSONObject getJsonObjectFromJhsApi(String jhsHost, String jhsHttpPort, String relativeUrl) throws AnkushException { String url = "http://" + jhsHost + ":" + jhsHttpPort + Hadoop2Monitor.RELATIVE_URL_JHS_REST_API + relativeUrl; LOG.info("Connecting to JobHistoryServer REST API - " + url, Constant.Component.Name.HADOOP); try { return HadoopUtils.getJsonObjectUsingCallable(url); } catch (AnkushException e) { throw e; } catch (Exception e) { String errMsg = "Could not fetch data from JobHistoryServer REST API - " + url; LOG.error(errMsg, Constant.Component.Name.HADOOP, jhsHost, e); throw new AnkushException(errMsg); } } public static String getHadoopLogsDir(List<String> runtimeArgumentsList) throws Exception { String propertyName = HadoopConstants.Hadoop.Keys.INPUT_ARG_HADOOP_LOG_DIR; return HadoopUtils.getInputArgumentPropertyValue(runtimeArgumentsList, propertyName); } /** * Gets the hadoop home dir from jmx data. * * @param nodeIp * the node ip * @param clientPort * the client port * @return the hadoop home dir from jmx data * @throws Exception * the exception */ public static String getHadoopHomeDir(List<String> runtimeArgumentsList) throws Exception { String propertyName = HadoopConstants.Hadoop.Keys.INPUT_ARG_HADOOP_HOME_DIR; return HadoopUtils.getInputArgumentPropertyValue(runtimeArgumentsList, propertyName); } /** * Gets the hadoop script dir. * * @param hadoopConfig * the hadoop config * @return the hadoop script dir */ public static String getHadoopScriptDir(ComponentConfig hadoopConfig) { return (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.DIR_SCRIPT); } /** * Gets the hadoop conf dir. * * @param hadoopConfig * the hadoop config * @return the hadoop conf dir */ public static String getHadoopConfDir(ComponentConfig hadoopConfig) { return (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.DIR_CONF); } /** * Gets the hadoop bin dir. * * @param hadoopConfig * the hadoop config * @return the hadoop bin dir */ public static String getHadoopBinDir(ComponentConfig hadoopConfig) { return (String) hadoopConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.DIR_BIN); } /** * Adds the property to file. * * @param clusterConfig * the cluster config * @param nodeConfig * the node config * @param filePath * the file path * @param propertyName * the property name * @param propertyValue * the property value * @return true, if successful */ public static boolean addPropertyToFile(ClusterConfig clusterConfig, NodeConfig nodeConfig, String filePath, String propertyName, String propertyValue) { try { // Configuration manager to save the property file change records. ConfigurationManager confManager = new ConfigurationManager(); Result res = null; AnkushTask addProperty = new AddConfProperty(propertyName, propertyValue, filePath, Constant.File_Extension.XML, clusterConfig.getAgentInstallDir()); res = nodeConfig.getConnection().exec(addProperty); if (!res.isSuccess) { LOG.error("Could not add " + propertyName + " to " + filePath, Constant.Component.Name.HADOOP, nodeConfig.getHost()); return false; } // saving the Add Event for Audit Trail confManager.saveConfiguration(clusterConfig.getClusterId(), clusterConfig.getCreatedBy(), FilenameUtils.getName(filePath), nodeConfig.getHost(), propertyName, propertyValue); } catch (Exception e) { LOG.error("Could not add " + propertyName + " to " + filePath, Constant.Component.Name.HADOOP, nodeConfig.getHost(), e); return false; } return true; } /** * Load hadoop config xml parameters. * * @param filePath * the file path * @param root * the root * @param subItems * the sub items * @return the hadoop config xml set */ private static CmpConfigMappingSet loadHadoopConfigXMLParameters(String filePath, String root, List<String> subItems) { // map of item. CmpConfigMappingSet hadoopConfigXmlSet = new CmpConfigMappingSet(); try { // creating sax builder obj. SAXBuilder builder = new SAXBuilder(); // getting file object. File xml = new File(filePath); // input file stream. InputStream inputStream = new FileInputStream(xml); // jdom document object. org.jdom.Document doc = builder.build(inputStream); // getting root element. Element elements = doc.getRootElement(); // getting child elements. List child = elements.getChildren(root); // iterating over the childs. for (int index = 0; index < child.size(); index++) { // getting element. Element e = (Element) child.get(index); String installationType = e.getAttribute(Attributes.INSTALLATION_TYPE).getValue(); String vendor = e.getAttribute(Attributes.VENDOR).getValue(); String version = e.getAttribute(Attributes.VERSION).getValue(); Map<String, String> classMap = new HashMap<String, String>(); // iterating over the element properties. for (String subItem : subItems) { // getting element values. String value = XmlUtil.getTagContent(e, subItem); // putting element value. classMap.put(subItem, value); } CmpConfigMapping configXmlObject = new CmpConfigMapping(installationType, vendor, version, classMap); // putting map against the attribute value. hadoopConfigXmlSet.add(configXmlObject); } // closing input stream. inputStream.close(); } catch (Exception e) { // printing stack trace. e.printStackTrace(); } // returning items. return hadoopConfigXmlSet; } /** * Sets the hadoop config classes. */ public static void setHadoopConfigClasses() { String classType = "hadoop"; List<String> subItems = new ArrayList<String>(); subItems.add(HadoopConstants.ConfigXmlKeys.ClassType.INSTALLER); subItems.add(HadoopConstants.ConfigXmlKeys.ClassType.CONFIGURATOR); subItems.add(HadoopConstants.ConfigXmlKeys.ClassType.SERVICE_MANAGER); subItems.add(HadoopConstants.ConfigXmlKeys.ClassType.COMMANDS_MANAGER); subItems.add(HadoopConstants.ConfigXmlKeys.ClassType.MONITOR); String filePath = AppStoreWrapper.getConfigClassNameFile(classType); CmpConfigMappingSet hadoopConfigXmlSet = null; if (filePath != null) { try { hadoopConfigXmlSet = HadoopUtils.loadHadoopConfigXMLParameters(filePath, HadoopConstants.ConfigXmlKeys.CLASS, subItems); AppStore.setObject(Constant.AppStore.CompConfigXmlMapping.HADOOP, hadoopConfigXmlSet); } catch (Exception e) { System.err.println(e.getMessage()); } } } /** * Gets the installer instance. * * @param clusterConfig * the cluster config * @param hadoopConfig * the hadoop config * @return the installer instance * @throws ClassNotFoundException * the class not found exception */ public static HadoopInstaller getInstallerInstance(ClusterConfig clusterConfig, ComponentConfig hadoopConfig) throws ClassNotFoundException { return (HadoopInstaller) HadoopUtils.getHadoopClassInstance(clusterConfig, hadoopConfig, HadoopConstants.ConfigXmlKeys.ClassType.INSTALLER); } /** * Gets the commands manager instance. * * @param clusterConfig * the cluster config * @param hadoopConfig * the hadoop config * @return the commands manager instance * @throws ClassNotFoundException * the class not found exception */ public static HadoopCommandsManager getCommandsManagerInstance(ClusterConfig clusterConfig, ComponentConfig hadoopConfig) throws ClassNotFoundException { return (HadoopCommandsManager) HadoopUtils.getHadoopClassInstance(clusterConfig, hadoopConfig, HadoopConstants.ConfigXmlKeys.ClassType.COMMANDS_MANAGER); } /** * Gets the service manager instance. * * @param clusterConfig * the cluster config * @param compConfig * the comp config * @return the service manager instance * @throws ClassNotFoundException * the class not found exception */ public static HadoopServiceManager getServiceManagerInstance(ClusterConfig clusterConfig, ComponentConfig compConfig) throws ClassNotFoundException { return (HadoopServiceManager) HadoopUtils.getHadoopClassInstance(clusterConfig, compConfig, HadoopConstants.ConfigXmlKeys.ClassType.SERVICE_MANAGER); } public static HadoopServiceManager getServiceManagerInstance(ClusterConfig clusterConfig, ComponentConfig compConfig, boolean saveLogsInDb) throws ClassNotFoundException { return (HadoopServiceManager) HadoopUtils.getHadoopClassInstance(clusterConfig, compConfig, HadoopConstants.ConfigXmlKeys.ClassType.SERVICE_MANAGER); } /** * Gets the configurator instance. * * @param clusterConfig * the cluster config * @param compConfig * the comp config * @return the configurator instance * @throws ClassNotFoundException * the class not found exception */ public static HadoopConfigurator getConfiguratorInstance(ClusterConfig clusterConfig, ComponentConfig compConfig) throws ClassNotFoundException { return (HadoopConfigurator) HadoopUtils.getHadoopClassInstance(clusterConfig, compConfig, HadoopConstants.ConfigXmlKeys.ClassType.CONFIGURATOR); } /** * Gets the monitor instance. * * @param clusterConfig * the cluster config * @param compConfig * the comp config * @return the monitor instance * @throws ClassNotFoundException * the class not found exception */ public static HadoopMonitor getMonitorInstance(ClusterConfig clusterConfig, ComponentConfig compConfig) throws ClassNotFoundException { return (HadoopMonitor) HadoopUtils.getHadoopClassInstance(clusterConfig, compConfig, HadoopConstants.ConfigXmlKeys.ClassType.MONITOR, false); } /** * Gets the hadoop class instance. * * @param clusterConfig * the cluster config * @param compConfig * the comp config * @param classType * the class type * @return the hadoop class instance * @throws ClassNotFoundException * the class not found exception */ public static Object getHadoopClassInstance(ClusterConfig clusterConfig, ComponentConfig compConfig, String classType) throws ClassNotFoundException { CmpConfigMappingSet configXmlSet = AppStoreWrapper.getCmpConfigMapping(Constant.Component.Name.HADOOP); String className = configXmlSet.get(compConfig, false).getClassName(classType); ComponentConfigContext hadoopContextObject = (ComponentConfigContext) ReflectionUtil.getObject(className); hadoopContextObject.setClusterConfig(clusterConfig); hadoopContextObject.setComponentConfig(compConfig); hadoopContextObject.setLOG(Class.forName(className)); return hadoopContextObject; } public static Object getHadoopClassInstance(ClusterConfig clusterConfig, ComponentConfig compConfig, String classType, boolean saveLogsInDb) throws ClassNotFoundException { CmpConfigMappingSet configXmlSet = AppStoreWrapper.getCmpConfigMapping(Constant.Component.Name.HADOOP); String className = configXmlSet.get(compConfig, false).getClassName(classType); ComponentConfigContext hadoopContextObject = (ComponentConfigContext) HadoopUtils .getHadoopClassInstance(clusterConfig, compConfig, classType); if (!saveLogsInDb) { AnkushLogger logger = new AnkushLogger(Class.forName(className)); logger.setClusterName(clusterConfig.getName()); hadoopContextObject.setLOG(logger); } return hadoopContextObject; } public static String convertMbIntoGb(long megaBytes) { String convertedVal = "0"; if (megaBytes == 0) { return convertedVal; } final double HUNDRED = 100.0; final long DIVEDEBY = 1024L; DecimalFormat df = new DecimalFormat("0.00"); if (megaBytes / DIVEDEBY > 0) { return (df.format(((megaBytes * HUNDRED / DIVEDEBY)) / HUNDRED) + "GB"); } else { return (String.valueOf(megaBytes) + "MB"); } } /** * Gets the input argument property value. * * @param inputArgumentsList * the input arguments list * @param propertyName * the property name * @return the input argument property value * @throws Exception * the exception */ private static String getInputArgumentPropertyValue(List<String> inputArgumentsList, String propertyName) throws Exception { String propName = "-D" + propertyName + "="; for (String inputArgument : inputArgumentsList) { if (inputArgument.startsWith(propName)) { return (inputArgument.split("="))[1]; } } return null; } /** * Checks if is hadoop2 config. * * @param hadoopConfig * the hadoop config * @return true, if is hadoop2 config */ public static boolean isHadoop2Config(ComponentConfig hadoopConfig) { return (hadoopConfig.getVersion().startsWith("2")); } public static boolean isNameNode(ComponentConfig hadoopConfig, String host) { if (HadoopUtils.isHdfsHaEnabled(hadoopConfig)) { return HadoopUtils.getHaNameNodeHosts(hadoopConfig).contains(host); } return HadoopUtils.getNameNodeHost(hadoopConfig).equals(host); } public static boolean isSecondaryNameNode(ComponentConfig hadoopConfig, String host) { return HadoopUtils.getSecondaryNameNodeHost(hadoopConfig).equals(host); } public static boolean isJobTrackerNode(ComponentConfig hadoopConfig, String host) { return HadoopUtils.getJobTrackerHost(hadoopConfig).equals(host); } public static boolean isResourceManagerNode(ComponentConfig hadoopConfig, String host) { return HadoopUtils.getResourceManagerHost(hadoopConfig).equals(host); } public static boolean isJournalNode(ComponentConfig hadoopConfig, String host) { return HadoopUtils.getJournalNodeHosts(hadoopConfig).contains(host); } public static boolean isJobHistoryServerNode(ComponentConfig hadoopConfig, String host) { return HadoopUtils.getJobHistoryServerHost(hadoopConfig).equals(host); } public static boolean isWebAppProxyServerNode(ComponentConfig hadoopConfig, String host) { return HadoopUtils.getWebAppProxyServerHost(hadoopConfig).equals(host); } public static Set<String> getSlavesListFromJmx(Map<String, Object> beanObjNameNodeInfo) throws RegisterClusterException { try { List<String> nodeTypeList = new ArrayList<String>(); nodeTypeList.add(HadoopConstants.JmxBeanKeys.DfsData.Nodes.LIVE); nodeTypeList.add(HadoopConstants.JmxBeanKeys.DfsData.Nodes.DEAD); nodeTypeList.add(HadoopConstants.JmxBeanKeys.DfsData.Nodes.DECOMMISION); Set<String> slavesList = new HashSet<String>(); for (String nodeType : nodeTypeList) { slavesList.addAll( getHostsFromJsonString(String.valueOf(beanObjNameNodeInfo.get(nodeType)), nodeType)); } return slavesList; } catch (RegisterClusterException e) { throw e; } catch (Exception e) { throw new RegisterClusterException("Could not fetch slaves information from NameNodeInfo bean."); } } private static Set<String> getHostsFromJsonString(String nodesInfoJson, String nodeType) throws RegisterClusterException { JSONObject jsonNodesInfo = JsonMapperUtil.objectFromString(nodesInfoJson, JSONObject.class); if (jsonNodesInfo != null) { return jsonNodesInfo.keySet(); } throw new RegisterClusterException( "Could not fetch " + nodeType + " nodes information from NameNodeInfo bean."); } /** * The Class RolePidFileMap. */ public static class RolePidFilePathMap { /** The dir pid hdfs. */ static String DIR_PID_HDFS = "/var/run/hadoop-hdfs/"; /** The prefix pid hdfs. */ static String PREFIX_PID_HDFS = "hadoop-hdfs-"; /** The dir pid mapreduce. */ static String DIR_PID_MAPREDUCE = "/var/run/hadoop-mapreduce/"; /** The prefix pid mapreduce. */ static String PREFIX_PID_MAPREDUCE = "mapreduce-mapreduce-"; /** The DI r_ pi d_ m rv1. */ static String DIR_PID_MRv1 = "/var/run/hadoop-0.20-mapreduce/"; /** The prefix pid mrv1. */ static String PREFIX_PID_MRv1 = "hadoop-hadoop-"; /** The dir pid yarn. */ static String DIR_PID_YARN = "/var/run/hadoop-yarn/"; /** The prefix pid yarn. */ static String PREFIX_PID_YARN = "yarn-yarn-"; /** The pid extn. */ static String PID_EXTN = ".pid"; /** The process pid file map. */ static Map<String, String> ROLE_PID_FILE_PATH_MAP = new HashMap<String, String>(); static { Map<String, String> roleFilePathMap = new HashMap<String, String>(); roleFilePathMap.put(HadoopConstants.Roles.NAMENODE, RolePidFilePathMap.DIR_PID_HDFS + PREFIX_PID_HDFS + HadoopConstants.Roles.NAMENODE.toLowerCase() + RolePidFilePathMap.PID_EXTN); roleFilePathMap.put(HadoopConstants.Roles.DATANODE, RolePidFilePathMap.DIR_PID_HDFS + PREFIX_PID_HDFS + HadoopConstants.Roles.DATANODE.toLowerCase() + RolePidFilePathMap.PID_EXTN); roleFilePathMap.put(HadoopConstants.Roles.SECONDARYNAMENODE, RolePidFilePathMap.DIR_PID_HDFS + PREFIX_PID_HDFS + HadoopConstants.Roles.SECONDARYNAMENODE.toLowerCase() + RolePidFilePathMap.PID_EXTN); roleFilePathMap.put(HadoopConstants.Roles.DFSZKFAILOVERCONTROLLER, RolePidFilePathMap.DIR_PID_HDFS + PREFIX_PID_HDFS + HadoopConstants.Roles.DFSZKFAILOVERCONTROLLER.toLowerCase() + RolePidFilePathMap.PID_EXTN); roleFilePathMap.put(HadoopConstants.Roles.DFSZKFAILOVERCONTROLLER, RolePidFilePathMap.DIR_PID_HDFS + PREFIX_PID_HDFS + HadoopConstants.Roles.DFSZKFAILOVERCONTROLLER.toLowerCase() + RolePidFilePathMap.PID_EXTN); roleFilePathMap.put(HadoopConstants.Roles.JOBTRACKER, RolePidFilePathMap.DIR_PID_MRv1 + PREFIX_PID_MRv1 + HadoopConstants.Roles.JOBTRACKER.toLowerCase() + RolePidFilePathMap.PID_EXTN); roleFilePathMap.put(HadoopConstants.Roles.TASKTRACKER, RolePidFilePathMap.DIR_PID_MRv1 + PREFIX_PID_MRv1 + HadoopConstants.Roles.TASKTRACKER.toLowerCase() + RolePidFilePathMap.PID_EXTN); roleFilePathMap.put(HadoopConstants.Roles.JOBHISTORYSERVER, RolePidFilePathMap.DIR_PID_MAPREDUCE + PREFIX_PID_MAPREDUCE + "historyserver" + RolePidFilePathMap.PID_EXTN); roleFilePathMap.put(HadoopConstants.Roles.RESOURCEMANAGER, RolePidFilePathMap.DIR_PID_YARN + PREFIX_PID_YARN + HadoopConstants.Roles.RESOURCEMANAGER.toLowerCase() + RolePidFilePathMap.PID_EXTN); roleFilePathMap.put(HadoopConstants.Roles.NODEMANAGER, RolePidFilePathMap.DIR_PID_YARN + PREFIX_PID_YARN + HadoopConstants.Roles.NODEMANAGER.toLowerCase() + RolePidFilePathMap.PID_EXTN); roleFilePathMap.put(HadoopConstants.Roles.WEBAPPPROXYSERVER, RolePidFilePathMap.DIR_PID_YARN + PREFIX_PID_YARN + "proxyserver" + RolePidFilePathMap.PID_EXTN); ROLE_PID_FILE_PATH_MAP = roleFilePathMap; } /** * Gets the process pid file path. * * @param roleName * the role name * @return the process pid file path */ public static String getRolePidFilePath(String roleName) { return ROLE_PID_FILE_PATH_MAP.get(roleName); } } /** * The Enum Vendor. */ public enum Vendor { /** The Apache. */ Apache, /** The Cloudera. */ Cloudera, /** The hdp. */ HDP } /** * The Enum InstallationType. */ public enum InstallationType { /** The Bundle. */ Bundle, /** The Package. */ Package } /** * Implementation to get the Java Process Name for a given role. * * @author Akhil * */ public static class RoleCommandName { /** The role command map. */ static Map<String, String> ROLE_COMMAND_MAP = new HashMap<String, String>(); static { Map<String, String> roleCommandMap = new HashMap<String, String>(); // Common Roles roleCommandMap.put(HadoopConstants.Roles.NAMENODE, "namenode"); roleCommandMap.put(HadoopConstants.Roles.DATANODE, "datanode"); roleCommandMap.put(HadoopConstants.Roles.SECONDARYNAMENODE, "secondarynamenode"); // Roles for Hadoop 0.x & 1.x roleCommandMap.put(HadoopConstants.Roles.JOBTRACKER, "jobtracker"); roleCommandMap.put(HadoopConstants.Roles.TASKTRACKER, "tasktracker"); // Roles for Hadoop 2.x roleCommandMap.put(HadoopConstants.Roles.RESOURCEMANAGER, "resourcemanager"); roleCommandMap.put(HadoopConstants.Roles.WEBAPPPROXYSERVER, "proxyserver"); roleCommandMap.put(HadoopConstants.Roles.NODEMANAGER, "nodemanager"); roleCommandMap.put(HadoopConstants.Roles.JOURNALNODE, "journalnode"); roleCommandMap.put(HadoopConstants.Roles.JOBHISTORYSERVER, "historyserver"); roleCommandMap.put(HadoopConstants.Roles.DFSZKFAILOVERCONTROLLER, "zkfc"); ROLE_COMMAND_MAP = roleCommandMap; } /** * To get the comma separated services for the given role. * * @param role * the role * @return comma separated services. */ public static String getCommandName(String role) { return ROLE_COMMAND_MAP.get(role); } } /** * The Class RolePriorityMap. */ public static class RolePriorityMap { /** The Constant rolePriorityMap. */ static final Map<String, Integer> rolePriorityMap = new HashMap<String, Integer>() { { put(HadoopConstants.Roles.JOURNALNODE, 1); put(HadoopConstants.Roles.NAMENODE, 2); put(HadoopConstants.Roles.DFSZKFAILOVERCONTROLLER, 3); put(HadoopConstants.Roles.SECONDARYNAMENODE, 4); put(HadoopConstants.Roles.DATANODE, 5); put(HadoopConstants.Roles.JOBTRACKER, 6); put(HadoopConstants.Roles.RESOURCEMANAGER, 6); put(HadoopConstants.Roles.TASKTRACKER, 7); put(HadoopConstants.Roles.NODEMANAGER, 7); put(HadoopConstants.Roles.WEBAPPPROXYSERVER, 8); put(HadoopConstants.Roles.JOBHISTORYSERVER, 9); } }; /** * Gets the role priority. * * @param role * the role * @return the role priority */ public static int getRolePriority(String role) { Integer priority = rolePriorityMap.get(role); if (priority != null) { return priority.intValue(); } return 0; } } /** * The Class RoleProcessName. */ public static class RoleProcessName { /** The role java process map. */ static Map<String, String> ROLE_PROCESS_MAP = new HashMap<String, String>(); static { Map<String, String> roleProcessMap = new HashMap<String, String>(); // Common Roles roleProcessMap.put(HadoopConstants.Roles.NAMENODE, "NameNode"); roleProcessMap.put(HadoopConstants.Roles.DATANODE, "DataNode"); roleProcessMap.put(HadoopConstants.Roles.SECONDARYNAMENODE, "SecondaryNameNode"); // Roles for Hadoop 0.x & 1.x roleProcessMap.put(HadoopConstants.Roles.JOBTRACKER, "JobTracker"); roleProcessMap.put(HadoopConstants.Roles.TASKTRACKER, "TaskTracker"); // Roles for Hadoop 2.x roleProcessMap.put(HadoopConstants.Roles.RESOURCEMANAGER, "ResourceManager"); roleProcessMap.put(HadoopConstants.Roles.WEBAPPPROXYSERVER, "WebAppProxyServer"); roleProcessMap.put(HadoopConstants.Roles.NODEMANAGER, "NodeManager"); roleProcessMap.put(HadoopConstants.Roles.JOURNALNODE, "JournalNode"); roleProcessMap.put(HadoopConstants.Roles.JOBHISTORYSERVER, "JobHistoryServer"); roleProcessMap.put(HadoopConstants.Roles.DFSZKFAILOVERCONTROLLER, "DFSZKFailoverController"); ROLE_PROCESS_MAP = roleProcessMap; } /** * To get the comma separated services for the given role. * * @param role * the role * @return comma separated services. */ public static String getProcessName(String role) { return ROLE_PROCESS_MAP.get(role); } /** * Gets the role name. * * @param javaProcess * the java process * @return the role name */ public static String getRoleName(String javaProcess) { for (String role : ROLE_PROCESS_MAP.keySet()) { String process = ROLE_PROCESS_MAP.get(role); if (process.equalsIgnoreCase(javaProcess)) { return role; } } return ""; } } }