org.apache.hadoop.hdfs.server.namenode.ClusterJspHelper.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hdfs.server.namenode.ClusterJspHelper.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs.server.namenode;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import javax.management.MalformedObjectNameException;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DFSUtil.ConfiguredNNAddress;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
import org.apache.hadoop.util.StringUtils;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
import org.znerd.xmlenc.XMLOutputter;

import com.google.common.base.Charsets;

/**
 * This class generates the data that is needed to be displayed on cluster web 
 * console.
 */
@InterfaceAudience.Private
class ClusterJspHelper {
    private static final Log LOG = LogFactory.getLog(ClusterJspHelper.class);
    public static final String OVERALL_STATUS = "overall-status";
    public static final String DEAD = "Dead";
    private static final String JMX_QRY = "/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo";

    /**
     * JSP helper function that generates cluster health report.  When 
     * encountering exception while getting Namenode status, the exception will 
     * be listed on the page with corresponding stack trace.
     */
    ClusterStatus generateClusterHealthReport() {
        ClusterStatus cs = new ClusterStatus();
        Configuration conf = new Configuration();
        List<ConfiguredNNAddress> nns = null;
        try {
            nns = DFSUtil.flattenAddressMap(DFSUtil.getNNServiceRpcAddresses(conf));
        } catch (Exception e) {
            // Could not build cluster status
            cs.setError(e);
            return cs;
        }

        // Process each namenode and add it to ClusterStatus
        for (ConfiguredNNAddress cnn : nns) {
            InetSocketAddress isa = cnn.getAddress();
            NamenodeMXBeanHelper nnHelper = null;
            try {
                nnHelper = new NamenodeMXBeanHelper(isa, conf);
                String mbeanProps = queryMbean(nnHelper.httpAddress, conf);
                NamenodeStatus nn = nnHelper.getNamenodeStatus(mbeanProps);
                if (cs.clusterid.isEmpty() || cs.clusterid.equals("")) { // Set clusterid only once
                    cs.clusterid = nnHelper.getClusterId(mbeanProps);
                }
                cs.addNamenodeStatus(nn);
            } catch (Exception e) {
                // track exceptions encountered when connecting to namenodes
                cs.addException(isa.getHostName(), e);
                continue;
            }
        }
        return cs;
    }

    /**
     * Helper function that generates the decommissioning report.  Connect to each
     * Namenode over http via JmxJsonServlet to collect the data nodes status.
     */
    DecommissionStatus generateDecommissioningReport() {
        String clusterid = "";
        Configuration conf = new Configuration();
        List<ConfiguredNNAddress> cnns = null;
        try {
            cnns = DFSUtil.flattenAddressMap(DFSUtil.getNNServiceRpcAddresses(conf));
        } catch (Exception e) {
            // catch any exception encountered other than connecting to namenodes
            DecommissionStatus dInfo = new DecommissionStatus(clusterid, e);
            return dInfo;
        }

        // Outer map key is datanode. Inner map key is namenode and the value is 
        // decom status of the datanode for the corresponding namenode
        Map<String, Map<String, String>> statusMap = new HashMap<String, Map<String, String>>();

        // Map of exceptions encountered when connecting to namenode
        // key is namenode and value is exception
        Map<String, Exception> decommissionExceptions = new HashMap<String, Exception>();

        List<String> unreportedNamenode = new ArrayList<String>();
        for (ConfiguredNNAddress cnn : cnns) {
            InetSocketAddress isa = cnn.getAddress();
            NamenodeMXBeanHelper nnHelper = null;
            try {
                nnHelper = new NamenodeMXBeanHelper(isa, conf);
                String mbeanProps = queryMbean(nnHelper.httpAddress, conf);
                if (clusterid.equals("")) {
                    clusterid = nnHelper.getClusterId(mbeanProps);
                }
                nnHelper.getDecomNodeInfoForReport(statusMap, mbeanProps);
            } catch (Exception e) {
                // catch exceptions encountered while connecting to namenodes
                String nnHost = isa.getHostName();
                decommissionExceptions.put(nnHost, e);
                unreportedNamenode.add(nnHost);
                continue;
            }
        }
        updateUnknownStatus(statusMap, unreportedNamenode);
        getDecommissionNodeClusterState(statusMap);
        return new DecommissionStatus(statusMap, clusterid, getDatanodeHttpPort(conf), decommissionExceptions);
    }

    /**
     * Based on the state of the datanode at each namenode, marks the overall
     * state of the datanode across all the namenodes, to one of the following:
     * <ol>
     * <li>{@link DecommissionStates#DECOMMISSIONED}</li>
     * <li>{@link DecommissionStates#DECOMMISSION_INPROGRESS}</li>
     * <li>{@link DecommissionStates#PARTIALLY_DECOMMISSIONED}</li>
     * <li>{@link DecommissionStates#UNKNOWN}</li>
     * </ol>
     * 
     * @param statusMap
     *          map whose key is datanode, value is an inner map with key being
     *          namenode, value being decommission state.
     */
    private void getDecommissionNodeClusterState(Map<String, Map<String, String>> statusMap) {
        if (statusMap == null || statusMap.isEmpty()) {
            return;
        }

        // For each datanodes
        Iterator<Entry<String, Map<String, String>>> it = statusMap.entrySet().iterator();
        while (it.hasNext()) {
            // Map entry for a datanode:
            // key is namenode, value is datanode status at the namenode
            Entry<String, Map<String, String>> entry = it.next();
            Map<String, String> nnStatus = entry.getValue();
            if (nnStatus == null || nnStatus.isEmpty()) {
                continue;
            }

            boolean isUnknown = false;
            int unknown = 0;
            int decommissioned = 0;
            int decomInProg = 0;
            int inservice = 0;
            int dead = 0;
            DecommissionStates overallState = DecommissionStates.UNKNOWN;
            // Process a datanode state from each namenode
            for (Map.Entry<String, String> m : nnStatus.entrySet()) {
                String status = m.getValue();
                if (status.equals(DecommissionStates.UNKNOWN.toString())) {
                    isUnknown = true;
                    unknown++;
                } else if (status.equals(AdminStates.DECOMMISSION_INPROGRESS.toString())) {
                    decomInProg++;
                } else if (status.equals(AdminStates.DECOMMISSIONED.toString())) {
                    decommissioned++;
                } else if (status.equals(AdminStates.NORMAL.toString())) {
                    inservice++;
                } else if (status.equals(DEAD)) {
                    // dead
                    dead++;
                }
            }

            // Consolidate all the states from namenode in to overall state
            int nns = nnStatus.keySet().size();
            if ((inservice + dead + unknown) == nns) {
                // Do not display this data node. Remove this entry from status map.  
                it.remove();
            } else if (isUnknown) {
                overallState = DecommissionStates.UNKNOWN;
            } else if (decommissioned == nns) {
                overallState = DecommissionStates.DECOMMISSIONED;
            } else if ((decommissioned + decomInProg) == nns) {
                overallState = DecommissionStates.DECOMMISSION_INPROGRESS;
            } else if ((decommissioned + decomInProg < nns) && (decommissioned + decomInProg > 0)) {
                overallState = DecommissionStates.PARTIALLY_DECOMMISSIONED;
            } else {
                LOG.warn("Cluster console encounters a not handled situtation.");
            }

            // insert overall state
            nnStatus.put(OVERALL_STATUS, overallState.toString());
        }
    }

    /**
     * update unknown status in datanode status map for every unreported namenode
     */
    private void updateUnknownStatus(Map<String, Map<String, String>> statusMap, List<String> unreportedNn) {
        if (unreportedNn == null || unreportedNn.isEmpty()) {
            // no unreported namenodes
            return;
        }

        for (Map.Entry<String, Map<String, String>> entry : statusMap.entrySet()) {
            String dn = entry.getKey();
            Map<String, String> nnStatus = entry.getValue();
            for (String nn : unreportedNn) {
                nnStatus.put(nn, DecommissionStates.UNKNOWN.toString());
            }
            statusMap.put(dn, nnStatus);
        }
    }

    /**
     * Get datanode http port from configration
     */
    private int getDatanodeHttpPort(Configuration conf) {
        String address = conf.get(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY, "");
        if (address.equals("")) {
            return -1;
        }
        return Integer.parseInt(address.split(":")[1]);
    }

    /**
     * Class for connecting to Namenode over http via JmxJsonServlet 
     * to get JMX attributes exposed by the MXBean.  
     */
    static class NamenodeMXBeanHelper {
        private static final ObjectMapper mapper = new ObjectMapper();
        private final String host;
        private final URI httpAddress;

        NamenodeMXBeanHelper(InetSocketAddress addr, Configuration conf)
                throws IOException, MalformedObjectNameException {
            this.host = addr.getHostName();
            this.httpAddress = DFSUtil.getInfoServer(addr, conf, DFSUtil.getHttpClientScheme(conf));
        }

        /** Get the map corresponding to the JSON string */
        private static Map<String, Map<String, Object>> getNodeMap(String json) throws IOException {
            TypeReference<Map<String, Map<String, Object>>> type = new TypeReference<Map<String, Map<String, Object>>>() {
            };
            return mapper.readValue(json, type);
        }

        /**
         * Get the number of live datanodes.
         * 
         * @param json JSON string that contains live node status.
         * @param nn namenode status to return information in
         */
        private static void getLiveNodeCount(String json, NamenodeStatus nn) throws IOException {
            // Map of datanode host to (map of attribute name to value)
            Map<String, Map<String, Object>> nodeMap = getNodeMap(json);
            if (nodeMap == null || nodeMap.isEmpty()) {
                return;
            }

            nn.liveDatanodeCount = nodeMap.size();
            for (Entry<String, Map<String, Object>> entry : nodeMap.entrySet()) {
                // Inner map of attribute name to value
                Map<String, Object> innerMap = entry.getValue();
                if (innerMap != null) {
                    if (innerMap.get("adminState").equals(AdminStates.DECOMMISSIONED.toString())) {
                        nn.liveDecomCount++;
                    }
                }
            }
        }

        /**
         * Count the number of dead datanode.
         * 
         * @param nn namenode
         * @param json JSON string
         */
        private static void getDeadNodeCount(String json, NamenodeStatus nn) throws IOException {
            Map<String, Map<String, Object>> nodeMap = getNodeMap(json);
            if (nodeMap == null || nodeMap.isEmpty()) {
                return;
            }

            nn.deadDatanodeCount = nodeMap.size();
            for (Entry<String, Map<String, Object>> entry : nodeMap.entrySet()) {
                Map<String, Object> innerMap = entry.getValue();
                if (innerMap != null && !innerMap.isEmpty()) {
                    if (((Boolean) innerMap.get("decommissioned")).booleanValue() == true) {
                        nn.deadDecomCount++;
                    }
                }
            }
        }

        public String getClusterId(String props) throws IOException {
            return getProperty(props, "ClusterId").getTextValue();
        }

        public NamenodeStatus getNamenodeStatus(String props)
                throws IOException, MalformedObjectNameException, NumberFormatException {
            NamenodeStatus nn = new NamenodeStatus();
            nn.host = host;
            nn.filesAndDirectories = getProperty(props, "TotalFiles").getLongValue();
            nn.capacity = getProperty(props, "Total").getLongValue();
            nn.free = getProperty(props, "Free").getLongValue();
            nn.bpUsed = getProperty(props, "BlockPoolUsedSpace").getLongValue();
            nn.nonDfsUsed = getProperty(props, "NonDfsUsedSpace").getLongValue();
            nn.blocksCount = getProperty(props, "TotalBlocks").getLongValue();
            nn.missingBlocksCount = getProperty(props, "NumberOfMissingBlocks").getLongValue();
            nn.httpAddress = httpAddress.toURL();
            getLiveNodeCount(getProperty(props, "LiveNodes").asText(), nn);
            getDeadNodeCount(getProperty(props, "DeadNodes").asText(), nn);
            nn.softwareVersion = getProperty(props, "SoftwareVersion").getTextValue();
            return nn;
        }

        /**
         * Get the decommission node information.
         * @param statusMap data node status map
         * @param props string
         */
        private void getDecomNodeInfoForReport(Map<String, Map<String, String>> statusMap, String props)
                throws IOException, MalformedObjectNameException {
            getLiveNodeStatus(statusMap, host, getProperty(props, "LiveNodes").asText());
            getDeadNodeStatus(statusMap, host, getProperty(props, "DeadNodes").asText());
            getDecommissionNodeStatus(statusMap, host, getProperty(props, "DecomNodes").asText());
        }

        /**
         * Store the live datanode status information into datanode status map and
         * DecommissionNode.
         * 
         * @param statusMap Map of datanode status. Key is datanode, value
         *          is an inner map whose key is namenode, value is datanode status.
         *          reported by each namenode.
         * @param namenodeHost host name of the namenode
         * @param json JSON string contains datanode status
         * @throws IOException
         */
        private static void getLiveNodeStatus(Map<String, Map<String, String>> statusMap, String namenodeHost,
                String json) throws IOException {
            Map<String, Map<String, Object>> nodeMap = getNodeMap(json);
            if (nodeMap != null && !nodeMap.isEmpty()) {
                List<String> liveDecommed = new ArrayList<String>();
                for (Map.Entry<String, Map<String, Object>> entry : nodeMap.entrySet()) {
                    Map<String, Object> innerMap = entry.getValue();
                    String dn = entry.getKey();
                    if (innerMap != null) {
                        if (innerMap.get("adminState").equals(AdminStates.DECOMMISSIONED.toString())) {
                            liveDecommed.add(dn);
                        }
                        // the inner map key is namenode, value is datanode status.
                        Map<String, String> nnStatus = statusMap.get(dn);
                        if (nnStatus == null) {
                            nnStatus = new HashMap<String, String>();
                        }
                        nnStatus.put(namenodeHost, (String) innerMap.get("adminState"));
                        // map whose key is datanode, value is the inner map.
                        statusMap.put(dn, nnStatus);
                    }
                }
            }
        }

        /**
         * Store the dead datanode information into datanode status map and
         * DecommissionNode.
         * 
         * @param statusMap map with key being datanode, value being an
         *          inner map (key:namenode, value:decommisionning state).
         * @param host datanode hostname
         * @param json String
         * @throws IOException
         */
        private static void getDeadNodeStatus(Map<String, Map<String, String>> statusMap, String host, String json)
                throws IOException {
            Map<String, Map<String, Object>> nodeMap = getNodeMap(json);
            if (nodeMap == null || nodeMap.isEmpty()) {
                return;
            }
            List<String> deadDn = new ArrayList<String>();
            List<String> deadDecommed = new ArrayList<String>();
            for (Entry<String, Map<String, Object>> entry : nodeMap.entrySet()) {
                deadDn.add(entry.getKey());
                Map<String, Object> deadNodeDetailMap = entry.getValue();
                String dn = entry.getKey();
                if (deadNodeDetailMap != null && !deadNodeDetailMap.isEmpty()) {
                    // NN - status
                    Map<String, String> nnStatus = statusMap.get(dn);
                    if (nnStatus == null) {
                        nnStatus = new HashMap<String, String>();
                    }
                    if (((Boolean) deadNodeDetailMap.get("decommissioned")).booleanValue() == true) {
                        deadDecommed.add(dn);
                        nnStatus.put(host, AdminStates.DECOMMISSIONED.toString());
                    } else {
                        nnStatus.put(host, DEAD);
                    }
                    // dn-nn-status
                    statusMap.put(dn, nnStatus);
                }
            }
        }

        /**
         * Get the decommisioning datanode information.
         * 
         * @param dataNodeStatusMap map with key being datanode, value being an
         *          inner map (key:namenode, value:decommisionning state).
         * @param host datanode
         * @param json String
         */
        private static void getDecommissionNodeStatus(Map<String, Map<String, String>> dataNodeStatusMap,
                String host, String json) throws IOException {
            Map<String, Map<String, Object>> nodeMap = getNodeMap(json);
            if (nodeMap == null || nodeMap.isEmpty()) {
                return;
            }
            List<String> decomming = new ArrayList<String>();
            for (Entry<String, Map<String, Object>> entry : nodeMap.entrySet()) {
                String dn = entry.getKey();
                decomming.add(dn);
                // nn-status
                Map<String, String> nnStatus = new HashMap<String, String>();
                if (dataNodeStatusMap.containsKey(dn)) {
                    nnStatus = dataNodeStatusMap.get(dn);
                }
                nnStatus.put(host, AdminStates.DECOMMISSION_INPROGRESS.toString());
                // dn-nn-status
                dataNodeStatusMap.put(dn, nnStatus);
            }
        }
    }

    /**
     * This class contains cluster statistics.
     */
    static class ClusterStatus {
        /** Exception indicates failure to get cluster status */
        Exception error = null;

        /** Cluster status information */
        String clusterid = "";
        long total_sum = 0;
        long free_sum = 0;
        long clusterDfsUsed = 0;
        long nonDfsUsed_sum = 0;
        long totalFilesAndDirectories = 0;

        /** List of namenodes in the cluster */
        final List<NamenodeStatus> nnList = new ArrayList<NamenodeStatus>();

        /** Map of namenode host and exception encountered when getting status */
        final Map<String, Exception> nnExceptions = new HashMap<String, Exception>();

        public void setError(Exception e) {
            error = e;
        }

        public void addNamenodeStatus(NamenodeStatus nn) {
            nnList.add(nn);

            // Add namenode status to cluster status
            totalFilesAndDirectories += nn.filesAndDirectories;
            total_sum += nn.capacity;
            free_sum += nn.free;
            clusterDfsUsed += nn.bpUsed;
            nonDfsUsed_sum += nn.nonDfsUsed;
        }

        public void addException(String host, Exception e) {
            nnExceptions.put(host, e);
        }

        public void toXML(XMLOutputter doc) throws IOException {
            if (error != null) {
                // general exception, only print exception message onto web page.
                createGeneralException(doc, clusterid, StringUtils.stringifyException(error));
                doc.getWriter().flush();
                return;
            }

            int size = nnList.size();
            long total = 0L, free = 0L, nonDfsUsed = 0l;
            float dfsUsedPercent = 0.0f, dfsRemainingPercent = 0.0f;
            if (size > 0) {
                total = total_sum / size;
                free = free_sum / size;
                nonDfsUsed = nonDfsUsed_sum / size;
                dfsUsedPercent = DFSUtil.getPercentUsed(clusterDfsUsed, total);
                dfsRemainingPercent = DFSUtil.getPercentRemaining(free, total);
            }

            doc.startTag("cluster");
            doc.attribute("clusterId", clusterid);

            doc.startTag("storage");

            toXmlItemBlock(doc, "Total Files And Directories", Long.toString(totalFilesAndDirectories));

            toXmlItemBlock(doc, "Configured Capacity", StringUtils.byteDesc(total));

            toXmlItemBlock(doc, "DFS Used", StringUtils.byteDesc(clusterDfsUsed));

            toXmlItemBlock(doc, "Non DFS Used", StringUtils.byteDesc(nonDfsUsed));

            toXmlItemBlock(doc, "DFS Remaining", StringUtils.byteDesc(free));

            // dfsUsedPercent
            toXmlItemBlock(doc, "DFS Used%", DFSUtil.percent2String(dfsUsedPercent));

            // dfsRemainingPercent
            toXmlItemBlock(doc, "DFS Remaining%", DFSUtil.percent2String(dfsRemainingPercent));

            doc.endTag(); // storage

            doc.startTag("namenodes");
            // number of namenodes
            toXmlItemBlock(doc, "NamenodesCount", Integer.toString(size));

            for (NamenodeStatus nn : nnList) {
                doc.startTag("node");
                toXmlItemBlockWithLink(doc, nn.host, nn.httpAddress, "NameNode");
                toXmlItemBlock(doc, "Blockpool Used", StringUtils.byteDesc(nn.bpUsed));
                toXmlItemBlock(doc, "Blockpool Used%",
                        DFSUtil.percent2String(DFSUtil.getPercentUsed(nn.bpUsed, total)));
                toXmlItemBlock(doc, "Files And Directories", Long.toString(nn.filesAndDirectories));
                toXmlItemBlock(doc, "Blocks", Long.toString(nn.blocksCount));
                toXmlItemBlock(doc, "Missing Blocks", Long.toString(nn.missingBlocksCount));
                toXmlItemBlockWithLink(doc, nn.liveDatanodeCount + " (" + nn.liveDecomCount + ")",
                        new URL(nn.httpAddress, "/dfsnodelist.jsp?whatNodes=LIVE"),
                        "Live Datanode (Decommissioned)");
                toXmlItemBlockWithLink(doc, nn.deadDatanodeCount + " (" + nn.deadDecomCount + ")",
                        new URL(nn.httpAddress, "/dfsnodelist.jsp?whatNodes=DEAD"),
                        "Dead Datanode (Decommissioned)");
                toXmlItemBlock(doc, "Software Version", nn.softwareVersion);
                doc.endTag(); // node
            }
            doc.endTag(); // namenodes

            createNamenodeExceptionMsg(doc, nnExceptions);
            doc.endTag(); // cluster
            doc.getWriter().flush();
        }
    }

    /**
     * This class stores namenode statistics to be used to generate cluster
     * web console report.
     */
    static class NamenodeStatus {
        String host = "";
        long capacity = 0L;
        long free = 0L;
        long bpUsed = 0L;
        long nonDfsUsed = 0L;
        long filesAndDirectories = 0L;
        long blocksCount = 0L;
        long missingBlocksCount = 0L;
        int liveDatanodeCount = 0;
        int liveDecomCount = 0;
        int deadDatanodeCount = 0;
        int deadDecomCount = 0;
        URL httpAddress = null;
        String softwareVersion = "";
    }

    /**
     * cluster-wide decommission state of a datanode
     */
    public enum DecommissionStates {
        /*
         * If datanode state is decommissioning at one or more namenodes and 
         * decommissioned at the rest of the namenodes.
         */
        DECOMMISSION_INPROGRESS("Decommission In Progress"),

        /* If datanode state at all the namenodes is decommissioned */
        DECOMMISSIONED("Decommissioned"),

        /*
         * If datanode state is not decommissioning at one or more namenodes and 
         * decommissioned/decommissioning at the rest of the namenodes.
         */
        PARTIALLY_DECOMMISSIONED("Partially Decommissioning"),

        /*
         * If datanode state is not known at a namenode, due to problems in getting
         * the datanode state from the namenode.
         */
        UNKNOWN("Unknown");

        final String value;

        DecommissionStates(final String v) {
            this.value = v;
        }

        @Override
        public String toString() {
            return value;
        }
    }

    /**
     * This class consolidates the decommissioning datanodes information in the
     * cluster and generates decommissioning reports in XML.
     */
    static class DecommissionStatus {
        /** Error when set indicates failure to get decomission status*/
        final Exception error;

        /** Map of dn host <-> (Map of NN host <-> decommissioning state) */
        final Map<String, Map<String, String>> statusMap;
        final String clusterid;
        final int httpPort;
        int decommissioned = 0; // total number of decommissioned nodes
        int decommissioning = 0; // total number of decommissioning datanodes
        int partial = 0; // total number of partially decommissioned nodes

        /** Map of namenode and exception encountered when getting decom status */
        Map<String, Exception> exceptions = new HashMap<String, Exception>();

        private DecommissionStatus(Map<String, Map<String, String>> statusMap, String cid, int httpPort,
                Map<String, Exception> exceptions) {
            this(statusMap, cid, httpPort, exceptions, null);
        }

        public DecommissionStatus(String cid, Exception e) {
            this(null, cid, -1, null, e);
        }

        private DecommissionStatus(Map<String, Map<String, String>> statusMap, String cid, int httpPort,
                Map<String, Exception> exceptions, Exception error) {
            this.statusMap = statusMap;
            this.clusterid = cid;
            this.httpPort = httpPort;
            this.exceptions = exceptions;
            this.error = error;
        }

        /**
         * Generate decommissioning datanode report in XML format
         * 
         * @param doc
         *          , xmloutputter
         * @throws IOException
         */
        public void toXML(XMLOutputter doc) throws IOException {
            if (error != null) {
                createGeneralException(doc, clusterid, StringUtils.stringifyException(error));
                doc.getWriter().flush();
                return;
            }
            if (statusMap == null || statusMap.isEmpty()) {
                // none of the namenodes has reported, print exceptions from each nn.
                doc.startTag("cluster");
                createNamenodeExceptionMsg(doc, exceptions);
                doc.endTag();
                doc.getWriter().flush();
                return;
            }
            doc.startTag("cluster");
            doc.attribute("clusterId", clusterid);

            doc.startTag("decommissioningReport");
            countDecommissionDatanodes();
            toXmlItemBlock(doc, DecommissionStates.DECOMMISSIONED.toString(), Integer.toString(decommissioned));

            toXmlItemBlock(doc, DecommissionStates.DECOMMISSION_INPROGRESS.toString(),
                    Integer.toString(decommissioning));

            toXmlItemBlock(doc, DecommissionStates.PARTIALLY_DECOMMISSIONED.toString(), Integer.toString(partial));

            doc.endTag(); // decommissioningReport

            doc.startTag("datanodes");
            Set<String> dnSet = statusMap.keySet();
            for (String dnhost : dnSet) {
                Map<String, String> nnStatus = statusMap.get(dnhost);
                if (nnStatus == null || nnStatus.isEmpty()) {
                    continue;
                }
                String overallStatus = nnStatus.get(OVERALL_STATUS);
                // check if datanode is in decommission states
                if (overallStatus != null && (overallStatus.equals(AdminStates.DECOMMISSION_INPROGRESS.toString())
                        || overallStatus.equals(AdminStates.DECOMMISSIONED.toString())
                        || overallStatus.equals(DecommissionStates.PARTIALLY_DECOMMISSIONED.toString())
                        || overallStatus.equals(DecommissionStates.UNKNOWN.toString()))) {
                    doc.startTag("node");
                    // dn
                    toXmlItemBlockWithLink(doc, dnhost, new URL("http", dnhost, httpPort, ""), "DataNode");

                    // overall status first
                    toXmlItemBlock(doc, OVERALL_STATUS, overallStatus);

                    for (Map.Entry<String, String> m : nnStatus.entrySet()) {
                        String nn = m.getKey();
                        if (nn.equals(OVERALL_STATUS)) {
                            continue;
                        }
                        // xml
                        toXmlItemBlock(doc, nn, nnStatus.get(nn));
                    }
                    doc.endTag(); // node
                }
            }
            doc.endTag(); // datanodes

            createNamenodeExceptionMsg(doc, exceptions);

            doc.endTag();// cluster
        } // toXML

        /**
         * Count the total number of decommissioned/decommission_inprogress/
         * partially decommissioned datanodes.
         */
        private void countDecommissionDatanodes() {
            for (String dn : statusMap.keySet()) {
                Map<String, String> nnStatus = statusMap.get(dn);
                String status = nnStatus.get(OVERALL_STATUS);
                if (status.equals(DecommissionStates.DECOMMISSIONED.toString())) {
                    decommissioned++;
                } else if (status.equals(DecommissionStates.DECOMMISSION_INPROGRESS.toString())) {
                    decommissioning++;
                } else if (status.equals(DecommissionStates.PARTIALLY_DECOMMISSIONED.toString())) {
                    partial++;
                }
            }
        }
    }

    /**
     * Generate a XML block as such, <item label=key value=value/>
     */
    private static void toXmlItemBlock(XMLOutputter doc, String key, String value) throws IOException {
        doc.startTag("item");
        doc.attribute("label", key);
        doc.attribute("value", value);
        doc.endTag();
    }

    /**
     * Generate a XML block as such, <item label="Node" value="hostname"
     * link="http://hostname:50070" />
     */
    private static void toXmlItemBlockWithLink(XMLOutputter doc, String value, URL url, String label)
            throws IOException {
        doc.startTag("item");
        doc.attribute("label", label);
        doc.attribute("value", value);
        doc.attribute("link", url.toString());
        doc.endTag(); // item
    }

    /**
     * create the XML for exceptions that we encountered when connecting to
     * namenode.
     */
    private static void createNamenodeExceptionMsg(XMLOutputter doc, Map<String, Exception> exceptionMsg)
            throws IOException {
        if (exceptionMsg.size() > 0) {
            doc.startTag("unreportedNamenodes");
            for (Map.Entry<String, Exception> m : exceptionMsg.entrySet()) {
                doc.startTag("node");
                doc.attribute("name", m.getKey());
                doc.attribute("exception", StringUtils.stringifyException(m.getValue()));
                doc.endTag();// node
            }
            doc.endTag(); // unreportedNamnodes
        }
    }

    /**
     * create XML block from general exception.
     */
    private static void createGeneralException(XMLOutputter doc, String clusterid, String eMsg) throws IOException {
        doc.startTag("cluster");
        doc.attribute("clusterId", clusterid);
        doc.startTag("message");
        doc.startTag("item");
        doc.attribute("msg", eMsg);
        doc.endTag(); // item
        doc.endTag(); // message
        doc.endTag(); // cluster
    }

    /**
     * Read in the content from a URL
     * @param url URL To read
     * @return the text from the output
     * @throws IOException if something went wrong
     */
    private static String readOutput(URL url) throws IOException {
        StringBuilder out = new StringBuilder();
        URLConnection connection = url.openConnection();
        BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(), Charsets.UTF_8));
        String inputLine;
        while ((inputLine = in.readLine()) != null) {
            out.append(inputLine);
        }
        in.close();
        return out.toString();
    }

    private static String queryMbean(URI httpAddress, Configuration conf) throws IOException {
        /**
         * Although the other namenode might support HTTPS, it is fundamentally
         * broken to get the JMX via an HTTPS connection inside the namenode,
         * because in HTTPS set up the principal of the client and the one of
         * the namenode differs. Therefore, there is no guarantees that the
         * HTTPS connection can be set up.
         *
         * As a result, we just hard code the connection as an HTTP connection.
         */
        URL url = new URL(httpAddress.toURL(), JMX_QRY);
        return readOutput(url);
    }

    /**
     * In order to query a namenode mxbean, a http connection in the form of
     * "http://hostname/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo"
     * is sent to namenode.  JMX attributes are exposed via JmxJsonServelet on 
     * the namenode side.
     */
    private static JsonNode getProperty(String props, String propertyname) throws IOException {
        if (props == null || props.equals("") || propertyname == null || propertyname.equals("")) {
            return null;
        }
        ObjectMapper m = new ObjectMapper();
        JsonNode rootNode = m.readValue(props, JsonNode.class);
        JsonNode jn = rootNode.get("beans").get(0).get(propertyname);
        return jn;
    }
}