esg.node.components.monitoring.MonitorDAO.java Source code

Java tutorial

Introduction

Here is the source code for esg.node.components.monitoring.MonitorDAO.java

Source

/***************************************************************************
 *                                                                          *
 *  Organization: Lawrence Livermore National Lab (LLNL)                    *
 *   Directorate: Computation                                               *
 *    Department: Computing Applications and Research                       *
 *      Division: S&T Global Security                                       *
 *        Matrix: Atmospheric, Earth and Energy Division                    *
 *       Program: PCMDI                                                     *
 *       Project: Earth Systems Grid (ESG) Data Node Software Stack         *
 *  First Author: Gavin M. Bell (gavin@llnl.gov)                            *
 *                                                                          *
 ****************************************************************************
 *                                                                          *
 *   Copyright (c) 2009, Lawrence Livermore National Security, LLC.         *
 *   Produced at the Lawrence Livermore National Laboratory                 *
 *   Written by: Gavin M. Bell (gavin@llnl.gov)                             *
 *   LLNL-CODE-420962                                                       *
 *                                                                          *
 *   All rights reserved. This file is part of the:                         *
 *   Earth System Grid (ESG) Data Node Software Stack, Version 1.0          *
 *                                                                          *
 *   For details, see http://esgf.org/esg-node/                    *
 *   Please also read this link                                             *
 *    http://esgf.org/LICENSE                                      *
 *                                                                          *
 *   * Redistribution and use in source and binary forms, with or           *
 *   without modification, are permitted provided that the following        *
 *   conditions are met:                                                    *
 *                                                                          *
 *   * Redistributions of source code must retain the above copyright       *
 *   notice, this list of conditions and the disclaimer below.              *
 *                                                                          *
 *   * Redistributions in binary form must reproduce the above copyright    *
 *   notice, this list of conditions and the disclaimer (as noted below)    *
 *   in the documentation and/or other materials provided with the          *
 *   distribution.                                                          *
 *                                                                          *
 *   Neither the name of the LLNS/LLNL nor the names of its contributors    *
 *   may be used to endorse or promote products derived from this           *
 *   software without specific prior written permission.                    *
 *                                                                          *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS    *
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT      *
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS      *
 *   FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE    *
 *   LIVERMORE NATIONAL SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR     *
 *   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,           *
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT       *
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF       *
 *   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND    *
 *   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,     *
 *   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT     *
 *   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF     *
 *   SUCH DAMAGE.                                                           *
 *                                                                          *
 ***************************************************************************/

/**
   Description:
   Perform sql query to find out all the people who
   Return Tuple of info needed (dataset_id, recipients/(user), names of updated files)
       
**/
package esg.node.components.monitoring;

import java.util.List;
import java.util.ArrayList;
import java.util.Set;
import java.util.HashSet;
import java.util.Map;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Calendar;
import java.util.Properties;
import java.util.regex.*;
import java.io.Serializable;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import javax.sql.DataSource;
import java.nio.*;
import java.nio.charset.*;
import java.nio.channels.*;

import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.ResultSetHandler;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.*;

import esg.common.Utils;
import esg.common.ESGInvalidObjectStateException;

public class MonitorDAO implements Serializable {

    private static final String markTimeQuery = "UPDATE esgf_node_manager.monitor_run_log SET last_run_time = ? WHERE id = ?";
    private static final String regCheckEntryQuery = "SELECT COUNT(*) FROM esgf_node_manager.monitor_run_log WHERE id = ?";
    private static final String regAddEntryQuery = "INSERT INTO esgf_node_manager.monitor_run_log (id, last_run_time) VALUES ( ? , ? )";

    private static final Log log = LogFactory.getLog(MonitorDAO.class);

    private Properties props = null;
    private DataSource dataSource = null;
    private QueryRunner queryRunner = null;
    private String nodeID = null;

    //Used for fetching disk usage information
    private Pattern disk_info_dsroot_pat = null;
    private Pattern disk_info_dsroot_keyval_pat = null;
    private ByteBuffer disk_info_byte_buffer = null;

    //Used for fetching system memory information
    private InfoResources memInfoResource = null;
    private Pattern memInfoPattern_memTotal = null;
    private Pattern memInfoPattern_memFree = null;
    private Pattern memInfoPattern_swapTotal = null;
    private Pattern memInfoPattern_swapFree = null;

    //Used for fetching system cpu information
    private InfoResources cpuInfoResource = null;
    private Pattern cpuInfoPattern = null;

    //Used for fetching uptime information
    private InfoResources uptimeInfoResource = null;
    private Pattern uptimeInfoPattern = null;
    private InfoResources loadAvgInfoResource = null;
    private Pattern loadAvgInfoPattern = null;

    private long startTime = 0L;

    public MonitorDAO(DataSource dataSource, String nodeID, Properties props) {
        this.setDataSource(dataSource);
        this.setNodeID(nodeID);
        this.setProperties(props);
        init();
    }

    /**
       Not preferred constructor.  Uses default node id value...
    */
    public MonitorDAO(DataSource dataSource) {
        this(dataSource, Utils.getNodeID(), new Properties());
    }

    /**
       Not preferred constructor but here for serialization requirement.
    */
    public MonitorDAO() {
        this(null, null, new Properties());
    }

    //Initialize result set handlers...
    public void init() {
        log.trace("Setting up result handlers");
        registerWithMonitorRunLog();
        startTime = System.currentTimeMillis() / 1000;
        log.trace("Node Start Time: " + startTime);
        loadDiskInfoResource();
        loadMemInfoResource();
        loadCPUInfoResource();
        loadUptimeInfoResource();
    }

    public void setProperties(Properties props) {
        this.props = props;
    }

    public void setDataSource(DataSource dataSource) {
        log.trace("Setting Up Monitor DAO's Pooled Data Source");
        this.dataSource = dataSource;
        this.queryRunner = new QueryRunner(dataSource);
    }

    private void setNodeID(String nodeID) {
        log.trace("Monitor DAO's nodeID: " + nodeID);
        this.nodeID = nodeID;
    }

    private String getNodeID() {
        if (nodeID == null)
            throw new ESGInvalidObjectStateException("NodeID cannot be NULL!");
        return nodeID;
    }

    //------------------------------------
    //Query function calls...
    //------------------------------------

    public int markLastCompletionTime() {
        int ret = -1;
        try {
            long now = System.currentTimeMillis() / 1000;
            ret = queryRunner.update(markTimeQuery, now, getNodeID());
        } catch (SQLException ex) {
            log.error(ex);
        }
        return ret;
    }

    private int registerWithMonitorRunLog() {
        int ret = -1;
        try {
            log.trace("Registering this node [" + getNodeID() + "] into database");
            int count = queryRunner.query(regCheckEntryQuery, new ResultSetHandler<Integer>() {
                public Integer handle(ResultSet rs) throws SQLException {
                    if (!rs.next()) {
                        return -1;
                    }
                    return rs.getInt(1);
                }
            }, MonitorDAO.this.getNodeID());

            if (count > 0) {
                log.info("Yes, " + MonitorDAO.this.getNodeID() + " exists in monitor run log table");
            } else {
                log.info("No, " + MonitorDAO.this.getNodeID() + " does NOT exist in monitor run log table");
                ret = queryRunner.update(regAddEntryQuery, MonitorDAO.this.getNodeID(),
                        System.currentTimeMillis() / 1000);
            }

        } catch (SQLException ex) {
            log.error(ex);
        }
        return ret;
    }

    //------------------------------------

    //slightly less GC and memory friendly...
    public MonitorInfo getMonitorInfo() {
        return this.setMonitorInfo(null);
    }

    //more GC and memory friendly if info is != null
    public MonitorInfo setMonitorInfo(MonitorInfo info) {
        if (null == info)
            info = new MonitorInfo();
        log.trace("setting up monitor information");
        this.setDiskInfo(info);
        this.setMemInfo(info);
        this.setCPUInfo(info);
        this.setUptimeInfo(info);
        this.setXferInfo(info);
        log.trace(infoAsString(info));
        return info;
    }

    //This method should be called once during initialization 
    //This sets up the resources used for the setDiskInfo call (below)
    private void loadDiskInfoResource() {
        disk_info_dsroot_pat = Pattern.compile("thredds_dataset_roots\\s*=\\s*(.*?)\\s*\\w+\\s*?=");
        disk_info_dsroot_keyval_pat = Pattern.compile("\\s*(\\w+)\\s*\\|\\s*(\\S+)\\s*");

        String filename = props.getProperty("monitor.esg.ini",
                System.getenv().get("ESG_USER_HOME") + "/.esgcet/esg.ini");
        File iniFile = new File(filename);
        if (!iniFile.exists()) {
            log.warn("ESG publisher config file [" + filename + "] not found! Cannot provide disk info!");
            return;
        }

        log.debug("Scanning for drives specified in: " + filename);

        try {
            FileInputStream fis = new FileInputStream(iniFile);
            FileChannel fc = fis.getChannel();
            disk_info_byte_buffer = fc.map(FileChannel.MapMode.READ_ONLY, 0, (int) fc.size());
        } catch (FileNotFoundException e) {
            log.error(e);
        } catch (IOException e) {
            log.error(e);
        }
    }

    private void setDiskInfo(MonitorInfo info) {
        if (info.diskInfo == null) {
            info.diskInfo = new HashMap<String, Map<String, String>>();
        }

        info.diskInfo.clear();

        if (disk_info_byte_buffer == null) {
            log.warn("Disk Info Byte Buffer is : [" + disk_info_byte_buffer + "] cannot provide disk information!");
            return;
        }

        //"thredds_dataset_roots"
        Map<String, File> datasetRoots = new HashMap<String, File>();

        try {
            Charset cs = Charset.forName("8859_1");
            CharBuffer cb = cs.newDecoder().decode(disk_info_byte_buffer);

            //Flatten the file...
            String data = cb.toString().replaceAll("[\n]+", " ");

            Matcher m = disk_info_dsroot_pat.matcher(data);
            String key_vals = null;
            Matcher m2 = null;

            while (m.find()) {
                key_vals = m.group(1);
                m2 = disk_info_dsroot_keyval_pat.matcher(key_vals);
                while (m2.find()) {
                    log.trace("Checking... dataset_root [" + m2.group(1) + "] dir [" + m2.group(2) + "]");
                    datasetRoots.put(m2.group(1), new File(m2.group(2)));
                }
            }
            disk_info_byte_buffer.flip();

            Map<String, String> statsMap = null;
            for (String rootLabel : datasetRoots.keySet()) {
                Long total, free = 0L;
                statsMap = new HashMap<String, String>();
                log.trace("rootLabel: " + rootLabel);
                File f = datasetRoots.get(rootLabel);
                log.trace("value : " + f);
                statsMap.put(MonitorInfo.TOTAL_SPACE,
                        "" + (total = (datasetRoots.get(rootLabel).getTotalSpace()) / 1024L));
                statsMap.put(MonitorInfo.FREE_SPACE,
                        "" + (free = (datasetRoots.get(rootLabel).getFreeSpace()) / 1024L));
                statsMap.put(MonitorInfo.USED_SPACE, "" + (total - free));
                info.diskInfo.put(rootLabel, statsMap);
            }
        } catch (java.nio.charset.CharacterCodingException e) {
            log.error(e);
        }
    }

    //This method should be called once during initialization 
    //This sets up the resources used for the setMemInfo call (below)
    private void loadMemInfoResource() {
        memInfoResource = new InfoResources("/proc/meminfo",
                Integer.parseInt(props.getProperty("monitor.buffer.meminfo", "-1")));
        memInfoPattern_memTotal = Pattern.compile("(?:MemTotal)\\s*:\\s*(\\d*)\\s*kB.*", Pattern.CASE_INSENSITIVE);
        memInfoPattern_memFree = Pattern.compile("(?:MemFree)\\s*:\\s*(\\d*)\\s*kB.*", Pattern.CASE_INSENSITIVE);
        memInfoPattern_swapTotal = Pattern.compile("(?:SwapTotal)\\s*:\\s*(\\d*)\\s*kB.*",
                Pattern.CASE_INSENSITIVE);
        memInfoPattern_swapFree = Pattern.compile("(?:SwapFree)\\s*:\\s*(\\d*)\\s*kB.*", Pattern.CASE_INSENSITIVE);
    }

    private void setMemInfo(MonitorInfo info) {
        if (info.memInfo == null) {
            info.memInfo = new HashMap<String, String>();
        }
        java.nio.CharBuffer cb = null;
        try {
            //TODO read the /proc/meminfo file pull out values
            int totalMem = -1;
            int freeMem = -1;
            int totalSwap = -1;
            int freeSwap = -1;
            cb = memInfoResource.scan();
            //log.warn("WHAT I AM LOOKING AT: (if first two lines are garbled this is a kernel mmap bug)\n["+cb+"]");
            Matcher m_memTotal = memInfoPattern_memTotal.matcher(cb);
            Matcher m_memFree = memInfoPattern_memFree.matcher(cb);
            Matcher m_swapTotal = memInfoPattern_swapTotal.matcher(cb);
            Matcher m_swapFree = memInfoPattern_swapFree.matcher(cb);
            if (m_memTotal.find()) {
                info.memInfo.put(MonitorInfo.TOTAL_MEMORY, "" + (totalMem = Integer.parseInt(m_memTotal.group(1))));
            } else {
                info.memInfo.put(MonitorInfo.TOTAL_MEMORY, "-1");
            }
            if (m_memFree.find()) {
                info.memInfo.put(MonitorInfo.FREE_MEMORY, "" + (freeMem = Integer.parseInt(m_memFree.group(1))));
            } else {
                info.memInfo.put(MonitorInfo.FREE_MEMORY, "-1");
            }
            if (m_swapTotal.find()) {
                info.memInfo.put(MonitorInfo.TOTAL_SWAP, "" + (totalSwap = Integer.parseInt(m_swapTotal.group(1))));
            } else {
                info.memInfo.put(MonitorInfo.TOTAL_SWAP, "-1");
            }
            if (m_swapFree.find()) {
                info.memInfo.put(MonitorInfo.FREE_SWAP, "" + (freeSwap = Integer.parseInt(m_swapFree.group(1))));
            } else {
                info.memInfo.put(MonitorInfo.FREE_SWAP, "-1");
            }

            //derived values...
            info.memInfo.put(MonitorInfo.USED_MEMORY, "" + (totalMem - freeMem));
            info.memInfo.put(MonitorInfo.USED_SWAP, "" + (totalSwap - freeSwap));

        } catch (Exception e) {
            e.printStackTrace();
            log.error(e);
        }
    }

    //This method should be called once during initialization 
    //This sets up the resources used for the setMemInfo call (below)
    private void loadCPUInfoResource() {
        cpuInfoResource = new InfoResources("/proc/cpuinfo",
                Integer.parseInt(props.getProperty("monitor.buffer.cpuinfo", "-1")));
        cpuInfoPattern = Pattern.compile("cpu MHz\\s*:\\s*(\\d+\\.\\d*)", Pattern.CASE_INSENSITIVE);
    }

    private void setCPUInfo(MonitorInfo info) {
        if (info.cpuInfo == null) {
            info.cpuInfo = new HashMap<String, String>();
        }

        try {

            Matcher m = cpuInfoPattern.matcher(cpuInfoResource.scan());
            if (m.find())
                info.cpuInfo.put(MonitorInfo.CLOCK_SPEED, m.group(1));

            info.cpuInfo.put(MonitorInfo.CORES, "" + Runtime.getRuntime().availableProcessors());

        } catch (Exception e) {
            e.printStackTrace();
            log.error(e);
        }
    }

    private void loadUptimeInfoResource() {
        uptimeInfoResource = new InfoResources("/proc/uptime",
                Integer.parseInt(props.getProperty("monitor.buffer.uptime", "-1")));
        uptimeInfoPattern = Pattern.compile("(\\d*(\\.\\d*))\\s*");
        loadAvgInfoResource = new InfoResources("/proc/loadavg",
                Integer.parseInt(props.getProperty("monitor.buffer.loadavg", "-1")));
        loadAvgInfoPattern = Pattern.compile("\\d\\.\\d*");
    }

    private void setUptimeInfo(MonitorInfo info) {
        if (info.uptimeInfo == null) {
            info.uptimeInfo = new HashMap<String, String>();
        }

        try {
            Matcher m = null;

            //TODO read the /proc/cpuinfo file pull out values
            m = uptimeInfoPattern.matcher(uptimeInfoResource.scan());
            if (m.find())
                info.uptimeInfo.put(MonitorInfo.HOST_UPTIME, m.group(0));

            m = loadAvgInfoPattern.matcher(loadAvgInfoResource.scan());
            if (m.find())
                info.uptimeInfo.put(MonitorInfo.LOAD_AVG1, m.group(0));
            if (m.find())
                info.uptimeInfo.put(MonitorInfo.LOAD_AVG2, m.group(0));
            if (m.find())
                info.uptimeInfo.put(MonitorInfo.LOAD_AVG3, m.group(0));

            info.uptimeInfo.put(MonitorInfo.DNM_UPTIME, "" + ((System.currentTimeMillis() / 1000) - startTime));

        } catch (Exception e) {
            log.error(e);
        }
    }

    private void setXferInfo(MonitorInfo info) {
        if (info.xferInfo == null) {
            info.xferInfo = new HashMap<String, String>();
        }
        info.xferInfo.put(MonitorInfo.XFER_AVG, "-1");
    }

    private String infoAsString(MonitorInfo info) {
        if (info == null) {
            log.warn("MonitorInfo object is null: [" + info + "]");
            return null;
        }

        StringBuilder out = new StringBuilder();
        out.append("MonitorInfo Object:\n");
        out.append(" diskInfo: " + info.diskInfo + "\n");
        out.append(" memInfo: " + info.memInfo + "\n");
        out.append(" cpuInfo: " + info.cpuInfo + "\n");
        out.append(" uptime: " + info.uptimeInfo + "\n");
        out.append(" xfer: " + info.xferInfo + "\n");
        out.append(" components: " + info.componentList + "\n");
        //System.out.println(out.toString());
        return out.toString();
    }

    //------------------------------------

    public String toString() {
        StringBuilder out = new StringBuilder();
        out.append("DAO:(1)[" + this.getClass().getName() + "] - [Q:" + regCheckEntryQuery + "] "
                + ((dataSource == null) ? "[OK]" : "[INVALID]\n"));
        out.append("DAO:(1)[" + this.getClass().getName() + "] - [Q:" + regAddEntryQuery + "] "
                + ((dataSource == null) ? "[OK]" : "[INVALID]\n"));
        out.append("DAO:(1)[" + this.getClass().getName() + "] - [Q:" + markTimeQuery + "] "
                + ((dataSource == null) ? "[OK]" : "[INVALID]"));
        return out.toString();
    }
}