ddf.metrics.reporting.internal.rrd4j.JmxCollector.java Source code

Java tutorial

Introduction

Here is the source code for ddf.metrics.reporting.internal.rrd4j.JmxCollector.java

Source

/**
 * Copyright (c) Codice Foundation
 *
 * This is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either
 * version 3 of the License, or any later version. 
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU Lesser General Public License for more details. A copy of the GNU Lesser General Public License is distributed along with this program and can be found at
 * <http://www.gnu.org/licenses/lgpl.html>.
 *
 **/
package ddf.metrics.reporting.internal.rrd4j;

import java.io.File;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.List;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import javax.management.MBeanServer;
import javax.management.ObjectName;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.log4j.Logger;
import org.rrd4j.ConsolFun;
import org.rrd4j.DsType;
import org.rrd4j.core.RrdDb;
import org.rrd4j.core.RrdDbPool;
import org.rrd4j.core.RrdDef;
import org.rrd4j.core.Sample;

import ddf.metrics.reporting.internal.Collector;
import ddf.metrics.reporting.internal.CollectorException;

/**
 * Periodically collects a metrics data from its associated JMX MBean attribute and
 * stores this data in an RRD (Round Robin Database) file.
 * 
 * NOTE: This is an internal class (as noted in its package name) that is not intended 
 * for external use by third party developers.
 * 
 * The configuration parameters for a JmxCollector are typically configured in the
 * {@code <config>} stanzas of the catalog-core-app features.xml file.
 * 
 * Sample config stanza for a JmxCollector:
 * <pre>
 * {@code
 *  <config name="ddf.metrics.reporting.internal.rrd4j.JmxCollector-UsageThreshold">
 *          mbeanName = java.lang:type=MemoryPool,name=Code Cache
 *          mbeanAttributeName = UsageThreshold
 *          rrdPath = usageThreshold.rrd
 *          rrdDataSourceName = data
 *          rrdDataSourceType = COUNTER
 * </config>
 * }
 * </pre>
 *  
 * @author rodgersh
 * @author ddf.isgs@lmco.com
 *
 */
public class JmxCollector implements Collector {
    private static final int FIVE_MINUTES_MILLIS = 300000;

    private static final Logger LOGGER = Logger.getLogger(JmxCollector.class);

    public static final String DEFAULT_METRICS_DIR = "data/metrics/";

    public static final String COUNTER_DATA_SOURCE_TYPE = "COUNTER";

    public static final String GAUGE_DATA_SOURCE_TYPE = "GAUGE";

    public static final int ONE_YEAR_IN_15_MINUTE_STEPS = 4 * 24 * 365;

    public static final int TEN_YEARS_IN_HOURS = ONE_YEAR_IN_15_MINUTE_STEPS * 10;

    /** 
     * Name of the JMX MBean that contains the metric being collected. 
     * (Should be set by <config> stanza in metrics-reporting-app features.xml file)
     */
    private String mbeanName;

    /** 
     * Name of the JMX MBean attribute that maps to the metric being collected. 
     * (Should be set by <config> stanza in metrics-reporting-app features.xml file)
     */
    private String mbeanAttributeName;

    /** 
     * Name of the RRD file to store the metric's data being collected. The
     * DDF metrics base directory will be prepended to this filename.
     * (Should be set by <config> stanza in metrics-reporting-app features.xml file)
     */
    private String rrdPath;

    /** 
     * The name of the RRD data source to use for the metric being collected. This
     * can (and should) be the same for all metrics configured, e.g., "data", since
     * there should only be one data source per RRD file they do not need to be unique
     * data source names.
     * (Should be set by <config> stanza in metrics-reporting-app features.xml file)
     */
    private String rrdDataSourceName;

    /** 
     * Type of RRD data source to use for the metric's data being collected. A
     * COUNTER type is used for metrics that always increment, e.g., query count.
     * A GAUGE is used for metrics whose value can vary up or down at any time, e.g.,
     * query response time.
     * (Should be set by <config> stanza in metrics-reporting-app features.xml file)
     */
    private String rrdDataSourceType;

    private String metricsDir;
    private int sampleRate;
    private int rrdStep;
    private MBeanServer localMBeanServer;
    private final RrdDbPool pool;
    private RrdDb rrdDb;
    private Sample sample = null;
    private ScheduledThreadPoolExecutor executor;
    private long mbeanTimeoutMillis = FIVE_MINUTES_MILLIS;

    public JmxCollector() {
        metricsDir = DEFAULT_METRICS_DIR;
        localMBeanServer = getLocalMBeanServer();

        // Only expose this value via setter/getter methods for unit
        // testing purposes so that unit tests can run in seconds vs. minutes
        // by using a faster (lower value) sample rate.
        this.sampleRate = 60;

        // Should always be the same as the sample rate
        rrdStep = this.sampleRate;
        pool = RrdDbPool.getInstance();

        // Set to default - should be overridden by the <config> in the 
        // metrics-reporting-app features.xml file, which will call this
        // attribute's setter method.
        this.rrdDataSourceType = COUNTER_DATA_SOURCE_TYPE;

        LOGGER.trace("EXITING: JmxCollector default constructor");
    }

    /**
     * Initialization when the JmxCollector is created. Called by blueprint.
     */
    public void init() throws IOException, CollectorException {
        LOGGER.trace("ENTERING: init()");

        if (!isMbeanAccessible()) {
            LOGGER.warn("MBean attribute " + mbeanAttributeName
                    + " is not accessible - no collector will be configured for it.");
            throw new CollectorException("MBean attribute " + mbeanAttributeName
                    + " is not accessible - no collector will be configured for it.");
        }

        if (rrdDataSourceType == null) {
            throw new CollectorException(
                    "Data Source type for the RRD file cnnot be null - must be either COUNTER or GAUGE.");
        }

        createRrdFile(rrdPath, rrdDataSourceName, DsType.valueOf(rrdDataSourceType));

        updateSamples();

        LOGGER.trace("EXITING: init()");
    }

    /**
     * Cleanup when the JmxCollector is destroyed. Called by blueprint.
     */
    public void destroy() {
        LOGGER.trace("ENTERING: destroy()");

        // Shutdown the scheduled threaded executor that is polling the MBean attribute (metric)
        if (executor != null) {
            List<Runnable> tasks = executor.shutdownNow();
            if (tasks != null) {
                LOGGER.debug("Num tasks awaiting execution = " + tasks.size());
            } else {
                LOGGER.debug("No tasks awaiting execution");
            }
        }

        // Close the RRD DB
        try {
            if (rrdDb != null) {
                rrdDb.close();
                pool.release(rrdDb);
            }
        } catch (IOException e) {
            LOGGER.warn("Unable to close RRD DB", e);
        }

        LOGGER.trace("EXITING: destroy()");
    }

    /**
     * Verify MBean and its attribute exists and can be collected, 
     * i.e., is numeric data (vs. CompositeData)
     * 
     * @return true if MBean can be accessed, false otherwise
     */
    private boolean isMbeanAccessible() {
        Object attr = null;
        long startTime = System.currentTimeMillis();
        while (attr == null && (System.currentTimeMillis() - startTime < mbeanTimeoutMillis)) {
            try {
                attr = localMBeanServer.getAttribute(new ObjectName(mbeanName), mbeanAttributeName);

                if (!isNumeric(attr)) {
                    LOGGER.debug(mbeanAttributeName + " from MBean " + mbeanName + " has non-numeric data");
                    return false;
                }

                if (!(attr instanceof Integer) && !(attr instanceof Long) && !(attr instanceof Float)
                        && !(attr instanceof Double)) {
                    return false;
                }
            } catch (Exception e) {
                try {
                    LOGGER.trace("MBean [" + mbeanName + "] not found, sleeping...");
                    Thread.sleep(1000);
                } catch (InterruptedException ie) {
                    // Ignore this
                }
            }
        }

        return attr != null;
    }

    /**
     * Create an RRD file based on the metric's name (path) in the DDF metrics sub-directory.
     * An RRD DB instance is created from this RRD file's definition (if the RRD file did not already
     * exist, which can occur if the RRD file is created then DDF is restarted and this method is called).
     * If the RRD file already exists, then just create an RRD DB instance based on the existing
     * RRD file.
     * 
     * @param path path where the RRD file is to be created. This is required.
     * @param dsName data source name for the RRD file. This is required.
     * @param dsType data source type, i.e., COUNTER or GAUGE (This is required.)
     *       (DERIVE and ABSOLUTE are not currently supported)
     *       
     * @throws IOException
     * @throws CollectorException
     */
    private void createRrdFile(final String path, final String dsName, final DsType dsType)
            throws IOException, CollectorException {
        LOGGER.trace("ENTERING: createRrdFile");

        if (StringUtils.isEmpty(path)) {
            throw new CollectorException("Path where RRD file is to be created must be specified.");
        } else {
            rrdPath = metricsDir + path;
        }

        if (StringUtils.isEmpty(dsName)) {
            throw new CollectorException("The name of the data source used in the RRD file must be specified.");
        }

        if (!dsType.equals(DsType.COUNTER) && !dsType.equals(DsType.GAUGE)) {
            throw new CollectorException("Data Source type for the RRD file must be either COUNTER or GAUGE.");
        }

        File file = new File(rrdPath);
        if (!file.exists()) {
            // Create necessary parent directories
            if (!file.getParentFile().exists()) {
                file.getParentFile().mkdirs();
            }

            LOGGER.debug("Creating new RRD file " + rrdPath);

            RrdDef def = new RrdDef(rrdPath, rrdStep);

            // NOTE: Currently restrict each RRD file to only have one data source
            // (even though RRD supports multiple data sources in a single RRD file)
            def.addDatasource(dsName, dsType, 90, 0, Double.NaN);

            // NOTE: Separate code segments based on dsType in case in future
            // we want more or less archivers based on data source type.

            // Use a COUNTER for continuous incrementing counters, e.g., number of queries
            if (dsType == DsType.COUNTER) {
                // 1 minute resolution for last 60 minutes
                def.addArchive(ConsolFun.TOTAL, 0.5, 1, 60);

                // 15 minute resolution for the last year
                def.addArchive(ConsolFun.TOTAL, 0.5, 15, ONE_YEAR_IN_15_MINUTE_STEPS);

                // 1 minute resolution for last 60 minutes
                def.addArchive(ConsolFun.AVERAGE, 0.5, 1, 60);

                // 15 minute resolution for the last year
                def.addArchive(ConsolFun.AVERAGE, 0.5, 15, ONE_YEAR_IN_15_MINUTE_STEPS);

                // 1 minute resolution for last 60 minutes
                def.addArchive(ConsolFun.MAX, 0.5, 1, 60);

                // 15 minute resolution for the last year
                def.addArchive(ConsolFun.MAX, 0.5, 15, ONE_YEAR_IN_15_MINUTE_STEPS);

                // 1 minute resolution for last 60 minutes
                def.addArchive(ConsolFun.MIN, 0.5, 1, 60);

                // 15 minute resolution for the last year
                def.addArchive(ConsolFun.MIN, 0.5, 15, ONE_YEAR_IN_15_MINUTE_STEPS);
            }

            // Use a GAUGE to store the values we measure directly as they are,
            // e.g., response time for an ingest or query
            else if (dsType == DsType.GAUGE) {
                // If you want to know the amount, look at the averages. 
                // If you want to know the rate, look at the maximum.

                // 1 minute resolution for last 60 minutes
                def.addArchive(ConsolFun.TOTAL, 0.5, 1, 60);

                // 15 minute resolution for the last year
                def.addArchive(ConsolFun.TOTAL, 0.5, 15, ONE_YEAR_IN_15_MINUTE_STEPS);

                // 1 minute resolution for last 60 minutes
                def.addArchive(ConsolFun.AVERAGE, 0.5, 1, 60);

                // 15 minute resolution for the last year
                def.addArchive(ConsolFun.AVERAGE, 0.5, 15, ONE_YEAR_IN_15_MINUTE_STEPS);

                // 1 minute resolution for last 60 minutes
                def.addArchive(ConsolFun.MAX, 0.5, 1, 60);

                // 15 minute resolution for the last year
                def.addArchive(ConsolFun.MAX, 0.5, 15, ONE_YEAR_IN_15_MINUTE_STEPS);

                // 1 minute resolution for last 60 minutes
                def.addArchive(ConsolFun.MIN, 0.5, 1, 60);

                // 15 minute resolution for the last year
                def.addArchive(ConsolFun.MIN, 0.5, 15, ONE_YEAR_IN_15_MINUTE_STEPS);
            }

            // Create RRD file based on the RRD file definition
            rrdDb = pool.requestRrdDb(def);
        } else {
            LOGGER.debug("rrd file " + path + " already exists - absolute path = " + file.getAbsolutePath());
            rrdDb = pool.requestRrdDb(rrdPath);
        }

        LOGGER.trace("EXITING: createRrdFile");
    }

    /**
     * Configures a scheduled threaded executor to poll the metric's MBean periodically
     * and add a sample to the RRD file with the metric's current value.
     * 
     * @throws CollectorException
     */
    public void updateSamples() throws CollectorException {
        LOGGER.trace("ENTERING: updateSamples");

        if (executor == null) {
            executor = new ScheduledThreadPoolExecutor(1);
        }

        final Runnable updater = new Runnable() {
            public void run() {
                Object attr = null;
                try {
                    attr = localMBeanServer.getAttribute(new ObjectName(mbeanName), mbeanAttributeName);

                    LOGGER.trace("Sampling attribute " + mbeanAttributeName + " from MBean " + mbeanName);

                    // Cast the metric's sampled value to the appropriate data type
                    double val = 0;
                    if (attr instanceof Integer) {
                        val = (Integer) attr;
                    } else if (attr instanceof Long) {
                        val = ((Long) attr).intValue();
                    } else if (attr instanceof Float) {
                        val = ((Float) attr);
                    } else if (attr instanceof Double) {
                        val = ((Double) attr);
                    } else {
                        throw new IllegalArgumentException(
                                "Unsupported type " + attr + " for attribute " + mbeanAttributeName);
                    }

                    LOGGER.trace("MBean attribute " + mbeanAttributeName + " has value = " + val);

                    // If first time this metric has been sampled, then need to create a
                    // sample in the RRD file
                    if (sample == null) {
                        sample = rrdDb.createSample();
                    }

                    try {
                        // Add metric's sample to RRD file with current timestamp (i.e., "NOW")
                        sample.setAndUpdate("NOW:" + val);
                    } catch (IllegalArgumentException iae) {
                        LOGGER.error("Dropping sample of datasource " + rrdDataSourceName, iae);
                    }
                } catch (Exception e) {
                    LOGGER.warn("Problems getting MBean attribute " + mbeanAttributeName, e);
                }
            }
        };

        // Setup threaded scheduler to retrieve this MBean attribute's value
        // at the specified sample rate
        LOGGER.debug("Setup ScheduledThreadPoolExecutor for MBean " + mbeanName);
        executor.scheduleWithFixedDelay(updater, 0, sampleRate, TimeUnit.SECONDS);

        LOGGER.trace("EXITING: updateSamples");
    }

    /**
     * @return local MBean server
     */
    private MBeanServer getLocalMBeanServer() {
        if (localMBeanServer == null) {
            localMBeanServer = ManagementFactory.getPlatformMBeanServer();
        }

        return localMBeanServer;
    }

    /**
     * Determines whether an object's value is a numeric type or a String with
     * a numeric value.
     * 
     * @param value the Object to be tested whether it has a numeric value
     * 
     * @return true if object's value is numeric, false otherwise
     */
    public static boolean isNumeric(Object value) {
        return ((value instanceof Number) || ((value instanceof String) && NumberUtils.isNumber((String) value)));
    }

    public void setMbeanName(String mbeanName) {
        LOGGER.trace("Setting mbeanName to " + mbeanName);

        this.mbeanName = mbeanName;
    }

    public String getMbeanAttributeName() {
        return mbeanAttributeName;
    }

    public void setMbeanAttributeName(String mbeanAttributeName) {
        this.mbeanAttributeName = mbeanAttributeName;
    }

    public String getMetricsDir() {
        return metricsDir;
    }

    public void setMetricsDir(String metricsDir) {
        this.metricsDir = metricsDir;
    }

    public String getRrdPath() {
        return rrdPath;
    }

    public void setRrdPath(String rrdPath) {
        this.rrdPath = rrdPath;
    }

    public String getRrdDataSourceName() {
        return rrdDataSourceName;
    }

    public void setRrdDataSourceName(String rrdDataSourceName) {
        this.rrdDataSourceName = rrdDataSourceName;
    }

    public String getRrdDataSourceType() {
        return rrdDataSourceType;
    }

    public void setRrdDataSourceType(String rrdDataSourceType) {
        this.rrdDataSourceType = rrdDataSourceType;
        LOGGER.debug("rrdDataSourceType = " + rrdDataSourceType);
    }

    protected int getSampleRate() {
        return sampleRate;
    }

    protected void setSampleRate(int sampleRate) {
        this.sampleRate = sampleRate;
        this.rrdStep = this.sampleRate;
    }

    void setMbeanTimeoutMillis(long mbeanTimeoutMillis) {
        this.mbeanTimeoutMillis = mbeanTimeoutMillis;
    }

}