password.pwm.svc.stats.StatisticsManager.java Source code

Java tutorial

Introduction

Here is the source code for password.pwm.svc.stats.StatisticsManager.java

Source

/*
 * Password Management Servlets (PWM)
 * http://www.pwm-project.org
 *
 * Copyright (c) 2006-2009 Novell, Inc.
 * Copyright (c) 2009-2017 The PWM Project
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

package password.pwm.svc.stats;

import org.apache.commons.csv.CSVPrinter;
import password.pwm.PwmApplication;
import password.pwm.PwmConstants;
import password.pwm.config.option.DataStorageMethod;
import password.pwm.error.PwmException;
import password.pwm.health.HealthRecord;
import password.pwm.http.PwmRequest;
import password.pwm.svc.PwmService;
import password.pwm.util.AlertHandler;
import password.pwm.util.java.JavaHelper;
import password.pwm.util.java.JsonUtil;
import password.pwm.util.java.TimeDuration;
import password.pwm.util.localdb.LocalDB;
import password.pwm.util.localdb.LocalDBException;
import password.pwm.util.logging.PwmLogger;

import java.io.IOException;
import java.io.OutputStream;
import java.math.BigDecimal;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.TimeZone;
import java.util.TimerTask;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

public class StatisticsManager implements PwmService {

    private static final PwmLogger LOGGER = PwmLogger.forClass(StatisticsManager.class);

    private static final int DB_WRITE_FREQUENCY_MS = 60 * 1000; // 1 minutes

    private static final String DB_KEY_VERSION = "STATS_VERSION";
    private static final String DB_KEY_CUMULATIVE = "CUMULATIVE";
    private static final String DB_KEY_INITIAL_DAILY_KEY = "INITIAL_DAILY_KEY";
    private static final String DB_KEY_PREFIX_DAILY = "DAILY_";
    private static final String DB_KEY_TEMP = "TEMP_KEY";

    private static final String DB_VALUE_VERSION = "1";

    public static final String KEY_CURRENT = "CURRENT";
    public static final String KEY_CUMULATIVE = "CUMULATIVE";

    private LocalDB localDB;

    private DailyKey currentDailyKey = new DailyKey(new Date());
    private DailyKey initialDailyKey = new DailyKey(new Date());

    private ScheduledExecutorService executorService;

    private final StatisticsBundle statsCurrent = new StatisticsBundle();
    private StatisticsBundle statsDaily = new StatisticsBundle();
    private StatisticsBundle statsCummulative = new StatisticsBundle();
    private Map<String, EventRateMeter> epsMeterMap = new HashMap<>();

    private PwmApplication pwmApplication;

    private STATUS status = STATUS.NEW;

    private final Map<String, StatisticsBundle> cachedStoredStats = new LinkedHashMap<String, StatisticsBundle>() {
        @Override
        protected boolean removeEldestEntry(final Map.Entry<String, StatisticsBundle> eldest) {
            return this.size() > 50;
        }
    };

    public StatisticsManager() {
    }

    public synchronized void incrementValue(final Statistic statistic) {
        statsCurrent.incrementValue(statistic);
        statsDaily.incrementValue(statistic);
        statsCummulative.incrementValue(statistic);
    }

    public synchronized void updateAverageValue(final Statistic statistic, final long value) {
        statsCurrent.updateAverageValue(statistic, value);
        statsDaily.updateAverageValue(statistic, value);
        statsCummulative.updateAverageValue(statistic, value);
    }

    public Map<String, String> getStatHistory(final Statistic statistic, final int days) {
        final Map<String, String> returnMap = new LinkedHashMap<>();
        DailyKey loopKey = currentDailyKey;
        int counter = days;
        while (counter > 0) {
            final StatisticsBundle bundle = getStatBundleForKey(loopKey.toString());
            if (bundle != null) {
                final String key = (new SimpleDateFormat("MMM dd")).format(loopKey.calendar().getTime());
                final String value = bundle.getStatistic(statistic);
                returnMap.put(key, value);
            }
            loopKey = loopKey.previous();
            counter--;
        }
        return returnMap;
    }

    public StatisticsBundle getStatBundleForKey(final String key) {
        if (key == null || key.length() < 1 || KEY_CUMULATIVE.equals(key)) {
            return statsCummulative;
        }

        if (KEY_CURRENT.equals(key)) {
            return statsCurrent;
        }

        if (currentDailyKey.toString().equals(key)) {
            return statsDaily;
        }

        if (cachedStoredStats.containsKey(key)) {
            return cachedStoredStats.get(key);
        }

        if (localDB == null) {
            return null;
        }

        try {
            final String storedStat = localDB.get(LocalDB.DB.PWM_STATS, key);
            final StatisticsBundle returnBundle;
            if (storedStat != null && storedStat.length() > 0) {
                returnBundle = StatisticsBundle.input(storedStat);
            } else {
                returnBundle = new StatisticsBundle();
            }
            cachedStoredStats.put(key, returnBundle);
            return returnBundle;
        } catch (LocalDBException e) {
            LOGGER.error("error retrieving stored stat for " + key + ": " + e.getMessage());
        }

        return null;
    }

    public Map<DailyKey, String> getAvailableKeys(final Locale locale) {
        final DateFormat dateFormatter = SimpleDateFormat.getDateInstance(SimpleDateFormat.DEFAULT, locale);
        final Map<DailyKey, String> returnMap = new LinkedHashMap<DailyKey, String>();

        // add current time;
        returnMap.put(currentDailyKey, dateFormatter.format(new Date()));

        // if now historical data then we're done
        if (currentDailyKey.equals(initialDailyKey)) {
            return returnMap;
        }

        DailyKey loopKey = currentDailyKey;
        int safetyCounter = 0;
        while (!loopKey.equals(initialDailyKey) && safetyCounter < 5000) {
            final Calendar c = loopKey.calendar();
            final String display = dateFormatter.format(c.getTime());
            returnMap.put(loopKey, display);
            loopKey = loopKey.previous();
            safetyCounter++;
        }
        return returnMap;
    }

    public String toString() {
        final StringBuilder sb = new StringBuilder();

        for (final Statistic m : Statistic.values()) {
            sb.append(m.toString());
            sb.append("=");
            sb.append(statsCurrent.getStatistic(m));
            sb.append(", ");
        }

        if (sb.length() > 2) {
            sb.delete(sb.length() - 2, sb.length());
        }

        return sb.toString();
    }

    public void init(final PwmApplication pwmApplication) throws PwmException {
        for (final Statistic.EpsType type : Statistic.EpsType.values()) {
            for (final Statistic.EpsDuration duration : Statistic.EpsDuration.values()) {
                epsMeterMap.put(type.toString() + duration.toString(),
                        new EventRateMeter(duration.getTimeDuration()));
            }
        }

        status = STATUS.OPENING;
        this.localDB = pwmApplication.getLocalDB();
        this.pwmApplication = pwmApplication;

        if (localDB == null) {
            LOGGER.error("LocalDB is not available, will remain closed");
            status = STATUS.CLOSED;
            return;
        }

        {
            final String storedCummulativeBundleStr = localDB.get(LocalDB.DB.PWM_STATS, DB_KEY_CUMULATIVE);
            if (storedCummulativeBundleStr != null && storedCummulativeBundleStr.length() > 0) {
                try {
                    statsCummulative = StatisticsBundle.input(storedCummulativeBundleStr);
                } catch (Exception e) {
                    LOGGER.warn("error loading saved stored statistics: " + e.getMessage());
                }
            }
        }

        {
            for (final Statistic.EpsType loopEpsType : Statistic.EpsType.values()) {
                for (final Statistic.EpsType loopEpsDuration : Statistic.EpsType.values()) {
                    final String key = "EPS-" + loopEpsType.toString() + loopEpsDuration.toString();
                    final String storedValue = localDB.get(LocalDB.DB.PWM_STATS, key);
                    if (storedValue != null && storedValue.length() > 0) {
                        try {
                            final EventRateMeter eventRateMeter = JsonUtil.deserialize(storedValue,
                                    EventRateMeter.class);
                            epsMeterMap.put(loopEpsType.toString() + loopEpsDuration.toString(), eventRateMeter);
                        } catch (Exception e) {
                            LOGGER.error("unexpected error reading last EPS rate for " + loopEpsType
                                    + " from LocalDB: " + e.getMessage());
                        }
                    }
                }
            }

        }

        {
            final String storedInitialString = localDB.get(LocalDB.DB.PWM_STATS, DB_KEY_INITIAL_DAILY_KEY);
            if (storedInitialString != null && storedInitialString.length() > 0) {
                initialDailyKey = new DailyKey(storedInitialString);
            }
        }

        {
            currentDailyKey = new DailyKey(new Date());
            final String storedDailyStr = localDB.get(LocalDB.DB.PWM_STATS, currentDailyKey.toString());
            if (storedDailyStr != null && storedDailyStr.length() > 0) {
                statsDaily = StatisticsBundle.input(storedDailyStr);
            }
        }

        try {
            localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_TEMP, JavaHelper.toIsoDate(new Date()));
        } catch (IllegalStateException e) {
            LOGGER.error("unable to write to localDB, will remain closed, error: " + e.getMessage());
            status = STATUS.CLOSED;
            return;
        }

        localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_VERSION, DB_VALUE_VERSION);
        localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_INITIAL_DAILY_KEY, initialDailyKey.toString());

        { // setup a timer to roll over at 0 Zula and one to write current stats every 10 seconds
            executorService = JavaHelper.makeSingleThreadExecutorService(pwmApplication, this.getClass());
            executorService.scheduleAtFixedRate(new FlushTask(), 10 * 1000, DB_WRITE_FREQUENCY_MS,
                    TimeUnit.MILLISECONDS);
            final TimeDuration delayTillNextZulu = TimeDuration.fromCurrent(JavaHelper.nextZuluZeroTime());
            executorService.scheduleAtFixedRate(new NightlyTask(), delayTillNextZulu.getTotalMilliseconds(),
                    TimeUnit.DAYS.toMillis(1), TimeUnit.MILLISECONDS);
        }

        status = STATUS.OPEN;
    }

    private void writeDbValues() {
        if (localDB != null) {
            try {
                localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_CUMULATIVE, statsCummulative.output());
                localDB.put(LocalDB.DB.PWM_STATS, currentDailyKey.toString(), statsDaily.output());

                for (final Statistic.EpsType loopEpsType : Statistic.EpsType.values()) {
                    for (final Statistic.EpsDuration loopEpsDuration : Statistic.EpsDuration.values()) {
                        final String key = "EPS-" + loopEpsType.toString();
                        final String mapKey = loopEpsType.toString() + loopEpsDuration.toString();
                        final String value = JsonUtil.serialize(this.epsMeterMap.get(mapKey));
                        localDB.put(LocalDB.DB.PWM_STATS, key, value);
                    }
                }
            } catch (LocalDBException e) {
                LOGGER.error("error outputting pwm statistics: " + e.getMessage());
            }
        }

    }

    private void resetDailyStats() {
        try {
            final Map<String, String> emailValues = new LinkedHashMap<>();
            for (final Statistic statistic : Statistic.values()) {
                final String key = statistic.getLabel(PwmConstants.DEFAULT_LOCALE);
                final String value = statsDaily.getStatistic(statistic);
                emailValues.put(key, value);
            }

            AlertHandler.alertDailyStats(pwmApplication, emailValues);
        } catch (Exception e) {
            LOGGER.error("error while generating daily alert statistics: " + e.getMessage());
        }

        currentDailyKey = new DailyKey(new Date());
        statsDaily = new StatisticsBundle();
        LOGGER.debug("reset daily statistics");
    }

    public STATUS status() {
        return status;
    }

    public void close() {
        try {
            writeDbValues();
        } catch (Exception e) {
            LOGGER.error("unexpected error closing: " + e.getMessage());
        }

        JavaHelper.closeAndWaitExecutor(executorService, new TimeDuration(3, TimeUnit.SECONDS));

        status = STATUS.CLOSED;
    }

    public List<HealthRecord> healthCheck() {
        return Collections.emptyList();
    }

    private class NightlyTask extends TimerTask {
        public void run() {
            writeDbValues();
            resetDailyStats();
        }
    }

    private class FlushTask extends TimerTask {
        public void run() {
            writeDbValues();
        }
    }

    public static class DailyKey {
        int year;
        int day;

        public DailyKey(final Date date) {
            final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("Zulu"));
            calendar.setTime(date);
            year = calendar.get(Calendar.YEAR);
            day = calendar.get(Calendar.DAY_OF_YEAR);
        }

        public DailyKey(final String value) {
            final String strippedValue = value.substring(DB_KEY_PREFIX_DAILY.length(), value.length());
            final String[] splitValue = strippedValue.split("_");
            year = Integer.valueOf(splitValue[0]);
            day = Integer.valueOf(splitValue[1]);
        }

        private DailyKey() {
        }

        @Override
        public String toString() {
            return DB_KEY_PREFIX_DAILY + String.valueOf(year) + "_" + String.valueOf(day);
        }

        public DailyKey previous() {
            final Calendar calendar = calendar();
            calendar.add(Calendar.HOUR, -24);
            final DailyKey newKey = new DailyKey();
            newKey.year = calendar.get(Calendar.YEAR);
            newKey.day = calendar.get(Calendar.DAY_OF_YEAR);
            return newKey;
        }

        public Calendar calendar() {
            final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("Zulu"));
            calendar.set(Calendar.YEAR, year);
            calendar.set(Calendar.DAY_OF_YEAR, day);
            return calendar;
        }

        @Override
        public boolean equals(final Object o) {
            if (this == o) {
                return true;
            }
            if (o == null || getClass() != o.getClass()) {
                return false;
            }

            final DailyKey key = (DailyKey) o;

            if (day != key.day) {
                return false;
            }
            if (year != key.year) {
                return false;
            }

            return true;
        }

        @Override
        public int hashCode() {
            int result = year;
            result = 31 * result + day;
            return result;
        }
    }

    public void updateEps(final Statistic.EpsType type, final int itemCount) {
        for (final Statistic.EpsDuration duration : Statistic.EpsDuration.values()) {
            epsMeterMap.get(type.toString() + duration.toString()).markEvents(itemCount);
        }
    }

    public BigDecimal readEps(final Statistic.EpsType type, final Statistic.EpsDuration duration) {
        return epsMeterMap.get(type.toString() + duration.toString()).readEventRate();
    }

    public int outputStatsToCsv(final OutputStream outputStream, final Locale locale, final boolean includeHeader)
            throws IOException {
        LOGGER.trace("beginning output stats to csv process");
        final Instant startTime = Instant.now();

        final StatisticsManager statsManger = pwmApplication.getStatisticsManager();
        final CSVPrinter csvPrinter = JavaHelper.makeCsvPrinter(outputStream);

        if (includeHeader) {
            final List<String> headers = new ArrayList<>();
            headers.add("KEY");
            headers.add("YEAR");
            headers.add("DAY");
            for (final Statistic stat : Statistic.values()) {
                headers.add(stat.getLabel(locale));
            }
            csvPrinter.printRecord(headers);
        }

        int counter = 0;
        final Map<StatisticsManager.DailyKey, String> keys = statsManger
                .getAvailableKeys(PwmConstants.DEFAULT_LOCALE);
        for (final StatisticsManager.DailyKey loopKey : keys.keySet()) {
            counter++;
            final StatisticsBundle bundle = statsManger.getStatBundleForKey(loopKey.toString());
            final List<String> lineOutput = new ArrayList<>();
            lineOutput.add(loopKey.toString());
            lineOutput.add(String.valueOf(loopKey.year));
            lineOutput.add(String.valueOf(loopKey.day));
            for (final Statistic stat : Statistic.values()) {
                lineOutput.add(bundle.getStatistic(stat));
            }
            csvPrinter.printRecord(lineOutput);
        }

        csvPrinter.flush();
        LOGGER.trace("completed output stats to csv process; output " + counter + " records in "
                + TimeDuration.fromCurrent(startTime).asCompactString());
        return counter;
    }

    public ServiceInfoBean serviceInfo() {
        if (status() == STATUS.OPEN) {
            return new ServiceInfoBean(Collections.singletonList(DataStorageMethod.LOCALDB));
        } else {
            return new ServiceInfoBean(Collections.<DataStorageMethod>emptyList());
        }
    }

    public static void incrementStat(final PwmRequest pwmRequest, final Statistic statistic) {
        incrementStat(pwmRequest.getPwmApplication(), statistic);
    }

    public static void incrementStat(final PwmApplication pwmApplication, final Statistic statistic) {
        if (pwmApplication == null) {
            LOGGER.error("skipping requested statistic increment of " + statistic + " due to null pwmApplication");
            return;
        }

        final StatisticsManager statisticsManager = pwmApplication.getStatisticsManager();
        if (statisticsManager == null) {
            LOGGER.error(
                    "skipping requested statistic increment of " + statistic + " due to null statisticsManager");
            return;
        }

        if (statisticsManager.status() != STATUS.OPEN) {
            LOGGER.trace("skipping requested statistic increment of " + statistic
                    + " due to StatisticsManager being closed");
            return;
        }

        statisticsManager.incrementValue(statistic);
    }

}