Java tutorial
/* Copyright IBM Corp. 2015, 2016 This file is part of Anomaly Detection Engine for Linux Logs (ADE). ADE is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. ADE is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with ADE. If not, see <http://www.gnu.org/licenses/>. */ package org.openmainframe.ade.ext.stats; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.TimeZone; import org.openmainframe.ade.Ade; import org.openmainframe.ade.exceptions.AdeException; import org.openmainframe.ade.ext.AdeExt; import org.openmainframe.ade.ext.service.AdeExtUsageException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Days; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; /** * This class keep track of the Interval Message Rate. Interval in this class has a different meaning from * Ade. * * Interval is the time that data are kept. Within this interval, the data are kept for each 10 minutes. * Statistics will be generated by combining a subset of these 10 minutes interval. * * For example, for 120 minutes interval, we might want statistics for every 10, 20, 30, 60 and 120 minutes. */ public class MessageRateStats { /** * A map from Source to Analysis Group */ private static Map<String, String> s_sourceToAnalysisGroupMap = new HashMap<String, String>(); public static void addSourceAndAnalysisGroup(String sourceName, String analysisGroupName) { s_sourceToAnalysisGroupMap.put(sourceName, analysisGroupName); } /** * A map containing the message Rate Stats */ private static Map<String, MessageRateStats> s_sourceToMsgRatesStatsMap = new HashMap<String, MessageRateStats>(); public static MessageRateStats getMessageRateStatsForSource(String source) throws AdeException { MessageRateStats stats; String name; /* Override the source name to "AllSource" if we want to merge all the sources */ if (AdeExt.getAdeExt().getConfigProperties().isMsgRateMergeSource()) { name = s_sourceToAnalysisGroupMap.get(source); if (name == null) { name = "[ALL_SOURCES]"; } } else { name = source; } stats = s_sourceToMsgRatesStatsMap.get(name); if (stats == null) { stats = new MessageRateStats(name); s_sourceToMsgRatesStatsMap.put(name, stats); } return stats; } /** * Generate a for all sources * @throws AdeException */ public static void generateReportForAllSources() throws AdeException { final Collection<MessageRateStats> statsSourceCollection = s_sourceToMsgRatesStatsMap.values(); for (MessageRateStats statsForASource : statsSourceCollection) { statsForASource.generateReport(); } } /** * ENUM with the reporting frequency */ public enum ReportFrequency { MONTHLY, DAYS(10); private int m_days; /** * Number of days * @param days */ private ReportFrequency(int days) { m_days = days; } /** * This is used for non-days based frequency */ private ReportFrequency() { } /** * Return the days * @return */ public void setDays(int days) { m_days = days; } /** * Return the days * @return */ public int getDays() { return m_days; } @Override public String toString() { String str; switch (this) { case DAYS: str = this.name() + "(" + getDays() + ")"; break; case MONTHLY: str = this.name(); break; default: str = "ERROR"; } return str; } } /** * 10 minutes */ private static final long TEN_MINUTES = (long) 10 * 60 * 1000; /** * Output format */ private static final DateTimeFormatter s_dateTimeFormatter = DateTimeFormat.forPattern("MM/dd/yyyy HH:mm:ss"); /** * The outputTimeZone defined in the conf file. */ private static DateTimeZone s_outTimeZone; /** * The frequency where reports are generated */ private ReportFrequency m_reportFrequency; /** * The different time used for reporting. */ private DateTime m_lastReportDateTimeBegin; private DateTime m_processingStartDateTime; /** * The input time of the most recent message */ private DateTime m_messageInputDateTime; /** * Logger */ private static final Logger logger = LoggerFactory.getLogger(MessageRateStats.class); /** * Logger for statistics */ private static final Logger statsLogger = LoggerFactory.getLogger(MessageRateStats.class); /** * The length of the data to keep. This will be used to determine the size of array for * tracking the message count. */ private short m_numberOf10MinutesSlotsToKeep; /** * The interval size statistics to keep */ private short[] m_subIntervalSizeList; private static final short[] SUB_INTERVAL_SIZE_LIST_TWO_HOURS = { 1, 2, 3, 6, 12 }; /** * Each index of this array is map to the subIntervalSizeList. * * Each index represent the overall statistics for a subinterval size (10, 20 minutes etc) */ private OverallStats[] m_overallStatsForAllIntervals; /** * Beginning of interval, initialize to 0 to indicate it's not initialized yet. */ private long m_beginOfInterval = 0; private long m_beginOfNextInterval = 0; /** * The current index of the 10MinutesMsgCountArray */ private int m_currentIndex10MinutesMsgCountArray = 0; /** * A map from Msg ID to the MsgStats object. */ private Map<String, MessageStats> m_msgIdToMsgStatsMap = new HashMap<String, MessageRateStats.MessageStats>(); private final static int MAX_MESSAGE_STATS_TO_KEEP = 1000; private static int s_maxMsgToKeep = 1000; /** * The source this stats object represents */ private String m_source; /** * A default constructor * @param source * @throws AdeException */ private MessageRateStats(String source) throws AdeException { /* Default the interval size to keep 12 ten minutes data. */ this(source, (short) 12, SUB_INTERVAL_SIZE_LIST_TWO_HOURS); } /** * Constructor * @param source * @throws AdeException */ private MessageRateStats(String source, short numberOf10MinutesIntervalToKeep, short[] intervalSizeList) throws AdeException { m_source = source; getConfiguration(numberOf10MinutesIntervalToKeep, intervalSizeList); if (intervalSizeList.length == 0) { final String msg = "tenMinutesIntervalSizeList has a size of 0"; logger.warn(msg); throw new AdeExtUsageException(msg); } /* Verify if the number of interval to keep is divisible by intervalSizeList */ for (short intervalSize : m_subIntervalSizeList) { final int reminder = m_numberOf10MinutesSlotsToKeep % intervalSize; if (reminder != 0) { final String msg = "m_numberOf10MinutesIntervalToKeep(" + m_numberOf10MinutesSlotsToKeep + ") is not divisible by tenMinutesIntervalSizeList value: " + intervalSize; logger.warn(msg); throw new AdeExtUsageException(msg); } } /* Initialize the OverStats object for each requested interval size. */ initOverallStatsForAllIntervals(); } /** * Retrieve the configuration parameters * @throws AdeException */ private void getConfiguration(short numberOf10MinutesSlotsToKeep, short[] intervalSizeList) throws AdeException { final TimeZone outJavaTimeZone = Ade.getAde().getConfigProperties().getOutputTimeZone(); s_outTimeZone = DateTimeZone.forOffsetMillis(outJavaTimeZone.getRawOffset()); /* Set the number of messages to keep */ s_maxMsgToKeep = AdeExt.getAdeExt().getConfigProperties().getMsgRateMsgToKeep(); if (s_maxMsgToKeep == -1) { s_maxMsgToKeep = MAX_MESSAGE_STATS_TO_KEEP; } /* Set the report frequency */ final String reportFreqStr = AdeExt.getAdeExt().getConfigProperties().getMsgRateReportFreq(); if (reportFreqStr == null || reportFreqStr.length() == 0) { m_reportFrequency = ReportFrequency.DAYS; m_reportFrequency.setDays(10); } else if (reportFreqStr.equalsIgnoreCase(ReportFrequency.MONTHLY.name())) { m_reportFrequency = ReportFrequency.MONTHLY; } else { final int reportFreq = Integer.parseInt(reportFreqStr); m_reportFrequency = ReportFrequency.DAYS; m_reportFrequency.setDays(reportFreq); } /* Set the number of 10 minutes interval to keep */ m_numberOf10MinutesSlotsToKeep = AdeExt.getAdeExt().getConfigProperties().getMsgRate10MinSlotsToKeep(); if (m_numberOf10MinutesSlotsToKeep == -1) { m_numberOf10MinutesSlotsToKeep = numberOf10MinutesSlotsToKeep; } /* Set the subinterval list */ m_subIntervalSizeList = AdeExt.getAdeExt().getConfigProperties().getMsgRate10MinSubIntervals(); if (m_subIntervalSizeList == null) { m_subIntervalSizeList = intervalSizeList; } /* Output the trace settings */ StringBuilder bldtrace = new StringBuilder(""); for (short interval : m_subIntervalSizeList) { bldtrace.append(interval + ","); } logger.info("Tracking Message Rate for " + m_source + " for " + m_numberOf10MinutesSlotsToKeep + " ten minutes slots" + " reportFreq=" + m_reportFrequency.toString() + " maxMsgToKeep=" + s_maxMsgToKeep + " 10MinIntervals=" + bldtrace.toString()); } /** * Add a message to the collection * @throws AdeException */ public void markLoggerStarting(long nextMessageInputTime) throws AdeException { if (m_messageInputDateTime == null) { /* This is the first time we process a process for this source, and this message indicate the logger * just started. No logging needed. */ return; } /* Find the starting of the next ten minutes interval for the previous message. */ long prevMessageInputTime = m_messageInputDateTime.getMillis(); if (prevMessageInputTime % TEN_MINUTES > 0) { prevMessageInputTime = TEN_MINUTES * (prevMessageInputTime / TEN_MINUTES) + TEN_MINUTES; } final DateTime prevMessageDateTime = new DateTime(prevMessageInputTime).withZone(s_outTimeZone); /* Find the end of the 10 minutes interval before the next message */ nextMessageInputTime = TEN_MINUTES * (nextMessageInputTime / TEN_MINUTES); final DateTime nextMessageDateTime = new DateTime(nextMessageInputTime).withZone(s_outTimeZone); /* Find the number of 10 minutes interval being skipped */ final long skippedInterval = (nextMessageInputTime - prevMessageInputTime) / TEN_MINUTES; /* Note: Skipped Interval could be less than 0, if the next Message and pre Message are within * the same 10 minutes interval. */ if (skippedInterval > 0) { /* Write out the message, using the previous message's timestamp */ final DateTime dateTime = m_messageInputDateTime; final String dateStr = s_dateTimeFormatter.print(dateTime); final String reportString = m_source + ", " + dateStr + ", Logger Unavailable For (10 min intervals)=" + skippedInterval + ", emptyIntervalStart=" + s_dateTimeFormatter.print(prevMessageDateTime) + ", emptyIntervalEnd=" + s_dateTimeFormatter.print(nextMessageDateTime); statsLogger.info(reportString); } } /** * Add a message to the collection * @throws AdeException */ public void addMessage(String msgId, long inputTime, boolean isWrapperMessage) throws AdeException { m_messageInputDateTime = new DateTime(inputTime).withZone(s_outTimeZone); /* Update the time/index as needed */ determineAndProcessEndOfInterval(inputTime); /* Look up the msgRateStats for the msgId. If not exist, add it */ MessageStats msgRateStats = m_msgIdToMsgStatsMap.get(msgId); if (msgRateStats == null) { /* The number of entry allowed to add to the HashMap is limited, * this is to control memory consumption. * * The goal for this Message Rate tracker is to determine * if there are "enough" unique messages, if there are more than * 1000 in 2 hours, it already satisfied our needs.Was */ if (m_msgIdToMsgStatsMap.size() >= s_maxMsgToKeep) { return; } msgRateStats = new MessageStats(msgId, m_numberOf10MinutesSlotsToKeep, isWrapperMessage); m_msgIdToMsgStatsMap.put(msgId, msgRateStats); } /* The inputTime is guarantee to be in the current interval or before. * If inputTime is less than beginning of current interval (indexOf10MinutesMsgCountArray will be negative), * or it's less than current 10 minutes index (i.e. message's time moved backward), * Then, use m_currentIndex10MinutesMsgCountArray set previously.*/ final int indexOf10MinutesMsgCountArray = (int) ((inputTime - m_beginOfInterval) / TEN_MINUTES); if (indexOf10MinutesMsgCountArray > m_currentIndex10MinutesMsgCountArray) { m_currentIndex10MinutesMsgCountArray = indexOf10MinutesMsgCountArray; } /* Indicate that the message has occured */ if (inputTime >= m_beginOfInterval) { /* Only if the message is > begin of interval, then we will add it. * This will allow message go back in time at most 2 hours. */ msgRateStats.addMessage(m_currentIndex10MinutesMsgCountArray); } } /** * Manage the time, and set the proper index. * @throws AdeException */ private void determineAndProcessEndOfInterval(long inputTime) throws AdeException { if (m_beginOfInterval == 0) { /* Determine the beginning of interval */ m_beginOfInterval = (inputTime / m_numberOf10MinutesSlotsToKeep / TEN_MINUTES) * m_numberOf10MinutesSlotsToKeep * TEN_MINUTES; m_beginOfNextInterval = m_beginOfInterval + m_numberOf10MinutesSlotsToKeep * TEN_MINUTES; /* Keep the last report time and when this process started */ m_lastReportDateTimeBegin = new DateTime(m_beginOfInterval).withZone(s_outTimeZone); m_lastReportDateTimeBegin = m_lastReportDateTimeBegin.withTimeAtStartOfDay(); m_processingStartDateTime = new DateTime(m_lastReportDateTimeBegin.getMillis()).withZone(s_outTimeZone); } else { boolean isNewInterval = false; /* Message's input time is greater than current interval */ if (inputTime >= m_beginOfNextInterval) { endOfIntervalProcessing(inputTime); isNewInterval = true; m_beginOfInterval = m_beginOfNextInterval; m_beginOfNextInterval = m_beginOfInterval + m_numberOf10MinutesSlotsToKeep * TEN_MINUTES; /* Reset the count array index to 0 */ m_currentIndex10MinutesMsgCountArray = 0; } /* if the inputTime is still greater than the next interval, * continue to find the beginning of next interval. */ while (inputTime >= m_beginOfNextInterval) { endOfIntervalProcessing(inputTime); /* We don't call endOfIntervalProcessing() here, because those interval missed * are considered no data and shouldn't affect the stddev calculation. */ m_beginOfInterval = m_beginOfNextInterval; m_beginOfNextInterval = m_beginOfInterval + m_numberOf10MinutesSlotsToKeep * TEN_MINUTES; } if (isNewInterval) { /* If it's time to generate report, then generate the report */ generateReportIfNeeded(inputTime); } } } /** * Handle end of interval processing. * * During end of a 120 minutes interval, we will consolidate all the stats collected * for each message (i.e. for a specific 10, 20, 30 minutes interval, how many unique * message is there). The consolidated stats for 10 minutes interval will be added * to the "overallStats" object for 10 minutes interval (same for 20, 30 minutes). * * These overallStats will get reset once a report is generated and outputted. * @param inputTime * @throws AdeException */ private void endOfIntervalProcessing(long inputTime) throws AdeException { /* Initialize an array to keep track of the subInterval statistics */ final long[][] arrayOfSubIntervalMsg1UniqueMsgIdCount = new long[m_subIntervalSizeList.length][]; final long[][] arrayOfSubIntervalMsg1TotalMsgCount = new long[m_subIntervalSizeList.length][]; final long[][] arrayOfSubIntervalMsg2UniqueMsgIdCount = new long[m_subIntervalSizeList.length][]; final long[][] arrayOfSubIntervalMsg2TotalMsgCount = new long[m_subIntervalSizeList.length][]; for (int i = 0; i < m_subIntervalSizeList.length; i++) { final short intervalCount = m_subIntervalSizeList[i]; arrayOfSubIntervalMsg1UniqueMsgIdCount[i] = new long[m_numberOf10MinutesSlotsToKeep / intervalCount]; arrayOfSubIntervalMsg1TotalMsgCount[i] = new long[m_numberOf10MinutesSlotsToKeep / intervalCount]; arrayOfSubIntervalMsg2UniqueMsgIdCount[i] = new long[m_numberOf10MinutesSlotsToKeep / intervalCount]; arrayOfSubIntervalMsg2TotalMsgCount[i] = new long[m_numberOf10MinutesSlotsToKeep / intervalCount]; } /* Go through all messages */ final Collection<MessageStats> msgStatsCollection = m_msgIdToMsgStatsMap.values(); for (MessageStats msgStats : msgStatsCollection) { for (int i = 0; i < m_subIntervalSizeList.length; i++) { final short subIntervalSize = m_subIntervalSizeList[i]; final long[] intervalCountArray = msgStats.getCountBasedOnIntervalSize(subIntervalSize); /* Determine number of the subInterval has message */ for (int j = 0; j < intervalCountArray.length; j++) { final long intervalCount = intervalCountArray[j]; if (msgStats.isMsg1()) { /* Total Message Count */ arrayOfSubIntervalMsg1TotalMsgCount[i][j] += intervalCount; /* Unique Message Count */ if (intervalCount > 0) { /* Increase the count array, once per message per interval if * the message appears at least once. */ arrayOfSubIntervalMsg1UniqueMsgIdCount[i][j]++; } } else { /* Total Message Count */ arrayOfSubIntervalMsg2TotalMsgCount[i][j] += intervalCount; /* Unique Message Count */ if (intervalCount > 0) { /* Increase the count array, once per message per interval if * the message appears at least once. */ arrayOfSubIntervalMsg2UniqueMsgIdCount[i][j]++; } } } } } /* Clear the list of msg that we are tracking. */ m_msgIdToMsgStatsMap.clear(); /* Add the statistics from the interval to the overallStats*/ for (int i = 0; i < m_subIntervalSizeList.length; i++) { m_overallStatsForAllIntervals[i].addStats(arrayOfSubIntervalMsg1UniqueMsgIdCount[i], arrayOfSubIntervalMsg1TotalMsgCount[i], arrayOfSubIntervalMsg2UniqueMsgIdCount[i], arrayOfSubIntervalMsg2TotalMsgCount[i]); } } /** * Initialize the overallStats for all interval size. If overallStats array already * exist, it will be overwritten. */ private void initOverallStatsForAllIntervals() { m_overallStatsForAllIntervals = new OverallStats[m_subIntervalSizeList.length]; for (int i = 0; i < m_subIntervalSizeList.length; i++) { final short subIntervalSize = m_subIntervalSizeList[i]; m_overallStatsForAllIntervals[i] = new OverallStats(subIntervalSize); } } /** * Output the overallStats to logger * @throws AdeException */ private void generateReportIfNeeded(long inputTime) throws AdeException { final DateTime inputDateTimeStartOfDay = new DateTime(inputTime).withZone(s_outTimeZone) .withTimeAtStartOfDay(); final int daysSinceLastReported = Days.daysBetween(m_lastReportDateTimeBegin, inputDateTimeStartOfDay) .getDays(); if (daysSinceLastReported < 1) { /* Do not need report if the days is less than 1 day */ return; } boolean createReport = false; boolean resetData = false; switch (m_reportFrequency) { case MONTHLY: { final boolean monthChangedSinceLastReported = m_lastReportDateTimeBegin .getMonthOfYear() != inputDateTimeStartOfDay.getMonthOfYear() || m_lastReportDateTimeBegin.getYear() != inputDateTimeStartOfDay.getYear(); if (monthChangedSinceLastReported) { createReport = true; resetData = true; } else { resetData = false; /* If this processing is within the same month as processing start, * output every day */ final boolean monthChangedSinceProcessingStarted = m_processingStartDateTime .getMonthOfYear() != inputDateTimeStartOfDay.getMonthOfYear() || m_processingStartDateTime.getYear() != inputDateTimeStartOfDay.getYear(); if (!monthChangedSinceProcessingStarted) { createReport = true; } } break; } case DAYS: { if (daysSinceLastReported >= m_reportFrequency.getDays()) { createReport = true; resetData = true; } else { resetData = false; /* If this processing just started less than or equal to 10 days, output every day */ final int daysSinceStarted = Days.daysBetween(m_processingStartDateTime, inputDateTimeStartOfDay) .getDays(); if (daysSinceStarted <= m_reportFrequency.getDays()) { createReport = true; } } break; } default: break; } /* Write out the report */ if (createReport) { generateReport(getYesterdayEndOfDay(inputDateTimeStartOfDay)); m_lastReportDateTimeBegin = inputDateTimeStartOfDay; /* Reset all the data */ if (resetData) { initOverallStatsForAllIntervals(); } } } /** * Return the end of day of yesterday * @param dateTime * @return */ private DateTime getYesterdayEndOfDay(DateTime dateTime) { dateTime = dateTime.withTimeAtStartOfDay(); dateTime = dateTime.minusMillis(1); return dateTime; } /** * Generate a report * @throws AdeException */ public void generateReport() throws AdeException { /* Note: generateReport without any input time is called when EndOfStream is reached for a reader. * There could be more stream coming in, or there could be no more. * * When this is called, the timestamp of the message last seen will be used. */ if (m_messageInputDateTime != null) { generateReport(m_messageInputDateTime); } else { generateReport("EndOfFile_No_Date"); } } /** * * @param dateStr * @throws AdeException */ public void generateReport(DateTime dateTime) throws AdeException { final String dateStr = s_dateTimeFormatter.print(dateTime); generateReport(dateStr); } /** * Generate a report * @throws AdeException */ private void generateReport(String dateStr) throws AdeException { /* Write the log for parsing errors */ MessagesWithParseErrorStats.getParserErrorStats().writeToLog(); final String reportString = m_source + ", " + dateStr + ", "; for (OverallStats overallStats : m_overallStatsForAllIntervals) { final String trace = reportString + " " + overallStats.toString(); statsLogger.info(trace); } } @Override public String toString() { final DateTime beginOfIntervalDateTime = new DateTime(m_beginOfInterval).withZone(s_outTimeZone); String trace = "source=" + m_source + " curIndex=" + m_currentIndex10MinutesMsgCountArray; trace += " inputDateTime=" + m_messageInputDateTime; trace += " lastReportDateTimeStartOfDay=" + m_lastReportDateTimeBegin; trace += "\n beginProcessingDateTime=" + m_processingStartDateTime; trace += " beginInterval=" + beginOfIntervalDateTime; return trace; } private static class MessageStats { private String m_msgId; private int[] m_10MinutesMsgCountArray; private boolean m_isMsg2; /** * Create a new object for a message */ public MessageStats(String msgId, int numberOf10MinutesInterval, boolean isMsg2) { m_msgId = msgId; m_10MinutesMsgCountArray = new int[numberOf10MinutesInterval]; m_isMsg2 = isMsg2; } /** * Indicate a new occurrance of this message */ public void addMessage(int index) { /* Limit the count to MAX, so that it will not wrap over */ if (m_10MinutesMsgCountArray[index] < Integer.MAX_VALUE) { m_10MinutesMsgCountArray[index]++; } } /** * Given a requested interval size (10, 20 minutes), generate a count of occurrence for * that interval. * * @return */ public long[] getCountBasedOnIntervalSize(short numberOf10MinutesSlotPerSubInterval) { final int numberOfSubIntervals = m_10MinutesMsgCountArray.length / numberOf10MinutesSlotPerSubInterval; final long[] subIntervals = new long[numberOfSubIntervals]; /* Sum up every N entry in the array */ for (int i = 0; i < subIntervals.length; i++) { long value = 0; for (int j = 0; j < numberOf10MinutesSlotPerSubInterval; j++) { final int tenMinutesArrayIndex = i * numberOf10MinutesSlotPerSubInterval + j; value += m_10MinutesMsgCountArray[tenMinutesArrayIndex]; } subIntervals[i] = value; } return subIntervals; } /** * Whether this message is 2nd message * @return */ public boolean isMsg1() { return !m_isMsg2; } @Override public String toString() { StringBuilder bldstr = new StringBuilder(m_msgId); for (int i = 0; i < m_10MinutesMsgCountArray.length; i++) { final int count = m_10MinutesMsgCountArray[i]; bldstr.append(", i" + i + "=" + count); } return bldstr.toString(); } } static class OverallStats { /** * The size of Interval this OverallStats object present */ private short m_intervalSizeRepresented; /** * The total number of intervals */ private long m_numberOfIntervals; /** * The total number of intervals */ private long m_intervalWithZeroCounts; /** * The sum of count all the intervals */ private long m_msg1TotalCount; private long m_msg2TotalCount; /** * The sum of count all the intervals */ private long m_sumOfMsg1UniqueMsgIdCount; private long m_sumOfMsg2UniqueMsgIdCount; /** * The sum of square of the interval data */ private long m_sumOfMsg1UniqueMsgIdCountSquare; /** * The smallest count */ private long m_minMsg1UniqueMsgIdCount = Long.MAX_VALUE; /** * The largest count */ private long m_maxMsg1UniqueMsgIdCount = 0; /** * Constructor * @param intervalSize */ public OverallStats(short intervalSizeRepresented) { m_intervalSizeRepresented = intervalSizeRepresented; } /** * Add an array of stats * @param intervalSize */ public void addStats(long[] msg1UniqueMsgIdStats, long[] msg1TotalStats, long[] msg2UniqueMsgIdStats, long[] msg2TotalStats) { m_numberOfIntervals += msg1UniqueMsgIdStats.length; for (int i = 0; i < msg1UniqueMsgIdStats.length; i++) { final long msg1UniqueMsgIdStat = msg1UniqueMsgIdStats[i]; final long msg1TotalStat = msg1TotalStats[i]; final long msg2UniqueMsgIdStat = msg2UniqueMsgIdStats[i]; final long msg2TotalStat = msg2TotalStats[i]; if (msg1UniqueMsgIdStat == 0) { /* Only need to check Msg1, since Msg2 won't exist unless there is a Msg1 */ m_intervalWithZeroCounts++; } else { /* Do not make zero as the min, just in case we have partial interval contains no data * - startup, shutdown etc */ m_minMsg1UniqueMsgIdCount = Math.min(m_minMsg1UniqueMsgIdCount, msg1UniqueMsgIdStat); } m_maxMsg1UniqueMsgIdCount = Math.max(m_maxMsg1UniqueMsgIdCount, msg1UniqueMsgIdStat); m_sumOfMsg1UniqueMsgIdCount += msg1UniqueMsgIdStat; m_sumOfMsg2UniqueMsgIdCount += msg2UniqueMsgIdStat; m_sumOfMsg1UniqueMsgIdCountSquare += msg1UniqueMsgIdStat * msg1UniqueMsgIdStat; m_msg1TotalCount += msg1TotalStat; m_msg2TotalCount += msg2TotalStat; } } /** * Return the total count * @return */ public long getMsg1UniqueMsgIdCount() { return m_sumOfMsg1UniqueMsgIdCount; } /** * Return the total count * @return */ public long getMsg2UniqueMsgIdCount() { return m_sumOfMsg2UniqueMsgIdCount; } /** * Return the total count * @return */ public long getMsg1TotalCount() { return m_msg1TotalCount; } /** * Return the total count * @return */ public long getMsg2TotalCount() { return m_msg2TotalCount; } /** * Return the interval size * @return */ public int getIntervalSize() { return m_intervalSizeRepresented; } /** * Return the min * @return */ public long getMsg1UniqueMsgIdMin() { return m_minMsg1UniqueMsgIdCount; } /** * Return the max * @return */ public long getMsg1UniqueMsgIdMax() { return m_maxMsg1UniqueMsgIdCount; } /** * Return the mean value */ public double getMsg1UniqueMsgIdMean() { return (double) m_sumOfMsg1UniqueMsgIdCount / (double) m_numberOfIntervals; } /** * Return the mean value */ public double getMsg2UniqueMsgIdMean() { return (double) m_sumOfMsg2UniqueMsgIdCount / (double) m_numberOfIntervals; } /** * Return the total number of intervals * @return */ public long getNumberOfIntervals() { return m_numberOfIntervals; } /** * Return the total number of intervals * @return */ public long getIntervalsWithZeroCount() { return m_intervalWithZeroCounts; } /** * Return the mean value */ public double getMsg1UniqueMsgIdVariance() { return (double) m_sumOfMsg1UniqueMsgIdCountSquare / (double) m_numberOfIntervals - -((double) getMsg1UniqueMsgIdMean() * (double) getMsg1UniqueMsgIdMean()); } /** * Return the mean value */ public double getMsg1UniqueMsgIdStandardVariation() { final double variance = getMsg1UniqueMsgIdVariance(); return Math.sqrt(variance); } @Override public String toString() { String trace = "IntervalSize=" + getIntervalSize(); trace += String.format(", msg1UMIDAvgCount=%.2f", getMsg1UniqueMsgIdMean()); trace += String.format(", msg2UMIDAvgCount=%.2f", getMsg2UniqueMsgIdMean()); trace += ", msg1UMIDCount=" + getMsg1UniqueMsgIdCount(); trace += ", msg2UMIDCount=" + getMsg2UniqueMsgIdCount(); trace += ", numOfIntervals=" + getNumberOfIntervals(); trace += String.format(", msg1UMIDStdDev=%.2f", getMsg1UniqueMsgIdStandardVariation()); trace += ", msg1UMIDMin=" + getMsg1UniqueMsgIdMin(); trace += ", msg1UMIDMax=" + getMsg1UniqueMsgIdMax(); trace += ", zeroCountIntervals=" + getIntervalsWithZeroCount(); trace += ", msg1TotalCount=" + getMsg1TotalCount(); trace += ", msg2TotalCount=" + getMsg2TotalCount(); return trace; } } }