Java tutorial
/** * Copyright 2014 Groupon.com * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.arpnetworking.tsdcore.limiter.legacy; import com.arpnetworking.tsdcore.model.AggregatedData; import com.arpnetworking.utility.OvalBuilder; import com.google.common.base.MoreObjects; import com.google.common.collect.Maps; import net.sf.oval.constraint.Min; import net.sf.oval.constraint.NotNull; import org.joda.time.DateTime; import org.joda.time.Duration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Closeable; import java.util.Collections; import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; /** * Limit the number of aggregations that will be emitted. * <p> * <ul> * <li><code>addToAggregations()</code> should be called when a new TSData is added * to the TSAggregations.</li> * <li><code>mark()</code> should be called each time a datum is added to a TSData.</li> * </ul> * There is a lot of machinery: * <ul> * <li>The _marks map keeps track of the last time a metric was emitted and the * number of aggregations for that metric.</li> * <li>_nAggregations is the sum of the aggregation counts in the _marks map</li> * <li>Since _marks and _nAggregations have to updated together, operations on * them live in a critical section controlled by _updateMarksMutex</li> * <li>The _marks list is persisted across program runs</li> * <li>The _marks list written to disk when a new metric is seen, and periodically * (so that the timestamps stay relatively fresh)</li> * <li>Metrics can age out of the _marks list</li> * </ul> * * There are two entry points for the limiter and both are called from line * processor. <code>addToAggregions()</code> is called when a new * <code>TSData</code> is added to the <code>_aggregations</code> map and * <code>mark()</code> is called when a data point is added to a * <code>TSData</code> to record the last time a metric was written. The code * was structured this way way because it is expected that the MetricsLimiter * will only be used for a short time. * * A better solution if the <code>MetricsLimiter</code> is to be permanent is * to merge the limiter and the <code>_aggregations</code> into a single object * so the updating of the metrics limit list and the last-written times is * hidden inside the <code>_aggregations</code> object. * * @author Joe Frisbie (jfrisbie at groupon dot com) */ public final class DefaultMetricsLimiter implements Closeable { /** * Consider whether a <code>AggregatedData</code> instance should be * processed further given predefined limits on the number of unique * instances. * * @param data <code>AggregatedData</code> instance to consider. * @param time The current date and time. * @return True if and only if the data was accepted. */ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "DM_EXIT") public boolean offer(final AggregatedData data, final DateTime time) { final String key = data.getFQDSN().getCluster() + "-" + data.getFQDSN().getService() + "-" + data.getFQDSN().getMetric() + "-" + data.getFQDSN().getStatistic().getName() + "-" + data.getPeriod(); final long now = time.getMillis(); final int nNewAggregations = 1; // We're updating the marks list and it's size together, so need a mutex section synchronized (_updateMarksMutex) { // If the metric is in the marks, list, it is already incorporated in _nAggregations so just add it // to the aggregations if (_marks.containsKey(key)) { _marks.get(key)._time = now; return true; } // Age out metrics to free up some slots if we're going exceed the maximum if (_nAggregations + nNewAggregations > _maxAggregations) { ageOutAggregations(now); } // If we now have enough room, create a marks entry, update _nAggregations and add the aggregations if (_nAggregations + nNewAggregations <= _maxAggregations) { _marks.putIfAbsent(key, new Mark(nNewAggregations, now)); _nAggregations += nNewAggregations; _stateManager.requestWrite(); return true; } } // If we get here, there was no room for the new aggregations, log it (but not too often) and then ignore final Long lastLogged = _lastLogged.get(key); if (lastLogged == null || now - lastLogged.longValue() >= _loggingInterval.getMillis()) { LOGGER.error(String.format("Limited aggregate %s; already aggregating %d", key, Integer.valueOf(_nAggregations))); _lastLogged.put(key, Long.valueOf(now)); } return false; } /** * {@inheritDoc} */ @Override public void close() { if (_enableStateAutoWriter) { _stateManager.stopAutoWriter(true); } } // Stuff below here has package scope for test access /** * Remove aggregations that are older than _ageOutThresholdMs if nNewAggregations would exceed _maxAggregations. * Returns true if aggregations were removed (and therefore the list of aggregations should be flushed to disk). * <p/> * Note: since _nAggregations and aggregations are being updated together, this needs to run within a critical * section guarded by _updateMetricsListMutex. */ boolean ageOutAggregations(final long nowMs) { boolean removedSome = false; // Loop through removing as many as we can final Iterator<Map.Entry<String, Mark>> entries = _marks.entrySet().iterator(); while (entries.hasNext()) { final Map.Entry<String, Mark> aggregationMarkTime = entries.next(); if (nowMs - aggregationMarkTime.getValue()._time >= _ageOutThreshold.getMillis()) { final String key = aggregationMarkTime.getKey(); entries.remove(); _nAggregations -= aggregationMarkTime.getValue()._count; LOGGER.info(String.format("Aggregation %s aged out", key)); removedSome = true; } } return removedSome; } MetricsLimiterStateManager getStateManager() { return _stateManager; } void updateMarksAndAggregationCount(final Map<String, Mark> marks, final long nowMs) { synchronized (_updateMarksMutex) { for (final Map.Entry<String, Mark> metricMark : marks.entrySet()) { // If the mark is too old, ignore it if (metricMark.getValue()._time < nowMs - _ageOutThreshold.getMillis()) { continue; } final Mark oldMark = _marks.putIfAbsent(metricMark.getKey(), metricMark.getValue()); if (oldMark == null) { // We just added a mark for a metric not previously seen in this session _nAggregations += metricMark.getValue()._count; continue; } // There is already a mark for the metric, so make a new mark with the max of the historical count // and the most recent timestamp final Mark newMark = new Mark(Math.max(oldMark._count, metricMark.getValue()._count), Math.max(oldMark._time, metricMark.getValue()._time)); _marks.put(metricMark.getKey(), newMark); } } } Map<String, Mark> getMarks() { return Collections.unmodifiableMap(_marks); } int getNAggregations() { return _nAggregations; } private DefaultMetricsLimiter(final Builder builder) { _maxAggregations = builder._maxAggregations.longValue(); _loggingInterval = builder._loggingInterval; _ageOutThreshold = builder._ageOutThreshold; _stateManager = builder._stateManagerBuilder.build(_marks); // Load the persisted limit state & enable periodic state writes updateMarksAndAggregationCount(_stateManager.readState(), System.currentTimeMillis()); _enableStateAutoWriter = builder._enableStateAutoWriter.booleanValue(); if (_enableStateAutoWriter) { _stateManager.startAutoWriter(); } } private final long _maxAggregations; private final Duration _loggingInterval; private final Duration _ageOutThreshold; private final boolean _enableStateAutoWriter; // Used to synchronize updates to _nAggregations & aggregations (from the LineProcessor) private final Object _updateMarksMutex = new Object(); // Records the last time a metric had tsd data recorded, typically includes data from prior runs of tsdAgg private final ConcurrentHashMap<String, Mark> _marks = new ConcurrentHashMap<>(); // Keeps track of the last time a message was logged saying that a particular metric was being // dropped because too many metrics are in use. private final Map<String, Long> _lastLogged = Maps.newHashMap(); private final MetricsLimiterStateManager _stateManager; // The current number of aggregations that have ad tsd data recorded private int _nAggregations; private static final long DEFAULT_MAX_AGGREGATIONS = 1000; private static final Duration DEFAULT_LOGGING_INTERVAL = Duration.standardMinutes(5); private static final Duration DEFAULT_AGE_OUT_THRESHOLD = Duration.standardDays(7); private static final Logger LOGGER = LoggerFactory.getLogger(DefaultMetricsLimiter.class); /** * Hold the number of aggregations and the last time a metric produced a * data point. */ public static class Mark { /** * Public constructor. * * @param count The count. * @param time The timestamp in milliseconds since epoch. */ public Mark(final long count, final long time) { this._count = count; this._time = time; } public long getCount() { return _count; } public long getTime() { return _time; } /** * {@inheritDoc} */ @Override public String toString() { return MoreObjects.toStringHelper(Mark.class).add("Count", _count).add("Timer", _time).toString(); } /** * {@inheritDoc} */ @Override public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } final Mark mark = (Mark) o; return _count == mark._count && _time == mark._time; } /** * {@inheritDoc} */ @Override public int hashCode() { int result = (int) (_time ^ (_time >>> 32)); result = 31 * result + ((int) _count); return result; } private final long _count; private long _time; } /** * Builder for a MetricsLimiter. */ public static final class Builder extends OvalBuilder<DefaultMetricsLimiter> { /** * Public constructor. */ public Builder() { super(DefaultMetricsLimiter.class); } /** * Set the max aggregations. * * @param maxAggregations The max aggregations. * @return This instance of <code>Builder</code>. */ public Builder setMaxAggregations(final Long maxAggregations) { _maxAggregations = maxAggregations; return this; } /** * Set the logging interval. * * @param loggingInterval The logging interval. Optional. The default * is five minutes. * @return This instance of <code>Builder</code>. */ public Builder setLoggingInterval(final Duration loggingInterval) { _loggingInterval = loggingInterval; return this; } /** * Set the age out threshold. Optional. The default is seven days. * * @param ageOutThreshold The age out threshold. * @return This instance of <code>Builder</code>. */ public Builder setAgeOutThreshold(final Duration ageOutThreshold) { _ageOutThreshold = ageOutThreshold; return this; } /** * Set the <code>MetricsLimiterStateManager</code> instance. * * @param stateManagerBuilder The <code>MetricsLimiterStateManager</code> * instance. * @return This instance of <code>Builder</code>. */ public Builder setStateManagerBuilder(final MetricsLimiterStateManager.Builder stateManagerBuilder) { _stateManagerBuilder = stateManagerBuilder; return this; } /** * Set whether the state auto writer is enabled. Optional. The default * is true. * * @param enableStateAutoWriter Whether the state auto writer is enabled. * @return This instance of <code>Builder</code>. */ public Builder setEnableStateAutoWriter(final Boolean enableStateAutoWriter) { _enableStateAutoWriter = enableStateAutoWriter; return this; } @NotNull @Min(0) private Long _maxAggregations; @NotNull private MetricsLimiterStateManager.Builder _stateManagerBuilder; @NotNull private Duration _loggingInterval = DEFAULT_LOGGING_INTERVAL; @NotNull private Duration _ageOutThreshold = DEFAULT_AGE_OUT_THRESHOLD; @NotNull private Boolean _enableStateAutoWriter = Boolean.TRUE; } }