Java tutorial
/** * Licensed to Odiago, Inc. under one or more contributor license * agreements. See the NOTICE.txt file distributed with this work for * additional information regarding copyright ownership. Odiago, Inc. * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the * License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations * under the License. */ package com.odiago.flumebase.exec; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.PriorityBlockingQueue; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.cloudera.util.Pair; import com.odiago.flumebase.exec.local.TimerFlowElemContext; import com.odiago.flumebase.lang.TimeSpan; import com.odiago.flumebase.parser.AliasedExpr; import com.odiago.flumebase.parser.Expr; import com.odiago.flumebase.parser.FnCallExpr; import com.odiago.flumebase.parser.TypedField; import com.odiago.flumebase.parser.WindowSpec; import com.odiago.flumebase.plan.AggregateNode; import com.odiago.flumebase.plan.PlanNode; import com.odiago.flumebase.util.IterableIterator; import com.odiago.flumebase.util.PairLeftRightComparator; import com.odiago.flumebase.util.concurrent.SelectableQueue; @SuppressWarnings("rawtypes") /** * Perform aggregation functions over time series data divided into * a fixed number of buckets over the aggregation time interval. */ public class BucketedAggregationElement extends AvroOutputElementImpl { private static final Logger LOG = LoggerFactory.getLogger(BucketedAggregationElement.class.getName()); /** Configuration key for the number of buckets that subdivide the aggregation time interval. */ private static final String NUM_BUCKETS_KEY = "flumebase.aggregation.buckets"; private static final int DEFAULT_NUM_BUCKETS = 100; /** * Configuration key specifying whether continuous output should be used. * If true, output should be generated for every bucket interval, even if no new data * is available in that bucket; if false, only generate output when the input * condition changes. */ private static final String CONTINUOUS_OUTPUT_KEY = "flumebase.aggregation.continuous.output"; private static final boolean DEFAULT_CONTINUOUS_OUTPUT = false; /** * Configuration key specifying the amount of slack time we tolerate between * events that should occur at the same time. */ public static final String SLACK_INTERVAL_KEY = "flumebase.slack.time"; public static final int DEFAULT_SLACK_INTERVAL = 200; /** * Configuration key specifying how far in the past we emit as output when * an insertion forces a close/emit of prior time windows. * If there's a massive stall, don't worry about data that is more than * this many milliseconds old. */ private static final String MAX_PRIOR_EMIT_INTERVAL_KEY = "flumebase.aggregation.max.prior.interval"; private static final long DEFAULT_MAX_PRIOR_EMIT_INTERVAL = 5000; /** The number of buckets that subdivide the aggregation time interval. */ private final int mNumBuckets; /** Indicates whether continuous output is enabled. */ private final boolean mContinuousOutput; /** How far into the past we will look for windows to close when catching up to the present. */ private final long mMaxPriorEmitInterval; private final List<TypedField> mGroupByFields; /** The window specification over which we're aggregating. */ private final WindowSpec mWindowSpec; /** The actual time interval over which we're aggregating. (Derived from mWindowSpec) */ private final TimeSpan mTimeSpan; /** * The "width" of each bucket, in milliseconds. Specifies how we round the * true timestamps for events off, into the timestamps associated with * buckets in the time interval. */ private final long mTimeModulus; /** * The maximum lateness (specified in milliseconds) we will tolerate for an * event. */ private final long mSlackTime; /** * The set of aliased expressions describing the aggregation functions to run * over records we receive, and what alias to assign to their outputs. */ private final List<AliasedExpr> mAggregateExprs; private final List<TypedField> mPropagateFields; /** * Map that returns a set of Bucket objects. Each bucket object * contains the state associated with a single aggregation function. * The key is a pair consisting of the timestamp (as a Long) and a HashedEvent: * an object that implements equals() and hashCode() based on a subset of the * fields of an EventWrapper. */ private Map<Pair<Long, HashedEvent>, List<Bucket>> mBucketMap; /** * The same set of buckets as mBucketMap, organized as time-ordered lists * arranged by the group-by columns. */ private Map<HashedEvent, List<Pair<Long, List<Bucket>>>> mBucketsByGroup; /** * Timestamp associated with the newest buckets in the pipeline. * This is used for auto-closing old windows when newer ones arrive. */ private long mHeadBucketTime = 0; /** * Timestamp associated with the oldest bucket that can act as a window head * in the pipeline. */ private long mTailBucketTime = 0; /** Timestamp of the most recent wakeup call enqueued. */ private long mLastEnqueuedWakeup = 0; /** * SelectableQueue for the downstream timer element, which our eviction thread * enqueues into. */ private SelectableQueue<Object> mTimerQueue = null; private EvictionThread mEvictionThread; public BucketedAggregationElement(FlowElementContext ctxt, AggregateNode aggregateNode) { super(ctxt, (Schema) aggregateNode.getAttr(PlanNode.OUTPUT_SCHEMA_ATTR)); Configuration conf = aggregateNode.getConf(); assert null != conf; mNumBuckets = conf.getInt(NUM_BUCKETS_KEY, DEFAULT_NUM_BUCKETS); mContinuousOutput = conf.getBoolean(CONTINUOUS_OUTPUT_KEY, DEFAULT_CONTINUOUS_OUTPUT); mMaxPriorEmitInterval = conf.getLong(MAX_PRIOR_EMIT_INTERVAL_KEY, DEFAULT_MAX_PRIOR_EMIT_INTERVAL); int slackTime = conf.getInt(SLACK_INTERVAL_KEY, DEFAULT_SLACK_INTERVAL); if (slackTime < 0) { mSlackTime = DEFAULT_SLACK_INTERVAL; } else { mSlackTime = slackTime; } assert mMaxPriorEmitInterval > 0; assert mMaxPriorEmitInterval > mSlackTime; List<TypedField> groupByFields = aggregateNode.getGroupByFields(); if (null == groupByFields) { mGroupByFields = Collections.emptyList(); } else { mGroupByFields = groupByFields; } mAggregateExprs = aggregateNode.getAggregateExprs(); assert mAggregateExprs != null; mPropagateFields = aggregateNode.getPropagateFields(); Expr windowExpr = aggregateNode.getWindowExpr(); assert windowExpr.isConstant(); try { mWindowSpec = (WindowSpec) windowExpr.eval(new EmptyEventWrapper()); assert mWindowSpec.getRangeSpec().isConstant(); mTimeSpan = (TimeSpan) mWindowSpec.getRangeSpec().eval(new EmptyEventWrapper()); } catch (IOException ioe) { // The only way this can be thrown is if the window expr isn't actually constant. // This should not happen due to the assert above.. LOG.error("Got IOException when calculating window width: " + ioe); throw new RuntimeException(ioe); } mBucketMap = new HashMap<Pair<Long, HashedEvent>, List<Bucket>>(mNumBuckets); mBucketsByGroup = new HashMap<HashedEvent, List<Pair<Long, List<Bucket>>>>(); // Calculate the width of each bucket. mTimeModulus = mTimeSpan.getWidth() / mNumBuckets; if (mTimeModulus * mNumBuckets != mTimeSpan.getWidth()) { LOG.warn("Aggregation time step does not cleanly divide the time interval; " + "results may be inaccurate. Set " + NUM_BUCKETS_KEY + " to a better divisor."); } } /** {@inheritDoc} */ @Override public void open() throws IOException, InterruptedException { TimerFlowElemContext timerContext = (TimerFlowElemContext) getContext(); // Start the auto-closing thread. Initialize the reference to the queue it populates // from our timer context. mTimerQueue = timerContext.getTimerQueue(); mEvictionThread = new EvictionThread(); mEvictionThread.start(); super.open(); } /** {@inheritDoc} */ @Override public void close() throws IOException, InterruptedException { // We've got no new elements coming in; expire all buckets immediately. LOG.debug("Immediately expiring all buckets to mHeadBucketTime=" + mHeadBucketTime); closeUntil(mHeadBucketTime, mHeadBucketTime, getContext()); mEvictionThread.finish(); mEvictionThread = null; super.close(); } /** * Initialize the list of Bucket entries that are associated with a new * timestamp -> bucket mapping. This is typically done just before inserting * a value in a new bucket at the head of a new time window. * @return the list of initialized Bucket objects for this time subrange. */ private List<Bucket> initBuckets(Pair<Long, HashedEvent> bucketKey) { List<Bucket> newBuckets = new ArrayList<Bucket>(mAggregateExprs.size()); for (int i = 0; i < mAggregateExprs.size(); i++) { // Put in a new bucket instance for each aggregation funtion we're going to run. newBuckets.add(new Bucket()); } assert null == mBucketMap.get(bucketKey); mBucketMap.put(bucketKey, newBuckets); // Put this into the map organized by group, as well. // Get the set of (time, bucketlist) pairs for the group. List<Pair<Long, List<Bucket>>> bucketsByTime = mBucketsByGroup.get(bucketKey.getRight()); if (null == bucketsByTime) { bucketsByTime = new LinkedList<Pair<Long, List<Bucket>>>(); mBucketsByGroup.put(bucketKey.getRight(), bucketsByTime); } bucketsByTime.add(new Pair<Long, List<Bucket>>(bucketKey.getLeft(), newBuckets)); // Return the initialized set of Bucket objects back to the caller. return newBuckets; } /** * @return a key into our group-by map that is composed of the bucket * timestamp for the event, and a HashedEvent that reads the fields * of the event necessary to group by those fields. If we are not grouping * by any fields, this component of the pair is null. */ private Pair<Long, HashedEvent> getEventKey(EventWrapper e) { long eventTime = e.getEvent().getTimestamp(); long remainder = eventTime % mTimeModulus; Long bucketTime; // If we're on an interval boundary (e.g., t=100) we go into that bucket. // If we're off-boundary (e.g., t=103), we go into the closest "previous" bucket (t=100). bucketTime = Long.valueOf(eventTime - remainder); HashedEvent hashedEvent = new HashedEvent(e, mGroupByFields); return new Pair<Long, HashedEvent>(bucketTime, hashedEvent); } /** * Given a set of time buckets associated with a given group, * iterate over the time buckets for a specific time interval, * for a particular aggregation function. */ private static class BucketIterator<T> implements Iterator<Bucket<T>> { /** * Offset of the true Bucket object in the final List<Buckets> that * specifies buckest for each aggregation function we operate. */ private final int mFunctionId; /** Lowest timestamped bucket we return. */ private final long mLoTime; /** Highest timestamped bucket we return. */ private final long mHiTime; /** * Iterator over the outer list. We require this iterator * to return values in order. */ private final Iterator<Pair<Long, List<Bucket>>> mIterator; /** The next value we return. */ private Bucket<T> mNextBucket; /** Set to true if prepBucket() has been called, but not next(). */ private boolean mIsReady; /** The number of buckets in the time interval that were returned by this iterator. */ private int mYieldCount; public BucketIterator(int functionId, long loTime, long hiTime, List<Pair<Long, List<Bucket>>> inputList) { mFunctionId = functionId; mLoTime = loTime; mHiTime = hiTime; mIterator = inputList.iterator(); mYieldCount = 0; } /** * Scan ahead in the underlying iterator til we find the next element. * Set mNextBucket to the next value that next() should return, or null * if we cannot yield any more values. * Sets mIsReady to true. */ private void prepBucket() { assert !mIsReady; // This should not be called twice in a row. mIsReady = true; mNextBucket = null; while (mIterator.hasNext()) { Pair<Long, List<Bucket>> nextPair = mIterator.next(); long timestamp = nextPair.getLeft(); if (timestamp > mLoTime && timestamp <= mHiTime) { // We found the next one to return. mNextBucket = nextPair.getRight().get(mFunctionId); return; } } } public Bucket<T> next() { if (!mIsReady) { prepBucket(); } assert mIsReady; mIsReady = false; if (mNextBucket != null) { mYieldCount++; } return mNextBucket; } public boolean hasNext() { if (!mIsReady) { prepBucket(); } assert mIsReady; return mNextBucket != null; } public void remove() { throw new RuntimeException("Not implemented."); } int getYieldCount() { return mYieldCount; } } /** * Close the window ending with the bucket for 'closeTime'. * Remove any buckets that are older than closeTime - aggregationIntervalWidth. * since they will no longer contribute to any open windows. */ private void closeWindow(long closeTime, FlowElementContext context) throws IOException, InterruptedException { long loTime = closeTime - mTimeSpan.getWidth(); Long closeBucketTimestamp = Long.valueOf(closeTime); LOG.debug("Closing window for range: " + loTime + " -> " + closeTime); // For each group, emit an output record containing the aggregate values over // the whole time window. for (Map.Entry<HashedEvent, List<Pair<Long, List<Bucket>>>> entry : mBucketsByGroup.entrySet()) { HashedEvent group = entry.getKey(); // In non-continuous (demand-only) mode, check whether there's a bucket associated // with this window's closing time for this group. if (!mContinuousOutput && mBucketMap.get(new Pair<Long, HashedEvent>(closeBucketTimestamp, group)) == null) { continue; // Nothing to do. } GenericData.Record record = new GenericData.Record(getOutputSchema()); List<Pair<Long, List<Bucket>>> bucketsByTime = entry.getValue(); int numBucketsInRangeForGroup = 0; // Execute each aggregation function over the applicable subset of buckets // in bucketsByTime. for (int i = 0; i < mAggregateExprs.size(); i++) { BucketIterator aggIterator = new BucketIterator(i, loTime, closeTime, bucketsByTime); AliasedExpr aliasExpr = mAggregateExprs.get(i); FnCallExpr fnCall = (FnCallExpr) aliasExpr.getExpr(); Object result = fnCall.finishWindow(new IterableIterator(aggIterator)); numBucketsInRangeForGroup += aggIterator.getYieldCount(); record.put(aliasExpr.getAvroLabel(), result); } // If there are no buckets in bucketsByTime that are in our time range, // we should not emit anything for this group. Just silently continue. if (0 == numBucketsInRangeForGroup) { // Discard this output; we didn't actually calculate anything. continue; } // Copy the specified fields to propagate from the record used to define // the group, into the output record. EventWrapper groupWrapper = group.getEventWrapper(); for (TypedField propagateField : mPropagateFields) { record.put(propagateField.getAvroName(), groupWrapper.getField(propagateField)); } // Emit this as an output event! emitAvroRecord(record, groupWrapper.getEvent(), closeTime, context); } // Remove any buckets that are too old to be useful to any subsequent windows. // TODO(aaron): This is O(groups * num_buckets). We should actually use a TreeMap // instad of a list internally, so we can quickly cull the herd. That would be // O(groups * log(num_buckets)). Iterator<Map.Entry<HashedEvent, List<Pair<Long, List<Bucket>>>>> bucketsByGrpIter = mBucketsByGroup .entrySet().iterator(); while (bucketsByGrpIter.hasNext()) { Map.Entry<HashedEvent, List<Pair<Long, List<Bucket>>>> entry = bucketsByGrpIter.next(); HashedEvent group = entry.getKey(); List<Pair<Long, List<Bucket>>> bucketsByTime = entry.getValue(); Iterator<Pair<Long, List<Bucket>>> bucketsByTimeIter = bucketsByTime.iterator(); while (bucketsByTimeIter.hasNext()) { Pair<Long, List<Bucket>> timedBucket = bucketsByTimeIter.next(); Long timestamp = timedBucket.getLeft(); if (timestamp.longValue() < loTime) { bucketsByTimeIter.remove(); // Remove from bucketsByTime list. Pair<HashedEvent, Long> key = new Pair<HashedEvent, Long>(group, timestamp); mBucketMap.remove(key); // Remove from mBucketMap. } } if (bucketsByTime.size() == 0) { // We've removed the last time bucket for a given group from mBucketsByGroup. // Remove the group from that map. bucketsByGrpIter.remove(); } } } /** * Close all open windows up to and including the window that ends with the bucket * for time 'lastWindow'. */ private void closeUntil(long curBucketTime, long lastWindow, FlowElementContext context) throws IOException, InterruptedException { LOG.debug("Close until: cur=" + curBucketTime + ", lastWindow=" + lastWindow + ", mTailBucketTime=" + mTailBucketTime + ", mTimeMod=" + mTimeModulus + ", mMaxPrior=" + mMaxPriorEmitInterval); if (lastWindow <= mTailBucketTime) { return; // We've already closed this window. } // If mHeadBucketTime is too far back from the current time, // do a mass expiration and throw out old data. closeTime is bounded by // mMaxPriorEmitInterval. for (long closeTime = Math.max(mTailBucketTime, curBucketTime - mMaxPriorEmitInterval); closeTime <= lastWindow; closeTime += mTimeModulus) { LOG.debug("Close window: closeTime=" + closeTime); closeWindow(closeTime, context); } mTailBucketTime = lastWindow + mTimeModulus; } @Override public void takeEvent(EventWrapper e) throws IOException, InterruptedException { Pair<Long, HashedEvent> bucketKey = getEventKey(e); long curBucketTime = bucketKey.getLeft(); LOG.debug("Handling event time=" + curBucketTime); if (curBucketTime > mHeadBucketTime) { // We've just received an event that is newer than any others we've yet // received. This advances the sliding window to match this event's timestamp. // Emit any output groups that are older than this one by at least the // slack time interval. LOG.debug("New bucket: cur=" + curBucketTime + "; mHeadBucketTime=" + mHeadBucketTime); closeUntil(curBucketTime, curBucketTime - mSlackTime - mTimeModulus, getContext()); // Since we've already handled these, remove their wake-up calls.. mEvictionThread.discardUntil(mHeadBucketTime - mSlackTime); mHeadBucketTime = curBucketTime; // This insert advances our head bucket. } else if (curBucketTime < mHeadBucketTime - mMaxPriorEmitInterval) { // This event is too old -- ignore it. // TODO: Should this be mHeadBucketTiem - mSlackTime? LOG.debug("Dropping late event arriving at aggregator; HeadBucketTime=" + mHeadBucketTime + " and event is for bucket " + curBucketTime); return; } // Get the bucket for the (timestamp, group-by-fields) of this event. // Actually returns a list of Bucket objects, one per AggregateFunc to // execute. List<Bucket> buckets = mBucketMap.get(bucketKey); if (null == buckets) { // We're putting the first event into a new bucket. buckets = initBuckets(bucketKey); } // For each aggregation function we're performing, insert this event into // the bucket for the aggregate function. assert buckets.size() == mAggregateExprs.size(); for (int i = 0; i < mAggregateExprs.size(); i++) { AliasedExpr aliasExpr = mAggregateExprs.get(i); Expr expr = aliasExpr.getExpr(); assert expr instanceof FnCallExpr; FnCallExpr fnCall = (FnCallExpr) expr; Bucket bucket = buckets.get(i); fnCall.insertAggregate(e, bucket); } // Insert a callback into a queue to allow time to expire these windows. enqueueWakeup(curBucketTime); } /** * Enqueue a wakeup in the EvictionThread that closes the bucket with the * specified bucket timestamp. */ private void enqueueWakeup(long bucketTime) { if (bucketTime <= mLastEnqueuedWakeup) { // We've already enqueued a wakeup to close this bucket. return; } long curTime = System.currentTimeMillis(); long offset = mTimeModulus + mSlackTime; long closeTime = curTime + offset; // local time to close the bucket. LOG.debug("Insert wakeup call: " + bucketTime + " at time offset=" + offset); mEvictionThread.insert(new Pair<Long, Long>(closeTime, bucketTime)); mLastEnqueuedWakeup = bucketTime; } /** * Thread that sends notices to our coprocessor FlowElement when it is time to * close old windows based on elapsed local time. */ private class EvictionThread extends Thread { private final Logger LOG = LoggerFactory.getLogger(EvictionThread.class.getName()); /** * Set to true when it's time for the thread to go home. The thread * actually exits after this flag is set to true and the incoming queue * is empty. */ private boolean mIsFinished; /** * Priority queue (heap) of times when we should insert expiry-times in * the coprocessor FlowElement's input queue. * * <p>The queue holds tuples of two long values. The first is a local * time when this thread should wake up; this is what the queue is * ordered on. The latter is the window time that should be expired.</p> */ private PriorityBlockingQueue<Pair<Long, Long>> mQueue; // Maximum queue length == number of open windows + the newly-opening window // + the currently-closing window. final long mMaxQueueLen = 2 + (mSlackTime / mTimeModulus); public EvictionThread() { super("AggregatorEvictionThread"); mQueue = new PriorityBlockingQueue<Pair<Long, Long>>((int) mMaxQueueLen, new PairLeftRightComparator<Long, Long>()); } /** * Add a wake-up call to the queue. */ public void insert(Pair<Long, Long> wakeUpCall) { synchronized (this) { assert mQueue.size() < mMaxQueueLen; // This operation should never block. mQueue.put(wakeUpCall); this.notify(); } // Interrupt any wait that's going on, in case we are asleep and should // actually immediately service this wake-up call. this.interrupt(); } /** * Discard all wakeup calls up to time 'minTime'. * minTime is a 'bucket time', not a 'local time'. */ public void discardUntil(long minTime) { synchronized (this) { LOG.debug("discardUntil: " + minTime); Iterator<Pair<Long, Long>> iterator = mQueue.iterator(); while (iterator.hasNext()) { Pair<Long, Long> wakeUpCall = iterator.next(); if (wakeUpCall.getRight() < minTime) { LOG.debug("discard@ " + wakeUpCall); iterator.remove(); } } this.notify(); } } /** * Set the finished flag to true; try to get the thread to stop as * quickly as possible. */ public void finish() { synchronized (this) { this.mIsFinished = true; this.notify(); } this.interrupt(); // Interrupt any current sleep. } /** * Main loop of the thread. * Continually sleeps until the next timer event is ready to occur. */ public void run() { while (true) { Pair<Long, Long> wakeUpCall = null; long curTime; long nextWakeUp; synchronized (this) { while (mQueue.size() == 0) { try { if (this.mIsFinished) { // Parent is finished and we have drained our input queue. Go home. return; } this.wait(); } catch (InterruptedException ie) { // Interrupted while waiting for another wake-up call to enter our queue. // Try again, if we're not already finished. continue; } } assert mQueue.size() > 0; wakeUpCall = mQueue.peek(); } if (null == wakeUpCall) { continue; } curTime = System.currentTimeMillis(); nextWakeUp = wakeUpCall.getLeft(); if (nextWakeUp <= curTime) { // TODO(aaron): This section probably bears further deadlock analysis. // The put() into the timer queue can block (it has fixed length // LocalEnvironment.MAX_QUEUE_LEN) until the timer FE services its // existing list. // If we are interrupted doing this, it is because the main thread // has just inserted another wakeup call while we were blocking. // This thread's input queue must not block when being filled from // the main aggregation FE. I believe mMaxQueueLen should be sufficient // to guarantee this is the case, because before we call enqueueWakeup(), // we will have had to call closeUntil() in BucketedAggElem.takeEvent() // on enough windows to free up the slots in this queue. try { LOG.debug("Timer evicting at " + curTime + ": " + wakeUpCall); // Service this by injecting the getRight() into our outbound queue. mTimerQueue.put(new TimeoutEventWrapper(wakeUpCall.getRight())); } catch (InterruptedException ie) { // Not a problem. If we were interrupted doing the put into mTimerQueue, // then we'll service this again on the next go-around of the loop. // Just make sure we don't mark this as 'complete.' continue; } synchronized (this) { // Now actually remove this from the input queue. if (mQueue.peek() == wakeUpCall) { // O(1) fast path; no intervening push. mQueue.remove(); } else { // intervening push of an earlier wakeup (?). Slow path. mQueue.remove(wakeUpCall); } } } else { // If we're down here, we need to sleep until it is the next wake-up time. long napTime = nextWakeUp - curTime; try { Thread.sleep(napTime); } catch (InterruptedException ie) { // We were awoken early... this is expected (there may have been a // new enqueue, etc). } } } } } /** EventWrapper used to deliver the expiry time payload to the TimeoutEvictionElement. */ private static class TimeoutEventWrapper extends EmptyEventWrapper { /** The time window that should be expired. */ private final Long mExpireWindow; public TimeoutEventWrapper(Long expire) { mExpireWindow = expire; } @Override public Object getField(TypedField field) { return mExpireWindow; } } /** * Separate FlowElement that handles notifications from the EvictionThread; this * operates in the main thread, closing windows that cannot receive new events * because they are past the slack time interval. */ public class TimeoutEvictionElement extends AvroOutputElementImpl { private final Logger LOG = LoggerFactory.getLogger(TimeoutEvictionElement.class.getName()); private TimeoutEvictionElement(FlowElementContext ctxt, Schema outSchema) { super(ctxt, outSchema); } public void takeEvent(EventWrapper e) throws IOException, InterruptedException { assert e instanceof TimeoutEventWrapper; Long expireTime = (Long) e.getField(null); // TimeoutEventWrapper returns a single Long val LOG.debug("Handling in eviction element - timeout to: " + expireTime); closeUntil(expireTime, expireTime, getContext()); } } /** * Create a TimeoutEvictionElement coupled to this BucketedAggregationElement. */ public TimeoutEvictionElement getTimeoutElement(FlowElementContext timeoutContext) { return this.new TimeoutEvictionElement(timeoutContext, getOutputSchema()); } }