com.odiago.flumebase.exec.BucketedAggregationElement.java Source code

Java tutorial

Introduction

Here is the source code for com.odiago.flumebase.exec.BucketedAggregationElement.java

Source

/**
 * Licensed to Odiago, Inc. under one or more contributor license
 * agreements.  See the NOTICE.txt file distributed with this work for
 * additional information regarding copyright ownership.  Odiago, Inc.
 * licenses this file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the
 * License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */

package com.odiago.flumebase.exec;

import java.io.IOException;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import java.util.concurrent.PriorityBlockingQueue;

import org.apache.avro.Schema;

import org.apache.avro.generic.GenericData;

import org.apache.hadoop.conf.Configuration;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.cloudera.util.Pair;

import com.odiago.flumebase.exec.local.TimerFlowElemContext;

import com.odiago.flumebase.lang.TimeSpan;

import com.odiago.flumebase.parser.AliasedExpr;
import com.odiago.flumebase.parser.Expr;
import com.odiago.flumebase.parser.FnCallExpr;
import com.odiago.flumebase.parser.TypedField;
import com.odiago.flumebase.parser.WindowSpec;

import com.odiago.flumebase.plan.AggregateNode;
import com.odiago.flumebase.plan.PlanNode;

import com.odiago.flumebase.util.IterableIterator;
import com.odiago.flumebase.util.PairLeftRightComparator;

import com.odiago.flumebase.util.concurrent.SelectableQueue;

@SuppressWarnings("rawtypes")
/**
 * Perform aggregation functions over time series data divided into
 * a fixed number of buckets over the aggregation time interval.
 */
public class BucketedAggregationElement extends AvroOutputElementImpl {
    private static final Logger LOG = LoggerFactory.getLogger(BucketedAggregationElement.class.getName());

    /** Configuration key for the number of buckets that subdivide the aggregation time interval. */
    private static final String NUM_BUCKETS_KEY = "flumebase.aggregation.buckets";
    private static final int DEFAULT_NUM_BUCKETS = 100;

    /**
     * Configuration key specifying whether continuous output should be used.
     * If true, output should be generated for every bucket interval, even if no new data
     * is available in that bucket; if false, only generate output when the input
     * condition changes.
     */
    private static final String CONTINUOUS_OUTPUT_KEY = "flumebase.aggregation.continuous.output";
    private static final boolean DEFAULT_CONTINUOUS_OUTPUT = false;

    /**
     * Configuration key specifying the amount of slack time we tolerate between
     * events that should occur at the same time.
     */
    public static final String SLACK_INTERVAL_KEY = "flumebase.slack.time";
    public static final int DEFAULT_SLACK_INTERVAL = 200;

    /**
     * Configuration key specifying how far in the past we emit as output when
     * an insertion forces a close/emit of prior time windows.
     * If there's a massive stall, don't worry about data that is more than
     * this many milliseconds old.
     */
    private static final String MAX_PRIOR_EMIT_INTERVAL_KEY = "flumebase.aggregation.max.prior.interval";
    private static final long DEFAULT_MAX_PRIOR_EMIT_INTERVAL = 5000;

    /** The number of buckets that subdivide the aggregation time interval. */
    private final int mNumBuckets;

    /** Indicates whether continuous output is enabled. */
    private final boolean mContinuousOutput;

    /** How far into the past we will look for windows to close when catching up to the present. */
    private final long mMaxPriorEmitInterval;

    private final List<TypedField> mGroupByFields;

    /** The window specification over which we're aggregating. */
    private final WindowSpec mWindowSpec;

    /** The actual time interval over which we're aggregating. (Derived from mWindowSpec) */
    private final TimeSpan mTimeSpan;

    /**
     * The "width" of each bucket, in milliseconds. Specifies how we round the
     * true timestamps for events off, into the timestamps associated with
     * buckets in the time interval.
     */
    private final long mTimeModulus;

    /**
     * The maximum lateness (specified in milliseconds) we will tolerate for an
     * event.
     */
    private final long mSlackTime;

    /**
     * The set of aliased expressions describing the aggregation functions to run
     * over records we receive, and what alias to assign to their outputs.
     */
    private final List<AliasedExpr> mAggregateExprs;

    private final List<TypedField> mPropagateFields;

    /**
     * Map that returns a set of Bucket objects. Each bucket object
     * contains the state associated with a single aggregation function.
     * The key is a pair consisting of the timestamp (as a Long) and a HashedEvent:
     * an object that implements equals() and hashCode() based on a subset of the
     * fields of an EventWrapper.
     */
    private Map<Pair<Long, HashedEvent>, List<Bucket>> mBucketMap;

    /**
     * The same set of buckets as mBucketMap, organized as time-ordered lists
     * arranged by the group-by columns.
     */
    private Map<HashedEvent, List<Pair<Long, List<Bucket>>>> mBucketsByGroup;

    /**
     * Timestamp associated with the newest buckets in the pipeline.
     * This is used for auto-closing old windows when newer ones arrive.
     */
    private long mHeadBucketTime = 0;

    /**
     * Timestamp associated with the oldest bucket that can act as a window head
     * in the pipeline.
     */
    private long mTailBucketTime = 0;

    /** Timestamp of the most recent wakeup call enqueued. */
    private long mLastEnqueuedWakeup = 0;

    /**
     * SelectableQueue for the downstream timer element, which our eviction thread
     * enqueues into.
     */
    private SelectableQueue<Object> mTimerQueue = null;

    private EvictionThread mEvictionThread;

    public BucketedAggregationElement(FlowElementContext ctxt, AggregateNode aggregateNode) {
        super(ctxt, (Schema) aggregateNode.getAttr(PlanNode.OUTPUT_SCHEMA_ATTR));

        Configuration conf = aggregateNode.getConf();
        assert null != conf;
        mNumBuckets = conf.getInt(NUM_BUCKETS_KEY, DEFAULT_NUM_BUCKETS);
        mContinuousOutput = conf.getBoolean(CONTINUOUS_OUTPUT_KEY, DEFAULT_CONTINUOUS_OUTPUT);
        mMaxPriorEmitInterval = conf.getLong(MAX_PRIOR_EMIT_INTERVAL_KEY, DEFAULT_MAX_PRIOR_EMIT_INTERVAL);
        int slackTime = conf.getInt(SLACK_INTERVAL_KEY, DEFAULT_SLACK_INTERVAL);
        if (slackTime < 0) {
            mSlackTime = DEFAULT_SLACK_INTERVAL;
        } else {
            mSlackTime = slackTime;
        }

        assert mMaxPriorEmitInterval > 0;
        assert mMaxPriorEmitInterval > mSlackTime;

        List<TypedField> groupByFields = aggregateNode.getGroupByFields();
        if (null == groupByFields) {
            mGroupByFields = Collections.emptyList();
        } else {
            mGroupByFields = groupByFields;
        }

        mAggregateExprs = aggregateNode.getAggregateExprs();
        assert mAggregateExprs != null;
        mPropagateFields = aggregateNode.getPropagateFields();

        Expr windowExpr = aggregateNode.getWindowExpr();
        assert windowExpr.isConstant();
        try {
            mWindowSpec = (WindowSpec) windowExpr.eval(new EmptyEventWrapper());
            assert mWindowSpec.getRangeSpec().isConstant();
            mTimeSpan = (TimeSpan) mWindowSpec.getRangeSpec().eval(new EmptyEventWrapper());
        } catch (IOException ioe) {
            // The only way this can be thrown is if the window expr isn't actually constant.
            // This should not happen due to the assert above..
            LOG.error("Got IOException when calculating window width: " + ioe);
            throw new RuntimeException(ioe);
        }

        mBucketMap = new HashMap<Pair<Long, HashedEvent>, List<Bucket>>(mNumBuckets);
        mBucketsByGroup = new HashMap<HashedEvent, List<Pair<Long, List<Bucket>>>>();

        // Calculate the width of each bucket.
        mTimeModulus = mTimeSpan.getWidth() / mNumBuckets;
        if (mTimeModulus * mNumBuckets != mTimeSpan.getWidth()) {
            LOG.warn("Aggregation time step does not cleanly divide the time interval; "
                    + "results may be inaccurate. Set " + NUM_BUCKETS_KEY + " to a better divisor.");
        }
    }

    /** {@inheritDoc} */
    @Override
    public void open() throws IOException, InterruptedException {
        TimerFlowElemContext timerContext = (TimerFlowElemContext) getContext();
        // Start the auto-closing thread. Initialize the reference to the queue it populates
        // from our timer context.
        mTimerQueue = timerContext.getTimerQueue();
        mEvictionThread = new EvictionThread();
        mEvictionThread.start();
        super.open();
    }

    /** {@inheritDoc} */
    @Override
    public void close() throws IOException, InterruptedException {
        // We've got no new elements coming in; expire all buckets immediately.
        LOG.debug("Immediately expiring all buckets to mHeadBucketTime=" + mHeadBucketTime);
        closeUntil(mHeadBucketTime, mHeadBucketTime, getContext());
        mEvictionThread.finish();
        mEvictionThread = null;
        super.close();
    }

    /**
     * Initialize the list of Bucket entries that are associated with a new
     * timestamp -&gt; bucket mapping. This is typically done just before inserting
     * a value in a new bucket at the head of a new time window.
     * @return the list of initialized Bucket objects for this time subrange.
     */
    private List<Bucket> initBuckets(Pair<Long, HashedEvent> bucketKey) {
        List<Bucket> newBuckets = new ArrayList<Bucket>(mAggregateExprs.size());
        for (int i = 0; i < mAggregateExprs.size(); i++) {
            // Put in a new bucket instance for each aggregation funtion we're going to run.
            newBuckets.add(new Bucket());
        }

        assert null == mBucketMap.get(bucketKey);
        mBucketMap.put(bucketKey, newBuckets);

        // Put this into the map organized by group, as well.
        // Get the set of (time, bucketlist) pairs for the group.
        List<Pair<Long, List<Bucket>>> bucketsByTime = mBucketsByGroup.get(bucketKey.getRight());
        if (null == bucketsByTime) {
            bucketsByTime = new LinkedList<Pair<Long, List<Bucket>>>();
            mBucketsByGroup.put(bucketKey.getRight(), bucketsByTime);
        }
        bucketsByTime.add(new Pair<Long, List<Bucket>>(bucketKey.getLeft(), newBuckets));

        // Return the initialized set of Bucket objects back to the caller.
        return newBuckets;
    }

    /**
     * @return a key into our group-by map that is composed of the bucket
     * timestamp for the event, and a HashedEvent that reads the fields
     * of the event necessary to group by those fields. If we are not grouping
     * by any fields, this component of the pair is null.
     */
    private Pair<Long, HashedEvent> getEventKey(EventWrapper e) {
        long eventTime = e.getEvent().getTimestamp();
        long remainder = eventTime % mTimeModulus;
        Long bucketTime;

        // If we're on an interval boundary (e.g., t=100) we go into that bucket.
        // If we're off-boundary (e.g., t=103), we go into the closest "previous" bucket (t=100).
        bucketTime = Long.valueOf(eventTime - remainder);
        HashedEvent hashedEvent = new HashedEvent(e, mGroupByFields);
        return new Pair<Long, HashedEvent>(bucketTime, hashedEvent);
    }

    /**
     * Given a set of time buckets associated with a given group,
     * iterate over the time buckets for a specific time interval,
     * for a particular aggregation function.
     */
    private static class BucketIterator<T> implements Iterator<Bucket<T>> {
        /**
         * Offset of the true Bucket object in the final List<Buckets> that
         * specifies buckest for each aggregation function we operate.
         */
        private final int mFunctionId;

        /** Lowest timestamped bucket we return. */
        private final long mLoTime;

        /** Highest timestamped bucket we return. */
        private final long mHiTime;

        /**
         * Iterator over the outer list. We require this iterator
         * to return values in order.
         */
        private final Iterator<Pair<Long, List<Bucket>>> mIterator;

        /** The next value we return. */
        private Bucket<T> mNextBucket;

        /** Set to true if prepBucket() has been called, but not next(). */
        private boolean mIsReady;

        /** The number of buckets in the time interval that were returned by this iterator. */
        private int mYieldCount;

        public BucketIterator(int functionId, long loTime, long hiTime, List<Pair<Long, List<Bucket>>> inputList) {
            mFunctionId = functionId;
            mLoTime = loTime;
            mHiTime = hiTime;
            mIterator = inputList.iterator();
            mYieldCount = 0;
        }

        /**
         * Scan ahead in the underlying iterator til we find the next element.
         * Set mNextBucket to the next value that next() should return, or null
         * if we cannot yield any more values.
         * Sets mIsReady to true.
         */
        private void prepBucket() {
            assert !mIsReady; // This should not be called twice in a row.

            mIsReady = true;
            mNextBucket = null;

            while (mIterator.hasNext()) {
                Pair<Long, List<Bucket>> nextPair = mIterator.next();
                long timestamp = nextPair.getLeft();
                if (timestamp > mLoTime && timestamp <= mHiTime) {
                    // We found the next one to return.
                    mNextBucket = nextPair.getRight().get(mFunctionId);
                    return;
                }
            }
        }

        public Bucket<T> next() {
            if (!mIsReady) {
                prepBucket();
            }

            assert mIsReady;
            mIsReady = false;
            if (mNextBucket != null) {
                mYieldCount++;
            }
            return mNextBucket;
        }

        public boolean hasNext() {
            if (!mIsReady) {
                prepBucket();
            }

            assert mIsReady;
            return mNextBucket != null;
        }

        public void remove() {
            throw new RuntimeException("Not implemented.");
        }

        int getYieldCount() {
            return mYieldCount;
        }
    }

    /**
     * Close the window ending with the bucket for 'closeTime'.
     * Remove any buckets that are older than closeTime - aggregationIntervalWidth.
     * since they will no longer contribute to any open windows.
     */
    private void closeWindow(long closeTime, FlowElementContext context) throws IOException, InterruptedException {
        long loTime = closeTime - mTimeSpan.getWidth();
        Long closeBucketTimestamp = Long.valueOf(closeTime);

        LOG.debug("Closing window for range: " + loTime + " -> " + closeTime);

        // For each group, emit an output record containing the aggregate values over
        // the whole time window.
        for (Map.Entry<HashedEvent, List<Pair<Long, List<Bucket>>>> entry : mBucketsByGroup.entrySet()) {
            HashedEvent group = entry.getKey();

            // In non-continuous (demand-only) mode, check whether there's a bucket associated
            // with this window's closing time for this group.
            if (!mContinuousOutput
                    && mBucketMap.get(new Pair<Long, HashedEvent>(closeBucketTimestamp, group)) == null) {
                continue; // Nothing to do.
            }

            GenericData.Record record = new GenericData.Record(getOutputSchema());
            List<Pair<Long, List<Bucket>>> bucketsByTime = entry.getValue();

            int numBucketsInRangeForGroup = 0;
            // Execute each aggregation function over the applicable subset of buckets
            // in bucketsByTime.
            for (int i = 0; i < mAggregateExprs.size(); i++) {
                BucketIterator aggIterator = new BucketIterator(i, loTime, closeTime, bucketsByTime);
                AliasedExpr aliasExpr = mAggregateExprs.get(i);
                FnCallExpr fnCall = (FnCallExpr) aliasExpr.getExpr();
                Object result = fnCall.finishWindow(new IterableIterator(aggIterator));
                numBucketsInRangeForGroup += aggIterator.getYieldCount();
                record.put(aliasExpr.getAvroLabel(), result);
            }

            // If there are no buckets in bucketsByTime that are in our time range,
            // we should not emit anything for this group. Just silently continue.
            if (0 == numBucketsInRangeForGroup) {
                // Discard this output; we didn't actually calculate anything.
                continue;
            }

            // Copy the specified fields to propagate from the record used to define
            // the group, into the output record.
            EventWrapper groupWrapper = group.getEventWrapper();
            for (TypedField propagateField : mPropagateFields) {
                record.put(propagateField.getAvroName(), groupWrapper.getField(propagateField));
            }

            // Emit this as an output event!
            emitAvroRecord(record, groupWrapper.getEvent(), closeTime, context);
        }

        // Remove any buckets that are too old to be useful to any subsequent windows.
        // TODO(aaron): This is O(groups * num_buckets). We should actually use a TreeMap
        // instad of a list internally, so we can quickly cull the herd. That would be
        // O(groups * log(num_buckets)).
        Iterator<Map.Entry<HashedEvent, List<Pair<Long, List<Bucket>>>>> bucketsByGrpIter = mBucketsByGroup
                .entrySet().iterator();
        while (bucketsByGrpIter.hasNext()) {
            Map.Entry<HashedEvent, List<Pair<Long, List<Bucket>>>> entry = bucketsByGrpIter.next();
            HashedEvent group = entry.getKey();
            List<Pair<Long, List<Bucket>>> bucketsByTime = entry.getValue();
            Iterator<Pair<Long, List<Bucket>>> bucketsByTimeIter = bucketsByTime.iterator();
            while (bucketsByTimeIter.hasNext()) {
                Pair<Long, List<Bucket>> timedBucket = bucketsByTimeIter.next();
                Long timestamp = timedBucket.getLeft();
                if (timestamp.longValue() < loTime) {
                    bucketsByTimeIter.remove(); // Remove from bucketsByTime list.
                    Pair<HashedEvent, Long> key = new Pair<HashedEvent, Long>(group, timestamp);
                    mBucketMap.remove(key); // Remove from mBucketMap.
                }
            }

            if (bucketsByTime.size() == 0) {
                // We've removed the last time bucket for a given group from mBucketsByGroup.
                // Remove the group from that map.
                bucketsByGrpIter.remove();
            }
        }
    }

    /**
     * Close all open windows up to and including the window that ends with the bucket
     * for time 'lastWindow'.
     */
    private void closeUntil(long curBucketTime, long lastWindow, FlowElementContext context)
            throws IOException, InterruptedException {

        LOG.debug("Close until: cur=" + curBucketTime + ", lastWindow=" + lastWindow + ", mTailBucketTime="
                + mTailBucketTime + ", mTimeMod=" + mTimeModulus + ", mMaxPrior=" + mMaxPriorEmitInterval);
        if (lastWindow <= mTailBucketTime) {
            return; // We've already closed this window.
        }

        // If mHeadBucketTime is too far back from the current time,
        // do a mass expiration and throw out old data. closeTime is bounded by
        // mMaxPriorEmitInterval.
        for (long closeTime = Math.max(mTailBucketTime,
                curBucketTime - mMaxPriorEmitInterval); closeTime <= lastWindow; closeTime += mTimeModulus) {
            LOG.debug("Close window: closeTime=" + closeTime);
            closeWindow(closeTime, context);
        }

        mTailBucketTime = lastWindow + mTimeModulus;
    }

    @Override
    public void takeEvent(EventWrapper e) throws IOException, InterruptedException {
        Pair<Long, HashedEvent> bucketKey = getEventKey(e);

        long curBucketTime = bucketKey.getLeft();
        LOG.debug("Handling event time=" + curBucketTime);
        if (curBucketTime > mHeadBucketTime) {
            // We've just received an event that is newer than any others we've yet
            // received. This advances the sliding window to match this event's timestamp.
            // Emit any output groups that are older than this one by at least the
            // slack time interval.
            LOG.debug("New bucket: cur=" + curBucketTime + "; mHeadBucketTime=" + mHeadBucketTime);
            closeUntil(curBucketTime, curBucketTime - mSlackTime - mTimeModulus, getContext());
            // Since we've already handled these, remove their wake-up calls..
            mEvictionThread.discardUntil(mHeadBucketTime - mSlackTime);
            mHeadBucketTime = curBucketTime; // This insert advances our head bucket.
        } else if (curBucketTime < mHeadBucketTime - mMaxPriorEmitInterval) {
            // This event is too old -- ignore it.
            // TODO: Should this be mHeadBucketTiem - mSlackTime?
            LOG.debug("Dropping late event arriving at aggregator; HeadBucketTime=" + mHeadBucketTime
                    + " and event is for bucket " + curBucketTime);
            return;
        }

        // Get the bucket for the (timestamp, group-by-fields) of this event.
        // Actually returns a list of Bucket objects, one per AggregateFunc to
        // execute.
        List<Bucket> buckets = mBucketMap.get(bucketKey);
        if (null == buckets) {
            // We're putting the first event into a new bucket.
            buckets = initBuckets(bucketKey);
        }

        // For each aggregation function we're performing, insert this event into
        // the bucket for the aggregate function.
        assert buckets.size() == mAggregateExprs.size();
        for (int i = 0; i < mAggregateExprs.size(); i++) {
            AliasedExpr aliasExpr = mAggregateExprs.get(i);
            Expr expr = aliasExpr.getExpr();
            assert expr instanceof FnCallExpr;
            FnCallExpr fnCall = (FnCallExpr) expr;
            Bucket bucket = buckets.get(i);
            fnCall.insertAggregate(e, bucket);
        }

        // Insert a callback into a queue to allow time to expire these windows.
        enqueueWakeup(curBucketTime);
    }

    /**
     * Enqueue a wakeup in the EvictionThread that closes the bucket with the
     * specified bucket timestamp.
     */
    private void enqueueWakeup(long bucketTime) {
        if (bucketTime <= mLastEnqueuedWakeup) {
            // We've already enqueued a wakeup to close this bucket.
            return;
        }

        long curTime = System.currentTimeMillis();
        long offset = mTimeModulus + mSlackTime;
        long closeTime = curTime + offset; // local time to close the bucket.
        LOG.debug("Insert wakeup call: " + bucketTime + " at time offset=" + offset);
        mEvictionThread.insert(new Pair<Long, Long>(closeTime, bucketTime));
        mLastEnqueuedWakeup = bucketTime;
    }

    /**
     * Thread that sends notices to our coprocessor FlowElement when it is time to
     * close old windows based on elapsed local time.
     */
    private class EvictionThread extends Thread {
        private final Logger LOG = LoggerFactory.getLogger(EvictionThread.class.getName());

        /**
         * Set to true when it's time for the thread to go home. The thread
         * actually exits after this flag is set to true and the incoming queue
         * is empty.
         */
        private boolean mIsFinished;

        /**
         * Priority queue (heap) of times when we should insert expiry-times in
         * the coprocessor FlowElement's input queue.
         *
         * <p>The queue holds tuples of two long values. The first is a local
         * time when this thread should wake up; this is what the queue is
         * ordered on. The latter is the window time that should be expired.</p>
         */
        private PriorityBlockingQueue<Pair<Long, Long>> mQueue;

        // Maximum queue length == number of open windows + the newly-opening window
        //     + the currently-closing window.
        final long mMaxQueueLen = 2 + (mSlackTime / mTimeModulus);

        public EvictionThread() {
            super("AggregatorEvictionThread");

            mQueue = new PriorityBlockingQueue<Pair<Long, Long>>((int) mMaxQueueLen,
                    new PairLeftRightComparator<Long, Long>());
        }

        /**
         * Add a wake-up call to the queue.
         */
        public void insert(Pair<Long, Long> wakeUpCall) {
            synchronized (this) {
                assert mQueue.size() < mMaxQueueLen; // This operation should never block.
                mQueue.put(wakeUpCall);
                this.notify();
            }

            // Interrupt any wait that's going on, in case we are asleep and should
            // actually immediately service this wake-up call.
            this.interrupt();
        }

        /**
         * Discard all wakeup calls up to time 'minTime'.
         * minTime is a 'bucket time', not a 'local time'.
         */
        public void discardUntil(long minTime) {
            synchronized (this) {
                LOG.debug("discardUntil: " + minTime);
                Iterator<Pair<Long, Long>> iterator = mQueue.iterator();
                while (iterator.hasNext()) {
                    Pair<Long, Long> wakeUpCall = iterator.next();
                    if (wakeUpCall.getRight() < minTime) {
                        LOG.debug("discard@ " + wakeUpCall);
                        iterator.remove();
                    }
                }

                this.notify();
            }
        }

        /**
         * Set the finished flag to true; try to get the thread to stop as
         * quickly as possible.
         */
        public void finish() {
            synchronized (this) {
                this.mIsFinished = true;
                this.notify();
            }
            this.interrupt(); // Interrupt any current sleep.
        }

        /**
         * Main loop of the thread.
         * Continually sleeps until the next timer event is ready to occur.
         */
        public void run() {
            while (true) {
                Pair<Long, Long> wakeUpCall = null;
                long curTime;
                long nextWakeUp;

                synchronized (this) {
                    while (mQueue.size() == 0) {
                        try {
                            if (this.mIsFinished) {
                                // Parent is finished and we have drained our input queue. Go home.
                                return;
                            }
                            this.wait();
                        } catch (InterruptedException ie) {
                            // Interrupted while waiting for another wake-up call to enter our queue.
                            // Try again, if we're not already finished.
                            continue;
                        }
                    }

                    assert mQueue.size() > 0;
                    wakeUpCall = mQueue.peek();
                }

                if (null == wakeUpCall) {
                    continue;
                }

                curTime = System.currentTimeMillis();
                nextWakeUp = wakeUpCall.getLeft();
                if (nextWakeUp <= curTime) {
                    // TODO(aaron): This section probably bears further deadlock analysis.
                    // The put() into the timer queue can block (it has fixed length
                    // LocalEnvironment.MAX_QUEUE_LEN) until the timer FE services its
                    // existing list.
                    // If we are interrupted doing this, it is because the main thread
                    // has just inserted another wakeup call while we were blocking.
                    // This thread's input queue must not block when being filled from
                    // the main aggregation FE. I believe mMaxQueueLen should be sufficient
                    // to guarantee this is the case, because before we call enqueueWakeup(),
                    // we will have had to call closeUntil() in BucketedAggElem.takeEvent()
                    // on enough windows to free up the slots in this queue.
                    try {
                        LOG.debug("Timer evicting at " + curTime + ": " + wakeUpCall);
                        // Service this by injecting the getRight() into our outbound queue.
                        mTimerQueue.put(new TimeoutEventWrapper(wakeUpCall.getRight()));
                    } catch (InterruptedException ie) {
                        // Not a problem. If we were interrupted doing the put into mTimerQueue,
                        // then we'll service this again on the next go-around of the loop.
                        // Just make sure we don't mark this as 'complete.'
                        continue;
                    }

                    synchronized (this) {
                        // Now actually remove this from the input queue.
                        if (mQueue.peek() == wakeUpCall) {
                            // O(1) fast path; no intervening push.
                            mQueue.remove();
                        } else {
                            // intervening push of an earlier wakeup (?). Slow path.
                            mQueue.remove(wakeUpCall);
                        }
                    }
                } else {
                    // If we're down here, we need to sleep until it is the next wake-up time.
                    long napTime = nextWakeUp - curTime;
                    try {
                        Thread.sleep(napTime);
                    } catch (InterruptedException ie) {
                        // We were awoken early... this is expected (there may have been a
                        // new enqueue, etc).
                    }
                }
            }
        }
    }

    /** EventWrapper used to deliver the expiry time payload to the TimeoutEvictionElement. */
    private static class TimeoutEventWrapper extends EmptyEventWrapper {
        /** The time window that should be expired. */
        private final Long mExpireWindow;

        public TimeoutEventWrapper(Long expire) {
            mExpireWindow = expire;
        }

        @Override
        public Object getField(TypedField field) {
            return mExpireWindow;
        }
    }

    /**
     * Separate FlowElement that handles notifications from the EvictionThread; this
     * operates in the main thread, closing windows that cannot receive new events
     * because they are past the slack time interval.
     */
    public class TimeoutEvictionElement extends AvroOutputElementImpl {
        private final Logger LOG = LoggerFactory.getLogger(TimeoutEvictionElement.class.getName());

        private TimeoutEvictionElement(FlowElementContext ctxt, Schema outSchema) {
            super(ctxt, outSchema);
        }

        public void takeEvent(EventWrapper e) throws IOException, InterruptedException {
            assert e instanceof TimeoutEventWrapper;
            Long expireTime = (Long) e.getField(null); // TimeoutEventWrapper returns a single Long val
            LOG.debug("Handling in eviction element - timeout to: " + expireTime);
            closeUntil(expireTime, expireTime, getContext());
        }
    }

    /**
     * Create a TimeoutEvictionElement coupled to this BucketedAggregationElement.
     */
    public TimeoutEvictionElement getTimeoutElement(FlowElementContext timeoutContext) {
        return this.new TimeoutEvictionElement(timeoutContext, getOutputSchema());
    }
}