com.streamreduce.storm.spouts.EventSpout.java Source code

Java tutorial

Introduction

Here is the source code for com.streamreduce.storm.spouts.EventSpout.java

Source

/*
 * Copyright 2012 Nodeable Inc
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

package com.streamreduce.storm.spouts;

import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
import com.mongodb.BasicDBObject;
import com.streamreduce.storm.GroupingNameConstants;
import com.streamreduce.storm.MongoClient;
import com.streamreduce.storm.Visibility;
import org.apache.log4j.Logger;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;

/**
 * Extension of {@link AbstractScheduledDBCollectionSpout} that will
 * emit the {@link com.mongodb.BasicDBObject} representation for each Nodeable Event
 */
public class EventSpout extends AbstractScheduledDBCollectionSpout {

    private static Logger logger = Logger.getLogger(EventSpout.class);
    private static final long serialVersionUID = -3339407048421810230L;

    // Not final like all others due to EventSpoutTest needing to set mock/set the MongoClient
    private static MongoClient mongoClient = new MongoClient(MongoClient.MESSAGEDB_CONFIG_ID);

    private Date lastProcessedEventDate;

    /**
     * Constructor.  (Calls super constructor with a sleep duration of 15s for now.)
     */
    public EventSpout() {
        super(15000);
    }

    /**
     * Constructor.
     *
     * @param lastProcessedEventDate the date at which we should start polling events from
     */
    public EventSpout(Date lastProcessedEventDate) {
        this();

        this.lastProcessedEventDate = lastProcessedEventDate;
    }

    /**
     * {@inheritDoc}
     */
    public EventSpout(int sleepDuration) {
        super(sleepDuration);
    }

    /**
     * Constructor.
     *
     * @param sleepDuration          the sleep duration in between queue populations
     * @param lastProcessedEventDate the date at which we should start polling events from
     */
    public EventSpout(int sleepDuration, Date lastProcessedEventDate) {
        super(sleepDuration);

        this.lastProcessedEventDate = lastProcessedEventDate;
    }

    public Date getLastProcessedEventDate() {
        return lastProcessedEventDate;
    }

    public void setLastProcessedEventDate(Date lastProcessedEventDate) {
        this.lastProcessedEventDate = lastProcessedEventDate;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        // added the default stream because storm complained
        outputFieldsDeclarer.declare(new Fields());
        outputFieldsDeclarer.declareStream(GroupingNameConstants.ACCOUNT_GROUPING_NAME, new Fields("event"));
        outputFieldsDeclarer.declareStream(GroupingNameConstants.CONNECTION_GROUPING_NAME, new Fields("event"));
        outputFieldsDeclarer.declareStream(GroupingNameConstants.INVENTORY_ITEM_GROUPING_NAME, new Fields("event"));
        outputFieldsDeclarer.declareStream(GroupingNameConstants.USER_GROUPING_NAME, new Fields("event"));
        outputFieldsDeclarer.declareStream(GroupingNameConstants.MESSAGE_GROUPING_NAME, new Fields("event"));
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void handleDBEntry(SpoutOutputCollector collector, BasicDBObject entry) {
        BasicDBObject metadata = entry.containsField("metadata") ? (BasicDBObject) entry.get("metadata")
                : new BasicDBObject();
        String eventType = metadata.getString("targetType");

        // Just in case
        if (eventType == null) {
            // TODO: Figure out the best way to handle this

            // Log the inability to process further
            logger.error("Event with id of " + entry.get("_id") + " has no target type.  Unable to process.");

            // Early return to avoid emitting the event
            return;
        }

        String v = (String) metadata.get("targetVisibility");
        if (v != null) {
            Visibility visibility = Visibility.valueOf(v);
            if (visibility == Visibility.SELF) {
                // do not process private information
                return;
            }
        }

        MongoClient.mapMongoToPlainJavaTypes(entry);
        // Emit the entry to the type-specific stream
        collector.emit(eventType, new Values(entry.toMap()));
        ack(entry);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public List<BasicDBObject> getDBEntries() {
        try {

            //Get events in a one hour sliding window, or return all events if lastProcessedEventDate is null, which is
            //needed when dealing with a new bootstrapped message db, or a new eventStream.
            Date oneHourAfterLastProcessed = null;
            long lastProcessedEvent = mongoClient.readLastProcessedEventDate("EventSpout");
            if (lastProcessedEvent != -1) {
                lastProcessedEventDate = new Date(lastProcessedEvent);
            }

            logger.info("Determine last processed event date....");

            if (lastProcessedEventDate != null) {
                oneHourAfterLastProcessed = new Date(lastProcessedEventDate.getTime() + TimeUnit.HOURS.toMillis(1));
                logger.info("Retrieving all events between " + lastProcessedEventDate + " and "
                        + oneHourAfterLastProcessed);
            }

            logger.debug("Mongo geEvents Query Time, Begin:" + System.currentTimeMillis());
            List<BasicDBObject> events = mongoClient.getEvents(lastProcessedEventDate, oneHourAfterLastProcessed);
            logger.debug("Mongo geEvents Query Time, End:" + System.currentTimeMillis());

            logger.info("Number of events to be emitted:  " + events.size());

            persistLastProcessedEventDate(oneHourAfterLastProcessed, events);
            return events;
        } catch (Exception e) {
            logger.error("0 events emmitted due to failure in getDBEntries", e);
            return new ArrayList<>();
        }
    }

    /**
     * Persists the lastProcessedEventDate to the last entry of the passed in events if we had events or to the end of
     * the sliding window if it isn't null.
     *
     * @param endOfSlidingWindow
     * @param events
     */
    private void persistLastProcessedEventDate(Date endOfSlidingWindow, List<BasicDBObject> events) {
        if (events.size() > 0) {
            //If we had events to process, lastProcessedEventDate is the date of the last event.
            BasicDBObject mostRecentEntry = getMostRecentEntryFromEvents(events);

            //events.get(events.size() - 1);
            lastProcessedEventDate = new Date(mostRecentEntry.getLong("timestamp"));
        } else if (endOfSlidingWindow != null
                && endOfSlidingWindow.getTime() < (System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(1))) {
            //If we didn't have events, only update lastProcessEventDate to the end our hour sliding window
            //if it is before the current system + a buffer of one minute to avoid updating lastProcessedEventDate to
            //a value in the future.
            lastProcessedEventDate = endOfSlidingWindow;
        }

        if (lastProcessedEventDate != null) {
            mongoClient.updateLastProcessedEventDate("EventSpout", lastProcessedEventDate.getTime());
        }
    }

    private BasicDBObject getMostRecentEntryFromEvents(List<BasicDBObject> events) {
        if (events == null) {
            return null;
        }
        BasicDBObject mostRecent = null;
        for (BasicDBObject event : events) {
            if (event == null) {
                continue;
            }
            if (mostRecent == null) {
                mostRecent = event;
                continue;
            }
            if (event.getLong("timestamp") > mostRecent.getLong("timestamp")) {
                mostRecent = event;
            }
        }
        return mostRecent;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public BasicDBObject getDBEntry(String id) {
        try {
            return mongoClient.getEvent(id);
        } catch (Exception e) {
            logger.error("Unable to retrieve DBEntry due to unexpected failure", e);
            return null;
        }
    }
}