Java tutorial
/* * Copyright 2012 Nodeable Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.streamreduce.storm.spouts; import backtype.storm.spout.SpoutOutputCollector; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Values; import com.mongodb.BasicDBObject; import com.streamreduce.storm.GroupingNameConstants; import com.streamreduce.storm.MongoClient; import com.streamreduce.storm.Visibility; import org.apache.log4j.Logger; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.concurrent.TimeUnit; /** * Extension of {@link AbstractScheduledDBCollectionSpout} that will * emit the {@link com.mongodb.BasicDBObject} representation for each Nodeable Event */ public class EventSpout extends AbstractScheduledDBCollectionSpout { private static Logger logger = Logger.getLogger(EventSpout.class); private static final long serialVersionUID = -3339407048421810230L; // Not final like all others due to EventSpoutTest needing to set mock/set the MongoClient private static MongoClient mongoClient = new MongoClient(MongoClient.MESSAGEDB_CONFIG_ID); private Date lastProcessedEventDate; /** * Constructor. (Calls super constructor with a sleep duration of 15s for now.) */ public EventSpout() { super(15000); } /** * Constructor. * * @param lastProcessedEventDate the date at which we should start polling events from */ public EventSpout(Date lastProcessedEventDate) { this(); this.lastProcessedEventDate = lastProcessedEventDate; } /** * {@inheritDoc} */ public EventSpout(int sleepDuration) { super(sleepDuration); } /** * Constructor. * * @param sleepDuration the sleep duration in between queue populations * @param lastProcessedEventDate the date at which we should start polling events from */ public EventSpout(int sleepDuration, Date lastProcessedEventDate) { super(sleepDuration); this.lastProcessedEventDate = lastProcessedEventDate; } public Date getLastProcessedEventDate() { return lastProcessedEventDate; } public void setLastProcessedEventDate(Date lastProcessedEventDate) { this.lastProcessedEventDate = lastProcessedEventDate; } /** * {@inheritDoc} */ @Override public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { // added the default stream because storm complained outputFieldsDeclarer.declare(new Fields()); outputFieldsDeclarer.declareStream(GroupingNameConstants.ACCOUNT_GROUPING_NAME, new Fields("event")); outputFieldsDeclarer.declareStream(GroupingNameConstants.CONNECTION_GROUPING_NAME, new Fields("event")); outputFieldsDeclarer.declareStream(GroupingNameConstants.INVENTORY_ITEM_GROUPING_NAME, new Fields("event")); outputFieldsDeclarer.declareStream(GroupingNameConstants.USER_GROUPING_NAME, new Fields("event")); outputFieldsDeclarer.declareStream(GroupingNameConstants.MESSAGE_GROUPING_NAME, new Fields("event")); } /** * {@inheritDoc} */ @Override public void handleDBEntry(SpoutOutputCollector collector, BasicDBObject entry) { BasicDBObject metadata = entry.containsField("metadata") ? (BasicDBObject) entry.get("metadata") : new BasicDBObject(); String eventType = metadata.getString("targetType"); // Just in case if (eventType == null) { // TODO: Figure out the best way to handle this // Log the inability to process further logger.error("Event with id of " + entry.get("_id") + " has no target type. Unable to process."); // Early return to avoid emitting the event return; } String v = (String) metadata.get("targetVisibility"); if (v != null) { Visibility visibility = Visibility.valueOf(v); if (visibility == Visibility.SELF) { // do not process private information return; } } MongoClient.mapMongoToPlainJavaTypes(entry); // Emit the entry to the type-specific stream collector.emit(eventType, new Values(entry.toMap())); ack(entry); } /** * {@inheritDoc} */ @Override public List<BasicDBObject> getDBEntries() { try { //Get events in a one hour sliding window, or return all events if lastProcessedEventDate is null, which is //needed when dealing with a new bootstrapped message db, or a new eventStream. Date oneHourAfterLastProcessed = null; long lastProcessedEvent = mongoClient.readLastProcessedEventDate("EventSpout"); if (lastProcessedEvent != -1) { lastProcessedEventDate = new Date(lastProcessedEvent); } logger.info("Determine last processed event date...."); if (lastProcessedEventDate != null) { oneHourAfterLastProcessed = new Date(lastProcessedEventDate.getTime() + TimeUnit.HOURS.toMillis(1)); logger.info("Retrieving all events between " + lastProcessedEventDate + " and " + oneHourAfterLastProcessed); } logger.debug("Mongo geEvents Query Time, Begin:" + System.currentTimeMillis()); List<BasicDBObject> events = mongoClient.getEvents(lastProcessedEventDate, oneHourAfterLastProcessed); logger.debug("Mongo geEvents Query Time, End:" + System.currentTimeMillis()); logger.info("Number of events to be emitted: " + events.size()); persistLastProcessedEventDate(oneHourAfterLastProcessed, events); return events; } catch (Exception e) { logger.error("0 events emmitted due to failure in getDBEntries", e); return new ArrayList<>(); } } /** * Persists the lastProcessedEventDate to the last entry of the passed in events if we had events or to the end of * the sliding window if it isn't null. * * @param endOfSlidingWindow * @param events */ private void persistLastProcessedEventDate(Date endOfSlidingWindow, List<BasicDBObject> events) { if (events.size() > 0) { //If we had events to process, lastProcessedEventDate is the date of the last event. BasicDBObject mostRecentEntry = getMostRecentEntryFromEvents(events); //events.get(events.size() - 1); lastProcessedEventDate = new Date(mostRecentEntry.getLong("timestamp")); } else if (endOfSlidingWindow != null && endOfSlidingWindow.getTime() < (System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(1))) { //If we didn't have events, only update lastProcessEventDate to the end our hour sliding window //if it is before the current system + a buffer of one minute to avoid updating lastProcessedEventDate to //a value in the future. lastProcessedEventDate = endOfSlidingWindow; } if (lastProcessedEventDate != null) { mongoClient.updateLastProcessedEventDate("EventSpout", lastProcessedEventDate.getTime()); } } private BasicDBObject getMostRecentEntryFromEvents(List<BasicDBObject> events) { if (events == null) { return null; } BasicDBObject mostRecent = null; for (BasicDBObject event : events) { if (event == null) { continue; } if (mostRecent == null) { mostRecent = event; continue; } if (event.getLong("timestamp") > mostRecent.getLong("timestamp")) { mostRecent = event; } } return mostRecent; } /** * {@inheritDoc} */ @Override public BasicDBObject getDBEntry(String id) { try { return mongoClient.getEvent(id); } catch (Exception e) { logger.error("Unable to retrieve DBEntry due to unexpected failure", e); return null; } } }