Java tutorial
/** * Copyright 2014 Comcast Cable Communications Management, LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.comcast.viper.flume2storm.spout; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.configuration.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import backtype.storm.spout.SpoutOutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseRichSpout; import com.comcast.viper.flume2storm.F2SConfigurationException; import com.comcast.viper.flume2storm.connection.parameters.ConnectionParameters; import com.comcast.viper.flume2storm.connection.receptor.EventReceptor; import com.comcast.viper.flume2storm.connection.receptor.EventReceptorFactory; import com.comcast.viper.flume2storm.event.F2SEvent; import com.comcast.viper.flume2storm.location.LocationService; import com.comcast.viper.flume2storm.location.LocationServiceFactory; import com.comcast.viper.flume2storm.location.ServiceListener; import com.comcast.viper.flume2storm.location.ServiceProvider; import com.comcast.viper.flume2storm.location.ServiceProviderSerialization; import com.google.common.collect.ImmutableSet; /** * A Storm spout that ingests data from Flume, via the Flume2Storm connector. * This is one of the 2 main components of the Flume2Storm connector. On * reception of Flume2Storm events, it queues them in-memory, and emit them on * {@link #nextTuple()} call. * <p /> * In order to connect it to the rest of the Storm topology, use one or more * {@link F2SEventEmitter}. * * @param <CP> * The Connection Parameters class * @param <SP> * The Service Provider class */ public class FlumeSpout<CP extends ConnectionParameters, SP extends ServiceProvider<CP>> extends BaseRichSpout { private static final long serialVersionUID = -2858136141243917853L; /** Small sleep for storm spout */ protected static final int SPOUT_SLEEP_TIME = 10; protected static final Logger LOG = LoggerFactory.getLogger(FlumeSpout.class); protected final FlumeSpoutConfiguration configuration; protected final Map<String, EventReceptor<CP>> eventReceptors; protected final Set<F2SEventEmitter> emitters; protected SpoutOutputCollector collector; protected LocationServiceFactory<SP> locationServiceFactory; protected ServiceProviderSerialization<SP> serviceProviderSerialization; protected LocationService<SP> locationService; protected EventReceptorFactory<CP> eventReceptorFactory; protected ServiceListener<SP> serviceListener; private class MyServiceListener implements ServiceListener<SP> { protected MyServiceListener() { super(); } /** * @see com.comcast.viper.flume2storm.location.ServiceListener#onProviderRemoved(com.comcast.viper.flume2storm.location.ServiceProvider) */ @Override public void onProviderRemoved(SP serviceProvider) { // The associated EventReceptor is not removed at this point } /** * @see com.comcast.viper.flume2storm.location.ServiceListener#onProviderAdded(com.comcast.viper.flume2storm.location.ServiceProvider) */ @Override public void onProviderAdded(SP serviceProvider) { try { LOG.info("Added provider: {}", serviceProvider); EventReceptor<CP> eventReceptor = eventReceptorFactory .create(serviceProvider.getConnectionParameters(), configuration.get()); LOG.debug("Adding event receptor: {}", eventReceptor); eventReceptor.start(); while (!eventReceptor.getStats().isConnected()) { LOG.debug("Receptor not started... waiting..."); try { Thread.sleep(500); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } eventReceptors.put(serviceProvider.getConnectionParameters().getId(), eventReceptor); } catch (F2SConfigurationException e) { LOG.error("Failed to add service provider: " + serviceProvider, e); } } } /** * Constructs a new {@link FlumeSpout} * * @param emitters * The list of {@link F2SEventEmitter} * @param configuration * The configuration for the spout * @throws F2SConfigurationException * If the configuration is invalid */ public FlumeSpout(final Set<F2SEventEmitter> emitters, final Configuration configuration) throws F2SConfigurationException { this.emitters = emitters; eventReceptors = new ConcurrentHashMap<String, EventReceptor<CP>>(); this.configuration = FlumeSpoutConfiguration.from(configuration); } /** * Convenience constructor * * @param emitter * One {@link F2SEventEmitter} * @param configuration * The configuration for the spout * @throws F2SConfigurationException * If the configuration is invalid */ public FlumeSpout(final F2SEventEmitter emitter, final Configuration configuration) throws F2SConfigurationException { this(ImmutableSet.of(emitter), configuration); } /** * @see backtype.storm.spout.ISpout#open(java.util.Map, * backtype.storm.task.TopologyContext, * backtype.storm.spout.SpoutOutputCollector) */ @SuppressWarnings("unchecked") @Override public void open(@SuppressWarnings("rawtypes") final Map conf, final TopologyContext context, final SpoutOutputCollector collector) { try { LOG.debug("Opening..."); this.collector = collector; Class<? extends LocationServiceFactory<SP>> locationServiceFactoryClass = (Class<? extends LocationServiceFactory<SP>>) Class .forName(configuration.getLocationServiceFactoryClassName()); this.locationServiceFactory = locationServiceFactoryClass.newInstance(); Class<? extends ServiceProviderSerialization<SP>> serviceProviderSerializationClass = (Class<? extends ServiceProviderSerialization<SP>>) Class .forName(configuration.getServiceProviderSerializationClassName()); this.serviceProviderSerialization = serviceProviderSerializationClass.newInstance(); Class<? extends EventReceptorFactory<CP>> eventReceptorFactoryClass = (Class<? extends EventReceptorFactory<CP>>) Class .forName(configuration.getEventReceptorFactoryClassName()); this.eventReceptorFactory = eventReceptorFactoryClass.newInstance(); locationService = locationServiceFactory.create(configuration.get(), serviceProviderSerialization); serviceListener = new MyServiceListener(); locationService.addListener(serviceListener); locationService.start(); LOG.info("Opened"); } catch (Exception e) { LOG.error("Failed to open properly: " + e.getMessage(), e); } } /** * @see backtype.storm.topology.base.BaseRichSpout#close() */ @Override public void close() { LOG.debug("Closing..."); locationService.removeListener(serviceListener); locationService.stop(); for (EventReceptor<CP> eventReceptor : eventReceptors.values()) { eventReceptor.stop(); } eventReceptors.clear(); LOG.info("Clossed"); } protected void removeServiceProvider(String serviceProviderId) { EventReceptor<CP> eventReceptor = eventReceptors.remove(serviceProviderId); if (eventReceptor != null) { eventReceptor.stop(); LOG.debug("Removed service provider: {}", eventReceptor); } } /** * @see backtype.storm.spout.ISpout#nextTuple() */ @Override public void nextTuple() { try { if (collector != null) { // Emit any events we have queued for (final EventReceptor<CP> eventReceptor : eventReceptors.values()) { List<F2SEvent> events = eventReceptor.getEvents(); for (final F2SEvent event : events) { LOG.trace("Received F2S event: {}", event); for (final F2SEventEmitter emitter : emitters) { emitter.emitEvent(event, collector); } } // Removing EventReceptor if: // - it's disconnected from the EventSender // - the associated ServiceProvider is not registered // - it does not have queued up events if (!eventReceptor.getStats().isConnected() && !locationService .containsServiceProvider(eventReceptor.getConnectionParameters().getId()) && events.isEmpty()) { removeServiceProvider(eventReceptor.getConnectionParameters().getId()); } } } } catch (final Exception ex) { LOG.error("There was a problem emitting events: " + ex.getLocalizedMessage(), ex); collector.reportError(ex); } // TODO remove: this is not needed since storm 0.8.1 // Always sleep for a little bit try { Thread.sleep(SPOUT_SLEEP_TIME); } catch (final InterruptedException ex) { LOG.warn("Thread interupted: " + ex.getLocalizedMessage()); } } /** * @see backtype.storm.topology.IComponent#declareOutputFields(backtype.storm.topology.OutputFieldsDeclarer) */ @Override public void declareOutputFields(final OutputFieldsDeclarer declarer) { for (final F2SEventEmitter emitter : emitters) { emitter.declareOutputFields(declarer); } } }