streamflow.spout.twitter.TwitterSampleSpout.java Source code

Introduction

Here is the source code for streamflow.spout.twitter.TwitterSampleSpout.java

Source

/**
 * Copyright 2014 Lockheed Martin Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package streamflow.spout.twitter;

import backtype.storm.Config;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
import backtype.storm.utils.Utils;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;

import twitter4j.*;
import twitter4j.conf.ConfigurationBuilder;

public class TwitterSampleSpout extends BaseRichSpout {

    private SpoutOutputCollector collector;

    private Logger logger;

    private String consumerKey;
    private String consumerSecret;
    private String accessToken;
    private String accessTokenSecret;
    private String proxyHost;
    private int proxyPort;

    private final LinkedBlockingQueue<Status> queue = new LinkedBlockingQueue<Status>(100000);
    private TwitterStream twitterStream;

    @Inject
    public void setConsumerKey(@Named("oauth-consumer-key") String consumerKey) {
        this.consumerKey = consumerKey;
    }

    @Inject
    public void setConsumerSecret(@Named("oauth-consumer-secret") String consumerSecret) {
        this.consumerSecret = consumerSecret;
    }

    @Inject
    public void setAccessToken(@Named("oauth-access-token") String accessToken) {
        this.accessToken = accessToken;
    }

    @Inject
    public void setAccessTokenSecret(@Named("oauth-access-token-secret") String accessTokenSecret) {
        this.accessTokenSecret = accessTokenSecret;
    }

    @Inject
    public void setLogger(Logger logger) {
        this.logger = logger;
    }

    // @Inject(optional = true)
    public void setProxyHost(@Named("http.proxy.host") String proxyHost) {
        this.proxyHost = proxyHost;
    }

    // @Inject(optional = true)
    public void setProxyPort(@Named("http.proxy.port") int proxyPort) {
        this.proxyPort = proxyPort;
    }

    @Override
    public void open(Map config, TopologyContext context, SpoutOutputCollector collector) {
        this.collector = collector;

        logger.info(
                "Twitter Sampler Started: Consumer Key = " + consumerKey + ", Consumer Secret = " + consumerSecret
                        + ", Access Token = " + accessToken + ", Access Token Secret = " + accessTokenSecret);

        if (StringUtils.isNotBlank(consumerKey) && StringUtils.isNotBlank(consumerSecret)
                && StringUtils.isNotBlank(accessToken) && StringUtils.isNotBlank(accessTokenSecret)) {
            // Build the twitter config to authenticate the requests
            ConfigurationBuilder twitterConfig = new ConfigurationBuilder().setOAuthConsumerKey(consumerKey)
                    .setOAuthConsumerSecret(consumerSecret).setOAuthAccessToken(accessToken)
                    .setOAuthAccessTokenSecret(accessTokenSecret).setJSONStoreEnabled(true)
                    .setIncludeEntitiesEnabled(true).setIncludeEntitiesEnabled(true);

            // Add the proxy settings to the Twitter config if they were specified
            if (StringUtils.isNotBlank(proxyHost) && proxyPort > 0) {
                try {
                    twitterConfig.setHttpProxyPort(proxyPort).setHttpProxyHost(proxyHost);
                } catch (Exception ex) {
                }
            }

            // Status listener which handle the status events and add them to the queue
            StatusListener listener = new StatusListener() {
                @Override
                public void onStatus(Status status) {
                    queue.offer(status);
                }

                @Override
                public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
                    logger.debug("Twitter Deletion Notice: " + statusDeletionNotice.getUserId());
                }

                @Override
                public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
                    logger.debug("Twitter On Track Limitation Notice: Number Of Limited Statuses"
                            + numberOfLimitedStatuses);
                }

                @Override
                public void onScrubGeo(long userId, long upToStatusId) {
                    logger.debug("Twitter Scrub Geo: UserID = " + userId + ", UpToStatusId = " + upToStatusId);
                }

                @Override
                public void onException(Exception exception) {
                    logger.debug("Twitter Exception: " + exception.getMessage());
                }

                @Override
                public void onStallWarning(StallWarning stallWarning) {
                    logger.debug("Twitter Stall Warning: " + stallWarning.toString());
                }
            };

            TwitterStreamFactory twitterFactory = new TwitterStreamFactory(twitterConfig.build());
            twitterStream = twitterFactory.getInstance();
            twitterStream.addListener(listener);
            twitterStream.sample();

            logger.info("Twitter Sample Stream Initialized");

        } else {
            logger.info("Twitter Sampler missing required OAuth properties. "
                    + "Pleast check your settings and try again.");
        }
    }

    @Override
    public void nextTuple() {
        Status status = queue.poll();

        if (status == null) {
            Utils.sleep(50);
        } else {
            // Emit the twitter status as a JSON String
            collector.emit(new Values(status));
        }
    }

    @Override
    public void close() {
        if (twitterStream != null) {
            twitterStream.shutdown();
        }

        logger.info("Twitter Sampler Stopped");
    }

    @Override
    public Map<String, Object> getComponentConfiguration() {
        Config config = new Config();
        config.setMaxTaskParallelism(1);
        return config;
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields("tweet"));
    }
}