KinesisStormClickstreamApp.SampleTopology.java Source code

Introduction

Here is the source code for KinesisStormClickstreamApp.SampleTopology.java

Source

/*
 * Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Amazon Software License (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 * http://aws.amazon.com/asl/
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package KinesisStormClickstreamApp;

import java.io.FileInputStream;
import java.io.IOException;
import java.util.Properties;

import com.amazonaws.regions.Regions;
import com.amazonaws.services.kinesis.stormspout.*;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.StormSubmitter;
import backtype.storm.generated.AlreadyAliveException;
import backtype.storm.generated.InvalidTopologyException;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.tuple.Fields;

import com.amazonaws.ClientConfiguration;

public class SampleTopology {
    private static final Logger LOG = LoggerFactory.getLogger(SampleTopology.class);
    private static String topologyName = "SampleTopology";
    private static String streamName;
    private static InitialPositionInStream initialPositionInStream;
    private static String zookeeperEndpoint;
    private static String zookeeperPrefix;
    private static String elasticCacheRedisEndpoint;
    private static String regionName;

    public static void main(String[] args) throws IllegalArgumentException, KeeperException, InterruptedException,
            AlreadyAliveException, InvalidTopologyException, IOException {
        String propertiesFile = null;
        String mode = null;

        if (args.length != 2) {
            printUsageAndExit();
        } else {
            propertiesFile = args[0];
            mode = args[1];
        }

        configure(propertiesFile);

        final KinesisSpoutConfig config = new KinesisSpoutConfig(streamName, zookeeperEndpoint)
                .withZookeeperPrefix(zookeeperPrefix).withInitialPositionInStream(initialPositionInStream)
                .withRegion(Regions.fromName(regionName));

        final KinesisSpout spout = new KinesisSpout(config, new CustomCredentialsProviderChain(),
                new ClientConfiguration());
        TopologyBuilder builder = new TopologyBuilder();
        LOG.info("Using Kinesis stream: " + config.getStreamName());

        // Using number of shards as the parallelism hint for the spout.
        builder.setSpout("Kinesis", spout, 2);
        builder.setBolt("Parse", new ParseReferrerBolt(), 6).shuffleGrouping("Kinesis");
        builder.setBolt("Count", new RollingCountBolt(5, 2, elasticCacheRedisEndpoint), 6).fieldsGrouping("Parse",
                new Fields("referrer"));

        //builder.setBolt("Count", new CountReferrerBolt(), 12).fieldsGrouping("Parse", new Fields("referrer"));

        Config topoConf = new Config();
        topoConf.setFallBackOnJavaSerialization(true);
        topoConf.setDebug(false);

        if (mode.equals("LocalMode")) {
            LOG.info("Starting sample storm topology in LocalMode ...");
            new LocalCluster().submitTopology("test_spout", topoConf, builder.createTopology());
        } else if (mode.equals("RemoteMode")) {
            topoConf.setNumWorkers(1);
            topoConf.setMaxSpoutPending(5000);
            LOG.info("Submitting sample topology " + topologyName + " to remote cluster.");
            StormSubmitter.submitTopology(topologyName, topoConf, builder.createTopology());
        } else {
            printUsageAndExit();
        }

    }

    private static void configure(String propertiesFile) throws IOException {
        FileInputStream inputStream = new FileInputStream(propertiesFile);
        Properties properties = new Properties();
        try {
            properties.load(inputStream);
        } finally {
            inputStream.close();
        }

        String topologyNameOverride = properties.getProperty(ConfigKeys.TOPOLOGY_NAME_KEY);
        if (topologyNameOverride != null) {
            topologyName = topologyNameOverride;
        }
        LOG.info("Using topology name " + topologyName);

        String streamNameOverride = properties.getProperty(ConfigKeys.STREAM_NAME_KEY);
        if (streamNameOverride != null) {
            streamName = streamNameOverride;
        }
        LOG.info("Using stream name " + streamName);

        String initialPositionOverride = properties.getProperty(ConfigKeys.INITIAL_POSITION_IN_STREAM_KEY);
        if (initialPositionOverride != null) {
            initialPositionInStream = InitialPositionInStream.valueOf(initialPositionOverride);
        }
        LOG.info("Using initial position " + initialPositionInStream.toString()
                + " (if a checkpoint is not found).");

        String zookeeperEndpointOverride = properties.getProperty(ConfigKeys.ZOOKEEPER_ENDPOINT_KEY);
        if (zookeeperEndpointOverride != null) {
            zookeeperEndpoint = zookeeperEndpointOverride;
        }
        LOG.info("Using zookeeper endpoint " + zookeeperEndpoint);

        String zookeeperPrefixOverride = properties.getProperty(ConfigKeys.ZOOKEEPER_PREFIX_KEY);
        if (zookeeperPrefixOverride != null) {
            zookeeperPrefix = zookeeperPrefixOverride;
        }
        LOG.info("Using zookeeper prefix " + zookeeperPrefix);

        String elasticCacheRedisEndpointOverride = properties.getProperty(ConfigKeys.REDIS_ENDPOINT_KEY);
        if (elasticCacheRedisEndpointOverride != null) {
            elasticCacheRedisEndpoint = elasticCacheRedisEndpointOverride;
        }
        LOG.info("Using zookeeper prefix " + elasticCacheRedisEndpoint);

        String regionNameOverride = properties.getProperty(ConfigKeys.REGION_NAME);
        if (regionNameOverride != null) {
            regionName = regionNameOverride;
        }
        LOG.info("Using Region " + regionName);

    }

    private static void printUsageAndExit() {
        System.out.println(
                "Usage: " + SampleTopology.class.getName() + " <propertiesFile> <LocalMode or RemoteMode>");
        System.exit(-1);
    }

}