Example usage for java.util ArrayList addAll

Introduction

In this page you can find the example usage for java.util ArrayList addAll.

Prototype

public boolean addAll(Collection<? extends E> c)

Source Link

Document

Appends all of the elements in the specified collection to the end of this list, in the order that they are returned by the specified collection's Iterator.

Usage

From source file:ch.epfl.lsir.xin.test.UserAverageTest.java

/**
 * @param args//from   w  w w . j  av a 2  s.  co m
 */
public static void main(String[] args) throws Exception {
    // TODO Auto-generated method stub

    PrintWriter logger = new PrintWriter(".//results//UserAverage");
    PropertiesConfiguration config = new PropertiesConfiguration();
    config.setFile(new File(".//conf//UserAverage.properties"));
    try {
        config.load();
    } catch (ConfigurationException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Read rating data...");
    DataLoaderFile loader = new DataLoaderFile(".//data//MoveLens100k.txt");
    loader.readSimple();
    DataSetNumeric dataset = loader.getDataset();
    System.out.println("Number of ratings: " + dataset.getRatings().size() + " Number of users: "
            + dataset.getUserIDs().size() + " Number of items: " + dataset.getItemIDs().size());
    logger.println("Number of ratings: " + dataset.getRatings().size() + " Number of users: "
            + dataset.getUserIDs().size() + " Number of items: " + dataset.getItemIDs().size());
    logger.flush();

    double totalMAE = 0;
    double totalRMSE = 0;
    int F = 5;
    logger.println(F + "- folder cross validation.");
    ArrayList<ArrayList<NumericRating>> folders = new ArrayList<ArrayList<NumericRating>>();
    for (int i = 0; i < F; i++) {
        folders.add(new ArrayList<NumericRating>());
    }
    while (dataset.getRatings().size() > 0) {
        int index = new Random().nextInt(dataset.getRatings().size());
        int r = new Random().nextInt(F);
        folders.get(r).add(dataset.getRatings().get(index));
        dataset.getRatings().remove(index);
    }
    for (int folder = 1; folder <= F; folder++) {
        logger.println("Folder: " + folder);
        System.out.println("Folder: " + folder);
        ArrayList<NumericRating> trainRatings = new ArrayList<NumericRating>();
        ArrayList<NumericRating> testRatings = new ArrayList<NumericRating>();
        for (int i = 0; i < folders.size(); i++) {
            if (i == folder - 1)//test data
            {
                testRatings.addAll(folders.get(i));
            } else {//training data
                trainRatings.addAll(folders.get(i));
            }
        }

        //create rating matrix
        HashMap<String, Integer> userIDIndexMapping = new HashMap<String, Integer>();
        HashMap<String, Integer> itemIDIndexMapping = new HashMap<String, Integer>();
        for (int i = 0; i < dataset.getUserIDs().size(); i++) {
            userIDIndexMapping.put(dataset.getUserIDs().get(i), i);
        }
        for (int i = 0; i < dataset.getItemIDs().size(); i++) {
            itemIDIndexMapping.put(dataset.getItemIDs().get(i), i);
        }
        RatingMatrix trainRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < trainRatings.size(); i++) {
            trainRatingMatrix.set(userIDIndexMapping.get(trainRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(trainRatings.get(i).getItemID()), trainRatings.get(i).getValue());
        }
        trainRatingMatrix.calculateGlobalAverage();
        RatingMatrix testRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < testRatings.size(); i++) {
            testRatingMatrix.set(userIDIndexMapping.get(testRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(testRatings.get(i).getItemID()), testRatings.get(i).getValue());
        }
        System.out.println("Training: " + trainRatingMatrix.getTotalRatingNumber() + " vs Test: "
                + testRatingMatrix.getTotalRatingNumber());

        logger.println("Initialize a recommendation model based on user average method.");
        UserAverage algo = new UserAverage(trainRatingMatrix);
        algo.setLogger(logger);
        algo.build();
        algo.saveModel(".//localModels//" + config.getString("NAME"));
        logger.println("Save the model.");
        System.out.println(trainRatings.size() + " vs. " + testRatings.size());

        double RMSE = 0;
        double MAE = 0;
        int count = 0;
        for (int i = 0; i < testRatings.size(); i++) {
            NumericRating rating = testRatings.get(i);
            double prediction = algo.predict(userIDIndexMapping.get(rating.getUserID()),
                    itemIDIndexMapping.get(rating.getItemID()));
            if (Double.isNaN(prediction)) {
                System.out.println("no prediction");
                continue;
            }
            MAE = MAE + Math.abs(rating.getValue() - prediction);
            RMSE = RMSE + Math.pow((rating.getValue() - prediction), 2);
            count++;
        }
        MAE = MAE / count;
        RMSE = Math.sqrt(RMSE / count);

        logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " MAE: " + MAE
                + " RMSE: " + RMSE);
        logger.flush();
        totalMAE = totalMAE + MAE;
        totalRMSE = totalRMSE + RMSE;
    }

    System.out.println("MAE: " + totalMAE / F + " RMSE: " + totalRMSE / F);
    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Final results: MAE: "
            + totalMAE / F + " RMSE: " + totalRMSE / F);
    logger.flush();
    logger.close();
    //MAE: 0.8353035962363073 RMSE: 1.0422971886952053 (MovieLens 100k)
}

From source file:ch.epfl.lsir.xin.test.GlobalMeanTest.java

/**
 * @param args//www  .  jav  a2s. c om
 */
public static void main(String[] args) throws Exception {
    // TODO Auto-generated method stub

    PrintWriter logger = new PrintWriter(".//results//GlobalMean");

    PropertiesConfiguration config = new PropertiesConfiguration();
    config.setFile(new File("conf//GlobalMean.properties"));
    try {
        config.load();
    } catch (ConfigurationException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Read rating data...");
    DataLoaderFile loader = new DataLoaderFile(".//data//MoveLens100k.txt");
    loader.readSimple();
    DataSetNumeric dataset = loader.getDataset();
    System.out.println("Number of ratings: " + dataset.getRatings().size() + " Number of users: "
            + dataset.getUserIDs().size() + " Number of items: " + dataset.getItemIDs().size());
    logger.println("Number of ratings: " + dataset.getRatings().size() + ", Number of users: "
            + dataset.getUserIDs().size() + ", Number of items: " + dataset.getItemIDs().size());

    double totalMAE = 0;
    double totalRMSE = 0;
    int F = 5;
    logger.println(F + "- folder cross validation.");
    logger.flush();
    ArrayList<ArrayList<NumericRating>> folders = new ArrayList<ArrayList<NumericRating>>();
    for (int i = 0; i < F; i++) {
        folders.add(new ArrayList<NumericRating>());
    }
    while (dataset.getRatings().size() > 0) {
        int index = new Random().nextInt(dataset.getRatings().size());
        int r = new Random().nextInt(F);
        folders.get(r).add(dataset.getRatings().get(index));
        dataset.getRatings().remove(index);
    }
    for (int folder = 1; folder <= F; folder++) {
        System.out.println("Folder: " + folder);
        logger.println("Folder: " + folder);
        ArrayList<NumericRating> trainRatings = new ArrayList<NumericRating>();
        ArrayList<NumericRating> testRatings = new ArrayList<NumericRating>();
        for (int i = 0; i < folders.size(); i++) {
            if (i == folder - 1)//test data
            {
                testRatings.addAll(folders.get(i));
            } else {//training data
                trainRatings.addAll(folders.get(i));
            }
        }

        //create rating matrix
        HashMap<String, Integer> userIDIndexMapping = new HashMap<String, Integer>();
        HashMap<String, Integer> itemIDIndexMapping = new HashMap<String, Integer>();
        for (int i = 0; i < dataset.getUserIDs().size(); i++) {
            userIDIndexMapping.put(dataset.getUserIDs().get(i), i);
        }
        for (int i = 0; i < dataset.getItemIDs().size(); i++) {
            itemIDIndexMapping.put(dataset.getItemIDs().get(i), i);
        }
        RatingMatrix trainRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < trainRatings.size(); i++) {
            trainRatingMatrix.set(userIDIndexMapping.get(trainRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(trainRatings.get(i).getItemID()), trainRatings.get(i).getValue());
        }
        RatingMatrix testRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < testRatings.size(); i++) {
            testRatingMatrix.set(userIDIndexMapping.get(testRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(testRatings.get(i).getItemID()), testRatings.get(i).getValue());
        }
        System.out.println("Training: " + trainRatingMatrix.getTotalRatingNumber() + " vs Test: "
                + testRatingMatrix.getTotalRatingNumber());

        logger.println("Initialize a recommendation model based on global average method.");
        GlobalAverage algo = new GlobalAverage(trainRatingMatrix);
        algo.setLogger(logger);
        algo.build();
        algo.saveModel(".//localModels//" + config.getString("NAME"));
        logger.println("Save the model.");
        logger.flush();

        System.out.println(trainRatings.size() + " vs. " + testRatings.size());

        double RMSE = 0;
        double MAE = 0;
        int count = 0;
        for (int i = 0; i < testRatings.size(); i++) {
            NumericRating rating = testRatings.get(i);
            double prediction = algo.predict(rating.getUserID(), rating.getItemID());
            if (Double.isNaN(prediction)) {
                System.out.println("no prediction");
                continue;
            }
            MAE = MAE + Math.abs(rating.getValue() - prediction);
            RMSE = RMSE + Math.pow((rating.getValue() - prediction), 2);
            count++;
        }
        MAE = MAE / count;
        RMSE = Math.sqrt(RMSE / count);

        //         System.out.println("MAE: " + MAE + " RMSE: " + RMSE);
        logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " MAE: " + MAE
                + " RMSE: " + RMSE);
        logger.flush();
        totalMAE = totalMAE + MAE;
        totalRMSE = totalRMSE + RMSE;
    }

    System.out.println("MAE: " + totalMAE / F + " RMSE: " + totalRMSE / F);
    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Final results: MAE: "
            + totalMAE / F + " RMSE: " + totalRMSE / F);
    logger.flush();
    logger.close();
    //MAE: 0.9338607074893257 RMSE: 1.1170971131112037 (MovieLens1M)
    //MAE: 0.9446876509332618 RMSE: 1.1256517870920375 (MovieLens100K)

}

From source file:ch.epfl.lsir.xin.test.ItemAverageTest.java

/**
 * @param args//from  www. j ava2s.c om
 */
public static void main(String[] args) throws Exception {
    // TODO Auto-generated method stub

    PrintWriter logger = new PrintWriter(".//results//ItemAverage");
    PropertiesConfiguration config = new PropertiesConfiguration();
    config.setFile(new File(".//conf//ItemAverage.properties"));
    try {
        config.load();
    } catch (ConfigurationException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Read rating data...");
    DataLoaderFile loader = new DataLoaderFile(".//data//MoveLens100k.txt");
    loader.readSimple();
    DataSetNumeric dataset = loader.getDataset();
    System.out.println("Number of ratings: " + dataset.getRatings().size() + " Number of users: "
            + dataset.getUserIDs().size() + " Number of items: " + dataset.getItemIDs().size());
    logger.println("Number of ratings: " + dataset.getRatings().size() + ", Number of users: "
            + dataset.getUserIDs().size() + ", Number of items: " + dataset.getItemIDs().size());
    logger.flush();

    double totalMAE = 0;
    double totalRMSE = 0;
    int F = 5;
    logger.println(F + "- folder cross validation.");
    ArrayList<ArrayList<NumericRating>> folders = new ArrayList<ArrayList<NumericRating>>();
    for (int i = 0; i < F; i++) {
        folders.add(new ArrayList<NumericRating>());
    }
    while (dataset.getRatings().size() > 0) {
        int index = new Random().nextInt(dataset.getRatings().size());
        int r = new Random().nextInt(F);
        folders.get(r).add(dataset.getRatings().get(index));
        dataset.getRatings().remove(index);
    }
    for (int folder = 1; folder <= F; folder++) {
        logger.println("Folder: " + folder);
        logger.flush();
        System.out.println("Folder: " + folder);
        ArrayList<NumericRating> trainRatings = new ArrayList<NumericRating>();
        ArrayList<NumericRating> testRatings = new ArrayList<NumericRating>();
        for (int i = 0; i < folders.size(); i++) {
            if (i == folder - 1)//test data
            {
                testRatings.addAll(folders.get(i));
            } else {//training data
                trainRatings.addAll(folders.get(i));
            }
        }

        //create rating matrix
        HashMap<String, Integer> userIDIndexMapping = new HashMap<String, Integer>();
        HashMap<String, Integer> itemIDIndexMapping = new HashMap<String, Integer>();
        for (int i = 0; i < dataset.getUserIDs().size(); i++) {
            userIDIndexMapping.put(dataset.getUserIDs().get(i), i);
        }
        for (int i = 0; i < dataset.getItemIDs().size(); i++) {
            itemIDIndexMapping.put(dataset.getItemIDs().get(i), i);
        }
        RatingMatrix trainRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < trainRatings.size(); i++) {
            trainRatingMatrix.set(userIDIndexMapping.get(trainRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(trainRatings.get(i).getItemID()), trainRatings.get(i).getValue());
        }
        trainRatingMatrix.calculateGlobalAverage();
        RatingMatrix testRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(),
                dataset.getItemIDs().size());
        for (int i = 0; i < testRatings.size(); i++) {
            testRatingMatrix.set(userIDIndexMapping.get(testRatings.get(i).getUserID()),
                    itemIDIndexMapping.get(testRatings.get(i).getItemID()), testRatings.get(i).getValue());
        }
        System.out.println("Training: " + trainRatingMatrix.getTotalRatingNumber() + " vs Test: "
                + testRatingMatrix.getTotalRatingNumber());

        logger.println("Initialize a recommendation model based on item average method.");
        ItemAverage algo = new ItemAverage(trainRatingMatrix);
        algo.setLogger(logger);
        algo.build();
        algo.saveModel(".//localModels//" + config.getString("NAME"));
        logger.println("Save the model.");
        logger.flush();
        System.out.println(trainRatings.size() + " vs. " + testRatings.size());

        double RMSE = 0;
        double MAE = 0;
        int count = 0;
        for (int i = 0; i < testRatings.size(); i++) {
            NumericRating rating = testRatings.get(i);
            double prediction = algo.predict(userIDIndexMapping.get(rating.getUserID()),
                    itemIDIndexMapping.get(rating.getItemID()));
            if (Double.isNaN(prediction)) {
                System.out.println("no prediction");
                continue;
            }
            MAE = MAE + Math.abs(rating.getValue() - prediction);
            RMSE = RMSE + Math.pow((rating.getValue() - prediction), 2);
            count++;
        }
        MAE = MAE / count;
        RMSE = Math.sqrt(RMSE / count);

        logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " MAE: " + MAE
                + " RMSE: " + RMSE);
        logger.flush();
        //         System.out.println("MAE: " + MAE + " RMSE: " + RMSE);
        totalMAE = totalMAE + MAE;
        totalRMSE = totalRMSE + RMSE;
    }

    System.out.println("MAE: " + totalMAE / F + " RMSE: " + totalRMSE / F);
    logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Final results: MAE: "
            + totalMAE / F + " RMSE: " + totalRMSE / F);
    logger.flush();
    //MAE: 0.8173633324758338 RMSE: 1.0251973503888645 (MovieLens 100K)

}

From source file:deck36.storm.plan9.php.HighFiveBadgeTopology.java

public static void main(String[] args) throws Exception {

    String env = null;/*from www.  j a va 2 s  .co m*/

    if (args != null && args.length > 0) {
        env = args[0];
    }

    if (!"dev".equals(env))
        if (!"prod".equals(env)) {
            System.out.println("Usage: $0 (dev|prod)\n");
            System.exit(1);
        }

    // Topology config
    Config conf = new Config();

    // Load parameters and add them to the Config
    Map configMap = YamlLoader.loadYamlFromResource("config_" + env + ".yml");

    conf.putAll(configMap);

    log.info(JSONValue.toJSONString((conf)));

    // Set topology loglevel to DEBUG
    conf.put(Config.TOPOLOGY_DEBUG, JsonPath.read(conf, "$.deck36_storm.debug"));

    // Create Topology builder
    TopologyBuilder builder = new TopologyBuilder();

    // if there are not special reasons, start with parallelism hint of 1
    // and multiple tasks. By that, you can scale dynamically later on.
    int parallelism_hint = JsonPath.read(conf, "$.deck36_storm.default_parallelism_hint");
    int num_tasks = JsonPath.read(conf, "$.deck36_storm.default_num_tasks");

    // Create Stream from RabbitMQ messages
    // bind new queue with name of the topology
    // to the main plan9 exchange (from properties config)
    // consuming only CBT-related events by using the rounting key 'cbt.#'

    String badgeName = HighFiveBadgeTopology.class.getSimpleName();

    String rabbitQueueName = badgeName; // use topology class name as name for the queue
    String rabbitExchangeName = JsonPath.read(conf, "$.deck36_storm.HighFiveBolt.rabbitmq.exchange");
    String rabbitRoutingKey = JsonPath.read(conf, "$.deck36_storm.HighFiveBolt.rabbitmq.routing_key");

    // Get JSON deserialization scheme
    Scheme rabbitScheme = new SimpleJSONScheme();

    // Setup a Declarator to configure exchange/queue/routing key
    RabbitMQDeclarator rabbitDeclarator = new RabbitMQDeclarator(rabbitExchangeName, rabbitQueueName,
            rabbitRoutingKey);

    // Create Configuration for the Spout
    ConnectionConfig connectionConfig = new ConnectionConfig(
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.host"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.port"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.user"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.pass"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.vhost"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.heartbeat"));

    ConsumerConfig spoutConfig = new ConsumerConfigBuilder().connection(connectionConfig).queue(rabbitQueueName)
            .prefetch((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch")).requeueOnFail()
            .build();

    // add global parameters to topology config - the RabbitMQSpout will read them from there
    conf.putAll(spoutConfig.asMap());

    // For production, set the spout pending value to the same value as the RabbitMQ pre-fetch
    // see: https://github.com/ppat/storm-rabbitmq/blob/master/README.md
    if ("prod".equals(env)) {
        conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING,
                (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch"));
    }

    // Add RabbitMQ spout to topology
    builder.setSpout("incoming", new RabbitMQSpout(rabbitScheme, rabbitDeclarator), parallelism_hint)
            .setNumTasks((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.spout_tasks"));

    // construct command to invoke the external bolt implementation
    ArrayList<String> command = new ArrayList(15);

    // Add main execution program (php, hhvm, zend, ..) and parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.php.executor"));
    command.addAll((List<String>) JsonPath.read(conf, "$.deck36_storm.php.executor_params"));

    // Add main command to be executed (app/console, the phar file, etc.) and global context parameters (environment etc.)
    command.add((String) JsonPath.read(conf, "$.deck36_storm.php.main"));
    command.addAll((List<String>) JsonPath.read(conf, "$.deck36_storm.php.main_params"));

    // Add main route to be invoked and its parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.HighFiveBolt.main"));
    List boltParams = (List<String>) JsonPath.read(conf, "$.deck36_storm.HighFiveBolt.params");
    if (boltParams != null)
        command.addAll(boltParams);

    // Log the final command
    log.info("Command to start bolt for HighFive badge: " + Arrays.toString(command.toArray()));

    // Add constructed external bolt command to topology using MultilangAdapterBolt
    builder.setBolt("badge", new MultilangAdapterBolt(command, "badge"), parallelism_hint)
            .setNumTasks(num_tasks).shuffleGrouping("incoming");

    builder.setBolt("rabbitmq_router", new Plan9RabbitMQRouterBolt(
            (String) JsonPath.read(conf, "$.deck36_storm.HighFiveBolt.rabbitmq.target_exchange"), "HighFive" // RabbitMQ routing key
    ), parallelism_hint).setNumTasks(num_tasks).shuffleGrouping("badge");

    builder.setBolt("rabbitmq_producer", new Plan9RabbitMQPushBolt(), parallelism_hint).setNumTasks(num_tasks)
            .shuffleGrouping("rabbitmq_router");

    if ("dev".equals(env)) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(badgeName + System.currentTimeMillis(), conf, builder.createTopology());
        Thread.sleep(2000000);
    }

    if ("prod".equals(env)) {
        StormSubmitter.submitTopology(badgeName + "-" + System.currentTimeMillis(), conf,
                builder.createTopology());
    }

}

From source file:deck36.storm.plan9.php.PrimeCatBadgeTopology.java

public static void main(String[] args) throws Exception {

    String env = null;/*from   ww w.  j  av a2s  .  c om*/

    if (args != null && args.length > 0) {
        env = args[0];
    }

    if (!"dev".equals(env))
        if (!"prod".equals(env)) {
            System.out.println("Usage: $0 (dev|prod)\n");
            System.exit(1);
        }

    // Topology config
    Config conf = new Config();

    // Load parameters and add them to the Config
    Map configMap = YamlLoader.loadYamlFromResource("config_" + env + ".yml");

    conf.putAll(configMap);

    log.info(JSONValue.toJSONString((conf)));

    // Set topology loglevel to DEBUG
    conf.put(Config.TOPOLOGY_DEBUG, JsonPath.read(conf, "$.deck36_storm.debug"));

    // Create Topology builder
    TopologyBuilder builder = new TopologyBuilder();

    // if there are not special reasons, start with parallelism hint of 1
    // and multiple tasks. By that, you can scale dynamically later on.
    int parallelism_hint = JsonPath.read(conf, "$.deck36_storm.default_parallelism_hint");
    int num_tasks = JsonPath.read(conf, "$.deck36_storm.default_num_tasks");

    // Create Stream from RabbitMQ messages
    // bind new queue with name of the topology
    // to the main plan9 exchange (from properties config)
    // consuming only POINTS-related events by using the routing key 'points.#'

    String badgeName = PrimeCatBadgeTopology.class.getSimpleName();

    String rabbitQueueName = badgeName; // use topology class name as name for the queue
    String rabbitExchangeName = JsonPath.read(conf, "$.deck36_storm.PrimeCatBolt.rabbitmq.exchange");
    String rabbitRoutingKey = JsonPath.read(conf, "$.deck36_storm.PrimeCatBolt.rabbitmq.routing_key");

    // Get JSON deserialization scheme
    Scheme rabbitScheme = new SimpleJSONScheme();

    // Setup a Declarator to configure exchange/queue/routing key
    RabbitMQDeclarator rabbitDeclarator = new RabbitMQDeclarator(rabbitExchangeName, rabbitQueueName,
            rabbitRoutingKey);

    // Create Configuration for the Spout
    ConnectionConfig connectionConfig = new ConnectionConfig(
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.host"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.port"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.user"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.pass"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.vhost"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.heartbeat"));

    ConsumerConfig spoutConfig = new ConsumerConfigBuilder().connection(connectionConfig).queue(rabbitQueueName)
            .prefetch((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch")).requeueOnFail()
            .build();

    // add global parameters to topology config - the RabbitMQSpout will read them from there
    conf.putAll(spoutConfig.asMap());

    // For production, set the spout pending value to the same value as the RabbitMQ pre-fetch
    // see: https://github.com/ppat/storm-rabbitmq/blob/master/README.md
    if ("prod".equals(env)) {
        conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING,
                (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch"));
    }

    // Add RabbitMQ spout to topology
    builder.setSpout("incoming", new RabbitMQSpout(rabbitScheme, rabbitDeclarator), parallelism_hint)
            .setNumTasks((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.spout_tasks"));

    // construct command to invoke the external bolt implementation
    ArrayList<String> command = new ArrayList(15);

    // Add main execution program (php, hhvm, zend, ..) and parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.php.executor"));
    command.addAll((List<String>) JsonPath.read(conf, "$.deck36_storm.php.executor_params"));

    // Add main command to be executed (app/console, the phar file, etc.) and global context parameters (environment etc.)
    command.add((String) JsonPath.read(conf, "$.deck36_storm.php.main"));
    command.addAll((List<String>) JsonPath.read(conf, "$.deck36_storm.php.main_params"));

    // Add main route to be invoked and its parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.PrimeCatBolt.main"));
    List boltParams = (List<String>) JsonPath.read(conf, "$.deck36_storm.PrimeCatBolt.params");
    if (boltParams != null)
        command.addAll(boltParams);

    // Log the final command
    log.info("Command to start bolt for PrimeCat badge: " + Arrays.toString(command.toArray()));

    // Add constructed external bolt command to topology using MultilangAdapterBolt
    builder.setBolt("badge",
            new MultilangAdapterTickTupleBolt(command,
                    (Integer) JsonPath.read(conf, "$.deck36_storm.PrimeCatBolt.primecat_frequency"), "badge"),
            parallelism_hint).setNumTasks(num_tasks).shuffleGrouping("incoming");

    builder.setBolt("rabbitmq_router", new Plan9RabbitMQRouterBolt(
            (String) JsonPath.read(conf, "$.deck36_storm.PrimeCatBolt.rabbitmq.target_exchange"), "PrimeCat" // RabbitMQ routing key
    ), parallelism_hint).setNumTasks(num_tasks).shuffleGrouping("badge");

    builder.setBolt("rabbitmq_producer", new Plan9RabbitMQPushBolt(), parallelism_hint).setNumTasks(num_tasks)
            .shuffleGrouping("rabbitmq_router");

    if ("dev".equals(env)) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(badgeName + System.currentTimeMillis(), conf, builder.createTopology());
        Thread.sleep(2000000);
    }

    if ("prod".equals(env)) {
        StormSubmitter.submitTopology(badgeName + "-" + System.currentTimeMillis(), conf,
                builder.createTopology());
    }

}

From source file:deck36.storm.plan9.php.StatusLevelTopology.java

public static void main(String[] args) throws Exception {

    String env = null;// w  w w.j a  v  a 2  s. c o  m

    if (args != null && args.length > 0) {
        env = args[0];
    }

    if (!"dev".equals(env))
        if (!"prod".equals(env)) {
            System.out.println("Usage: $0 (dev|prod)\n");
            System.exit(1);
        }

    // Topology config
    Config conf = new Config();

    // Load parameters and add them to the Config
    Map configMap = YamlLoader.loadYamlFromResource("config_" + env + ".yml");

    conf.putAll(configMap);

    log.info(JSONValue.toJSONString((conf)));

    // Set topology loglevel to DEBUG
    conf.put(Config.TOPOLOGY_DEBUG, JsonPath.read(conf, "$.deck36_storm.debug"));

    // Create Topology builder
    TopologyBuilder builder = new TopologyBuilder();

    // if there are not special reasons, start with parallelism hint of 1
    // and multiple tasks. By that, you can scale dynamically later on.
    int parallelism_hint = JsonPath.read(conf, "$.deck36_storm.default_parallelism_hint");
    int num_tasks = JsonPath.read(conf, "$.deck36_storm.default_num_tasks");

    // Create Stream from RabbitMQ messages
    // bind new queue with name of the topology
    // to the main plan9 exchange (from properties config)
    // consuming only POINTS-related events by using the routing key 'points.#'

    String badgeName = StatusLevelTopology.class.getSimpleName();

    String rabbitQueueName = badgeName; // use topology class name as name for the queue
    String rabbitExchangeName = JsonPath.read(conf, "$.deck36_storm.StatusLevelBolt.rabbitmq.exchange");
    String rabbitRoutingKey = JsonPath.read(conf, "$.deck36_storm.StatusLevelBolt.rabbitmq.routing_key");

    // Get JSON deserialization scheme
    Scheme rabbitScheme = new SimpleJSONScheme();

    // Setup a Declarator to configure exchange/queue/routing key
    RabbitMQDeclarator rabbitDeclarator = new RabbitMQDeclarator(rabbitExchangeName, rabbitQueueName,
            rabbitRoutingKey);

    // Create Configuration for the Spout
    ConnectionConfig connectionConfig = new ConnectionConfig(
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.host"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.port"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.user"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.pass"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.vhost"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.heartbeat"));

    ConsumerConfig spoutConfig = new ConsumerConfigBuilder().connection(connectionConfig).queue(rabbitQueueName)
            .prefetch((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch")).requeueOnFail()
            .build();

    // add global parameters to topology config - the RabbitMQSpout will read them from there
    conf.putAll(spoutConfig.asMap());

    // For production, set the spout pending value to the same value as the RabbitMQ pre-fetch
    // see: https://github.com/ppat/storm-rabbitmq/blob/master/README.md
    if ("prod".equals(env)) {
        conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING,
                (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch"));
    }

    // Add RabbitMQ spout to topology
    builder.setSpout("incoming", new RabbitMQSpout(rabbitScheme, rabbitDeclarator), parallelism_hint)
            .setNumTasks((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.spout_tasks"));

    // construct command to invoke the external bolt implementation
    ArrayList<String> command = new ArrayList(15);

    // Add main execution program (php, hhvm, zend, ..) and parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.php.executor"));
    command.addAll((List<String>) JsonPath.read(conf, "$.deck36_storm.php.executor_params"));

    // Add main command to be executed (app/console, the phar file, etc.) and global context parameters (environment etc.)
    command.add((String) JsonPath.read(conf, "$.deck36_storm.php.main"));
    command.addAll((List<String>) JsonPath.read(conf, "$.deck36_storm.php.main_params"));

    // Add main route to be invoked and its parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.StatusLevelBolt.main"));
    List boltParams = (List<String>) JsonPath.read(conf, "$.deck36_storm.StatusLevelBolt.params");
    if (boltParams != null)
        command.addAll(boltParams);

    // Log the final command
    log.info("Command to start bolt for StatusLevel badges: " + Arrays.toString(command.toArray()));

    // Add constructed external bolt command to topology using MultilangAdapterBolt
    builder.setBolt("badge", new MultilangAdapterBolt(command, "badge"), 1).setNumTasks(1)
            .shuffleGrouping("incoming");

    builder.setBolt("rabbitmq_router",
            new Plan9RabbitMQRouterBolt(
                    (String) JsonPath.read(conf, "$.deck36_storm.StatusLevelBolt.rabbitmq.target_exchange"),
                    "StatusLevel" // RabbitMQ routing key
            ), parallelism_hint).setNumTasks(num_tasks).shuffleGrouping("badge");

    builder.setBolt("rabbitmq_producer", new Plan9RabbitMQPushBolt(), parallelism_hint).setNumTasks(num_tasks)
            .shuffleGrouping("rabbitmq_router");

    if ("dev".equals(env)) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(badgeName + System.currentTimeMillis(), conf, builder.createTopology());
        Thread.sleep(2000000);
    }

    if ("prod".equals(env)) {
        StormSubmitter.submitTopology(badgeName + "-" + System.currentTimeMillis(), conf,
                builder.createTopology());
    }

}

From source file:deck36.storm.plan9.php.StumbleBlunderBadgeTopology.java

public static void main(String[] args) throws Exception {

    String env = null;//from   ww  w. j a v  a2 s .c  o m

    if (args != null && args.length > 0) {
        env = args[0];
    }

    if (!"dev".equals(env))
        if (!"prod".equals(env)) {
            System.out.println("Usage: $0 (dev|prod)\n");
            System.exit(1);
        }

    // Topology config
    Config conf = new Config();

    // Load parameters and add them to the Config
    Map configMap = YamlLoader.loadYamlFromResource("config_" + env + ".yml");

    conf.putAll(configMap);

    log.info(JSONValue.toJSONString((conf)));

    // Set topology loglevel to DEBUG
    conf.put(Config.TOPOLOGY_DEBUG, JsonPath.read(conf, "$.deck36_storm.debug"));

    // Create Topology builder
    TopologyBuilder builder = new TopologyBuilder();

    // if there are not special reasons, start with parallelism hint of 1
    // and multiple tasks. By that, you can scale dynamically later on.
    int parallelism_hint = JsonPath.read(conf, "$.deck36_storm.default_parallelism_hint");
    int num_tasks = JsonPath.read(conf, "$.deck36_storm.default_num_tasks");

    // Create Stream from RabbitMQ messages
    // bind new queue with name of the topology
    // to the main plan9 exchange (from properties config)
    // consuming only CBT-related events by using the rounting key 'cbt.#'

    String badgeName = StumbleBlunderBadgeTopology.class.getSimpleName();

    String rabbitQueueName = badgeName; // use topology class name as name for the queue
    String rabbitExchangeName = JsonPath.read(conf, "$.deck36_storm.StumbleBlunderBolt.rabbitmq.exchange");
    String rabbitRoutingKey = JsonPath.read(conf, "$.deck36_storm.StumbleBlunderBolt.rabbitmq.routing_key");

    // Get JSON deserialization scheme
    Scheme rabbitScheme = new SimpleJSONScheme();

    // Setup a Declarator to configure exchange/queue/routing key
    RabbitMQDeclarator rabbitDeclarator = new RabbitMQDeclarator(rabbitExchangeName, rabbitQueueName,
            rabbitRoutingKey);

    // Create Configuration for the Spout
    ConnectionConfig connectionConfig = new ConnectionConfig(
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.host"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.port"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.user"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.pass"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.vhost"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.heartbeat"));

    ConsumerConfig spoutConfig = new ConsumerConfigBuilder().connection(connectionConfig).queue(rabbitQueueName)
            .prefetch((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch")).requeueOnFail()
            .build();

    // add global parameters to topology config - the RabbitMQSpout will read them from there
    conf.putAll(spoutConfig.asMap());

    // For production, set the spout pending value to the same value as the RabbitMQ pre-fetch
    // see: https://github.com/ppat/storm-rabbitmq/blob/master/README.md
    if ("prod".equals(env)) {
        conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING,
                (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch"));
    }

    // Add RabbitMQ spout to topology
    builder.setSpout("incoming", new RabbitMQSpout(rabbitScheme, rabbitDeclarator), parallelism_hint)
            .setNumTasks((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.spout_tasks"));

    // construct command to invoke the external bolt implementation
    ArrayList<String> command = new ArrayList(15);

    // Add main execution program (php, hhvm, zend, ..) and parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.php.executor"));
    command.addAll((List<String>) JsonPath.read(conf, "$.deck36_storm.php.executor_params"));

    // Add main command to be executed (app/console, the phar file, etc.) and global context parameters (environment etc.)
    command.add((String) JsonPath.read(conf, "$.deck36_storm.php.main"));
    command.addAll((List<String>) JsonPath.read(conf, "$.deck36_storm.php.main_params"));

    // Add main route to be invoked and its parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.StumbleBlunderBolt.main"));
    List boltParams = (List<String>) JsonPath.read(conf, "$.deck36_storm.StumbleBlunderBolt.params");
    if (boltParams != null)
        command.addAll(boltParams);

    // Log the final command
    log.info("Command to start bolt for StumbleBlunder badge: " + Arrays.toString(command.toArray()));

    // Add constructed external bolt command to topology using MultilangAdapterBolt
    builder.setBolt("badge", new MultilangAdapterBolt(command, "badge"), parallelism_hint)
            .setNumTasks(num_tasks).shuffleGrouping("incoming");

    builder.setBolt("rabbitmq_router",
            new Plan9RabbitMQRouterBolt(
                    (String) JsonPath.read(conf, "$.deck36_storm.StumbleBlunderBolt.rabbitmq.target_exchange"),
                    "StumbleBlunder" // RabbitMQ routing key
            ), parallelism_hint).setNumTasks(num_tasks).shuffleGrouping("badge");

    builder.setBolt("rabbitmq_producer", new Plan9RabbitMQPushBolt(), parallelism_hint).setNumTasks(num_tasks)
            .shuffleGrouping("rabbitmq_router");

    if ("dev".equals(env)) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(badgeName + System.currentTimeMillis(), conf, builder.createTopology());
        Thread.sleep(2000000);
    }

    if ("prod".equals(env)) {
        StormSubmitter.submitTopology(badgeName + "-" + System.currentTimeMillis(), conf,
                builder.createTopology());
    }

}

From source file:deck36.storm.plan9.php.DeludedKittenRobbersTopology.java

public static void main(String[] args) throws Exception {

    String env = null;/*from  w w  w  .j av a2 s . c  o  m*/

    if (args != null && args.length > 0) {
        env = args[0];
    }

    if (!"dev".equals(env))
        if (!"prod".equals(env)) {
            System.out.println("Usage: $0 (dev|prod)\n");
            System.exit(1);
        }

    // Topology config
    Config conf = new Config();

    // Load parameters and add them to the Config
    Map configMap = YamlLoader.loadYamlFromResource("config_" + env + ".yml");

    conf.putAll(configMap);

    log.info(JSONValue.toJSONString((conf)));

    // Set topology loglevel to DEBUG
    conf.put(Config.TOPOLOGY_DEBUG, JsonPath.read(conf, "$.deck36_storm.debug"));

    // Create Topology builder
    TopologyBuilder builder = new TopologyBuilder();

    // if there are not special reasons, start with parallelism hint of 1
    // and multiple tasks. By that, you can scale dynamically later on.
    int parallelism_hint = JsonPath.read(conf, "$.deck36_storm.default_parallelism_hint");
    int num_tasks = JsonPath.read(conf, "$.deck36_storm.default_num_tasks");

    // Create Stream from RabbitMQ messages
    // bind new queue with name of the topology
    // to the main plan9 exchange (from properties config)
    // consuming only CBT-related events by using the rounting key 'cbt.#'

    String badgeName = DeludedKittenRobbersTopology.class.getSimpleName();

    String rabbitQueueName = badgeName; // use topology class name as name for the queue
    String rabbitExchangeName = JsonPath.read(conf,
            "$.deck36_storm.DeludedKittenRobbersBolt.rabbitmq.exchange");
    String rabbitRoutingKey = JsonPath.read(conf,
            "$.deck36_storm.DeludedKittenRobbersBolt.rabbitmq.routing_key");

    // Get JSON deserialization scheme
    Scheme rabbitScheme = new SimpleJSONScheme();

    // Setup a Declarator to configure exchange/queue/routing key
    RabbitMQDeclarator rabbitDeclarator = new RabbitMQDeclarator(rabbitExchangeName, rabbitQueueName,
            rabbitRoutingKey);

    // Create Configuration for the Spout
    ConnectionConfig connectionConfig = new ConnectionConfig(
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.host"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.port"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.user"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.pass"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.vhost"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.heartbeat"));

    ConsumerConfig spoutConfig = new ConsumerConfigBuilder().connection(connectionConfig).queue(rabbitQueueName)
            .prefetch((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch")).requeueOnFail()
            .build();

    // add global parameters to topology config - the RabbitMQSpout will read them from there
    conf.putAll(spoutConfig.asMap());

    // For production, set the spout pending value to the same value as the RabbitMQ pre-fetch
    // see: https://github.com/ppat/storm-rabbitmq/blob/master/README.md
    if ("prod".equals(env)) {
        conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING,
                (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch"));
    }

    // Add RabbitMQ spout to topology
    builder.setSpout("incoming", new RabbitMQSpout(rabbitScheme, rabbitDeclarator), parallelism_hint)
            .setNumTasks((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.spout_tasks"));

    // construct command to invoke the external bolt implementation
    ArrayList<String> command = new ArrayList(15);

    // Add main execution program (php, hhvm, zend, ..) and parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.php.executor"));
    command.addAll((List<String>) JsonPath.read(conf, "$.deck36_storm.php.executor_params"));

    // Add main command to be executed (app/console, the phar file, etc.) and global context parameters (environment etc.)
    command.add((String) JsonPath.read(conf, "$.deck36_storm.php.main"));
    command.addAll((List<String>) JsonPath.read(conf, "$.deck36_storm.php.main_params"));

    // Add main route to be invoked and its parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.DeludedKittenRobbersBolt.main"));
    List boltParams = (List<String>) JsonPath.read(conf, "$.deck36_storm.DeludedKittenRobbersBolt.params");
    if (boltParams != null)
        command.addAll(boltParams);

    // Log the final command
    log.info("Command to start bolt for Deluded Kitten Robbers: " + Arrays.toString(command.toArray()));

    // Add constructed external bolt command to topology using MultilangAdapterTickTupleBolt
    builder.setBolt("badge", new MultilangAdapterTickTupleBolt(command,
            (Integer) JsonPath.read(conf, "$.deck36_storm.DeludedKittenRobbersBolt.attack_frequency_secs"),
            "badge"), parallelism_hint).setNumTasks(num_tasks).shuffleGrouping("incoming");

    builder.setBolt("rabbitmq_router", new Plan9RabbitMQRouterBolt(
            (String) JsonPath.read(conf, "$.deck36_storm.DeludedKittenRobbersBolt.rabbitmq.target_exchange"),
            "DeludedKittenRobbers" // RabbitMQ routing key
    ), parallelism_hint).setNumTasks(num_tasks).shuffleGrouping("badge");

    builder.setBolt("rabbitmq_producer", new Plan9RabbitMQPushBolt(), parallelism_hint).setNumTasks(num_tasks)
            .shuffleGrouping("rabbitmq_router");

    if ("dev".equals(env)) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(badgeName + System.currentTimeMillis(), conf, builder.createTopology());
        Thread.sleep(2000000);
    }

    if ("prod".equals(env)) {
        StormSubmitter.submitTopology(badgeName + "-" + System.currentTimeMillis(), conf,
                builder.createTopology());
    }

}

From source file:deck36.storm.plan9.php.RaiderOfTheKittenRobbersTopology.java

public static void main(String[] args) throws Exception {

    String env = null;/*from w  ww . j  a va 2 s  . c o m*/

    if (args != null && args.length > 0) {
        env = args[0];
    }

    if (!"dev".equals(env))
        if (!"prod".equals(env)) {
            System.out.println("Usage: $0 (dev|prod)\n");
            System.exit(1);
        }

    // Topology config
    Config conf = new Config();

    // Load parameters and add them to the Config
    Map configMap = YamlLoader.loadYamlFromResource("config_" + env + ".yml");

    conf.putAll(configMap);

    log.info(JSONValue.toJSONString((conf)));

    // Set topology loglevel to DEBUG
    conf.put(Config.TOPOLOGY_DEBUG, JsonPath.read(conf, "$.deck36_storm.debug"));

    // Create Topology builder
    TopologyBuilder builder = new TopologyBuilder();

    // if there are not special reasons, start with parallelism hint of 1
    // and multiple tasks. By that, you can scale dynamically later on.
    int parallelism_hint = JsonPath.read(conf, "$.deck36_storm.default_parallelism_hint");
    int num_tasks = JsonPath.read(conf, "$.deck36_storm.default_num_tasks");

    // Create Stream from RabbitMQ messages
    // bind new queue with name of the topology
    // to the main plan9 exchange (from properties config)
    // consuming only CBT-related events by using the rounting key 'cbt.#'

    String badgeName = RaiderOfTheKittenRobbersTopology.class.getSimpleName();

    String rabbitQueueName = badgeName; // use topology class name as name for the queue
    String rabbitExchangeName = (String) JsonPath.read(conf,
            "$.deck36_storm.RaiderOfTheKittenRobbersBolt.rabbitmq.target_exchange");
    String rabbitRoutingKey = "#";

    // Get JSON deserialization scheme
    Scheme rabbitScheme = new SimpleJSONScheme();

    // Setup a Declarator to configure exchange/queue/routing key
    RabbitMQDeclarator rabbitDeclarator = new RabbitMQDeclarator(rabbitExchangeName, rabbitQueueName,
            rabbitRoutingKey);

    // Create Configuration for the Spout
    ConnectionConfig connectionConfig = new ConnectionConfig(
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.host"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.port"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.user"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.pass"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.vhost"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.heartbeat"));

    ConsumerConfig spoutConfig = new ConsumerConfigBuilder().connection(connectionConfig).queue(rabbitQueueName)
            .prefetch((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch")).requeueOnFail()
            .build();

    // add global parameters to topology config - the RabbitMQSpout will read them from there
    conf.putAll(spoutConfig.asMap());

    // For production, set the spout pending value to the same value as the RabbitMQ pre-fetch
    // see: https://github.com/ppat/storm-rabbitmq/blob/master/README.md
    if ("prod".equals(env)) {
        conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING,
                (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch"));
    }

    // Add RabbitMQ spout to topology
    builder.setSpout("incoming", new RabbitMQSpout(rabbitScheme, rabbitDeclarator), parallelism_hint)
            .setNumTasks((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.spout_tasks"));

    // construct command to invoke the external bolt implementation
    ArrayList<String> command = new ArrayList(15);

    // Add main execution program (php, hhvm, zend, ..) and parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.php.executor"));
    command.addAll((List<String>) JsonPath.read(conf, "$.deck36_storm.php.executor_params"));

    // Add main command to be executed (app/console, the phar file, etc.) and global context parameters (environment etc.)
    command.add((String) JsonPath.read(conf, "$.deck36_storm.php.main"));
    command.addAll((List<String>) JsonPath.read(conf, "$.deck36_storm.php.main_params"));

    // Add main route to be invoked and its parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.RaiderOfTheKittenRobbersBolt.main"));
    List boltParams = (List<String>) JsonPath.read(conf, "$.deck36_storm.RaiderOfTheKittenRobbersBolt.params");
    if (boltParams != null)
        command.addAll(boltParams);

    // Log the final command
    log.info("Command to start bolt for RaiderOfTheKittenRobbers badge: " + Arrays.toString(command.toArray()));

    // Add constructed external bolt command to topology using MultilangAdapterBolt
    builder.setBolt("badge", new MultilangAdapterBolt(command, "badge"), parallelism_hint)
            .setNumTasks(num_tasks).shuffleGrouping("incoming");

    builder.setBolt("rabbitmq_router",
            new Plan9RabbitMQRouterBolt(
                    (String) JsonPath.read(conf,
                            "$.deck36_storm.RaiderOfTheKittenRobbersBolt.rabbitmq.target_exchange"),
                    "RaiderOfTheKittenRobbers" // RabbitMQ routing key
            ), parallelism_hint).setNumTasks(num_tasks).shuffleGrouping("badge");

    builder.setBolt("rabbitmq_producer", new Plan9RabbitMQPushBolt(), parallelism_hint).setNumTasks(num_tasks)
            .shuffleGrouping("rabbitmq_router");

    if ("dev".equals(env)) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(badgeName + System.currentTimeMillis(), conf, builder.createTopology());
        Thread.sleep(2000000);
    }

    if ("prod".equals(env)) {
        StormSubmitter.submitTopology(badgeName + "-" + System.currentTimeMillis(), conf,
                builder.createTopology());
    }

}

From source file:deck36.storm.plan9.php.RecordBreakerBadgeTopology.java

public static void main(String[] args) throws Exception {

    String env = null;/*from   ww  w . ja  v a 2 s .c  o m*/

    if (args != null && args.length > 0) {
        env = args[0];
    }

    if (!"dev".equals(env))
        if (!"prod".equals(env)) {
            System.out.println("Usage: $0 (dev|prod)\n");
            System.exit(1);
        }

    // Topology config
    Config conf = new Config();

    // Load parameters and add them to the Config
    Map configMap = YamlLoader.loadYamlFromResource("config_" + env + ".yml");

    conf.putAll(configMap);

    log.info(JSONValue.toJSONString((conf)));

    // Set topology loglevel to DEBUG
    conf.put(Config.TOPOLOGY_DEBUG, JsonPath.read(conf, "$.deck36_storm.debug"));

    // Create Topology builder
    TopologyBuilder builder = new TopologyBuilder();

    // if there are not special reasons, start with parallelism hint of 1
    // and multiple tasks. By that, you can scale dynamically later on.
    int parallelism_hint = JsonPath.read(conf, "$.deck36_storm.default_parallelism_hint");
    int num_tasks = JsonPath.read(conf, "$.deck36_storm.default_num_tasks");

    // Create Stream from RabbitMQ messages
    // bind new queue with name of the topology
    // to the main plan9 exchange (from properties config)
    // consuming only CBT-related events by using the rounting key 'cbt.#'

    String badgeName = RecordBreakerBadgeTopology.class.getSimpleName();

    String rabbitQueueName = badgeName; // use topology class name as name for the queue
    String rabbitExchangeName = JsonPath.read(conf, "$.deck36_storm.RecordBreakerBolt.rabbitmq.exchange");
    String rabbitRoutingKey = JsonPath.read(conf, "$.deck36_storm.RecordBreakerBolt.rabbitmq.routing_key");

    // Get JSON deserialization scheme
    Scheme rabbitScheme = new SimpleJSONScheme();

    // Setup a Declarator to configure exchange/queue/routing key
    RabbitMQDeclarator rabbitDeclarator = new RabbitMQDeclarator(rabbitExchangeName, rabbitQueueName,
            rabbitRoutingKey);

    // Create Configuration for the Spout
    ConnectionConfig connectionConfig = new ConnectionConfig(
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.host"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.port"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.user"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.pass"),
            (String) JsonPath.read(conf, "$.deck36_storm.rabbitmq.vhost"),
            (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.heartbeat"));

    ConsumerConfig spoutConfig = new ConsumerConfigBuilder().connection(connectionConfig).queue(rabbitQueueName)
            .prefetch((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch")).requeueOnFail()
            .build();

    // add global parameters to topology config - the RabbitMQSpout will read them from there
    conf.putAll(spoutConfig.asMap());

    // For production, set the spout pending value to the same value as the RabbitMQ pre-fetch
    // see: https://github.com/ppat/storm-rabbitmq/blob/master/README.md
    if ("prod".equals(env)) {
        conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING,
                (Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.prefetch"));
    }

    // Add RabbitMQ spout to topology
    builder.setSpout("incoming", new RabbitMQSpout(rabbitScheme, rabbitDeclarator), parallelism_hint)
            .setNumTasks((Integer) JsonPath.read(conf, "$.deck36_storm.rabbitmq.spout_tasks"));

    // construct command to invoke the external bolt implementation
    ArrayList<String> command = new ArrayList(15);

    // Add main execution program (php, hhvm, zend, ..) and parameters
    command.add((String) JsonPath.read(conf, "$.deck36_storm.php.executor"));
    command.addAll((List<String>) JsonPath.read(conf, "$.deck36_storm.php.executor_params"));

    // Add main command to be executed (app/console, the phar file, etc.) and global context parameters (environment etc.)
    command.add((String) JsonPath.read(conf, "$.deck36_storm.php.main"));
    command.addAll((List<String>) JsonPath.read(conf, "$.deck36_storm.php.main_params"));

    // create command to execute the RecordBreakerBolt
    ArrayList<String> recordBreakerBoltCommand = new ArrayList<String>(command);

    // Add main route to be invoked and its parameters
    recordBreakerBoltCommand.add((String) JsonPath.read(conf, "$.deck36_storm.RecordBreakerBolt.main"));
    List boltParams = (List<String>) JsonPath.read(conf, "$.deck36_storm.RecordBreakerBolt.params");
    if (boltParams != null)
        recordBreakerBoltCommand.addAll(boltParams);

    // create command to execute the RecordMasterBolt
    ArrayList<String> recordMasterBoltCommand = new ArrayList<String>(command);

    // Add main route to be invoked and its parameters
    recordMasterBoltCommand.add((String) JsonPath.read(conf, "$.deck36_storm.RecordMasterBolt.main"));
    boltParams = (List<String>) JsonPath.read(conf, "$.deck36_storm.RecordMasterBolt.params");
    if (boltParams != null)
        recordMasterBoltCommand.addAll(boltParams);

    // Log the final commands
    log.info("Command to start bolt for RecordBreaker badge: "
            + Arrays.toString(recordBreakerBoltCommand.toArray()));
    log.info("Command to start bolt for RecordMaster badge: "
            + Arrays.toString(recordMasterBoltCommand.toArray()));

    // Add constructed external bolt command to topology using MultilangAdapterBolt
    // The RecordBreaker reads the incoming messages from the game application, i.e. the "incoming" spout
    builder.setBolt("record_breaker", new MultilangAdapterBolt(recordBreakerBoltCommand, "badge"),
            parallelism_hint).setNumTasks(num_tasks).shuffleGrouping("incoming");

    // The RecordMaster reads the badge messages generated by the RecordBreakerBolt
    builder.setBolt("record_master", new MultilangAdapterBolt(recordMasterBoltCommand, "badge"),
            parallelism_hint).setNumTasks(num_tasks).shuffleGrouping("record_breaker");

    // the RabbitMQ router bolt can read messages from both, RecordBreakerBolt and RecordMasterBolt,
    // and forward those messages to the broker
    builder.setBolt("rabbitmq_router",
            new Plan9RabbitMQRouterBolt(
                    (String) JsonPath.read(conf, "$.deck36_storm.RecordBreakerBolt.rabbitmq.target_exchange"),
                    "RecordBreakerMaster" // RabbitMQ routing key
            ), parallelism_hint).setNumTasks(num_tasks).shuffleGrouping("record_breaker")
            .shuffleGrouping("record_master");

    builder.setBolt("rabbitmq_producer", new Plan9RabbitMQPushBolt(), parallelism_hint).setNumTasks(num_tasks)
            .shuffleGrouping("rabbitmq_router");

    if ("dev".equals(env)) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(badgeName + System.currentTimeMillis(), conf, builder.createTopology());
        Thread.sleep(2000000);
    }

    if ("prod".equals(env)) {
        StormSubmitter.submitTopology(badgeName + "-" + System.currentTimeMillis(), conf,
                builder.createTopology());
    }

}