org.apache.kylin.source.kafka.util.KafkaSampleProducer.java Source code

Introduction

Here is the source code for org.apache.kylin.source.kafka.util.KafkaSampleProducer.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.kylin.source.kafka.util;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Random;
import java.util.UUID;

import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.OptionsHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.databind.ObjectMapper;

/**
 * A sample producer which will create sample data to kafka topic
 */
public class KafkaSampleProducer {

    private static final Logger logger = LoggerFactory.getLogger(KafkaSampleProducer.class);
    @SuppressWarnings("static-access")
    private static final Option OPTION_TOPIC = OptionBuilder.withArgName("topic").hasArg().isRequired(true)
            .withDescription("Kafka topic").create("topic");
    private static final Option OPTION_BROKER = OptionBuilder.withArgName("broker").hasArg().isRequired(true)
            .withDescription("Kafka broker").create("broker");
    private static final Option OPTION_INTERVAL = OptionBuilder.withArgName("interval").hasArg().isRequired(false)
            .withDescription("Simulated message interval in mili-seconds, default 1000").create("interval");

    private static final ObjectMapper mapper = new ObjectMapper();

    public static void main(String[] args) throws Exception {
        logger.info("args: " + Arrays.toString(args));
        OptionsHelper optionsHelper = new OptionsHelper();
        Options options = new Options();
        String topic, broker;
        options.addOption(OPTION_TOPIC);
        options.addOption(OPTION_BROKER);
        options.addOption(OPTION_INTERVAL);
        optionsHelper.parseOptions(options, args);

        logger.info("options: '" + optionsHelper.getOptionsAsString() + "'");

        topic = optionsHelper.getOptionValue(OPTION_TOPIC);
        broker = optionsHelper.getOptionValue(OPTION_BROKER);

        long interval = 10;
        String intervalString = optionsHelper.getOptionValue(OPTION_INTERVAL);
        if (intervalString != null) {
            interval = Long.parseLong(intervalString);
        }

        List<String> countries = new ArrayList();
        countries.add("AUSTRALIA");
        countries.add("CANADA");
        countries.add("CHINA");
        countries.add("INDIA");
        countries.add("JAPAN");
        countries.add("KOREA");
        countries.add("US");
        countries.add("Other");
        List<String> category = new ArrayList();
        category.add("BOOK");
        category.add("TOY");
        category.add("CLOTH");
        category.add("ELECTRONIC");
        category.add("Other");
        List<String> devices = new ArrayList();
        devices.add("iOS");
        devices.add("Windows");
        devices.add("Andriod");
        devices.add("Other");

        List<String> genders = new ArrayList();
        genders.add("Male");
        genders.add("Female");

        Properties props = new Properties();
        props.put("bootstrap.servers", broker);
        props.put("acks", "all");
        props.put("retries", 0);
        props.put("batch.size", 16384);
        props.put("linger.ms", 1);
        props.put("buffer.memory", 33554432);
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");

        Producer<String, String> producer = new KafkaProducer<>(props);

        boolean alive = true;
        Random rnd = new Random();
        Map<String, Object> record = new HashMap();
        while (alive == true) {
            //add normal record
            record.put("order_time", (new Date().getTime()));
            record.put("country", countries.get(rnd.nextInt(countries.size())));
            record.put("category", category.get(rnd.nextInt(category.size())));
            record.put("device", devices.get(rnd.nextInt(devices.size())));
            record.put("qty", rnd.nextInt(10));
            record.put("currency", "USD");
            record.put("amount", rnd.nextDouble() * 100);
            //add embedded record
            Map<String, Object> user = new HashMap();
            user.put("id", UUID.randomUUID().toString());
            user.put("gender", genders.get(rnd.nextInt(2)));
            user.put("age", rnd.nextInt(20) + 10);
            record.put("user", user);
            //send message
            ProducerRecord<String, String> data = new ProducerRecord<>(topic, System.currentTimeMillis() + "",
                    mapper.writeValueAsString(record));
            System.out.println("Sending 1 message: " + JsonUtil.writeValueAsString(record));
            producer.send(data);
            Thread.sleep(interval);
        }
        producer.close();
    }

}