Simple example on how to read with apache Kafka consumer - Java Big Data

Java examples for Big Data:apache kafka

  1. HOME
  2. Java
  3. Big Data
  4. apache kafka

Description

Simple example on how to read with apache Kafka consumer

Demo Code


import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer082;
import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
import java.util.Properties;

/**/*  w w  w.j  a  v  a 2 s .co  m*/
 * Simple example on how to read with a Kafka consumer
 *
 * Note that the Kafka source is expecting the following parameters to be set
 *  - "bootstrap.servers" (comma separated list of kafka brokers)
 *  - "zookeeper.connect" (comma separated list of zookeeper servers)
 *  - "group.id" the id of the consumer group
 *  - "topic" the name of the topic to read data from.
 *
 * You can pass these required parameters using "--bootstrap.servers host:port,host1:port1 --zookeeper.connect host:port --topic testTopic"
 *
 * This is a valid input example:
 *     --topic test --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myGroup
 *
 *
 */
public class ReadFromKafka {

    public static void main(String[] args) throws Exception {
        // create execution environment
        StreamExecutionEnvironment env = StreamExecutionEnvironment
                .getExecutionEnvironment();

        // parse user parameters
        //    ParameterTool parameterTool = ParameterTool.fromArgs(args);

        //    DataStream<String> messageStream = env.addSource(new FlinkKafkaConsumer(parameterTool.getRequired("topic"), new SimpleStringSchema(), parameterTool.getProperties()));

        Properties properties = new Properties();
        properties.setProperty("bootstrap.servers", "node2:9092");
        properties.setProperty("zookeeper.connect", "node2:2181");
        properties.setProperty("group.id", "1");
        DataStream<String> messageStream = env
                .addSource(new FlinkKafkaConsumer082<>("demo",
                        new SimpleStringSchema(), properties)); //print();

        messageStream.print();
        System.out.print(messageStream + " Hello\n");

        // print() will write the contents of the stream to the TaskManager's standard out stream
        // the rebelance call is causing a repartitioning of the data so that all machines
        // see the messages (for example in cases when "num kafka partitions" < "num flink operators"
        //    messageStream.rebalance().map(new MapFunction<String, String>() {
        //      private static final long serialVersionUID = -6867736771747690202L;

        //      @Override
        //      public String map(String value) throws Exception {
        //        return "Kafka and Flink says: " + value;
        //      }
        //    }).print();

        env.execute("kafka consumer");
    }
}

Related Tutorials