Java tutorial
/* * Copyright 2016 Hortonworks. * <p> * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * <p> * http://www.apache.org/licenses/LICENSE-2.0 * <p> * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.hortonworks.registries.schemaregistry.examples.avro; import com.hortonworks.registries.schemaregistry.client.SchemaRegistryClient; import com.hortonworks.registries.schemaregistry.serdes.avro.kafka.KafkaAvroDeserializer; import com.hortonworks.registries.schemaregistry.serdes.avro.kafka.KafkaAvroSerializer; import org.apache.avro.Schema; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.io.DatumReader; import org.apache.avro.io.DecoderFactory; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionGroup; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileReader; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.UUID; import java.util.concurrent.TimeUnit; /** * Below class can be used to send messages to a given topic in kafka-producer.props like below. * * KafkaAvroSerDesApp -sm -d yelp_review_json -s yelp_review.avsc -p kafka-producer.props */ public class KafkaAvroSerDesApp { private static final Logger LOG = LoggerFactory.getLogger(KafkaAvroSerDesApp.class); public static final String MSGS_LIMIT = "msgsLimit"; public static final String TOPIC = "topic"; public static final String SCHEMA_REGISTRY_URL = "schema.registry.url"; public static final int DEFAULT_MSGS_LIMIT = 50; private String producerProps; private String schemaFile; private String consumerProps; public KafkaAvroSerDesApp(String producerProps, String schemaFile) { this.producerProps = producerProps; this.schemaFile = schemaFile; } public KafkaAvroSerDesApp(String consumerProps) { this.consumerProps = consumerProps; } public void sendMessages(String payloadJsonFile) throws Exception { Properties props = new Properties(); try (FileInputStream fileInputStream = new FileInputStream(this.producerProps)) { props.load(fileInputStream); } int limit = Integer.parseInt(props.getProperty(MSGS_LIMIT, DEFAULT_MSGS_LIMIT + "")); int current = 0; Schema schema = new Schema.Parser().parse(new File(this.schemaFile)); Map<String, Object> producerConfig = createProducerConfig(props); String topicName = props.getProperty(TOPIC); final Producer<String, Object> producer = new KafkaProducer<>(producerConfig); final Callback callback = new MyProducerCallback(); try (BufferedReader bufferedReader = new BufferedReader(new FileReader(payloadJsonFile))) { String line; while (current++ < limit && (line = bufferedReader.readLine()) != null) { // convert json to avro records Object avroMsg = jsonToAvro(line, schema); // send avro messages to given topic using KafkaAvroSerializer which registers payload schema if it does not exist // with schema name as "<topic-name>:v", type as "avro" and schemaGroup as "kafka". // schema registry should be running so that KafkaAvroSerializer can register the schema. LOG.info("Sending message: [{}] to topic: [{}]", avroMsg, topicName); ProducerRecord<String, Object> producerRecord = new ProducerRecord<>(topicName, avroMsg); producer.send(producerRecord, callback); } } finally { producer.flush(); LOG.info("All message are successfully sent to topic: [{}]", topicName); producer.close(5, TimeUnit.SECONDS); } } private Map<String, Object> createProducerConfig(Properties props) { String bootstrapServers = props.getProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG); Map<String, Object> config = new HashMap<>(); config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); config.putAll(Collections.singletonMap(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), props.get(SCHEMA_REGISTRY_URL))); config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName()); config.put(ProducerConfig.BATCH_SIZE_CONFIG, 1024); return config; } private Object jsonToAvro(String jsonString, Schema schema) throws Exception { DatumReader<Object> reader = new GenericDatumReader<Object>(schema); Object object = reader.read(null, DecoderFactory.get().jsonDecoder(schema, jsonString)); if (schema.getType().equals(Schema.Type.STRING)) { object = object.toString(); } return object; } private static class MyProducerCallback implements Callback { @Override public void onCompletion(RecordMetadata recordMetadata, Exception e) { LOG.info("#### received [{}], ex: [{}]", recordMetadata, e); } } public void consumeMessages() throws Exception { Properties props = new Properties(); try (FileInputStream inputStream = new FileInputStream(this.consumerProps);) { props.load(inputStream); } String topicName = props.getProperty(TOPIC); Map<String, Object> consumerConfig = createConsumerConfig(props); KafkaConsumer consumer = new KafkaConsumer<>(consumerConfig); consumer.subscribe(Collections.singletonList(topicName)); while (true) { ConsumerRecords<String, Object> records = consumer.poll(1000); LOG.info("records size " + records.count()); for (ConsumerRecord<String, Object> record : records) { LOG.info("Received message: (" + record.key() + ", " + record.value() + ") at offset " + record.offset()); } } } private Map<String, Object> createConsumerConfig(Properties props) { String bootstrapServers = props.getProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG); Map<String, Object> config = new HashMap<>(); config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); config.putAll(Collections.singletonMap(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), props.get(SCHEMA_REGISTRY_URL))); config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class.getName()); if (props.containsKey(ConsumerConfig.GROUP_ID_CONFIG)) { config.put(ConsumerConfig.GROUP_ID_CONFIG, props.getProperty(ConsumerConfig.GROUP_ID_CONFIG)); } else { config.put(ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString()); } config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); return config; } /** * Print the command line options help message and exit application. */ @SuppressWarnings("static-access") private static void showHelpMessage(String[] args, Options options) { Options helpOptions = new Options(); helpOptions.addOption(Option.builder("h").longOpt("help").desc("print this message").build()); try { CommandLine helpLine = new DefaultParser().parse(helpOptions, args, true); if (helpLine.hasOption("help") || args.length == 1) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("truck-events-kafka-ingest", options); System.exit(0); } } catch (ParseException ex) { LOG.error("Parsing failed. Reason: " + ex.getMessage()); System.exit(1); } } public static void main(String[] args) throws Exception { Option sendMessages = Option.builder("sm").longOpt("send-messages").desc("Send Messages to Kafka") .type(Boolean.class).build(); Option consumeMessages = Option.builder("cm").longOpt("consume-messages") .desc("Consume Messages from Kafka").type(Boolean.class).build(); Option dataFileOption = Option.builder("d").longOpt("data-file").hasArg().desc("Provide a data file") .type(String.class).build(); Option producerFileOption = Option.builder("p").longOpt("producer-config").hasArg() .desc("Provide a Kafka producer config file").type(String.class).build(); Option schemaOption = Option.builder("s").longOpt("schema-file").hasArg().desc("Provide a schema file") .type(String.class).build(); Option consumerFileOption = Option.builder("c").longOpt("consumer-config").hasArg() .desc("Provide a Kafka Consumer config file").type(String.class).build(); OptionGroup groupOpt = new OptionGroup(); groupOpt.addOption(sendMessages); groupOpt.addOption(consumeMessages); groupOpt.setRequired(true); Options options = new Options(); options.addOptionGroup(groupOpt); options.addOption(dataFileOption); options.addOption(producerFileOption); options.addOption(schemaOption); options.addOption(consumerFileOption); //showHelpMessage(args, options); CommandLineParser parser = new DefaultParser(); CommandLine commandLine; try { commandLine = parser.parse(options, args); if (commandLine.hasOption("sm")) { if (commandLine.hasOption("p") && commandLine.hasOption("d") && commandLine.hasOption("s")) { KafkaAvroSerDesApp kafkaAvroSerDesApp = new KafkaAvroSerDesApp(commandLine.getOptionValue("p"), commandLine.getOptionValue("s")); kafkaAvroSerDesApp.sendMessages(commandLine.getOptionValue("d")); } else { LOG.error("please provide following options for sending messages to Kafka"); LOG.error("-d or --data-file"); LOG.error("-s or --schema-file"); LOG.error("-p or --producer-config"); } } else if (commandLine.hasOption("cm")) { if (commandLine.hasOption("c")) { KafkaAvroSerDesApp kafkaAvroSerDesApp = new KafkaAvroSerDesApp(commandLine.getOptionValue("c")); kafkaAvroSerDesApp.consumeMessages(); } else { LOG.error("please provide following options for consuming messages from Kafka"); LOG.error("-c or --consumer-config"); } } } catch (ParseException e) { LOG.error("Please provide all the options ", e); } catch (Exception e) { LOG.error("Failed to send/receive messages ", e); } } }