Java tutorial
/* * Copyright 2014 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.suro.sink.kafka; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.netflix.servo.monitor.Monitors; import com.netflix.suro.message.Message; import com.netflix.suro.message.MessageContainer; import com.netflix.suro.queue.MemoryQueue4Sink; import com.netflix.suro.queue.MessageQueue4Sink; import com.netflix.suro.sink.QueuedSink; import com.netflix.suro.sink.Sink; import com.netflix.suro.sink.ThreadPoolQueuedSink; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.serialization.ByteArraySerializer; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicLong; /** * Kafka 0.8.2 Sink, using new Java-native producer, rather than Scala produer. * Requests are re-queued indefinitely if they fail. * * The configuration parameters for the new kafka producer are listed in: * http://kafka.apache.org/documentation.html#newproducerconfigs * * @author jbae * @author starzia */ public class KafkaSinkV2 extends ThreadPoolQueuedSink implements Sink { public final static String TYPE = "KafkaV2"; private String clientId; private final Map<String, String> keyTopicMap; private final KafkaProducer<byte[], byte[]> producer; private long msgId = 0; private AtomicLong receivedCount = new AtomicLong(0); private AtomicLong sentCount = new AtomicLong(0); private AtomicLong sentByteCount = new AtomicLong(0); /** number of times a message send failed without retrying */ private AtomicLong droppedCount = new AtomicLong(0); /** number of times a message send failed but was requeued */ private AtomicLong requeuedCount = new AtomicLong(0); @JsonCreator public KafkaSinkV2(@JsonProperty("queue4Sink") MessageQueue4Sink queue4Sink, @JsonProperty("client.id") String clientId, @JsonProperty("metadata.broker.list") String bootstrapServers, @JsonProperty("compression.codec") String codec, @JsonProperty("send.buffer.bytes") int sendBufferBytes, @JsonProperty("batchSize") int batchSize, @JsonProperty("batchTimeout") int batchTimeout, @JsonProperty("request.timeout.ms") int requestTimeout, @JsonProperty("kafka.etc") Properties etcProps, @JsonProperty("keyTopicMap") Map<String, String> keyTopicMap, @JsonProperty("jobQueueSize") int jobQueueSize, @JsonProperty("corePoolSize") int corePoolSize, @JsonProperty("maxPoolSize") int maxPoolSize, @JsonProperty("jobTimeout") long jobTimeout, @JsonProperty("pauseOnLongQueue") boolean pauseOnLongQueue) { super(jobQueueSize, corePoolSize, maxPoolSize, jobTimeout, KafkaSink.class.getSimpleName() + "-" + clientId); Preconditions.checkNotNull(bootstrapServers); Preconditions.checkNotNull(clientId); this.clientId = clientId; initialize("kafka_" + clientId, queue4Sink == null ? new MemoryQueue4Sink(10000) : queue4Sink, batchSize, batchTimeout, pauseOnLongQueue); Properties props = new Properties(); props.put("client.id", clientId); // metadata.broker.list was renamed to bootstrap.servers in the new kafka producer props.put("bootstrap.servers", bootstrapServers); if (codec != null) { props.put("compression.codec", codec); } if (sendBufferBytes > 0) { props.put("send.buffer.bytes", Integer.toString(sendBufferBytes)); } if (requestTimeout > 0) { props.put("request.timeout.ms", Integer.toString(requestTimeout)); } if (etcProps != null) { props.putAll(etcProps); } this.keyTopicMap = keyTopicMap != null ? keyTopicMap : Maps.<String, String>newHashMap(); producer = new KafkaProducer<>(props, new ByteArraySerializer(), new ByteArraySerializer()); Monitors.registerObject(clientId, this); } @Override public void writeTo(MessageContainer message) { long key = msgId++; if (!keyTopicMap.isEmpty()) { try { Map<String, Object> msgMap = message.getEntity(new TypeReference<Map<String, Object>>() { }); Object keyField = msgMap.get(keyTopicMap.get(message.getRoutingKey())); if (keyField != null) { key = keyField.hashCode(); } } catch (Exception e) { QueuedSink.log.error("Exception on getting key field: " + e.getMessage()); } } QueuedSink.log.trace("KafkaSink writeTo()"); receivedCount.incrementAndGet(); enqueue(new SuroKeyedMessage(key, message.getMessage())); } @Override public void open() { setName(KafkaSink.class.getSimpleName() + "-" + clientId); start(); } @Override protected void beforePolling() throws IOException { /*do nothing */} @Override protected void write(List<Message> msgList) { QueuedSink.log.trace("KafkaSink write() with {} messages", msgList.size()); // prepare "final" copies of the messages to be used in the anonymous class below final ArrayList<SuroKeyedMessage> msgCopies = new ArrayList<SuroKeyedMessage>(msgList.size()); for (Message m : msgList) { SuroKeyedMessage sKeyedMsg = (SuroKeyedMessage) m; msgCopies.add(new SuroKeyedMessage(sKeyedMsg.getKey(), new Message(m.getRoutingKey(), m.getPayload()))); } // The new KafkaProducer does not have interface for sending multiple messages, // so we loop and create lots of Runnables -- this seems inefficient, but the alternative // has its own problems. If we create one "big Runnable" that loops over messages we'll // drain the queue4sink too quickly -- all the messages will be queued in the in-memory // job queue storing the Runnables. for (final SuroKeyedMessage m : msgCopies) { senders.submit(new Runnable() { @Override public void run() { String topic = m.getRoutingKey(); // calculate the kafka partition, with backward compatibility with old kafka producer int numPartitions = producer.partitionsFor(topic).size(); int partition = Math.abs((int) (m.getKey() ^ (m.getKey() >>> 32))) % numPartitions; ProducerRecord r = new ProducerRecord(topic, partition, null, // don't store the key m.getPayload()); QueuedSink.log.trace("Will send message to Kafka"); long startTimeMs = System.currentTimeMillis(); // send Future<RecordMetadata> responseFtr = producer.send(r); QueuedSink.log.trace("Started aysnc producer"); boolean failure = true; boolean retry = true; if (responseFtr.isCancelled()) { QueuedSink.log.warn("Kafka producer request was cancelled"); // we assume that cancelled requests should not be retried. retry = false; } try { // wait for request to finish RecordMetadata response = responseFtr.get(); if (response.topic() == null) { QueuedSink.log.warn("Kafka producer got null topic in response"); } sentCount.incrementAndGet(); sentByteCount.addAndGet(m.getPayload().length); failure = false; retry = false; } catch (InterruptedException e) { // Assume that Interrupted means we're trying to shutdown so don't retry QueuedSink.log.warn("Caught InterruptedException: " + e); retry = false; } catch (UnknownTopicOrPartitionException e) { QueuedSink.log.warn("Caught UnknownTopicOrPartitionException for topic: " + m.getRoutingKey() + " This may be simply because KafkaProducer does not yet have information about the brokers." + " Request will be retried."); } catch (ExecutionException e) { QueuedSink.log.warn("Caught ExecutionException: " + e); } catch (Exception e) { QueuedSink.log.warn("Caught Exception: " + e); } long durationMs = System.currentTimeMillis() - startTimeMs; if (failure) { QueuedSink.log.warn("Kafka producer send failed after {} milliseconds", durationMs); requeuedCount.incrementAndGet(); if (retry) { enqueue(m); } else { QueuedSink.log.info("Dropped message"); droppedCount.incrementAndGet(); } } else { QueuedSink.log.trace("Kafka producer send succeeded after {} milliseconds", durationMs); } } }); } } @Override protected void innerClose() { super.innerClose(); producer.close(); } @Override public String recvNotice() { return null; } @Override public String getStat() { Map<MetricName, ? extends Metric> metrics = producer.metrics(); StringBuilder sb = new StringBuilder(); // add kafka producer stats, which are rates for (Map.Entry<MetricName, ? extends Metric> e : metrics.entrySet()) { sb.append("kafka.").append(e.getKey()).append(": ").append(e.getValue().value()).append('\n'); } // also report our counters sb.append("messages-in-queue4sink: ").append(this.queue4Sink.size()).append('\n'); sb.append("queued-jobs: ").append(this.jobQueue.size()).append('\n'); sb.append("active-threads: ").append(this.senders.getActiveCount()).append('\n'); sb.append("received-messages: ").append(this.receivedCount.get()).append('\n'); sb.append("sent-messages: ").append(this.sentCount.get()).append('\n'); sb.append("sent-bytes: ").append(this.sentByteCount.get()).append('\n'); sb.append("dropped-messages: ").append(this.droppedCount.get()).append('\n'); sb.append("requeued-messages: ").append(this.requeuedCount.get()).append('\n'); return sb.toString(); } }