com.ottogroup.bi.asap.server.emitter.kafka.KafkaTopicPartitionConsumer.java Source code

Introduction

Here is the source code for com.ottogroup.bi.asap.server.emitter.kafka.KafkaTopicPartitionConsumer.java

Source

/**
 * Copyright 2014 Otto (GmbH & Co KG)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.ottogroup.bi.asap.server.emitter.kafka;

import io.netty.util.CharsetUtil;

import java.io.UnsupportedEncodingException;

import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.message.MessageAndMetadata;

import org.apache.commons.lang3.CharUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;

import uk.co.real_logic.queues.OneToOneConcurrentArrayQueue3;

import com.ottogroup.bi.asap.message.StreamingDataMessage;

/**
 * Reads data from assigned {@link KafkaStream} and writes the data to a provided queue 
 * @author mnxfst
 * @since Sep 17, 2014
 */
public class KafkaTopicPartitionConsumer implements Runnable {

    private static final Logger logger = Logger.getLogger(KafkaTopicPartitionConsumer.class);

    private final String origin;
    private final OneToOneConcurrentArrayQueue3<StreamingDataMessage> messages;
    private final KafkaStream<byte[], byte[]> kafkaTopicPartitionStream;
    private boolean running = false;

    public KafkaTopicPartitionConsumer(final String origin,
            final KafkaStream<byte[], byte[]> kafkaTopicPartitionStream,
            final OneToOneConcurrentArrayQueue3<StreamingDataMessage> messages) {
        this.origin = origin;
        this.messages = messages;
        this.kafkaTopicPartitionStream = kafkaTopicPartitionStream;
    }

    /**
     * @see java.lang.Runnable#run()
     */
    public void run() {

        ConsumerIterator<byte[], byte[]> topicPartitionStreamIterator = this.kafkaTopicPartitionStream.iterator();

        this.running = true;
        // ensure that the partition iterator still has some elements left and the receiving
        // web socket producer still wants more
        while (running) {
            if (topicPartitionStreamIterator.hasNext()) {
                // next message must neither be null nor empty
                MessageAndMetadata<byte[], byte[]> message = topicPartitionStreamIterator.next();
                if (message != null && message.message() != null && message.message().length > 0) {
                    // try to convert the content into a UTF-8 encoded string
                    String messageContent = new String(message.message(), CharsetUtil.UTF_8);

                    // return only if the message is not empty or blank
                    // as we do not transport sender references it is forwarded with a reference to a dead letter box
                    if (StringUtils.isNotBlank(messageContent)) {
                        messages.add(new StreamingDataMessage(origin, messageContent, System.currentTimeMillis()));
                    }
                }
            } else {
                Thread.yield();
                // TODO wait strategy
            }
        }
    }

}