acromusashi.kafka.log.producer.LinuxLogTailExecutor.java Source code

Introduction

Here is the source code for acromusashi.kafka.log.producer.LinuxLogTailExecutor.java

Source

/**
* Copyright (c) Acroquest Technology Co, Ltd. All Rights Reserved.
* Please read the associated COPYRIGHTS file for more details.
*
* THE SOFTWARE IS PROVIDED BY Acroquest Technolog Co., Ltd.,
* WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
* BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDER BE LIABLE FOR ANY
* CLAIM, DAMAGES SUFFERED BY LICENSEE AS A RESULT OF USING, MODIFYING
* OR DISTRIBUTING THIS SOFTWARE OR ITS DERIVATIVES.
*/
package acromusashi.kafka.log.producer;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.List;

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;

import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import acromusashi.kafka.log.producer.util.KeyedMessageConverter;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;

/**
 * ???kafka producer???
 * 
 * @author hiroki
 */
public class LinuxLogTailExecutor implements Runnable {
    /**  */
    private static final Logger logger = LoggerFactory.getLogger(LinuxLogTailExecutor.class);

    /** ?? */
    private static final int DEFAULT_MAX_SEND_SIZE = 100;

    /** Kafka?Producer */
    private kafka.javaapi.producer.Producer<String, String> producer;

    /** ?(tail -F) */
    private String tailCommandStr;

    /** ???Kafka????????? */
    private String topic;

    /** apache? */
    private String apacheLogFormat;

    /** json????? */
    private String jsonDateFormatStr;

    /** Jackson??? */
    protected transient ObjectMapper objectMapper;

    /** producer??send??(32767??send????) */
    private int maxSendSize = DEFAULT_MAX_SEND_SIZE;

    /** LogAgent?????? */
    private String host = "defaultHost";

    /**  */
    private String encoding = "UTF-8";

    /**
     * ??
     * 
     * @param tailCommandStr ?
     * @param topic ???
     * @param apacheLogFormat apache??
     * @param jsonDateFormat json?????
     * @param hostName Producer??
     */
    public LinuxLogTailExecutor(String tailCommandStr, String topic, String apacheLogFormat, String jsonDateFormat,
            String hostName) {
        this.tailCommandStr = tailCommandStr;
        this.topic = topic;
        this.apacheLogFormat = apacheLogFormat;
        this.jsonDateFormatStr = jsonDateFormat;
        this.host = hostName;
    }

    /**
     * KafkaProducer?Config??KafkaProducer??
     * 
     * @param config KafkaProducerConfig
     */
    public void initialize(ProducerConfig config) {
        this.objectMapper = new ObjectMapper();
        this.producer = new Producer<>(config);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void run() {
        // ?????tail????
        while (true) {
            sendTailedLog();
        }
    }

    /**
     * ?Tail???KafkaBroker?????
     */
    protected void sendTailedLog() {
        String[] tailCommandArgs = this.tailCommandStr.split("\\s+");

        BufferedReader tailReader = null;
        Process tailProcess = null;

        try {
            tailProcess = new ProcessBuilder(tailCommandArgs).start();
            tailReader = new BufferedReader(new InputStreamReader(tailProcess.getInputStream(), this.encoding));

            String tailedLine = null;
            List<String> tailedLineList = Lists.newArrayList();
            int count = 0;

            while ((tailedLine = tailReader.readLine()) != null) {
                tailedLineList.add(tailedLine);
                count++;

                if (count >= this.maxSendSize) {
                    List<KeyedMessage<String, String>> messageList = getKeyedMessage(tailedLineList);
                    tailedLineList.clear();
                    this.producer.send(messageList);
                    count = 0;
                }

            }

            List<KeyedMessage<String, String>> messageList = getKeyedMessage(tailedLineList);
            tailedLineList.clear();
            this.producer.send(messageList);
        } catch (Exception e) {
            logger.error("Failed while running command: " + this.tailCommandStr, e);
            if (e instanceof InterruptedException) {
                Thread.currentThread().interrupt();
            }
        } finally {
            if (tailReader != null) {
                IOUtils.closeQuietly(tailReader);
            }
            if (tailProcess != null) {
                tailProcess.destroy();
                try {
                    tailProcess.waitFor();
                } catch (InterruptedException ex) {
                    Thread.currentThread().interrupt();
                }
            }
        }
    }

    /**
     * ?????keyedMessageList??
     * 
     * @param eachStr ??
     * @return list keyedMessage?
     */
    protected List<KeyedMessage<String, String>> getKeyedMessage(List<String> eachStr) {
        List<KeyedMessage<String, String>> list = Lists.newArrayList();
        for (String apacheLogStr : eachStr) {
            KeyedMessage<String, String> convertedMessage = null;

            try {
                convertedMessage = KeyedMessageConverter.convertToMessage(apacheLogStr, this.topic, this.host,
                        this.apacheLogFormat, this.jsonDateFormatStr);
            } catch (Exception ex) {
                logger.warn("Log convert failed. Dispose log message. Log=" + apacheLogStr, ex);
                continue;
            }

            list.add(convertedMessage);
        }

        return list;
    }
}