Java tutorial
/** * Copyright (c) Acroquest Technology Co, Ltd. All Rights Reserved. * Please read the associated COPYRIGHTS file for more details. * * THE SOFTWARE IS PROVIDED BY Acroquest Technolog Co., Ltd., * WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDER BE LIABLE FOR ANY * CLAIM, DAMAGES SUFFERED BY LICENSEE AS A RESULT OF USING, MODIFYING * OR DISTRIBUTING THIS SOFTWARE OR ITS DERIVATIVES. */ package acromusashi.kafka.log.producer; import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; import static java.nio.file.StandardWatchEventKinds.OVERFLOW; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.nio.file.FileSystems; import java.nio.file.Path; import java.nio.file.WatchEvent; import java.nio.file.WatchKey; import java.nio.file.WatchService; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import javax.swing.event.EventListenerList; import kafka.javaapi.producer.Producer; import kafka.producer.KeyedMessage; import kafka.producer.ProducerConfig; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.cli.PosixParser; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import acromusashi.kafka.log.producer.util.KeyedMessageConverter; import acromusashi.kafka.log.producer.util.ProducerConfigConverter; import acromusashi.kafka.log.producer.util.YamlReadUtil; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; /** * Windows?KafkaProducer * ???? * * @author hiroki */ public class WinApacheLogProducer extends Thread { /** Logger */ private static final Logger logger = LoggerFactory.getLogger(WinApacheLogProducer.class); /** ?? */ private static final int DEFAULT_ALLOCATE_SIZE = 65535; /** Kafka?Producer */ private kafka.javaapi.producer.Producer<String, String> producer; /** ???Kafka????????? */ private String topic; /** */ private EventListenerList listenerList = new EventListenerList(); /** ? */ private long tailPos; /** ??? */ private String path; /** LogAgent?????? */ private String host; /** ???? */ private List<String> newFileName = new ArrayList<String>(); /** ??? */ private File targetFile; /** ??? */ private String logPattern = "access_[0-9][0-9][0-9][0-9][0-9].*"; /** apache? */ private String apacheLogFormat; /** Jackson??? */ protected transient ObjectMapper objectMapper; /** ????????? */ private int retryNum = 1; /** json???? */ private String jsonDateFormatStr; /** */ private String encoding = "UTF-8"; /** * ????? */ public WinApacheLogProducer() { } /** * ?<br/> * <br/> * ?/?<br/> * <ul> * <li>-c WinApacheLogProducer(</li> * <li>-h </li> * </ul> * * @param args */ public static void main(String... args) { WinApacheLogProducer producer = new WinApacheLogProducer(); producer.startProducer(args); } /** * ???Producer? * * @param args */ protected void startProducer(String... args) { Options cliOptions = createOptions(); CommandLineParser parser = new PosixParser(); CommandLine commandLine = null; HelpFormatter help = new HelpFormatter(); try { commandLine = parser.parse(cliOptions, args); } catch (ParseException pex) { help.printHelp(WinApacheLogProducer.class.getName(), cliOptions, true); return; } if (commandLine.hasOption("h")) { // ?????????? help.printHelp(WinApacheLogProducer.class.getName(), cliOptions, true); return; } // ?? String confPath = commandLine.getOptionValue("c"); Map<String, Object> configMap = null; try { configMap = YamlReadUtil.readYaml(confPath); } catch (IOException ex) { // ??????? ex.printStackTrace(); return; } startTailLog(configMap); } /** * ????Log?Tail? * * @param configMap ?Map */ private void startTailLog(Map<String, Object> configMap) { this.path = configMap.get("tail.target.dir").toString(); this.topic = configMap.get("kafka.topic").toString(); this.apacheLogFormat = configMap.get("apachelog.format").toString(); this.jsonDateFormatStr = configMap.get("jsondate.format").toString(); this.host = "defaultHost"; try { this.host = InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException ex) { logger.warn("HostName resolve failed. Use default. : default=" + this.host, ex); } ProducerConfig producerConfig = ProducerConfigConverter.convertToProducerConfig(configMap); initialize(producerConfig); start(); } /** * KafkaProducer?Config??KafkaProducer?? * * @param config KafkaProducerConfig */ public void initialize(ProducerConfig config) { this.objectMapper = new ObjectMapper(); this.producer = new Producer<>(config); } /** * {@inheritDoc} */ @Override public void run() { while (true) { File accessLogFile = new File(this.path); tailRun(accessLogFile); } } /** * tail?? * * @param targetPath ? */ public void tailRun(File targetPath) { try (WatchService watcher = FileSystems.getDefault().newWatchService()) { Path targetDir = targetPath.toPath(); targetDir.register(watcher, ENTRY_MODIFY); targetDir.relativize(targetPath.toPath()); List<String> targetFileNames = getTargetLogFiles(Lists.newArrayList(targetDir.toFile().list())); Collections.sort(targetFileNames); int logFileNameSize = targetFileNames.size(); this.targetFile = new File(targetDir + "/" + targetFileNames.get(logFileNameSize - 1)); while (true) { WatchKey key = watcher.take(); for (WatchEvent<?> event : key.pollEvents()) { WatchEvent.Kind<?> kind = event.kind(); if (kind == OVERFLOW) { logger.warn("OVERFLOW"); continue; } byte[] tail = null; boolean noRetry = false; for (int retryCount = 0; retryCount < this.retryNum; retryCount++) { try { tail = getTail(this.targetFile); break; } catch (IOException ex) { if (retryCount == this.retryNum - 1) { noRetry = true; } } } // ????????????????? if (noRetry) { break; } List<String> allFileName = getTargetLogFiles(Arrays.asList(targetDir.toFile().list())); Collections.sort(allFileName); int allFileNameSize = allFileName.size(); if (tail.length > 0) { String inputStr = new String(tail, this.encoding); if (!allFileName.equals(targetFileNames)) { this.newFileName.add(allFileName.get(allFileNameSize - 1)); targetFileNames = allFileName; } List<String> eachStr = Arrays.asList(inputStr.split(System.getProperty("line.separator"))); List<KeyedMessage<String, String>> list = getKeyedMessage(eachStr); this.producer.send(list); } else { if (!allFileName.equals(targetFileNames)) { this.newFileName.add(allFileName.get(allFileNameSize - 1)); targetFileNames = allFileName; } if (this.newFileName.size() > 0) { this.targetFile = new File(targetDir + "/" + this.newFileName.get(0)); this.newFileName.remove(0); targetFileNames = allFileName; } } } boolean valid = key.reset(); if (!valid) { break; } } } catch (Exception ex) { // FindBugs?Java7????????FindBugs????????? logger.error("Failed start producer. ", ex); } } /** * ?????keyedMessageList?? * * @param eachStr ?? * @return list keyedMessage? */ protected List<KeyedMessage<String, String>> getKeyedMessage(List<String> eachStr) { List<KeyedMessage<String, String>> list = Lists.newArrayList(); for (String apacheLogStr : eachStr) { if (StringUtils.isBlank(apacheLogStr)) { continue; } KeyedMessage<String, String> convertedMessage = null; try { convertedMessage = KeyedMessageConverter.convertToMessage(apacheLogStr, this.topic, this.host, this.apacheLogFormat, this.jsonDateFormatStr); } catch (Exception ex) { logger.warn("Log convert failed. Dispose log message. Log=" + apacheLogStr, ex); continue; } list.add(convertedMessage); } return list; } /** * ??????? * * @param candidateFiles * @return returnLogFiles ? */ private List<String> getTargetLogFiles(List<String> candidateFiles) { int candidateFilesSize = candidateFiles.size(); List<String> returnLogFiles = new ArrayList<String>(); for (int index = 0; index < candidateFilesSize; index++) { String candidateFileName = candidateFiles.get(index); if (candidateFileName.matches(this.logPattern)) { returnLogFiles.add(candidateFileName); } } return returnLogFiles; } /** * ???? * * @param file ? * @return ????byte? * @throws IOException */ private byte[] getTail(File file) throws IOException { byte[] tail = new byte[0]; try (RandomAccessFile random = new RandomAccessFile(file, "r")) { if (this.tailPos < random.length()) { random.seek(this.tailPos); tail = readToEnd(random); } this.tailPos = random.length(); } return tail; } /** * ???? * * @param random * @return ????? * @throws IOException */ private byte[] readToEnd(RandomAccessFile random) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_ALLOCATE_SIZE); while ((random.getChannel().read(buffer)) != -1) { out.write(buffer.array()); buffer.clear(); } return out.toByteArray(); } /** * ??? * * @return ?? */ public static Options createOptions() { Options cliOptions = new Options(); // OptionBuilder.hasArg(true); OptionBuilder.withArgName("WinApacheLogProducer Conf Path"); OptionBuilder.withDescription("WinApacheLogProducer Conf Path"); OptionBuilder.isRequired(true); Option confPathOption = OptionBuilder.create("c"); // OptionBuilder.withDescription("show help"); Option helpOption = OptionBuilder.create("h"); cliOptions.addOption(confPathOption); cliOptions.addOption(helpOption); return cliOptions; } /** * tail?lisetener? * * @param listener */ public void addTailListener(WinTailEventListener listener) { this.listenerList.add(WinTailEventListener.class, listener); } /** * tail?listener??? * * @param listener */ public void removeTailListener(WinTailEventListener listener) { this.listenerList.remove(WinTailEventListener.class, listener); } }