acromusashi.stream.ml.common.spout.WatchTextBatchSpout.java Source code

Introduction

Here is the source code for acromusashi.stream.ml.common.spout.WatchTextBatchSpout.java

Source

/**
* Copyright (c) Acroquest Technology Co, Ltd. All Rights Reserved.
* Please read the associated COPYRIGHTS file for more details.
*
* THE SOFTWARE IS PROVIDED BY Acroquest Technolog Co., Ltd.,
* WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
* BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDER BE LIABLE FOR ANY
* CLAIM, DAMAGES SUFFERED BY LICENSEE AS A RESULT OF USING, MODIFYING
* OR DISTRIBUTING THIS SOFTWARE OR ITS DERIVATIVES.
*/
package acromusashi.stream.ml.common.spout;

import java.io.File;
import java.io.IOException;
import java.nio.file.FileSystem;
import java.nio.file.Path;
import java.nio.file.StandardWatchEventKinds;
import java.nio.file.WatchEvent;
import java.nio.file.WatchEvent.Kind;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.text.MessageFormat;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import storm.trident.operation.TridentCollector;
import storm.trident.spout.IBatchSpout;
import backtype.storm.task.TopologyContext;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;

/**
 * Text?????Spout<br>
 * 
 * @author kimura
 */
public class WatchTextBatchSpout implements IBatchSpout {
    /** serialVersionUID */
    private static final long serialVersionUID = -5097002059382826053L;

    /** logger */
    private static final Logger logger = LoggerFactory.getLogger(WatchTextBatchSpout.class);

    /** ? */
    private String dataFileDir;

    /** ?? */
    private String baseFileName;

    /** TaskIndex */
    private int taskIndex;

    /** ?? */
    private String dataFileName;

    /**  */
    private File targetFile = null;

    /** ??WatchService */
    private transient WatchService watcherService = null;

    /** ??Watcher */
    private transient WatchKey watchKey = null;

    /** ?????? */
    boolean isInitialReaded = false;

    /**
     * ?????
     */
    public WatchTextBatchSpout() {
    }

    /**
     * {@inheritDoc}
     */
    @SuppressWarnings({ "rawtypes" })
    @Override
    public void open(Map conf, TopologyContext context) {
        this.taskIndex = context.getThisTaskIndex();
        this.dataFileName = this.baseFileName + "_" + this.taskIndex;
        this.targetFile = new File(this.dataFileDir, this.dataFileName);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void emitBatch(long batchId, TridentCollector collector) {
        try {
            checkDataFile(collector);
        } catch (Exception ex) {
            String logFormat = "Check file failed, skip batch. FilePath={0}";
            logger.warn(MessageFormat.format(logFormat, this.targetFile.getAbsolutePath()), ex);
        }
    }

    /**
     * ???????????????????
     * 
     * @param collector Collector
     * @throws IOException 
     * @throws InterruptedException ?
     */
    @SuppressWarnings({ "rawtypes" })
    protected void checkDataFile(TridentCollector collector) throws IOException, InterruptedException {
        // ?????????????????
        if (this.isInitialReaded == false) {
            List<String> fileContents = FileUtils.readLines(this.targetFile);
            emitTuples(fileContents, collector);
            this.isInitialReaded = true;

            // 
            Path dirPath = new File(this.dataFileDir).toPath();
            FileSystem fileSystem = dirPath.getFileSystem();
            this.watcherService = fileSystem.newWatchService();
            this.watchKey = dirPath.register(this.watcherService,
                    new Kind[] { StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY });

            return;
        }

        // ????????
        WatchKey detectedKey = this.watcherService.poll(1, TimeUnit.SECONDS);

        // ???????????????????????
        if (detectedKey == null || detectedKey.equals(this.watchKey) == false) {
            return;
        }

        try {
            // ???????????????
            for (WatchEvent event : detectedKey.pollEvents()) {

                Path filePath = (Path) event.context();

                // ?????????????
                if (filePath == null
                        || this.targetFile.toPath().getFileName().equals(filePath.getFileName()) == false) {
                    continue;
                }

                List<String> fileContents = FileUtils.readLines(this.targetFile);
                emitTuples(fileContents, collector);
            }
        } finally {
            detectedKey.reset();
        }
    }

    /**
     * ???Tuple??
     * 
     * @param fileLines ?
     * @param collector Collector
     */
    protected void emitTuples(List<String> fileLines, TridentCollector collector) {
        for (String targetLine : fileLines) {
            collector.emit(new Values(targetLine));
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void ack(long batchId) {
        if (logger.isDebugEnabled() == true) {
            logger.debug("acked. taskIndex=" + this.taskIndex + ", batchId=" + batchId);
        }
    }

    @Override
    public void close() {
        // Do nothing.
    }

    /**
     * {@inheritDoc}
     */
    @SuppressWarnings("rawtypes")
    @Override
    public Map getComponentConfiguration() {
        return null;
    }

    @Override
    public Fields getOutputFields() {
        return new Fields("text");
    }

    /**
     * @return the dataFilePath
     */
    public String getDataFilePath() {
        return this.dataFileDir;
    }

    /**
     * @param dataFilePath the dataFilePath to set
     */
    public void setDataFilePath(String dataFilePath) {
        this.dataFileDir = dataFilePath;
    }

    /**
     * @return the baseFileName
     */
    public String getBaseFileName() {
        return this.baseFileName;
    }

    /**
     * @param baseFileName the baseFileName to set
     */
    public void setBaseFileName(String baseFileName) {
        this.baseFileName = baseFileName;
    }
}