com.dhenton9000.file.FileWatchers.java Source code

Introduction

Here is the source code for com.dhenton9000.file.FileWatchers.java

Source

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package com.dhenton9000.file;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 *
 * @author dhenton
 */
public class FileWatchers implements Runnable {

    protected static final Logger LOG = LoggerFactory.getLogger(FileWatchers.class);
    private boolean forever = true;
    private File inputDir = null;
    private File outputDir = null;
    private File tempDir = null;
    private ExecutorService fileProcessorService = Executors.newFixedThreadPool(3);
    private IReaderSink sink;

    private String fileExtension;

    public FileWatchers(String baseFolder, String fileExtension, IReaderSink sink) {
        //create the folders under neath the base folder
        outputDir = new File(baseFolder + "/output");
        tempDir = new File(baseFolder + "/tempDir");
        this.sink = sink;
        try {
            if (!outputDir.exists()) {
                FileUtils.forceMkdir(outputDir);

            }
            if (!tempDir.exists()) {
                FileUtils.forceMkdir(tempDir);

            }
        } catch (IOException ee) {
            throw new RuntimeException(ee.getMessage());
        }

        //input folder should exist already
        inputDir = new File(baseFolder + "/input");

        if (!inputDir.exists()) {
            throw new RuntimeException("cant find input folder");

        }

        this.fileExtension = fileExtension;

    }

    /**
     *
     * scan the directory and return the files you find
     *
     * @return batch of files
     */
    private List<File> findFiles() {

        String[] extensions = { this.fileExtension };
        Collection<File> fCollection = FileUtils.listFiles(inputDir, extensions, false);
        return new ArrayList<>(fCollection);
    }

    /**
     * generate a runnable that will process a file on a thread
     *
     * @param file file to process
     * @return the constructed runnable
     */
    private Runnable getRunnable(final File file, final IReaderSink sink) {
        return new Runnable() {
            @Override
            public void run() {
                //move from inputDir to tempDir

                File tempFile = new File(tempDir, FilenameUtils.getBaseName(file.getAbsolutePath()) + "_tmp.txt");
                String simpleName = FilenameUtils.getName(tempFile.getAbsolutePath());
                try {
                    FileUtils.copyFile(file, tempFile);
                    FileUtils.deleteQuietly(file);
                } catch (IOException ex) {
                    LOG.error("problem moving to temp " + ex.getMessage());
                }
                LOG.info("$#$move done " + simpleName);
                try {
                    LOG.info("$#$processing " + tempFile.getAbsolutePath());

                    FileReader in = new FileReader(tempFile);
                    BufferedReader bRead = new BufferedReader(in);

                    String line = bRead.readLine(); //skipping the header
                    //output the line
                    while (line != null) {
                        line = bRead.readLine();
                        if (line != null) {
                            LOG.debug(simpleName + " " + line.substring(0, 10) + "...");
                            if (sink != null) {
                                sink.report(line);

                            }
                        }

                    }

                    // parse each file into individual events
                    // 
                } catch (Exception e) {
                    LOG.error("$#$tried to process csv " + e.getClass().getName() + " " + e.getMessage());
                }
            }
        };

    }

    @Override
    public void run() {
        while (forever) {
            try {
                List<File> batch = findFiles();
                if (batch.size() > 0) {

                    Map<File, Future<?>> results = new HashMap<>(batch.size());
                    //load fileProcessorService with a thread per file
                    for (final File file : batch) {
                        results.put(file, fileProcessorService.submit(getRunnable(file, this.sink)));

                    }
                    // activate the per file threads
                    for (Map.Entry<File, Future<?>> result : results.entrySet()) {
                        try {
                            result.getValue().get();

                        } catch (InterruptedException | ExecutionException e) {
                            LOG.error("error processing file: " + result.getKey().getAbsolutePath(), e);
                        }
                    }

                } else {
                    Thread.sleep(1000);

                }

            } catch (Exception e) {

            }
        }

    }
    //http://stackoverflow.com/questions/15193977/how-to-scan-directory-with-multi-thread
    //http://winterbe.com/posts/2015/04/30/java8-concurrency-tutorial-synchronized-locks-examples/

    void cleanUp() {
        fileProcessorService.shutdownNow();
    }
}