org.fim.internal.StateGenerator.java Source code

Java tutorial

Introduction

Here is the source code for org.fim.internal.StateGenerator.java

Source

/*
 * This file is part of Fim - File Integrity Manager
 *
 * Copyright (C) 2015  Etienne Vrignaud
 *
 * Fim is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Fim is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Fim.  If not, see <http://www.gnu.org/licenses/>.
 */
package org.fim.internal;

import static org.fim.internal.hash.HashProgress.PROGRESS_DISPLAY_FILE_COUNT;
import static org.fim.model.HashMode.dontHash;
import static org.fim.util.HashModeUtil.hashModeToString;

import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.atteo.evo.inflector.English;
import org.fim.internal.hash.FileHasher;
import org.fim.internal.hash.HashProgress;
import org.fim.model.Context;
import org.fim.model.FileState;
import org.fim.model.FimIgnore;
import org.fim.model.State;
import org.fim.util.Console;
import org.fim.util.FileUtil;
import org.fim.util.Logger;

public class StateGenerator {
    public static final int FILES_QUEUE_CAPACITY = 500;

    private static Comparator<FileState> fileNameComparator = new FileState.FileNameComparator();

    private final Context context;
    private final HashProgress hashProgress;
    private final FimIgnoreManager fimIgnoreManager;

    private ExecutorService executorService;

    private Path rootDir;
    private BlockingDeque<Path> filesToHashQueue;
    private boolean fileHashersStarted;
    private List<FileHasher> fileHashers;
    private long overallTotalBytesHashed;

    public StateGenerator(Context context) {
        this.context = context;
        this.hashProgress = new HashProgress(context);
        this.fimIgnoreManager = new FimIgnoreManager(context);
    }

    public State generateState(String comment, Path rootDir, Path dirToScan) throws NoSuchAlgorithmException {
        this.rootDir = rootDir;

        int threadCount = context.getThreadCount();
        Logger.info(String.format("Scanning recursively local files, using '%s' mode and %d %s",
                hashModeToString(context.getHashMode()), threadCount, English.plural("thread", threadCount)));
        if (hashProgress.isProgressDisplayed()) {
            System.out.printf("(Hash progress legend for files grouped %d by %d: %s)%n",
                    PROGRESS_DISPLAY_FILE_COUNT, PROGRESS_DISPLAY_FILE_COUNT, hashProgress.hashLegend());
        }

        State state = new State();
        state.setComment(comment);
        state.setHashMode(context.getHashMode());

        long start = System.currentTimeMillis();
        hashProgress.outputInit();

        filesToHashQueue = new LinkedBlockingDeque<>(FILES_QUEUE_CAPACITY);
        initializeFileHashers();

        FimIgnore initialFimIgnore = fimIgnoreManager.loadInitialFimIgnore();
        scanFileTree(filesToHashQueue, dirToScan, initialFimIgnore);

        // In case the FileHashers have not already been started
        startFileHashers();

        waitAllFilesToBeHashed();

        overallTotalBytesHashed = 0;
        for (FileHasher fileHasher : fileHashers) {
            state.getFileStates().addAll(fileHasher.getFileStates());
            overallTotalBytesHashed += fileHasher.getTotalBytesHashed();
        }

        Collections.sort(state.getFileStates(), fileNameComparator);

        state.setIgnoredFiles(fimIgnoreManager.getIgnoredFiles());

        hashProgress.outputStop();
        displayStatistics(start, state);

        return state;
    }

    private void initializeFileHashers() {
        fileHashersStarted = false;
        fileHashers = new ArrayList<>();
        executorService = Executors.newFixedThreadPool(context.getThreadCount());
    }

    private void startFileHashers() throws NoSuchAlgorithmException {
        if (!fileHashersStarted) {
            String normalizedRootDir = FileUtil.getNormalizedFileName(rootDir);
            for (int index = 0; index < context.getThreadCount(); index++) {
                FileHasher hasher = new FileHasher(context, hashProgress, filesToHashQueue, normalizedRootDir);
                executorService.submit(hasher);
                fileHashers.add(hasher);
            }
            fileHashersStarted = true;
        }
    }

    private void waitAllFilesToBeHashed() {
        try {
            executorService.shutdown();
            executorService.awaitTermination(3, TimeUnit.DAYS);
        } catch (InterruptedException ex) {
            Logger.error("Exception while waiting for files to be hashed", ex, context.isDisplayStackTrace());
        }
    }

    private void displayStatistics(long start, State state) {
        long duration = System.currentTimeMillis() - start;

        String totalFileContentLengthStr = FileUtils.byteCountToDisplaySize(state.getFilesContentLength());
        String totalBytesHashedStr = FileUtils.byteCountToDisplaySize(overallTotalBytesHashed);
        String durationStr = DurationFormatUtils.formatDuration(duration, "HH:mm:ss");

        long durationSeconds = duration / 1000;
        if (durationSeconds <= 0) {
            durationSeconds = 1;
        }

        long globalThroughput = overallTotalBytesHashed / durationSeconds;
        String throughputStr = FileUtils.byteCountToDisplaySize(globalThroughput);

        if (context.getHashMode() == dontHash) {
            Logger.info(String.format("Scanned %d files (%s), during %s%n", state.getFileCount(),
                    totalFileContentLengthStr, durationStr));
        } else {
            Logger.info(
                    String.format("Scanned %d files (%s), hashed %s (avg %s/s), during %s%n", state.getFileCount(),
                            totalFileContentLengthStr, totalBytesHashedStr, throughputStr, durationStr));
        }
    }

    private void scanFileTree(BlockingDeque<Path> filesToHashQueue, Path directory, FimIgnore parentFimIgnore)
            throws NoSuchAlgorithmException {
        try (DirectoryStream<Path> stream = Files.newDirectoryStream(directory)) {
            FimIgnore fimIgnore = fimIgnoreManager.loadLocalIgnore(directory, parentFimIgnore);

            for (Path file : stream) {
                if (!fileHashersStarted && filesToHashQueue.size() > FILES_QUEUE_CAPACITY / 2) {
                    startFileHashers();
                }

                BasicFileAttributes attributes = Files.readAttributes(file, BasicFileAttributes.class,
                        LinkOption.NOFOLLOW_LINKS);
                String fileName = file.getFileName().toString();
                if (fimIgnoreManager.isIgnored(fileName, attributes, fimIgnore)) {
                    fimIgnoreManager.ignoreThisFiles(file, attributes);
                } else {
                    if (attributes.isRegularFile()) {
                        enqueueFile(filesToHashQueue, file);
                    } else if (attributes.isDirectory()) {
                        scanFileTree(filesToHashQueue, file, fimIgnore);
                    }
                }
            }
        } catch (IOException ex) {
            Console.newLine();
            Logger.error("Skipping - Error scanning directory '" + directory + "'", ex,
                    context.isDisplayStackTrace());
        }
    }

    private void enqueueFile(BlockingDeque<Path> filesToHashQueue, Path file) {
        try {
            filesToHashQueue.offer(file, 120, TimeUnit.MINUTES);
        } catch (InterruptedException ex) {
            Logger.error("Exception while enqueuing file '" + file + "'", ex, context.isDisplayStackTrace());
        }
    }
}