org.yamj.filescanner.ScannerManagementImpl.java Source code

Java tutorial

Introduction

Here is the source code for org.yamj.filescanner.ScannerManagementImpl.java

Source

/*
 *      Copyright (c) 2004-2013 YAMJ Members
 *      https://github.com/organizations/YAMJ/teams
 *
 *      This file is part of the Yet Another Media Jukebox (YAMJ).
 *
 *      YAMJ is free software: you can redistribute it and/or modify
 *      it under the terms of the GNU General Public License as published by
 *      the Free Software Foundation, either version 3 of the License, or
 *      any later version.
 *
 *      YAMJ is distributed in the hope that it will be useful,
 *      but WITHOUT ANY WARRANTY; without even the implied warranty of
 *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *      GNU General Public License for more details.
 *
 *      You should have received a copy of the GNU General Public License
 *      along with YAMJ.  If not, see <http://www.gnu.org/licenses/>.
 *
 *      Web: https://github.com/YAMJ/yamj-v3
 *
 */
package org.yamj.filescanner;

import org.yamj.common.cmdline.CmdLineParser;
import org.yamj.common.dto.StageDirectoryDTO;
import org.yamj.common.dto.StageFileDTO;
import org.yamj.common.remote.service.GitHubService;
import org.yamj.common.tools.PropertyTools;
import org.yamj.common.type.DirectoryType;
import org.yamj.common.type.ExitType;
import org.yamj.filescanner.comparator.FileTypeComparator;
import org.yamj.filescanner.model.Library;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yamj.filescanner.model.LibraryCollection;
import org.yamj.filescanner.model.StatType;
import org.yamj.filescanner.service.SystemInfoCore;
import org.yamj.filescanner.tools.DirectoryEnding;
import org.yamj.filescanner.tools.Watcher;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.concurrent.ConcurrentUtils;
import org.joda.time.DateTime;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.remoting.RemoteAccessException;
import org.springframework.remoting.RemoteConnectFailureException;
import org.springframework.util.CollectionUtils;
import org.yamj.common.model.YamjInfo;
import org.yamj.common.tools.StringTools;
import org.yamj.common.type.StatusType;
import org.yamj.common.util.KeywordMap;
import org.yamj.filescanner.model.TimeType;

/**
 * Performs an initial scan of the library location and then updates when changes occur.
 *
 * @author Stuart
 */
public class ScannerManagementImpl implements ScannerManagement {

    /*
     * TODO: choose between watcher process and simple re-scan
     * TODO: determine what files have changed between scans
     */
    private static final Logger LOG = LoggerFactory.getLogger(ScannerManagementImpl.class);
    // The default watched status
    private static final Boolean DEFAULT_WATCH_STATE = PropertyTools.getBooleanProperty("filescanner.watch.default",
            Boolean.FALSE);
    @Autowired
    private LibraryCollection libraryCollection;
    @Autowired
    private SystemInfoCore pingCore;
    @Autowired
    private GitHubService githubService;
    // ImportDTO constants
    private static final String DEFAULT_CLIENT = PropertyTools.getProperty("filescanner.default.client",
            "FileScanner");
    private static final String DEFAULT_PLAYER_PATH = PropertyTools.getProperty("filescanner.default.playerpath",
            "");
    private static final String DEFAULT_SPLIT = ",|;";
    private static final String FILE_MJBIGNORE = ".mjbignore";
    // Date check
    private static final int MAX_INSTALL_AGE = PropertyTools.getIntProperty("filescanner.installation.maxdays", 1);
    // Map of filenames & extensions that cause scanning of a directory to stop or a filename to be ignored
    private static final Map<String, List<String>> DIR_EXCLUSIONS = new HashMap<String, List<String>>();
    private static final List<Pattern> DIR_IGNORE_FILES;
    // YAMJ Information
    private static final YamjInfo YAMJ_INFO = new YamjInfo(ScannerManagementImpl.class);

    static {
        // Set up the break scanning list. A "null" for the list means all files.
        // Ensure all filenames and extensions are lowercase
        boolean nmjCompliant = PropertyTools.getBooleanProperty("filescanner.nmjCompliant", Boolean.FALSE);
        KeywordMap fsIgnore = PropertyTools.getKeywordMap("filescanner.ignore", "");

        DIR_EXCLUSIONS.put(FILE_MJBIGNORE, null);
        if (nmjCompliant) {
            DIR_EXCLUSIONS.put(".no_all.nmj", null);
        }

        List<String> keywordList = processKeywords(fsIgnore, "file");
        if (CollectionUtils.isEmpty(keywordList)) {
            DIR_IGNORE_FILES = Collections.emptyList();
        } else {
            DIR_IGNORE_FILES = new ArrayList<Pattern>(keywordList.size());
            for (String keyword : keywordList) {
                try {
                    String regex = keyword.replace("?", ".?").replace("*", ".*?");
                    LOG.debug("Replaced pattern '{}' with regex '{}'", keyword, regex);
                    DIR_IGNORE_FILES.add(Pattern.compile(regex));
                } catch (PatternSyntaxException ex) {
                    LOG.warn("Pattern '{}' not recognised. Error: {}", keyword, ex.getMessage());
                }
            }
        }

        keywordList = processKeywords(fsIgnore, "dir");
        if (!CollectionUtils.isEmpty(keywordList)) {
            for (String keyword : keywordList) {
                DIR_EXCLUSIONS.put(keyword.toLowerCase(), null);
            }
        }

        keywordList = processKeywords(fsIgnore, "video");
        if (!keywordList.isEmpty()) {
            DIR_EXCLUSIONS.put(".no_video.yamj", keywordList);
            if (nmjCompliant) {
                DIR_EXCLUSIONS.put(".no_video.nmj", keywordList);
            }
        }

        keywordList = processKeywords(fsIgnore, "image");
        if (!keywordList.isEmpty()) {
            DIR_EXCLUSIONS.put(".no_image.yamj", keywordList);
            if (nmjCompliant) {
                DIR_EXCLUSIONS.put(".no_photo.nmj", keywordList);
            }
        }

        keywordList = processKeywords(fsIgnore, "other");
        if (!keywordList.isEmpty()) {
            DIR_EXCLUSIONS.put(".no_other.yamj", keywordList);
        }
        LOG.debug("Directory exclusions: {}", DIR_EXCLUSIONS.toString());
        LOG.debug("File exclusions: {}", DIR_IGNORE_FILES);
    }

    private static List<String> processKeywords(KeywordMap fsIgnore, String keyName) {
        if (fsIgnore.containsKey(keyName) && StringUtils.isNotBlank(fsIgnore.get(keyName))) {
            return StringTools.splitList(fsIgnore.get(keyName), DEFAULT_SPLIT);
        } else {
            return Collections.emptyList();
        }
    }

    /**
     * Start the scanner and process the command line properties.
     *
     * @param parser
     * @return
     */
    @Override
    public ExitType runScanner(CmdLineParser parser) {
        checkGitHubStatus();
        libraryCollection.setDefaultClient(DEFAULT_CLIENT);
        libraryCollection.setDefaultPlayerPath(DEFAULT_PLAYER_PATH);
        pingCore.check(0, 0); // Do a quick check of the status of the connection

        String directoryProperty = parser.getParsedOptionValue("d");
        boolean watchEnabled = parseWatchStatus(parser.getParsedOptionValue("w"));
        String libraryFilename = parser.getParsedOptionValue("l");

        if (StringUtils.isNotBlank(libraryFilename)) {
            List<String> libraryList = Arrays.asList(libraryFilename.split(DEFAULT_SPLIT));
            libraryCollection.processLibraryList(libraryList, watchEnabled);
        }

        if (StringUtils.isNotBlank(directoryProperty)) {
            LOG.info("Adding directory from command line: {}", directoryProperty);
            libraryCollection.addLibraryDirectory(directoryProperty, watchEnabled);
        }

        LOG.info("Found {} libraries to process.", libraryCollection.size());
        if (libraryCollection.size() == 0) {
            return ExitType.NO_DIRECTORY;
        }

        //        String saveFilename="myLibrary.xml";
        //        LOG.info("Saving library to: {}",saveFilename);
        //        libraryCollection.saveLibraryFile(saveFilename);

        // Send all libraries to be scanned
        ExitType status = ExitType.SUCCESS;
        for (Library library : libraryCollection.getLibraries()) {
            library.getStatistics().setTime(TimeType.START);
            status = scan(library);
            library.getStatistics().setTime(TimeType.END);
            library.setScanningComplete(Boolean.TRUE);
            LOG.info("Scanning completed.");
        }

        // Wait for the libraries to be sent
        boolean allDone;
        do {
            allDone = Boolean.TRUE;
            for (Library library : libraryCollection.getLibraries()) {
                LOG.info("Library '{}' sending status: {}", library.getImportDTO().getBaseDirectory(),
                        library.isSendingComplete() ? "Done" : "Not Done");
                allDone = allDone && library.isSendingComplete();
            }

            if (!allDone) {
                try {
                    LOG.info("Waiting for library sending to complete...");
                    TimeUnit.SECONDS.sleep(10);
                } catch (InterruptedException ex) {
                    LOG.trace("Interrupted whilst waiting for threads to complete.");
                }
            }
        } while (!allDone);

        LOG.info(StringUtils.repeat("*", 50));
        LOG.info("Completed initial sending of all libraries ({} total).", libraryCollection.size());
        LOG.info("");
        LOG.info("Library statistics:");
        for (Library library : libraryCollection.getLibraries()) {
            LOG.info("Description: '{}'", library.getDescription());
            LOG.info("{}", library.getStatistics().generateStatistics(Boolean.TRUE));
        }

        if (watchEnabled) {
            Watcher wd = new Watcher();
            Boolean directoriesToWatch = Boolean.TRUE;

            for (Library library : libraryCollection.getLibraries()) {
                String dirToWatch = library.getImportDTO().getBaseDirectory();
                if (library.isWatch()) {
                    LOG.info("Watching directory '{}' for changes...", dirToWatch);
                    wd.addDirectory(dirToWatch);
                    directoriesToWatch = Boolean.TRUE;
                } else {
                    LOG.info("Watching skipped for directory '{}'", dirToWatch);
                }
            }

            if (directoriesToWatch) {
                wd.processEvents();
                LOG.info("Watching directory '{}' completed", directoryProperty);
            } else {
                LOG.info("No directories marked for watching.");
            }
        } else {
            LOG.info("Watching not enabled.");
        }

        LOG.info("Exiting with status {}", status);

        return status;
    }

    /**
     * Start scanning a library.
     *
     * @param library
     * @return
     */
    private ExitType scan(Library library) {
        library.getStatistics().setTime(TimeType.SCANNING_START);

        ExitType status = ExitType.SUCCESS;
        File baseDirectory = new File(library.getImportDTO().getBaseDirectory());
        LOG.info("Scanning library '{}'...", baseDirectory.getAbsolutePath());

        if (!baseDirectory.exists()) {
            LOG.info("Failed to read directory '{}'", baseDirectory.getAbsolutePath());
            return ExitType.NO_DIRECTORY;
        }

        scanDir(library, baseDirectory);

        library.getStatistics().setTime(TimeType.SCANNING_END);
        return status;
    }

    /**
     * Scan a directory (and recursively any other directories contained
     *
     * @param library
     * @param parentDto
     * @param directory
     */
    private StageDirectoryDTO scanDir(Library library, File directory) {
        DirectoryType dirType = DirectoryEnding.check(directory);
        StageDirectoryDTO stageDir;

        LOG.info("Scanning directory '{}', detected type - {}", library.getRelativeDir(directory), dirType);

        if (dirType == DirectoryType.BLURAY || dirType == DirectoryType.DVD) {
            // Don't scan BLURAY or DVD structures
            LOG.info("Skipping directory '{}' as its a {} type", directory.getAbsolutePath(), dirType);
            library.getStatistics().increment(dirType == DirectoryType.BLURAY ? StatType.BLURAY : StatType.DVD);
            stageDir = null;
        } else if (DIR_EXCLUSIONS.containsKey(directory.getName().toLowerCase())) {
            LOG.info("Skipping directory '{}' as its in the exclusion list.", directory.getAbsolutePath());
            stageDir = null;
        } else {
            try {
                if (FileUtils.directoryContains(directory, new File(directory, FILE_MJBIGNORE))) {
                    LOG.debug("Exclusion file '{}' found, skipping scanning of directory {}.", FILE_MJBIGNORE,
                            directory.getName());
                    return null;
                }
            } catch (IOException ex) {
                LOG.trace("Failed to seach for '{}' in the directory {}", FILE_MJBIGNORE, directory.getName());
            }

            stageDir = new StageDirectoryDTO();
            stageDir.setPath(directory.getAbsolutePath());
            stageDir.setDate(directory.lastModified());

            library.getStatistics().increment(StatType.DIRECTORY);

            List<File> currentFileList = Arrays.asList(directory.listFiles());
            FileTypeComparator comp = new FileTypeComparator(Boolean.FALSE);
            Collections.sort(currentFileList, comp);

            /*
             * We need to scan the directory and look for any of the exclusion filenames.
             *
             * We then build a list of those excluded extensions, so that when we scan the filename list we can exclude the unwanted files.
             */
            List<String> exclusions = new ArrayList<String>();
            for (File file : currentFileList) {
                if (file.isFile()) {
                    String lcFilename = file.getName().toLowerCase();
                    if (DIR_EXCLUSIONS.containsKey(lcFilename)) {
                        if (CollectionUtils.isEmpty(DIR_EXCLUSIONS.get(lcFilename))) {
                            // Because the value is null or empty we exclude the whole directory, so quit now.
                            LOG.debug("Exclusion file '{}' found, skipping scanning of directory {}.", lcFilename,
                                    file.getParent());
                            // All files to be excluded, so quit
                            return null;
                        } else {
                            // We found a match, so add it to our local copy
                            LOG.debug("Exclusion file '{}' found, will exclude all {} file types", lcFilename,
                                    DIR_EXCLUSIONS.get(lcFilename).toString());
                            exclusions.addAll(DIR_EXCLUSIONS.get(lcFilename));
                            // Skip to the next file, theres no need of further processing
                            continue;
                        }
                    }
                } else {
                    // First directory we find, we can stop (because we sorted the files first)
                    break;
                }
            }

            // Create a precompiled Matcher for use later (Doesn't matter what the values are)
            Matcher matcher = Pattern.compile(FILE_MJBIGNORE).matcher(FILE_MJBIGNORE);

            // Scan the directory properly
            for (File file : currentFileList) {
                boolean excluded = Boolean.FALSE;
                if (file.isFile()) {
                    String lcFilename = file.getName().toLowerCase();
                    if (exclusions.contains(FilenameUtils.getExtension(lcFilename))
                            || DIR_EXCLUSIONS.containsKey(lcFilename)) {
                        LOG.debug(
                                "File name '{}' excluded because it's listed in the exlusion list for this directory",
                                file.getName());
                        continue;
                    }

                    // Process the DIR_IGNORE_FILES
                    for (Pattern pattern : DIR_IGNORE_FILES) {
                        matcher.reset(lcFilename).usePattern(pattern);
                        if (matcher.matches()) {
                            // Found the file pattern, so skip the file
                            LOG.debug("File name '{}' excluded because it matches exlusion pattern '{}'",
                                    file.getName(), pattern.pattern());
                            excluded = Boolean.TRUE;
                            break;
                        }
                    }

                    if (!excluded) {
                        stageDir.addStageFile(scanFile(file));
                        library.getStatistics().increment(StatType.FILE);
                    }
                } else {
                    // First directory we find, we can stop (because we sorted the files first)
                    break;
                }
            }

            library.addDirectory(stageDir);
            queueForSending(library, stageDir);

            // Resort the files with directories first
            comp.setDirectoriesFirst(Boolean.TRUE);
            Collections.sort(currentFileList, comp);

            // Now scan the directories
            for (File scanDir : currentFileList) {
                if (scanDir.isDirectory()) {
                    if (scanDir(library, scanDir) == null) {
                        LOG.info("Not adding directory '{}', no files found or all excluded",
                                scanDir.getAbsolutePath());
                    }
                } else {
                    // First file we find, we can stop (because we are sorted directories first)
                    break;
                }
            }
        }
        return stageDir;
    }

    /**
     * Scan an individual file
     *
     * @param library
     * @param parentDto
     * @param file
     */
    private StageFileDTO scanFile(File file) {
        LOG.info("Scanning file '{}'", file.getName());
        return new StageFileDTO(file);
    }

    /**
     * Get the watched status from the command line property or return the default value.
     *
     * @param parsedOptionValue the property from the command line
     * @return
     */
    private boolean parseWatchStatus(String parsedOptionValue) {
        if (StringUtils.isBlank(parsedOptionValue)) {
            return DEFAULT_WATCH_STATE;
        }
        return Boolean.parseBoolean(parsedOptionValue);
    }

    private void checkGitHubStatus() {
        try {
            DateTime fsDate = YAMJ_INFO.getBuildDateTime();
            boolean installationOk = githubService.checkInstallationDate(fsDate, MAX_INSTALL_AGE);

            if (installationOk) {
                LOG.info("Installation is less than {} days old.", MAX_INSTALL_AGE);
            } else {
                LOG.error("***** Your installation is more than {} days old. You should consider updating! *****",
                        MAX_INSTALL_AGE);
            }
        } catch (RemoteConnectFailureException ex) {
            LOG.warn("Failed to get GitHub status, error: {}", ex.getMessage());
        } catch (RemoteAccessException ex) {
            LOG.warn("Failed to get GitHub status, error: {}", ex.getMessage());
        }
    }

    /**
     * Add the file to the library for sending to the core
     *
     * @param library
     * @param stageDir
     */
    private void queueForSending(Library library, StageDirectoryDTO stageDir) {
        library.addDirectoryStatus(stageDir.getPath(), ConcurrentUtils.constantFuture(StatusType.NEW));
    }
}