com.antelink.sourcesquare.client.scan.SourceSquareFSWalker.java Source code

Java tutorial

Introduction

Here is the source code for com.antelink.sourcesquare.client.scan.SourceSquareFSWalker.java

Source

/**
 * Copyright (C) 2009-2012 Antelink SAS
 *
 * This program is free software; you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License Version 3 as published
 * by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License Version 3
 * for more details.
 *
 * You should have received a copy of the GNU Affero General Public License Version
 * 3 along with this program. If not, see http://www.gnu.org/licenses/agpl.html
 *
 * Additional permission under GNU AGPL version 3 section 7
 *
 * If you modify this Program, or any covered work, by linking or combining it with
 * Eclipse Java development tools (JDT) or Jetty (or a modified version of these
 * libraries), containing parts covered by the terms of Eclipse Public License 1.0,
 * the licensors of this Program grant you additional permission to convey the
 * resulting work. Corresponding Source for a non-source form of such a combination
 * shall include the source code for the parts of Eclipse Java development tools
 * (JDT) or Jetty used as well as that of the covered work.
 */
package com.antelink.sourcesquare.client.scan;

import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import java.util.TreeSet;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.antelink.sourcesquare.TreemapNode;
import com.antelink.sourcesquare.event.base.EventBus;
import com.antelink.sourcesquare.event.events.ErrorEvent;
import com.antelink.sourcesquare.event.events.FilesIdentifiedEvent;
import com.antelink.sourcesquare.event.events.HiddenFileFoundEvent;
import com.antelink.sourcesquare.event.events.ScanCompleteEvent;
import com.antelink.sourcesquare.event.events.StartScanEvent;
import com.antelink.sourcesquare.event.handlers.FilesIdentifiedEventHandler;
import com.antelink.sourcesquare.event.handlers.StartScanEventHandler;
import com.antelink.sourcesquare.results.TreeMapBuilder;

public class SourceSquareFSWalker {

    private final static Log logger = LogFactory.getLog(SourceSquareFSWalker.class);

    private static final int MAX_FILE_PER_QUERY = 400;

    private static final int COMPUTE_WAIT_TIME = 1000;

    private final EventBus eventBus;

    private final TreeMapBuilder treemap;

    private int filePerQuery = 2;

    private final ArrayList<ProcessWorker> workers = new ArrayList<ProcessWorker>();

    /*contains number of folders per level*/
    private final ArrayList<Integer> levels;

    private final Object lock;

    private long total = 0;

    public SourceSquareFSWalker(SourceSquareEngine engine, EventBus eventBus, TreeMapBuilder treemap) {
        this.eventBus = eventBus;
        this.levels = new ArrayList<Integer>();
        this.lock = new Object();
        this.treemap = treemap;
        this.workers.add(new ProcessWorker(0, engine, this.lock));
        this.workers.add(new ProcessWorker(1, engine, this.lock));
        this.workers.add(new ProcessWorker(2, engine, this.lock));
    }

    public void bind() {
        this.eventBus.addHandler(StartScanEvent.TYPE, new StartScanEventHandler() {

            @Override
            public String getId() {
                return getClass().getCanonicalName() + ": " + StartScanEventHandler.class.getName();
            }

            @Override
            public void handle(File toScan) {
                logger.info("Counting files...");
                SourceSquareFSWalker.this.identifyFiles(toScan);
            }
        });

        this.eventBus.addHandler(FilesIdentifiedEvent.TYPE, new FilesIdentifiedEventHandler() {

            @Override
            public String getId() {
                return getClass().getCanonicalName() + ": " + StartScanEventHandler.class.getName();
            }

            @Override
            public void handle(TreeSet<File> fileSet) {
                try {
                    logger.info("Start scan for " + fileSet.size() + " files");
                    ScanStatus.INSTANCE.start();
                    SourceSquareFSWalker.this.queryFiles(fileSet);
                } catch (Exception e) {
                    logger.debug("Error handling tree identification", e);
                }

            }
        });
    }

    public synchronized void queryFiles(TreeSet<File> fileSet) throws InterruptedException {
        HashMap<String, String> toAnalyze = new HashMap<String, String>();
        Iterator<File> iterator = fileSet.iterator();
        logger.debug(fileSet.size() + " files to analyze");
        long count = 0;
        long timer = System.currentTimeMillis();
        while (iterator.hasNext()) {

            File file = iterator.next();
            logger.trace("adding analyze file to the pool: " + file.getAbsolutePath());
            try {
                String sha1 = FileAnalyzer.calculateHash("SHA-1", file);
                toAnalyze.put(file.getAbsolutePath(), sha1);
                count++;
            } catch (Exception e) {
                logger.error("skipping files " + file, e);
            }

            if (toAnalyze.size() == this.filePerQuery || System.currentTimeMillis() - timer > COMPUTE_WAIT_TIME) {
                // dispatch analysis
                timer = System.currentTimeMillis();
                analyzeMap(toAnalyze);
                this.filePerQuery = Math.min(MAX_FILE_PER_QUERY, this.filePerQuery * 2);
                logger.trace("new counter: " + count);

            }
        }
        analyzeMap(toAnalyze);

        while (!allProcessDone()) {
            synchronized (this.lock) {
                this.lock.wait();
            }

        }
        this.eventBus.fireEvent(new ScanCompleteEvent(this.levels));

        logger.info("Analysis done " + count);
    }

    private synchronized void analyzeMap(HashMap<String, String> tempMap) throws InterruptedException {
        if (tempMap == null || tempMap.isEmpty()) {
            return;
        }
        ProcessWorker worker = null;
        while ((worker = getAvailableProcessor()) == null) {
            synchronized (this.lock) {
                this.lock.wait();
            }

        }
        worker.process(new HashMap<String, String>(tempMap));
        tempMap.clear();
    }

    public void identifyFiles(File directory) {

        logger.debug("counting files for: " + directory.getAbsolutePath());
        TreeSet<File> fileSet = new TreeSet<File>();
        try {
            TreemapNode root = reccursiveIdentifyFiles(directory, fileSet, 0);
            this.treemap.setRoot(root);
            this.eventBus.fireEvent(new FilesIdentifiedEvent(fileSet));
        } catch (OutOfMemoryError e) {
            this.eventBus.fireEvent(new ErrorEvent(
                    "Out of memory: Try again with a smaller directory\nor change your JVM parameters."));
        }

    }

    private TreemapNode reccursiveIdentifyFiles(File directory, TreeSet<File> fileSet, int depth) {

        if (this.levels.size() < depth + 1) {
            this.levels.add(0);
        }

        logger.trace("Counting going down to directory : " + directory.getAbsolutePath());
        if (directory.isHidden()) {
            this.eventBus.fireEvent(new HiddenFileFoundEvent(directory));
            return null;
        }
        if (directory.isFile()) {
            fileSet.add(directory);
            return null;
        }
        if (!directory.isDirectory()) {
            return null;
        }
        // Protection if the directory forbids listing
        if (directory.listFiles() == null) {
            return null;
        }
        Set<TreemapNode> children = new HashSet<TreemapNode>();
        int nbFiles = 0;
        int nbDirs = 0;
        for (File child : directory.listFiles()) {
            if (child.isDirectory() || !child.isHidden()) {
                if (child.isDirectory()) {
                    nbDirs++;
                    TreemapNode childNode = reccursiveIdentifyFiles(child, fileSet, depth + 1);
                    if (childNode != null) {
                        children.add(childNode);
                    }
                } else if (child.isFile()) {
                    nbFiles++;
                    fileSet.add(child);
                }
            }
        }
        this.total = this.total + nbFiles;
        ScanStatus.INSTANCE.setNbFilesToScan((int) this.total);
        this.levels.set(depth, this.levels.get(depth) + nbDirs);
        return this.treemap.createTreeMapNode(directory.getAbsolutePath(), children, nbFiles);
    }

    private ProcessWorker getAvailableProcessor() {
        for (ProcessWorker worker : this.workers) {
            if (worker.isAvailable()) {
                return worker;
            }
        }
        return null;

    }

    private boolean allProcessDone() {
        for (ProcessWorker worker : this.workers) {
            if (!worker.isAvailable()) {
                return false;
            }
        }
        return true;
    }

}