org.dataconservancy.packaging.tool.impl.IPMServiceImpl.java Source code

Java tutorial

Introduction

Here is the source code for org.dataconservancy.packaging.tool.impl.IPMServiceImpl.java

Source

package org.dataconservancy.packaging.tool.impl;

/*
 * Copyright 2015 Johns Hopkins University
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import org.apache.commons.io.DirectoryWalker;
import org.dataconservancy.packaging.tool.api.IPMService;
import org.dataconservancy.packaging.tool.api.support.NodeComparison;
import org.dataconservancy.packaging.tool.impl.support.FilenameValidatorService;
import org.dataconservancy.packaging.tool.model.ipm.FileInfo;
import org.dataconservancy.packaging.tool.model.ipm.Node;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class IPMServiceImpl implements IPMService {
    private Map<URI, File> fileUris = new HashMap<>();
    private Map<File, Node> fileToNode = new HashMap<>();
    private Set<Node> visitedFiles = new HashSet<>();
    private final URIGenerator uriGenerator;
    private FilenameValidatorService validatorService;
    private final Logger log = LoggerFactory.getLogger(this.getClass());

    public IPMServiceImpl(URIGenerator uriGenerator) {
        this.uriGenerator = uriGenerator;
        this.validatorService = new FilenameValidatorService();
    }

    @Override
    public Node createTreeFromFileSystem(Path path) throws IOException {

        visitedFiles.clear();
        fileUris.clear();
        fileToNode.clear();

        Walker walker = new Walker();

        Thread walkerThread = new Thread(() -> {
            try {
                walker.doWalk(path.toFile(), visitedFiles);
            } catch (IOException e) {
                walker.cancel();
            }
        });

        walkerThread.start();

        while (walkerThread.isAlive()) {
            if (Thread.currentThread().isInterrupted()) {
                walker.cancel();
            }
            try {
                walkerThread.join(1000 * 5);
            } catch (InterruptedException e) {
                // ignore
            }
        }

        visitedFiles.parallelStream().forEach(visitedNode -> {
            final File file = fileUris.get(visitedNode.getIdentifier());
            final FileInfo fi;
            try {
                fi = new FileInfo(file.toPath().toRealPath());
                visitedNode.setFileInfo(fi);
            } catch (IOException e) {
                log.warn("Unable to resolve file path '{}': {}", file.toPath(), e.getMessage(), e);
            }
        });

        return walker.getRoot();
    }

    @Override
    public void ignoreNode(Node node, boolean status) {
        if (node.isIgnored() == status) {
            return;
        }

        if (node.isIgnored()) {
            node.setIgnored(false);

            // Unignore ancestors
            for (Node n = node.getParent(); n != null && n.isIgnored(); n = n.getParent()) {
                n.setIgnored(false);
            }

            // Unignore descendants
            if (node.getChildren() != null) {
                for (Node child : node.getChildren()) {
                    ignoreNode(child, false);
                }
            }
        } else {
            node.setIgnored(true);

            // Ignore descendants
            if (node.getChildren() != null) {
                for (Node child : node.getChildren()) {
                    ignoreNode(child, true);
                }
            }
        }
    }

    @Override
    public boolean mergeTree(Node existingTree, Map<Node, NodeComparison> comparisonResult) {

        //Handle the case where root node has been deleted.
        if (comparisonResult.get(existingTree) != null
                && comparisonResult.get(existingTree).getStatus() == NodeComparison.Status.DELETED) {
            //In the case where we're building a new tree we need to find the new root.
            Node newRoot = null;
            for (Node node : comparisonResult.keySet()) {
                NodeComparison comparison = comparisonResult.get(node);
                if (comparison.getStatus() == NodeComparison.Status.ADDED && comparison.getNode() == null
                        && !node.equals(existingTree)) {
                    newRoot = node;
                }
            }

            if (newRoot == null) {
                return false;
            }

            //Assign the new root to the old root
            existingTree.setIdentifier(newRoot.getIdentifier());
            existingTree.setFileInfo(newRoot.getFileInfo());
            existingTree.setDomainObject(newRoot.getDomainObject());
            existingTree.getChildren().clear();

            existingTree.setChildren(newRoot.getChildren());
            existingTree.setNodeType(newRoot.getNodeType());
            existingTree.setSubNodeTypes(newRoot.getSubNodeTypes());

        } else {
            applyTreeChanges(comparisonResult);
        }
        return true;
    }

    private void applyTreeChanges(Map<Node, NodeComparison> comparisonMap) {
        for (Node key : comparisonMap.keySet()) {
            NodeComparison comparison = comparisonMap.get(key);

            switch (comparison.getStatus()) {
            case ADDED:
                if (comparison.getNode() != null) {
                    comparison.getNode().addChild(key);
                }
                break;
            case DELETED:
                if (comparison.getNode() != null) {
                    comparison.getNode().removeChild(key);
                }
                break;
            case UPDATED:
                if (comparison.getNode() != null) {
                    comparison.getNode().setFileInfo(key.getFileInfo());
                }
                break;
            }
        }
    }

    @Override
    public Map<Node, NodeComparison> compareTree(Node existingTree, Node comparisonTree) {

        Map<Node, NodeComparison> nodeMap = new HashMap<>();

        //Generate maps of existing locations in the trees
        Map<URI, Node> existingLocationMap = new HashMap<>();
        Map<URI, Node> comparisonLocationMap = new HashMap<>();

        addNodeLocation(existingTree, existingLocationMap);
        addNodeLocation(comparisonTree, comparisonLocationMap);

        for (URI location : existingLocationMap.keySet()) {
            if (comparisonLocationMap.containsKey(location)) {
                Node existingNode = existingLocationMap.get(location);
                Node comparisonNode = comparisonLocationMap.get(location);

                //This is a specialized case that occurs when we create a node in the tree with no backing file entity
                if (existingNode.getParent() != null && existingNode.getParent().getFileInfo() == null) {
                    checkFileUpdate(location, existingNode, comparisonNode, nodeMap, comparisonLocationMap);
                } else if (existingNode.getParent() != null && comparisonNode.getParent() == null) {
                    //In the event we refreshed on a sub node, the sub node is the root of the comparison tree so it's options are limited we can simply check for update.
                    //Note that this is only true because in the GUI currently you can't refresh a node whose file content is missing.
                    checkFileUpdate(location, existingNode, comparisonNode, nodeMap, comparisonLocationMap);
                } else if ((existingNode.getParent() == null && comparisonNode.getParent() == null)
                        || existingNode.getParent().getFileInfo().getLocation()
                                .equals(comparisonNode.getParent().getFileInfo().getLocation())) {
                    //If both nodes are root or the node's parent location is the same then we consider it the same
                    checkFileUpdate(location, existingNode, comparisonNode, nodeMap, comparisonLocationMap);
                } else {
                    //The node has moved so we consider it a delete and add.
                    markNodesRemoved(existingNode, existingNode.getParent(), nodeMap);

                    //Determine what the parent of the new node will be it will either be already in the tree, or a new node being added.
                    Node parent;

                    //If we've added the comparison node as the parent it will be in the map
                    if (nodeMap.get(comparisonNode.getParent()) != null) {
                        parent = comparisonNode.getParent();
                    } else {
                        parent = existingNode.getParent();
                    }
                    nodeMap.put(comparisonNode, new NodeComparison(NodeComparison.Status.ADDED, parent));
                    comparisonLocationMap.remove(location);
                }
            } else {
                markNodesRemoved(existingLocationMap.get(location), existingLocationMap.get(location).getParent(),
                        nodeMap);
            }
        }

        //Anything remaining in the comparison location map should be added
        if (!comparisonLocationMap.isEmpty()) {
            for (URI location : comparisonLocationMap.keySet()) {
                //Determine what the parent of the new node will be it will either be already in the tree, or a new node being added.
                Node parent = null;

                Node newNode = comparisonLocationMap.get(location);
                if (newNode.getParent() != null) {
                    if (existingLocationMap.get(newNode.getParent().getFileInfo().getLocation()) != null) {
                        parent = existingLocationMap.get(newNode.getParent().getFileInfo().getLocation());
                    }

                    //If the parent isn't existing then we must have added it
                    if (parent == null) {
                        parent = comparisonLocationMap.get(newNode.getParent().getFileInfo().getLocation());
                    }
                }
                nodeMap.put(newNode, new NodeComparison(NodeComparison.Status.ADDED, parent));
            }
        }

        return nodeMap;
    }

    private void checkFileUpdate(URI location, Node existingNode, Node comparisonNode,
            Map<Node, NodeComparison> nodeMap, Map<URI, Node> comparisonLocationMap) {
        if (existingNode.getFileInfo().isFile() && comparisonNode.getFileInfo().isFile()
                && !existingNode.getFileInfo().getChecksum(FileInfo.Algorithm.MD5)
                        .equalsIgnoreCase(comparisonNode.getFileInfo().getChecksum(FileInfo.Algorithm.MD5))
                && !existingNode.getFileInfo().getChecksum(FileInfo.Algorithm.SHA1)
                        .equalsIgnoreCase(comparisonNode.getFileInfo().getChecksum(FileInfo.Algorithm.SHA1))) {

            //The checksums are different so we consider this an update
            nodeMap.put(comparisonNode, new NodeComparison(NodeComparison.Status.UPDATED, existingNode));
            comparisonLocationMap.remove(location);
        } else {
            //The file location is completely unchanged and not updated
            comparisonLocationMap.remove(location);
        }
    }

    private void markNodesRemoved(Node node, Node parent, Map<Node, NodeComparison> nodeMap) {
        nodeMap.put(node, new NodeComparison(NodeComparison.Status.DELETED, parent));
        if (node.getChildren() != null) {
            for (Node child : node.getChildren()) {
                markNodesRemoved(child, node, nodeMap);
            }
        }
    }

    private void addNodeLocation(Node node, Map<URI, Node> nodeLocationMap) {

        //If the node has no file information leave it out of the map
        if (node.getFileInfo() != null) {
            nodeLocationMap.put(node.getFileInfo().getLocation(), node);
        }

        if (node.getChildren() != null) {
            for (Node child : node.getChildren()) {
                addNodeLocation(child, nodeLocationMap);
            }
        }
    }

    @Override
    public boolean checkFileInfoIsAccessible(Node node) {
        boolean accessible = false;

        if (node != null && node.getFileInfo() != null) {
            FileInfo info = node.getFileInfo();
            if (info.getLocation() != null) {
                accessible = Paths.get(info.getLocation()).toFile().exists();
            }
        }
        return accessible;
    }

    @Override
    public void remapNode(Node node, Path newPath) throws IOException {

        Path oldPath = null;
        if (node.getFileInfo().getLocation() != null) {
            oldPath = Paths.get(node.getFileInfo().getLocation());
        }
        node.setFileInfo(new FileInfo(newPath));

        //If we have an old path try to remap children
        if (oldPath != null && node.getChildren() != null) {
            //If this path can't be relativized we won't automatically remap
            final Path finalOldPath = oldPath;
            node.getChildren().stream().filter(child -> child.getFileInfo() != null).forEach(child -> {
                try {
                    Path oldRelativePath = finalOldPath.relativize(Paths.get(child.getFileInfo().getLocation()));
                    Path newChildPath = newPath.resolve(oldRelativePath);
                    if (newChildPath.toFile().exists()) {
                        remapNode(child, newChildPath);
                    }
                } catch (IllegalArgumentException | IOException e) {
                    //If this path can't be relativized we won't automatically remap
                }
            });
        }
    }

    @Override
    public Map<Node, NodeComparison> refreshTreeContent(Node node) throws IOException {
        Node newTree = buildComparisonTree(node);
        return compareTree(node, newTree);
    }

    /**
     * Builds a tree from the current file system to compare with the existing file system.
     * @param node The node from the existing tree that will be the root of the comparison
     * @return The root of the new tree to compare
     * @throws IOException If there is a problem reading from the file system.
     */
    private Node buildComparisonTree(Node node) throws IOException {
        Node newTree = createTreeFromFileSystem(Paths.get(node.getFileInfo().getLocation()));
        buildContentRoots(node, newTree);

        return newTree;
    }

    /**
     * Loops through the existing tree to find any content locations different from their parent, it then builds a tree from the file system under that location.
     * @param node The node to check for different content locations
     * @param newTree The new tree to add the tree from the file system to
     * @throws IOException If there is a problem reading from the file system.
     */
    private void buildContentRoots(Node node, Node newTree) throws IOException {
        if (node.getChildren() != null && node.getFileInfo() != null) {
            for (Node child : node.getChildren()) {
                if (child.getFileInfo() != null && Paths.get(child.getFileInfo().getLocation()).toFile().exists()) {
                    if (!Paths.get(child.getFileInfo().getLocation())
                            .startsWith(Paths.get(node.getFileInfo().getLocation()))) {
                        Node newTreeParent = getNewTreeNodeForExistingNode(node, newTree);
                        if (newTreeParent != null) {
                            newTreeParent.addChild(buildComparisonTree(child));
                        } else {
                            newTree.addChild(buildComparisonTree(child));
                        }
                    } else if (child.getChildren() != null) {
                        buildContentRoots(child, newTree);
                    }
                } else if (child.getChildren() != null) {
                    buildContentRoots(child, newTree);
                }
            }
        }
    }

    /**
     * Finds nodes in the new comparison tree that correspond to nodes in the existing tree.
     * This is used to ensure new content locations are placed in the correct spot in the tree.
     * @param node The node to find in the new tree.
     * @param newTree The new tree to search for the node.
     * @return The node from the new tree or false if none exists
     */
    private Node getNewTreeNodeForExistingNode(Node node, Node newTree) {
        Node foundNode = null;
        if (node.getFileInfo() != null && newTree.getFileInfo() != null
                && node.getFileInfo().getLocation().equals(newTree.getFileInfo().getLocation())) {
            foundNode = newTree;
        } else if (newTree.getChildren() != null) {
            for (Node newTreeChild : newTree.getChildren()) {
                foundNode = getNewTreeNodeForExistingNode(node, newTreeChild);
                if (foundNode != null) {
                    break;
                }
            }
        }

        return foundNode;
    }

    /**
     * This method will check filenames in the given path for validity against the Cata Conservancy BagIt profile
     * specification, version 1.0
     * @param path The path to check filenames for
     * @throws IOException
     */
    private void validateFileNames(Path path) throws IOException {
        List<String> invalidNamesList = validatorService.findInvalidFilenames(path);
        if (invalidNamesList != null && !invalidNamesList.isEmpty()) {
            String invalidNames = "";
            for (int i = 0; i < invalidNamesList.size(); i++) {
                invalidNames += invalidNamesList.get(i);

                if (i + 1 < invalidNamesList.size()) {
                    invalidNames += "\n";
                }
            }
            throw new IOException(
                    "Error creating package tree. The following names were invalid:\n\n" + invalidNames);
        }
    }

    private class Walker extends DirectoryWalker<Node> {

        private boolean verifyExists = true;

        private boolean ignoreHidden = true;

        private boolean ignoreDot = true;

        private Node root = null;

        private Path rootPath = null;

        private volatile boolean cancelled = false;

        private Walker() {

        }

        private Walker(boolean verifyExists, boolean ignoreHidden, boolean ignoreDot) {
            this.verifyExists = verifyExists;
            this.ignoreHidden = ignoreHidden;
            this.ignoreDot = ignoreDot;
        }

        private void doWalk(File baseDir, Collection<Node> results) throws IOException {
            super.walk(baseDir, results);
        }

        @Override
        protected boolean handleDirectory(File directory, int depth, Collection<Node> results) throws IOException {
            return handle(directory, depth, results);
        }

        @Override
        protected void handleFile(File file, int depth, Collection<Node> results) throws IOException {
            handle(file, depth, results);
        }

        private boolean handle(File file, int depth, Collection<Node> results) throws IOException {
            final Path fileAsPath = file.toPath();

            if (Files.isSymbolicLink(fileAsPath) && fileAsPath.toRealPath().startsWith(rootPath)) {
                // ignore: the symbolic link is targeting a file underneath the root
                return false;
            }

            final URI nodeUri = uriGenerator.generateNodeURI();
            final Node node = new Node(nodeUri);

            fileUris.put(nodeUri, file);
            fileToNode.put(file, node);
            results.add(node);

            if (root == null) {
                root = node;
                rootPath = fileAsPath.toRealPath();
            } else {
                Node parent = fileToNode.get(file.getParentFile());
                node.setParent(parent);
                parent.addChild(node);
            }

            if (Files.isHidden(fileAsPath) && ignoreHidden) {
                node.setIgnored(true);
            }

            if (fileAsPath.getFileName().toString().startsWith(".") && ignoreDot) {
                node.setIgnored(true);
            }

            if (node.getParent() != null && node.getParent().isIgnored()) {
                node.setIgnored(node.getParent().isIgnored());
            }

            if (!verifyExists || file.exists()) {
                results.add(node);
                return true;
            }

            return false;
        }

        private void cancel() {
            cancelled = true;
        }

        @Override
        protected void handleCancelled(File startDirectory, Collection<Node> results, CancelException cancel)
                throws IOException {
            visitedFiles.clear();
            fileUris.clear();
            fileToNode.clear();
        }

        @Override
        protected boolean handleIsCancelled(File file, int depth, Collection<Node> results) throws IOException {
            return cancelled;
        }

        private Node getRoot() {
            return root;
        }
    }
}