org.roda.core.common.monitor.TransferredResourcesScanner.java Source code

Java tutorial

Introduction

Here is the source code for org.roda.core.common.monitor.TransferredResourcesScanner.java

Source

/**
 * The contents of this file are subject to the license and copyright
 * detailed in the LICENSE file at the root of the source
 * tree and available online at
 *
 * https://github.com/keeps/roda
 */
package org.roda.core.common.monitor;

import java.io.IOException;
import java.io.InputStream;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.FileVisitOption;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.apache.commons.lang3.StringUtils;
import org.roda.core.RodaCoreFactory;
import org.roda.core.common.iterables.CloseableIterable;
import org.roda.core.data.common.RodaConstants;
import org.roda.core.data.exceptions.AlreadyExistsException;
import org.roda.core.data.exceptions.GenericException;
import org.roda.core.data.exceptions.IsStillUpdatingException;
import org.roda.core.data.exceptions.NotFoundException;
import org.roda.core.data.exceptions.RequestNotValidException;
import org.roda.core.data.v2.LiteRODAObject;
import org.roda.core.data.v2.common.OptionalWithCause;
import org.roda.core.data.v2.index.filter.Filter;
import org.roda.core.data.v2.index.filter.SimpleFilterParameter;
import org.roda.core.data.v2.ip.TransferredResource;
import org.roda.core.index.IndexService;
import org.roda.core.model.LiteRODAObjectFactory;
import org.roda.core.storage.ContentPayload;
import org.roda.core.storage.fs.FSUtils;
import org.roda.core.util.IdUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TransferredResourcesScanner {
    private static final Logger LOGGER = LoggerFactory.getLogger(TransferredResourcesScanner.class);
    private static final List<String> fieldsToReturn = Arrays.asList(RodaConstants.INDEX_UUID,
            RodaConstants.TRANSFERRED_RESOURCE_RELATIVEPATH);

    private final Path basePath;
    private IndexService index;

    public TransferredResourcesScanner(Path basePath, IndexService index) {
        this.basePath = basePath;
        this.index = index;
    }

    public void commit() throws GenericException {
        index.commit(TransferredResource.class);
    }

    public Path getBasePath() {
        return basePath;
    }

    public TransferredResource createFolder(String parentUUID, String folderName)
            throws GenericException, RequestNotValidException, NotFoundException {
        Path parentPath;
        if (parentUUID != null) {
            TransferredResource parent = index.retrieve(TransferredResource.class, parentUUID, fieldsToReturn);
            parentPath = basePath.resolve(parent.getRelativePath());
        } else {
            parentPath = basePath;
        }

        try {
            Path createdPath = Files.createDirectories(parentPath.resolve(folderName));
            BasicFileAttributes attrs = Files.readAttributes(createdPath, BasicFileAttributes.class);
            TransferredResource resource = createTransferredResource(createdPath, attrs, 0L, basePath, new Date());
            index.create(TransferredResource.class, resource);
            return resource;
        } catch (IOException e) {
            LOGGER.error("Cannot create folder", e);
            throw new GenericException("Cannot create folder", e);
        }
    }

    public TransferredResource createFile(String parentUUID, String fileName, InputStream inputStream)
            throws GenericException, RequestNotValidException, NotFoundException, AlreadyExistsException {
        Path parentPath;
        if (StringUtils.isNotBlank(parentUUID)) {
            TransferredResource parent = index.retrieve(TransferredResource.class, parentUUID, fieldsToReturn);
            parentPath = basePath.resolve(parent.getRelativePath());
        } else {
            parentPath = basePath;
        }

        Path file = parentPath.resolve(fileName);
        try {
            try {
                Files.createDirectories(parentPath);
            } catch (FileAlreadyExistsException e) {
                // do nothing and carry on
            }

            Files.copy(inputStream, file);
            BasicFileAttributes attrs = Files.readAttributes(file, BasicFileAttributes.class);
            TransferredResource resource = createTransferredResource(file, attrs, attrs.size(), basePath,
                    new Date());
            index.create(TransferredResource.class, resource);
            return resource;
        } catch (FileAlreadyExistsException e) {
            LOGGER.error("Cannot create file", e);
            throw new AlreadyExistsException(file.toString());
        } catch (IOException e) {
            LOGGER.error("Cannot create file", e);
            throw new GenericException("Cannot create file", e);
        }
    }

    public InputStream retrieveFile(String path)
            throws NotFoundException, RequestNotValidException, GenericException {
        InputStream ret;
        Path p = basePath.resolve(path);
        if (!FSUtils.exists(p)) {
            throw new NotFoundException("File not found: " + path);
        } else if (!FSUtils.isFile(p)) {
            throw new RequestNotValidException("Requested file is not a regular file: " + path);
        } else {
            try {
                ret = Files.newInputStream(p);
            } catch (IOException e) {
                throw new GenericException("Could not create input stream: " + e.getMessage());
            }
        }
        return ret;
    }

    protected static TransferredResource createTransferredResource(Path resourcePath, BasicFileAttributes attr,
            long size, Path basePath, Date lastScanDate) {
        Date d = new Date(attr.creationTime().toMillis());

        TransferredResource tr = instantiateTransferredResource(resourcePath, basePath);
        tr.setSize(size);
        tr.setCreationDate(d);
        tr.setLastScanDate(lastScanDate);

        return tr;
    }

    public static TransferredResource instantiateTransferredResource(Path resourcePath, Path basePath) {
        Path relativeToBase = basePath.relativize(resourcePath);
        TransferredResource tr = new TransferredResource();

        tr.setFile(!FSUtils.isDirectory(resourcePath));
        tr.setFullPath(resourcePath.toString());
        String id = relativeToBase.toString();
        tr.setId(id);
        tr.setUUID(IdUtils.getTransferredResourceUUID(relativeToBase));
        tr.setName(resourcePath.getFileName().toString());

        tr.setRelativePath(relativeToBase.toString());
        if (relativeToBase.getParent() != null) {
            String parentId = relativeToBase.getParent().toString();
            tr.setParentId(parentId);
            tr.setParentUUID(IdUtils.createUUID(parentId));
        }

        List<String> ancestors = new ArrayList<>();

        StringBuilder temp = new StringBuilder();
        Iterator<Path> pathIterator = relativeToBase.iterator();
        while (pathIterator.hasNext()) {
            temp.append(pathIterator.next().toString());
            ancestors.add(temp.toString());
            temp.append("/");
        }
        ancestors.remove(ancestors.size() - 1);
        tr.setAncestorsPaths(ancestors);

        return tr;
    }

    public void deleteTransferredResource(List<String> ids)
            throws NotFoundException, GenericException, RequestNotValidException {
        for (String uuid : ids) {
            TransferredResource tr = index.retrieve(TransferredResource.class, uuid, fieldsToReturn);
            Path relative = Paths.get(tr.getRelativePath());
            Path fullPath = basePath.resolve(relative);
            if (FSUtils.exists(fullPath)) {
                FSUtils.deletePath(fullPath);

                Filter filter = new Filter(new SimpleFilterParameter(RodaConstants.TRANSFERRED_RESOURCE_ANCESTORS,
                        relative.toString()));
                index.delete(TransferredResource.class, filter);
            } else {
                throw new NotFoundException("Path does not exist: " + fullPath);
            }
        }
        index.delete(TransferredResource.class, ids);
        index.commit(TransferredResource.class);
    }

    public void updateTransferredResources(Optional<String> folderRelativePath, boolean waitToFinish)
            throws IsStillUpdatingException, GenericException {
        if (!RodaCoreFactory.getTransferredResourcesScannerUpdateStatus(folderRelativePath)) {
            if (index != null) {
                ReindexTransferredResourcesRunnable reindexRunnable = new ReindexTransferredResourcesRunnable(index,
                        basePath, folderRelativePath);

                if (waitToFinish) {
                    reindexRunnable.run();
                } else {
                    Thread threadReindex = new Thread(reindexRunnable, "ReindexThread");
                    threadReindex.start();
                }
            } else {
                throw new GenericException(
                        "Could not update transferred resources because index was not initialized");
            }
        } else {
            LOGGER.warn("Could not update transferred resources because it is still updating");
            throw new IsStillUpdatingException();
        }
    }

    public void updateTransferredResource(Optional<String> folderRelativePath, ContentPayload payload, String name,
            boolean waitToFinish)
            throws NotFoundException, GenericException, IOException, IsStillUpdatingException {
        if (folderRelativePath.isPresent()) {
            Path path = basePath.resolve(folderRelativePath.get());
            Path parent = path.getParent();
            Path parentToBase = basePath.relativize(parent);
            FSUtils.deletePath(path);

            payload.writeToPath(parent.resolve(name));
            updateTransferredResources(Optional.ofNullable(parentToBase.toString()), waitToFinish);
        }
    }

    public String renameTransferredResource(TransferredResource resource, String newName, boolean replaceExisting,
            boolean reindexResources)
            throws AlreadyExistsException, GenericException, IsStillUpdatingException, NotFoundException {

        if (FSUtils.exists(Paths.get(resource.getFullPath()))) {
            Path resourcePath = Paths.get(resource.getFullPath());
            Path newPath = resourcePath.getParent().resolve(newName);
            FSUtils.move(resourcePath, newPath, replaceExisting);

            if (reindexResources) {
                if (resource.getParentUUID() != null) {
                    try {
                        TransferredResource parent = index.retrieve(TransferredResource.class,
                                resource.getParentUUID(), fieldsToReturn);
                        if (parent != null) {
                            updateTransferredResources(Optional.of(parent.getRelativePath()), true);
                        } else {
                            updateTransferredResources(Optional.empty(), true);
                        }
                    } catch (GenericException | NotFoundException e) {
                        LOGGER.error("Could not reindex transferred resources after renaming");
                    }
                } else {
                    updateTransferredResources(Optional.empty(), true);
                }
            }

            Path relativeToBase = basePath.relativize(resourcePath.getParent().resolve(newName));
            return IdUtils.createUUID(relativeToBase.toString());
        } else {
            throw new NotFoundException("Transferred resource was moved or does not exist");
        }
    }

    public Map<String, String> moveTransferredResource(String newRelativePath, List<String> resourcesUUIDs,
            boolean replaceExisting)
            throws AlreadyExistsException, GenericException, IsStillUpdatingException, NotFoundException {
        List<TransferredResource> resources = Collections.emptyList();
        try {
            List<String> resourceFields = Arrays.asList(RodaConstants.INDEX_UUID,
                    RodaConstants.TRANSFERRED_RESOURCE_FULLPATH, RodaConstants.TRANSFERRED_RESOURCE_RELATIVEPATH,
                    RodaConstants.TRANSFERRED_RESOURCE_NAME);
            resources = index.retrieve(TransferredResource.class, resourcesUUIDs, resourceFields);
        } catch (NotFoundException e) {
            // do nothing and pass it an empty list
        }
        return moveTransferredResource(resources, newRelativePath, replaceExisting, false, true);
    }

    public Map<String, String> moveTransferredResource(List<TransferredResource> resources, String newRelativePath,
            boolean replaceExisting, boolean reindexResources)
            throws AlreadyExistsException, GenericException, IsStillUpdatingException, NotFoundException {
        return moveTransferredResource(resources, newRelativePath, replaceExisting, reindexResources, false);
    }

    public Map<String, String> moveTransferredResource(List<TransferredResource> resources, String newRelativePath,
            boolean replaceExisting, boolean reindexResources, boolean addOldRelativePathToNewRelativePath)
            throws AlreadyExistsException, GenericException, IsStillUpdatingException, NotFoundException {

        Map<String, String> oldToNewTransferredResourceIds = new HashMap<>();
        List<TransferredResource> resourcesToIndex = new ArrayList<>();
        boolean notFoundResources = false;

        String baseFolder = RodaCoreFactory.getRodaConfiguration().getString("core.ingest.processed.base_folder",
                "PROCESSED");
        String successFolder = RodaCoreFactory.getRodaConfiguration()
                .getString("core.ingest.processed.successfully_ingested", "SUCCESSFULLY_INGESTED");
        String unsuccessFolder = RodaCoreFactory.getRodaConfiguration()
                .getString("core.ingest.processed.unsuccessfully_ingested", "UNSUCCESSFULLY_INGESTED");

        for (TransferredResource resource : resources) {
            if (FSUtils.exists(Paths.get(resource.getFullPath()))) {
                Path newResourcePath = basePath.resolve(newRelativePath);
                if (addOldRelativePathToNewRelativePath) {
                    newResourcePath = newResourcePath
                            .resolve(resource.getRelativePath().replace(baseFolder + "/" + successFolder + "/", "")
                                    .replace(baseFolder + "/" + unsuccessFolder + "/", ""));
                } else {
                    newResourcePath = newResourcePath.resolve(resource.getName());
                }

                FSUtils.move(Paths.get(resource.getFullPath()), newResourcePath, replaceExisting);

                // create & index transferred resource in the new location
                TransferredResource newResource = instantiateTransferredResource(newResourcePath, basePath);
                Date creationDate = resource.getCreationDate();
                try {
                    BasicFileAttributes attr = Files.readAttributes(newResourcePath, BasicFileAttributes.class);
                    creationDate = new Date(attr.creationTime().toMillis());
                } catch (IOException e) {
                    creationDate = new Date();
                }

                newResource.setCreationDate(creationDate);
                newResource.setSize(resource.getSize());
                newResource.setLastScanDate(new Date());
                try {
                    index.create(TransferredResource.class, newResource);
                } catch (RequestNotValidException e) {
                    // do nothing
                }

                oldToNewTransferredResourceIds.put(resource.getUUID(), newResource.getUUID());
                resourcesToIndex.add(resource);
            } else {
                notFoundResources = true;
            }
        }

        if (reindexResources) {
            updateTransferredResources(Optional.of(newRelativePath), true);
        }
        reindexOldResourcesParentsAfterMove(resourcesToIndex);

        // doing the throw after the moving process to reindex the moved ones
        if (notFoundResources) {
            throw new NotFoundException("Some transferred resources were moved or do not exist");
        }

        return oldToNewTransferredResourceIds;
    }

    public void reindexOldResourcesParentsAfterMove(List<TransferredResource> resources)
            throws IsStillUpdatingException, GenericException {

        try {
            List<String> resourceUUIDs = resources.stream().map(tr -> tr.getUUID()).collect(Collectors.toList());
            index.delete(TransferredResource.class, resourceUUIDs);
        } catch (RequestNotValidException e) {
            LOGGER.error("Could not delete old transferred resources");
        }
    }

    public CloseableIterable<OptionalWithCause<LiteRODAObject>> listTransferredResources() {
        CloseableIterable<OptionalWithCause<LiteRODAObject>> resources = null;

        try {
            final Stream<Path> files = Files.walk(basePath, FileVisitOption.FOLLOW_LINKS)
                    .filter(path -> !path.equals(basePath));
            final Iterator<Path> fileIterator = files.iterator();

            resources = new CloseableIterable<OptionalWithCause<LiteRODAObject>>() {
                @Override
                public void close() throws IOException {
                    files.close();
                }

                @Override
                public Iterator<OptionalWithCause<LiteRODAObject>> iterator() {

                    return new Iterator<OptionalWithCause<LiteRODAObject>>() {
                        @Override
                        public boolean hasNext() {
                            return fileIterator.hasNext();
                        }

                        @Override
                        public OptionalWithCause<LiteRODAObject> next() {
                            Path file = fileIterator.next();
                            Optional<LiteRODAObject> liteResource = LiteRODAObjectFactory
                                    .get(TransferredResource.class, Arrays.asList(file.toString()), false);
                            return OptionalWithCause.of(liteResource);
                        }
                    };
                }
            };
        } catch (IOException e) {
            LOGGER.error("Errored when file walking to list transferred resources");
        }

        return resources;
    }

}