com.fluidops.iwb.api.WikiStorageBulkServiceImpl.java Source code

Java tutorial

Introduction

Here is the source code for com.fluidops.iwb.api.WikiStorageBulkServiceImpl.java

Source

/*
 * Copyright (C) 2008-2013, fluid Operations AG
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
    
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
    
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */

package com.fluidops.iwb.api;

import static com.google.common.collect.Iterables.filter;
import static com.google.common.collect.Iterables.isEmpty;
import static java.lang.String.format;

import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.rmi.RemoteException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Set;

import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.openrdf.model.URI;

import com.fluidops.iwb.api.Context.ContextLabel;
import com.fluidops.iwb.api.Context.ContextType;
import com.fluidops.iwb.api.solution.SolutionService;
import com.fluidops.iwb.model.Vocabulary;
import com.fluidops.iwb.ui.editor.SemWikiUtil;
import com.fluidops.iwb.util.Config;
import com.fluidops.iwb.util.DateTimeUtil;
import com.fluidops.iwb.util.IWBFileUtil;
import com.fluidops.iwb.wiki.WikiBot;
import com.fluidops.iwb.wiki.WikiStorage;
import com.fluidops.iwb.wiki.WikiStorage.WikiRevision;
import com.fluidops.iwb.wiki.Wikimedia;
import com.fluidops.util.FileUtil;
import com.fluidops.util.GenUtil;
import com.fluidops.util.StringUtil;
import com.fluidops.util.ZipUtil;
import com.google.common.base.Predicate;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;

/**
 * API for access to wiki storage providing various convenience functions.
 * 
 * @author as
 */
public class WikiStorageBulkServiceImpl implements WikiStorageBulkService {
    public static final String WIKIEXPORT_STORAGE = IWBFileUtil.DATA_DIRECTORY + "/solutions/";
    public static final String WIKIBOOTSTRAP_REL_PATH = IWBFileUtil.DATA_DIRECTORY;

    private static final Logger logger = Logger.getLogger(WikiStorageBulkServiceImpl.class);
    private static Logger installLogger = Logger.getLogger(SolutionService.INSTALL_LOGGER_NAME + ".wiki");

    /**
     * Return the location of the wiki export storage
     * 
     * @return
     */
    public static File getWikiExportStorageFolder() {
        return new File(Config.getConfig().getWorkingDir() + WIKIEXPORT_STORAGE);
    }

    private static final FilenameFilter BOOTSTRAP_DIRS = new FilenameFilter() {
        public boolean accept(File dir, String name) {
            return name.startsWith(WikiStorageBulkService.BOOTSTRAP_DIRNAME_PREFIX)
                    && new File(dir, name).isDirectory();
        }
    };
    private static final Predicate<WikiRevision> IS_BOOTSTRAP = new Predicate<WikiRevision>() {
        @Override
        public boolean apply(WikiRevision revision) {
            return revision != null && revision.isBootstrapRevision();
        }
    };
    private final WikiStorage ws;

    public WikiStorageBulkServiceImpl(WikiStorage wikiStorage) {
        this.ws = wikiStorage;
    }

    /**
     * Return all wikipages and associated metadata for the most recent
     * revision.
     * 
     * @return
     */
    public List<WikiPageMeta> getAllWikipages() {
        return getWikipagesInternal(new VoidFilter());
    }

    /**
     * Return all wikipages and associated metadata that are user edited for the
     * most recent version
     * 
     * @return
     */
    public List<WikiPageMeta> getAllWikipagesNotFromBootstrap() {
        return getWikipagesInternal(new NotFromBootstrapFilter());
    }

    /**
     * Return all user pages that shadow at least one bootstrap page, i.e.
     * there exists a bootstrap page in the revision history of a user page
     * @return
     */
    public List<WikiPageMeta> getAllUserPagesShadowingBootstrap() {
        return getWikipagesInternal(new UserPageShadowsBootstrapFilter());
    }

    /**
     * Return all wikipages and associated metadata that are user loaded for the
     * most recent version
     * 
     * @return
     */
    public List<WikiPageMeta> getAllWikipagesFromBootstrap() {
        return getWikipagesInternal(new FromBootstrapFilter());
    }

    /**
     * Return all wikipages applying some filter
     * 
     * @param filter
     * @return
     */
    public List<WikiPageMeta> getAllWikipages(Filter filter) {
        return getWikipagesInternal(filter);
    }

    /**
     * Create a wiki bootstrap called wikibootstrapTIMESTAMP.zip that contains the
     * most recent revision of the selected pages. The internal structure of the
     * zip file is such that it can be used by the solution service, i.e. pages
     * are in the sub directory "data/wikiBootstrap/".
     * 
     * The Zip file is written to {@link #WIKIEXPORT_STORAGE}
     * 
     * @param selectedPages
     * @return the filename, e.g. wikiBootstrapTIMESTAMP.zip
     * @throws Exception
     *             if an I/O error occurs
     * @throws IllegalArgumentException
     *             if the selections are empty or null
     * @throws IllegalStateException
     *             if the wiki directory could not be created
     */
    public String createWikiBootstrap(List<WikiPageMeta> selectedPages)
            throws IllegalArgumentException, IllegalStateException, Exception {

        if (selectedPages == null || selectedPages.size() == 0)
            throw new IllegalArgumentException("No wiki pages selected.");

        File storagePath = getWikiExportStorageFolder();
        if (!storagePath.exists())
            GenUtil.mkdirs(storagePath);

        File tempDir = GenUtil.createTmpDir("wikiExport");

        File wikiBootstrapFolder = wikiStorageRelFolder(tempDir);
        GenUtil.mkdirs(wikiBootstrapFolder);

        try {
            WikiStorage ws = Wikimedia.getWikiStorage();
            for (WikiPageMeta m : selectedPages) {
                File wikiFile = new File(wikiBootstrapFolder, StringUtil.urlEncode(m.getPageUri().stringValue()));
                String content = ws.getRawWikiContent(m.getPageUri(), m.getRevision().date);
                FileUtil.writeContentToFile(content, wikiFile.getAbsolutePath());
            }

            File zipFile = new File(storagePath,
                    wikiBootstrapFolder.getName() + System.currentTimeMillis() + ".zip");

            // zip files and delete temp files
            ZipUtil.doZipOutput(zipFile, tempDir, wikiBootstrapFolder.listFiles());

            logger.info("Successfully created wiki export to " + zipFile.getAbsolutePath());
            return zipFile.getName();
        } finally {
            // clean up temporary folder, i.e. keep zip files only
            GenUtil.deleteRec(tempDir);
        }
    }

    /**
     * Restore the latest bootstrap version iff
     * 
     * a) the current page is a user page
     * b) there exists some bootstrap version
     * c) the content of the latest version and the bootstrap are actually different
     * 
     * This method also writes the semantic link differences into
     * the triple store
     * 
     * @param name
     * @throws IOException
     * @throws {@link IllegalStateException} if the content of current and bootstrap are identical
     * @return true if the restore operation was successful, false if the operation was not performed
     *          as one of the conditions above was not met
     * 
     */
    public boolean restoreLatestBootstrap(URI name) throws IOException {

        List<WikiRevision> revs = ws.getWikiRevisions(name);
        if (revs == null || revs.size() <= 1)
            return false;

        WikiRevision last = revs.get(revs.size() - 1);
        if (last.isBootstrapRevision())
            return false;

        WikiRevision lastBootstrap = null;
        for (int i = revs.size() - 2; i >= 0; i--) {
            WikiRevision current = revs.get(i);
            if (current.isBootstrapRevision()) {
                lastBootstrap = current;
                break;
            }
        }

        if (lastBootstrap == null)
            return false;

        // TODO unify with SemWiki#saveWiki and also consider RevisionTable
        String content = ws.getWikiContent(name, lastBootstrap);
        String prevContent = ws.getWikiContent(name, last);
        if (content.equals(prevContent))
            throw new IllegalStateException("Current content identical for page " + name.stringValue());

        logger.debug("Restoring latest bootstrap version of resource " + name.stringValue());
        ws.storeWikiContent(name, content, "Restored revision from " + lastBootstrap.comment,
                DateTimeUtil.getDateSafe());
        SemWikiUtil.saveSemanticLinkDiff(prevContent, content, name,
                Context.getFreshUserContext(ContextLabel.WIKI));
        return true;
    }

    /**
     * Create a wiki bootstrap called wikibootstrapTIMESTAMP that contains all
     * user edited pages (i.e. excluding the ones that are imported by another
     * bootstrap)
     * 
     * @throws Exception
     *             if an I/O error occurs
     * @throws IllegalArgumentException
     *             if the selections are empty or null
     * @throws IllegalStateException
     *             if the wiki directory could not be created
     */
    public String createWikiBootstrap() throws IllegalArgumentException, IllegalStateException, Exception {
        return createWikiBootstrap(getAllWikipagesNotFromBootstrap());
    }

    @Override
    public void bootstrapWiki(File workingDir, String artifactVersion) {
        bootstrapWiki(workingDir, false, artifactVersion);
    }

    /**
     * the wiki storage folder relative to baseDir,
     * e.g. %baseDir%/data/wikiBootstrap/
     * @param baseDir
     * @return
     */
    static File wikiStorageRelFolder(File baseDir) {
        return new File(new File(baseDir, WIKIBOOTSTRAP_REL_PATH), BOOTSTRAP_DIRNAME_PREFIX);
    }

    public void bootstrapWikiAndRemove(File workingDir) {
        bootstrapWiki(workingDir, true, applicationVersion());
    }

    private String applicationVersion() {
        try {
            return EndpointImpl.api().version();
        } catch (RemoteException e) {
            throw Throwables.propagate(e);
        }
    }

    private void bootstrapWiki(File workingDir, boolean removeDirAfterImport, String artifactVersion) {
        for (File dir : workingDir.listFiles(BOOTSTRAP_DIRS)) {
            installLogger.info("bootstrap all wiki pages from folder: " + dir);

            for (File wikiFile : dir.listFiles()) {
                if (wikiFile.isFile() && wikiFile.canRead())
                    importFile(wikiFile, dir.getName(), artifactVersion);
            }
            if (removeDirAfterImport)
                FileUtil.deleteDirectory(dir);
        }
    }

    private URI importFile(File wikiFile, String bootstrapName, String artifactVersion) {
        try {
            URI subject = EndpointImpl.api().getNamespaceService()
                    .guessURI(StringUtil.urlDecode(wikiFile.getName()));
            Date date = bootstrapTimestampFor(subject);
            logger.trace("bootstrapping wiki file: " + wikiFile);

            // store the old content of the current version
            WikiRevision prevRevision = ws.getLatestRevision(subject);
            String oldContent = prevRevision == null ? "" : ws.getWikiContent(subject, prevRevision);

            if (prevRevision != null && !prevRevision.isBootstrapRevision())
                installLogger.warn("Bootstrap page '" + subject.stringValue() + "' is shadowed by a user page.");

            // update the content
            String content = FileUtils.readFileToString(wikiFile);
            ws.storeWikiContent(subject, content, format("Bootstrap %s (%s, v%03d)", artifactVersion, bootstrapName,
                    date.getTime() - WikiStorage.IWB_BOOTSTRAP_EPOCH), date);

            // store the new content of the current version
            WikiRevision curRevision = ws.getLatestRevision(subject);
            String newContent = curRevision == null ? "" : ws.getWikiContent(subject, curRevision);

            // if the current version has changed, we need to perform an update
            if (!oldContent.equals(newContent)) {
                // store the semantic link difference between the previous 
                // version (if any) and the current version
                Context context = Context.getFreshPublishedContext(ContextType.USER, null, Vocabulary.SYSTEM.USER,
                        null, null, true, ContextLabel.WIKI);
                SemWikiUtil.saveSemanticLinkDiff(oldContent, content, subject, context);
            }
            return subject;
        } catch (Exception e) {
            throw new RuntimeException("Error while processing file " + wikiFile + ": " + e.getMessage(), e);
        }
    }

    private Date bootstrapTimestampFor(URI subject) {
        Iterable<WikiRevision> bootstrapRevisions = filter(ws.getWikiRevisions(subject), IS_BOOTSTRAP);
        if (isEmpty(bootstrapRevisions))
            return new Date(WikiStorage.IWB_BOOTSTRAP_EPOCH);
        WikiRevision latesBootstrapRevision = max(bootstrapRevisions);
        return ws.nextBootstrapVersion(latesBootstrapRevision.date);
    }

    private WikiRevision max(Iterable<WikiRevision> bootstrapRevisions) {
        WikiRevision latesBootstrapRevision = bootstrapRevisions.iterator().next();
        for (WikiRevision wikiRevision : bootstrapRevisions) {
            if (latesBootstrapRevision.date.getTime() < wikiRevision.date.getTime())
                latesBootstrapRevision = wikiRevision;
        }
        return latesBootstrapRevision;
    }

    /**
     * Return all available wikipages after applying the provided filter. Uses
     * {@link Wikimedia#getWikiStorage()} as underlying API.
     * 
     * @param filter
     * @return
     */
    protected List<WikiPageMeta> getWikipagesInternal(Filter filter) {

        List<WikiPageMeta> res = new ArrayList<WikiPageMeta>();

        for (URI wikiPage : ws.getAllWikiURIs()) {
            WikiRevision revision = ws.getLatestRevision(wikiPage);

            // a revision might currently be null if there exist multiple pages
            // for a URI with different capitalization. In Windows the encoding
            // of such files is different than in Linux, thus after migrating
            // the wiki pages are in an inconsistent state. See bug 11378 for 
            // details
            if (revision == null)
                throw new IllegalStateException("Wiki revision could not be loaded for resource "
                        + wikiPage.stringValue() + ". Problems with wiki storage need to be resolved manually.");

            int numberOfRevisions = ws.getRevisionCount(wikiPage);
            // try to find shadowing user pages:
            // there exists a bootstrap page that is shadowed by a user page
            boolean shadows = false;
            if (!revision.isBootstrapRevision() && numberOfRevisions > 1) {
                List<WikiRevision> revs = ws.getWikiRevisions(wikiPage);
                for (WikiRevision r : revs) {
                    if (r.isBootstrapRevision()) {
                        shadows = true;
                    }
                }
            }
            WikiPageMeta wikiPageMeta = new WikiPageMeta(wikiPage, revision, numberOfRevisions, shadows);
            if (filter.keep(wikiPageMeta))
                res.add(wikiPageMeta);
        }

        return res;
    }

    public static class WikiPageMeta {
        private final WikiRevision revision;

        private final URI pageUri;

        private final int numberOfRevisions;

        private final boolean shadows;

        public WikiPageMeta(URI pageUri, WikiRevision revision, int numberOfRevisions, boolean shadows) {
            this.revision = revision;
            this.pageUri = pageUri;
            this.numberOfRevisions = numberOfRevisions;
            this.shadows = shadows;
        }

        /**
         * Returns true iff this revision is a user page and there
         * exists a shadowed bootstrap version
         * @return
         */
        public boolean shadowsBootstrapPage() {
            return shadows;
        }

        /**
         * Returns the latest version of the wiki page content
         */
        public String getContent() {
            return Wikimedia.getWikiContent(pageUri, null);
        }

        public WikiRevision getRevision() {
            return revision;
        }

        public URI getPageUri() {
            return pageUri;
        }

        public boolean isBootstrap() {
            return revision.bootstrapVersion() != null;
        }

        public int getNumberOfRevisions() {
            return numberOfRevisions;
        }

        @Override
        public String toString() {
            return "[pageUri=" + pageUri + "]";
        }

        @Override
        public int hashCode() {
            final int prime = 31;
            int result = 1;
            result = prime * result + ((pageUri == null) ? 0 : pageUri.hashCode());
            return result;
        }

        @Override
        public boolean equals(Object obj) {
            if (this == obj)
                return true;
            if (obj == null)
                return false;
            if (getClass() != obj.getClass())
                return false;
            WikiPageMeta other = (WikiPageMeta) obj;
            if (pageUri == null) {
                if (other.pageUri != null)
                    return false;
            } else if (!pageUri.equals(other.pageUri))
                return false;
            return true;
        }

    }

    /**
     * Implementations can filter wiki pages.
     * 
     * @author as
     */
    public static interface Filter {
        /**
         * Implementations can decide if the provided wikpage shall be kept
         * (true) or not (false)
         * 
         * @param wikiPage
         *            true if this wikipage shall be kept
         * @return
         */
        public boolean keep(WikiPageMeta wikiPage);
    }

    public static class VoidFilter implements Filter {
        @Override
        public boolean keep(WikiPageMeta wikiPage) {
            return true;
        }
    }

    /**
     * Filter if loaded from bootstrap
     */
    public static class NotFromBootstrapFilter implements Filter {
        @Override
        public boolean keep(WikiPageMeta wikiPage) {
            return !wikiPage.isBootstrap();
        }
    }

    /**
     * Filter if loaded from bootstrap
     */
    public static class FromBootstrapFilter implements Filter {
        @Override
        public boolean keep(WikiPageMeta wikiPage) {
            return wikiPage.isBootstrap();
        }
    }

    /**
     * Filter to show those user pages that shadow a bootstrap page
     */
    public static class UserPageShadowsBootstrapFilter implements Filter {
        @Override
        public boolean keep(WikiPageMeta wikiPage) {
            return wikiPage.shadowsBootstrapPage();
        }
    }

    /**
     * Filter if imported from {@link WikiBot}, keeps item if
     * comment equals {@value WikiBot#WIKIBOT_WIKIREVISION_COMMENT}
     */
    public static class ImportedFromWikibotFilter implements Filter {
        @Override
        public boolean keep(WikiPageMeta wikiPage) {
            return wikiPage.revision.comment.equals(WikiBot.WIKIBOT_WIKIREVISION_COMMENT);
        }
    }

    /**
     * Filter anything that does not have the provided namespace as prefix, i.e.
     * return all those wikipages that have the prefix
     */
    public static class NamespacePrefixFilter implements Filter {
        private final String nsPrefix;

        public NamespacePrefixFilter(String nsPrefix) {
            this.nsPrefix = nsPrefix;
        }

        @Override
        public boolean keep(WikiPageMeta wikiPage) {
            return wikiPage.getPageUri().stringValue().startsWith(nsPrefix);
        }
    }

    /**
     * Filter anything that is not in the provided set.
     */
    public static class StringSetFilter implements Filter {
        private final Set<String> keepSet;

        public StringSetFilter(Set<String> keepSet) {
            this.keepSet = keepSet;
        }

        @Override
        public boolean keep(WikiPageMeta wikiPage) {
            return keepSet.contains(wikiPage.getPageUri().stringValue());
        }
    }

    /**
     * Filter based on a regular expression. If the passed regexp is null
     * or empty, then the regexp filter has no effect.
     */
    public static class RegexpFilter implements Filter {
        private final String regexp;

        public RegexpFilter(String regexp) {
            this.regexp = regexp;
        }

        @Override
        public boolean keep(WikiPageMeta wikiPage) {
            // if no regexp is provided, the filter has no effect
            if (StringUtil.isNullOrEmpty(regexp))
                return true;

            return wikiPage.getPageUri().toString().matches(regexp);
        }
    }

    /**
     * Filters template pages.
     * 
     * @author marlon.braun
     */
    public static class TemplateFilter implements Filter {

        @Override
        public boolean keep(WikiPageMeta wikiPage) {
            return wikiPage.getPageUri().stringValue()
                    .startsWith(EndpointImpl.api().getNamespaceService().templateNamespace());
        }

    }

    /**
     * Filter consisting of several filters
     */
    public static class MultiFilter implements Filter {
        private final List<Filter> filters;

        public MultiFilter(List<Filter> filters) {
            this.filters = filters;
        }

        @Override
        public boolean keep(WikiPageMeta wikiPage) {
            for (Filter filter : filters) {
                if (!filter.keep(wikiPage))
                    return false;
            }

            return true;
        }
    }

    /**
     * Keep a number of latest revisions and delete the rest (check if the bootstrap revisions should be kept)
     * @param wikiPageURI
     * @param latestRevisionsToKeep
     * @param keepBootstrap
     */
    public void deleteRevisions(URI wikiPageURI, int latestRevisionsToKeep, boolean keepBootstrap) {
        if (latestRevisionsToKeep < 0)
            throw new IllegalArgumentException("Latest revisions to keep must be a positive integer");

        List<WikiRevision> revs = ws.getWikiRevisions(wikiPageURI);
        for (int i = 0; i < revs.size() - latestRevisionsToKeep; i++) {
            WikiRevision wr = revs.get(i);
            if (keepBootstrap && wr.isBootstrapRevision())
                continue;
            wr.delete(wikiPageURI);
        }
    }

    /**
     * Delete a number of latest revisions (check if the bootstrap revisions should be kept)
     * @param wikiPageURI
     * @param revisionsToDelete
     * @param keepBootstrap
     */
    public void deleteLatestRevisions(URI wikiPageURI, int revisionsToDelete, boolean keepBootstrap) {
        if (revisionsToDelete < 0)
            throw new IllegalArgumentException("Revisions to delete must be a positive integer");

        List<WikiRevision> revs = ws.getWikiRevisions(wikiPageURI);

        int allRevsNumber = revs.size();
        List<WikiRevision> toDelete = Lists.newArrayList();
        for (int i = 1; i <= revisionsToDelete && i <= allRevsNumber; i++) {
            WikiRevision wr = revs.get(allRevsNumber - i);
            if (keepBootstrap && wr.isBootstrapRevision())
                continue;
            toDelete.add(wr);
        }
        // impl note: we delete the wiki pages in reverse order to not
        // trigger keyword index updates on each deletion
        for (WikiRevision wr : Lists.reverse(toDelete))
            wr.delete(wikiPageURI);
    }
}