com.jaeksoft.searchlib.ClientCatalog.java Source code

Java tutorial

Introduction

Here is the source code for com.jaeksoft.searchlib.ClientCatalog.java

Source

/**   
 * License Agreement for OpenSearchServer
 *
 * Copyright (C) 2008-2014 Emmanuel Keller / Jaeksoft
 * 
 * http://www.open-search-server.com
 * 
 * This file is part of OpenSearchServer.
 *
 * OpenSearchServer is free software: you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 * OpenSearchServer is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with OpenSearchServer. 
 *  If not, see <http://www.gnu.org/licenses/>.
 **/

package com.jaeksoft.searchlib;

import java.io.File;
import java.io.FileFilter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.ThreadLocalRandom;

import javax.naming.NamingException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.xpath.XPathExpressionException;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.DirectoryFileFilter;
import org.apache.lucene.search.BooleanQuery;
import org.xml.sax.SAXException;
import org.zkoss.zk.ui.WebApp;

import com.jaeksoft.searchlib.cluster.ClusterInstance;
import com.jaeksoft.searchlib.cluster.ClusterManager;
import com.jaeksoft.searchlib.cluster.ClusterNotification;
import com.jaeksoft.searchlib.cluster.ClusterNotification.Type;
import com.jaeksoft.searchlib.config.ConfigFileRotation;
import com.jaeksoft.searchlib.config.ConfigFiles;
import com.jaeksoft.searchlib.crawler.cache.CrawlCacheManager;
import com.jaeksoft.searchlib.index.IndexType;
import com.jaeksoft.searchlib.ocr.OcrManager;
import com.jaeksoft.searchlib.renderer.RendererResults;
import com.jaeksoft.searchlib.replication.ReplicationMerge;
import com.jaeksoft.searchlib.template.TemplateAbstract;
import com.jaeksoft.searchlib.template.TemplateList;
import com.jaeksoft.searchlib.user.Role;
import com.jaeksoft.searchlib.user.User;
import com.jaeksoft.searchlib.user.UserList;
import com.jaeksoft.searchlib.util.IOUtils;
import com.jaeksoft.searchlib.util.LastModifiedAndSize;
import com.jaeksoft.searchlib.util.ReadWriteLock;
import com.jaeksoft.searchlib.util.ThreadUtils;
import com.jaeksoft.searchlib.util.XPathParser;
import com.jaeksoft.searchlib.util.XmlWriter;
import com.jaeksoft.searchlib.web.StartStopListener;
import com.jaeksoft.searchlib.web.controller.PushEvent;

/**
 * This class handles a list of indexes stored in a given directory.
 * 
 * 
 */
public class ClientCatalog {

    private static transient volatile TreeMap<File, Client> CLIENTS = new TreeMap<File, Client>();

    private static transient volatile TreeSet<File> OLD_CLIENTS = new TreeSet<File>();

    private static final ReadWriteLock clientsLock = new ReadWriteLock();

    private static final ReadWriteLock usersLock = new ReadWriteLock();

    private static UserList userList = null;

    private static final ConfigFiles configFiles = new ConfigFiles();

    private static final RendererResults rendererResults = new RendererResults();

    private static final ThreadGroup threadGroup = new ThreadGroup("Catalog");

    /**
     * This method should be called first before using any other function of
     * OpenSearchServer. It will initialize all internal resources. The
     * data_directory is the folder which will contain all the indexes (data and
     * configuration).
     * 
     * @param data_directory
     *            The directory which contain the indexes
     * @throws IOException
     */
    public static final void init(File data_directory) throws IOException {
        StartStopListener.start(data_directory);
    }

    /**
     * Close OpenSearchServer. This method closes all indexes and stops any
     * running task..
     */
    public static final void close() {
        StartStopListener.shutdown();
    }

    private static final boolean isOldClient(File indexDirectory) {
        clientsLock.r.lock();
        try {
            return OLD_CLIENTS.contains(indexDirectory);
        } finally {
            clientsLock.r.unlock();
        }
    }

    private static final Client getClient(File indexDirectory, boolean openIfNotLoaded) throws SearchLibException {
        clientsLock.r.lock();
        try {
            Client client = CLIENTS.get(indexDirectory);
            if (client != null)
                return client;
        } finally {
            clientsLock.r.unlock();
        }
        int i = 60;
        while (isOldClient(indexDirectory) && i > 0) {
            ThreadUtils.sleepMs(500);
            i--;
        }
        if (i == 0)
            throw new SearchLibException("Time out while getting " + indexDirectory);
        if (!openIfNotLoaded)
            return null;
        clientsLock.w.lock();
        try {
            Client client = CLIENTS.get(indexDirectory);
            if (client != null)
                return client;
            client = ClientFactory.INSTANCE.newClient(indexDirectory, true, false);
            CLIENTS.put(indexDirectory, client);
            return client;
        } finally {
            clientsLock.w.unlock();
        }
    }

    public static final void closeAll() {
        synchronized (ClientCatalog.class) {
            clientsLock.w.lock();
            try {
                for (Client client : CLIENTS.values()) {
                    if (client == null)
                        continue;
                    Logging.info("OSS unloads index " + client.getIndexName());
                    client.close();
                }
                CLIENTS.clear();
            } finally {
                clientsLock.w.unlock();
            }
            rendererResults.release();
        }
    }

    public static final long countAllDocuments() throws IOException, SearchLibException {
        long count = 0;
        clientsLock.r.lock();
        try {
            for (Client client : CLIENTS.values()) {
                if (client.isTrueReplicate())
                    continue;
                count += client.getStatistics().getNumDocs();
            }
        } finally {
            clientsLock.r.unlock();
        }
        return count;
    }

    private static volatile long lastInstanceSize = 0;

    public static final long calculateInstanceSize() throws SearchLibException {
        if (StartStopListener.OPENSEARCHSERVER_DATA_FILE == null)
            return 0;
        lastInstanceSize = new LastModifiedAndSize(StartStopListener.OPENSEARCHSERVER_DATA_FILE, false).getSize();
        return lastInstanceSize;
    }

    public static long getInstanceSize() throws SearchLibException {
        if (lastInstanceSize != 0)
            return lastInstanceSize;
        return calculateInstanceSize();
    }

    private static final File getClientDir(String indexName) throws SearchLibException {
        if (!isValidIndexName(indexName))
            throw new SearchLibException("The name '" + indexName + "' is not allowed");
        return new File(StartStopListener.OPENSEARCHSERVER_DATA_FILE, indexName);
    }

    public static final LastModifiedAndSize getLastModifiedAndSize(String indexName) throws SearchLibException {
        File file = getClientDir(indexName);
        if (!file.exists())
            return null;
        return new LastModifiedAndSize(file, false);
    }

    public static final Client getLoadedClient(String indexName) throws SearchLibException {
        return getClient(getClientDir(indexName), false);
    }

    public static final Client getClient(String indexName) throws SearchLibException {
        return getClient(getClientDir(indexName), true);
    }

    public static final void closeClient(String indexName) throws SearchLibException {
        Client client = null;
        clientsLock.w.lock();
        try {
            File indexDirectory = getClientDir(indexName);
            client = CLIENTS.get(indexName);
            if (client == null)
                return;
            System.out.println("Closing client " + indexDirectory.getName());
            client.close();
            CLIENTS.remove(indexDirectory);
        } finally {
            clientsLock.w.unlock();
        }
        if (client != null)
            PushEvent.eventClientSwitch.publish(client);
    }

    public static final Set<ClientCatalogItem> getClientCatalog(User user) throws SearchLibException {
        File[] files = StartStopListener.OPENSEARCHSERVER_DATA_FILE
                .listFiles((FileFilter) DirectoryFileFilter.DIRECTORY);
        Set<ClientCatalogItem> set = new TreeSet<ClientCatalogItem>();
        if (files == null)
            return null;
        for (File file : files) {
            if (!file.isDirectory())
                continue;
            String indexName = file.getName();
            if (!isValidIndexName(indexName))
                continue;
            if (user == null || user.hasAnyRole(indexName, Role.GROUP_INDEX))
                set.add(new ClientCatalogItem(indexName));
        }
        return set;
    }

    /**
     * Tests if an index exists
     * 
     * @param indexName
     *            The name of an index
     * @return
     * @throws SearchLibException
     */
    public static final boolean exists(String indexName) throws SearchLibException {
        return exists(null, indexName);
    }

    public static final boolean exists(User user, String indexName) throws SearchLibException {
        if (user != null && !user.isAdmin())
            throw new SearchLibException("Operation not permitted");
        if (!isValidIndexName(indexName))
            throw new SearchLibException("The name '" + indexName + "' is not allowed");
        return getClientCatalog(null).contains(new ClientCatalogItem(indexName));
    }

    public static final CrawlCacheManager getCrawlCacheManager() throws SearchLibException {
        return CrawlCacheManager.getInstance();
    }

    public static synchronized final OcrManager getOcrManager() throws SearchLibException {
        return OcrManager.getInstance();
    }

    public static synchronized final ClusterManager getClusterManager() throws SearchLibException {
        return ClusterManager.getInstance();
    }

    public static ClusterInstance getAnyClusterInstance(String indexName) throws SearchLibException {
        File clientDir = getClientDir(indexName);
        ClusterManager clusterManager = getClusterManager();
        String[] instanceIds = clusterManager.getClientInstances(clientDir);
        if (instanceIds == null || instanceIds.length == 0)
            return null;
        return clusterManager.getInstance(instanceIds[ThreadLocalRandom.current().nextInt(instanceIds.length)]);
    }

    final private static boolean isValidIndexName(String name) {
        if (name.startsWith("."))
            return false;
        if ("logs".equals(name))
            return false;
        return true;
    }

    /**
     * Create a new index.
     * 
     * @param indexName
     *            The name of the index.
     * @param template
     *            The name of the template (EMPTY_INDEX, WEB_CRAWLER,
     *            FILE_CRAWLER)
     * @throws IOException
     * @throws SearchLibException
     */
    public static void createIndex(String indexName, String templateName, IndexType indexType, URI remoteURI)
            throws SearchLibException, IOException {
        TemplateAbstract template = TemplateList.findTemplate(templateName);
        if (template == null)
            throw new SearchLibException("Template not found: " + templateName);
        createIndex(null, indexName, template, indexType, remoteURI);
    }

    public static void createIndex(User user, String indexName, TemplateAbstract template, IndexType indexType,
            URI remoteURI) throws SearchLibException, IOException {
        if (user != null && !user.isAdmin())
            throw new SearchLibException("Operation not permitted");
        ClientFactory.INSTANCE.properties.checkMaxIndexNumber();
        if (!isValidIndexName(indexName))
            throw new SearchLibException("The name '" + indexName + "' is not allowed");
        synchronized (ClientCatalog.class) {
            File indexDir = new File(StartStopListener.OPENSEARCHSERVER_DATA_FILE, indexName);
            if (indexDir.exists())
                throw new SearchLibException("directory " + indexName + " already exists");
            template.createIndex(indexDir, indexType, remoteURI);
        }
    }

    /**
     * Delete an index.
     * 
     * @param indexName
     *            The name of the index
     * @throws SearchLibException
     * @throws NamingException
     * @throws IOException
     */
    public static void eraseIndex(String indexName) throws SearchLibException, NamingException, IOException {
        eraseIndex(null, indexName);
    }

    public static void eraseIndex(User user, String indexName)
            throws SearchLibException, NamingException, IOException {
        if (user != null && !user.isAdmin())
            throw new SearchLibException("Operation not permitted");
        File indexDir = getClientDir(indexName);
        Client client = null;
        synchronized (ClientCatalog.class) {
            clientsLock.r.lock();
            try {
                client = CLIENTS.get(indexDir);
            } finally {
                clientsLock.r.unlock();
            }
            if (client != null) {
                client.close();
                client.delete();
            } else
                FileUtils.deleteDirectory(indexDir);
            if (client != null) {
                clientsLock.w.lock();
                try {
                    CLIENTS.remove(client.getDirectory());
                } finally {
                    clientsLock.w.unlock();
                }
                PushEvent.eventClientSwitch.publish(client);
            }
        }
    }

    public static UserList getUserList() throws SearchLibException {
        usersLock.r.lock();
        try {
            if (userList == null) {
                File userFile = new File(StartStopListener.OPENSEARCHSERVER_DATA_FILE, "users.xml");
                if (userFile.exists()) {
                    XPathParser xpp = new XPathParser(userFile);
                    userList = UserList.fromXml(xpp, xpp.getNode("/" + UserList.usersElement));
                } else
                    userList = new UserList();
            }
            return userList;
        } catch (ParserConfigurationException e) {
            throw new SearchLibException(e);
        } catch (SAXException e) {
            throw new SearchLibException(e);
        } catch (IOException e) {
            throw new SearchLibException(e);
        } catch (XPathExpressionException e) {
            throw new SearchLibException(e);
        } finally {
            usersLock.r.unlock();
        }
    }

    public static void flushPrivileges() {
        usersLock.w.lock();
        try {
            userList = null;
        } finally {
            usersLock.w.unlock();
        }
    }

    private static void saveUserListWithoutLock()
            throws TransformerConfigurationException, SAXException, IOException, SearchLibException {
        ConfigFileRotation cfr = configFiles.get(StartStopListener.OPENSEARCHSERVER_DATA_FILE, "users.xml");
        ClientFactory.INSTANCE.versionFile.lock();
        try {
            XmlWriter xmlWriter = new XmlWriter(cfr.getTempPrintWriter("UTF-8"), "UTF-8");
            getUserList().writeXml(xmlWriter);
            xmlWriter.endDocument();
            cfr.rotate(ClientFactory.INSTANCE.versionFile);
            ClusterManager.notify(new ClusterNotification(Type.RELOAD_USER));
        } finally {
            cfr.abort();
            ClientFactory.INSTANCE.versionFile.release();
        }
    }

    public static void saveUserList() throws SearchLibException {
        usersLock.w.lock();
        try {
            saveUserListWithoutLock();
        } catch (IOException e) {
            throw new SearchLibException(e);
        } catch (TransformerConfigurationException e) {
            throw new SearchLibException(e);
        } catch (SAXException e) {
            throw new SearchLibException(e);
        } finally {
            usersLock.w.unlock();
        }
    }

    public static User authenticate(String login, String password) throws SearchLibException {
        usersLock.r.lock();
        try {
            User user = getUserList().get(login);
            if (user == null)
                return null;
            if (!user.authenticate(password))
                return null;
            return user;
        } finally {
            usersLock.r.unlock();
        }
    }

    public static User authenticateKey(String login, String key) throws SearchLibException {
        usersLock.r.lock();
        try {
            User user = getUserList().get(login);
            if (user == null)
                return null;
            if (!user.authenticateKey(key))
                return null;
            return user;
        } finally {
            usersLock.r.unlock();
        }
    }

    private static final File getTempReceiveDir(Client client) {
        File clientDir = client.getDirectory();
        return new File(clientDir.getParentFile(), '.' + clientDir.getName());
    }

    private static final File getTrashReceiveDir(Client client) {
        File clientDir = client.getDirectory();
        return new File(clientDir.getParentFile(), "._" + clientDir.getName());
    }

    public static final void receive_init(Client client) throws IOException, SearchLibException {
        ClientFactory.INSTANCE.properties.checkMaxStorageLimit();
        File rootDir = getTempReceiveDir(client);
        FileUtils.deleteDirectory(rootDir);
        rootDir.mkdir();
    }

    private static void lockClientDir(File clientDir) {
        clientsLock.w.lock();
        try {
            CLIENTS.remove(clientDir);
            OLD_CLIENTS.add(clientDir);
        } finally {
            clientsLock.w.unlock();
        }
    }

    private static void unlockClientDir(File clientDir, Client newClient) {
        clientsLock.w.lock();
        try {
            if (newClient != null)
                CLIENTS.put(clientDir, newClient);
            OLD_CLIENTS.remove(clientDir);
        } finally {
            clientsLock.w.unlock();
        }
    }

    public static void receive_switch(WebApp webapp, Client client)
            throws SearchLibException, NamingException, IOException {
        File trashDir = getTrashReceiveDir(client);
        File clientDir = client.getDirectory();
        if (trashDir.exists())
            FileUtils.deleteDirectory(trashDir);
        Client newClient = null;
        lockClientDir(clientDir);
        try {
            client.trash(trashDir);
            getTempReceiveDir(client).renameTo(clientDir);
            newClient = ClientFactory.INSTANCE.newClient(clientDir, true, true);
            newClient.writeReplCheck();
        } finally {
            unlockClientDir(clientDir, newClient);
        }
        PushEvent.eventClientSwitch.publish(client);
        FileUtils.deleteDirectory(trashDir);
    }

    public static void receive_merge(WebApp webapp, Client client) throws SearchLibException, IOException {
        File tempDir = getTempReceiveDir(client);
        File clientDir = client.getDirectory();
        Client newClient = null;
        lockClientDir(clientDir);
        try {
            client.close();
            new ReplicationMerge(tempDir, clientDir);
            newClient = ClientFactory.INSTANCE.newClient(clientDir, true, true);
            newClient.writeReplCheck();
        } finally {
            unlockClientDir(clientDir, newClient);
        }
        PushEvent.eventClientSwitch.publish(client);
        FileUtils.deleteDirectory(tempDir);
    }

    public static void receive_abort(Client client) throws IOException {
        File tempDir = getTempReceiveDir(client);
        File trashDir = getTrashReceiveDir(client);
        synchronized (ClientCatalog.class) {
            if (tempDir.exists())
                FileUtils.deleteDirectory(tempDir);
            if (trashDir.exists())
                FileUtils.deleteDirectory(trashDir);
        }
    }

    public static final void receive_dir(Client client, String filePath) throws IOException {
        File rootDir = getTempReceiveDir(client);
        File targetFile = new File(rootDir, filePath);
        targetFile.mkdir();
    }

    public static final boolean receive_file_exists(Client client, String filePath, long lastModified, long length)
            throws IOException {
        File existsFile = new File(client.getDirectory(), filePath);
        if (!existsFile.exists())
            return false;
        if (existsFile.lastModified() != lastModified)
            return false;
        if (existsFile.length() != length)
            return false;
        File rootDir = getTempReceiveDir(client);
        File targetFile = new File(rootDir, filePath);
        FileUtils.copyFile(existsFile, targetFile);
        return true;
    }

    public static final void receive_file(Client client, String filePath, long lastModified, InputStream is)
            throws IOException {
        File rootDir = getTempReceiveDir(client);
        File targetFile = new File(rootDir, filePath);
        targetFile.createNewFile();
        FileOutputStream fos = null;
        try {
            fos = new FileOutputStream(targetFile);
            int len;
            byte[] buffer = new byte[131072];
            while ((len = is.read(buffer)) != -1)
                fos.write(buffer, 0, len);
        } catch (IOException e) {
            throw e;
        } finally {
            IOUtils.close(fos);
        }
        targetFile.setLastModified(lastModified);
    }

    public static int getMaxClauseCount() {
        return BooleanQuery.getMaxClauseCount();
    }

    public static void setMaxClauseCount(int value) {
        BooleanQuery.setMaxClauseCount(value);
    }

    public final static RendererResults getRendererResults() {
        return rendererResults;
    }

    public static ThreadGroup getThreadGroup() {
        return threadGroup;
    }

}