org.zenoss.zep.index.impl.MultiBackendEventIndexDao.java Source code

Java tutorial

Introduction

Here is the source code for org.zenoss.zep.index.impl.MultiBackendEventIndexDao.java

Source

/*****************************************************************************
 *
 * Copyright (C) Zenoss, Inc. 2014, all rights reserved.
 *
 * This content is made available according to terms specified in
 * License.zenoss under the directory where your Zenoss product is installed.
 *
 ****************************************************************************/
package org.zenoss.zep.index.impl;

import com.google.common.base.Predicates;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.springframework.scheduling.TaskScheduler;
import org.zenoss.protobufs.zep.Zep.*;
import org.zenoss.zep.Messages;
import org.zenoss.zep.UUIDGenerator;
import org.zenoss.zep.ZepException;
import org.zenoss.zep.dao.EventBatch;
import org.zenoss.zep.dao.EventBatchParams;
import org.zenoss.zep.dao.EventSummaryBaseDao;
import org.zenoss.zep.index.EventIndexBackend;
import org.zenoss.zep.index.SavedSearchProcessor;
import org.zenoss.zep.index.WorkQueue;
import org.zenoss.zep.index.WorkQueueBuilder;
import org.zenoss.zep.utils.KeyValueStore;

import java.io.IOException;
import java.io.Serializable;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

public class MultiBackendEventIndexDao extends BaseEventIndexDaoImpl<MultiBackendSavedSearch> {

    protected boolean useRedis;
    private String readerBackendId;
    private boolean disableRebuilders = false;
    private boolean enableAsyncProcessing = true;
    private final WorkQueueBuilder queueBuilder;
    private final List<EventIndexBackendConfiguration> initialBackendConfigurations;
    private final Map<String, EventIndexBackendConfiguration> backends;
    private final Map<String, WorkQueue> workQueues;
    private final Map<String, WorkerThread> workers;
    private final Map<String, RebuilderThread> rebuilders;
    private final KeyValueStore store;
    private final EventSummaryBaseDao eventDao;
    private final ReadWriteLock backendsLock = new ReentrantReadWriteLock();
    private final Lock backendsUse = backendsLock.readLock();
    private final Lock backendsModify = backendsLock.writeLock();

    public MultiBackendEventIndexDao(String name, EventSummaryBaseDao eventDao, WorkQueueBuilder queueBuilder,
            KeyValueStore store, Messages messages, TaskScheduler scheduler, UUIDGenerator uuidGenerator) {
        super(name, messages, scheduler, uuidGenerator);
        this.store = store;
        this.eventDao = eventDao;
        this.queueBuilder = queueBuilder;
        backends = Maps.newLinkedHashMap();
        workQueues = Maps.newConcurrentMap();
        workers = Maps.newConcurrentMap();
        rebuilders = Maps.newConcurrentMap();
        initialBackendConfigurations = Lists.newArrayList();
    }

    public final synchronized void disableAsyncProcessing() {
        this.enableAsyncProcessing = false;
        backendsUse.lock();
        try {
            for (String backendId : Lists.newArrayList(workers.keySet()))
                stopBackendWorker(backendId);
        } finally {
            backendsUse.unlock();
        }
    }

    /** Only call this within a backendsUse.lock() block. */
    private void processTasks(String backendId, List<EventIndexBackendTask> tasks, WorkQueue q)
            throws ZepException {
        final EventIndexBackendConfiguration configuration = backends.get(backendId);
        if (configuration == null)
            throw new ZepException("Tried to process tasks for unknown backend: " + backendId);
        final EventIndexBackend backend = configuration.getBackend();
        if (backend == null)
            throw new ZepException("Tried to process tasks for unknown backend: " + backendId);
        logger.debug("Processing {} tasks for backend {}", tasks.size(), backendId);

        final Set<EventIndexBackendTask> flushes = Sets.newHashSet();
        final Map<String, EventIndexBackendTask> indexTasks = Maps.newHashMapWithExpectedSize(tasks.size());
        final Set<EventSummary> toIndex = Sets.newHashSetWithExpectedSize(tasks.size());

        for (EventIndexBackendTask task : tasks) {
            switch (task.op) {
            case FLUSH:
                flushes.add(task);
                break;
            case INDEX_EVENT:
                indexTasks.put(task.uuid, task);
                toIndex.add(EventSummary.newBuilder().setUuid(task.uuid).setLastSeenTime(task.lastSeen).build());
                break;
            default:
                logger.error("UNEXPECTED TASK OPERATION: {}", task.op);
                q.complete(task);
            }
        }

        try {
            if (!toIndex.isEmpty()) {
                logger.debug(String.format("Looking up %d events by primary key", toIndex.size()));
                List<EventSummary> events = eventDao.findByKey(toIndex);
                if (events.size() != toIndex.size())
                    logger.info("Found {} of {} events by primary key", events.size(), toIndex.size());
                else
                    logger.debug("Found {} of {} events by primary key", events.size(), toIndex.size());
                try {
                    backend.index(events);
                    logger.debug("Indexed {} events", events.size());
                } catch (ZepException e) {
                    if (logger.isDebugEnabled())
                        logger.warn(String.format("failed to process task to index events (%d) for backend %s",
                                events.size(), backendId), e);
                    else
                        logger.warn(String.format("failed to process task to index events (%d) for backend %s",
                                events.size(), backendId));
                }
                List<EventIndexBackendTask> completedTasks = Lists.newArrayListWithExpectedSize(events.size());
                for (EventSummary event : events) {
                    EventIndexBackendTask task = indexTasks.remove(event.getUuid());
                    if (task != null) // should always be true
                        completedTasks.add(task);
                }
                q.completeAll(completedTasks);

                if (!indexTasks.isEmpty()) {
                    try {
                        if (configuration.isHonorDeletes()) {
                            logger.debug(
                                    "Removing {} events from the index since they weren't found by primary key in the database",
                                    indexTasks.size());
                            backend.delete(indexTasks.keySet());
                        }
                        q.completeAll(indexTasks.values());
                    } catch (ZepException e) {
                        if (logger.isDebugEnabled())
                            logger.warn(String.format("failed to delete %d events from backend %s", toIndex.size(),
                                    backendId), e);
                        else
                            logger.warn(String.format("failed to delete %d events from backend %s", toIndex.size(),
                                    backendId));
                    }
                }
            }

            if (!flushes.isEmpty()) {
                try {
                    logger.debug("flushing backend");
                    backend.flush();
                    q.completeAll(flushes);
                } catch (ZepException e) {
                    if (logger.isDebugEnabled())
                        logger.warn(String.format("failed to process tasks %s for backend %s", flushes, backendId),
                                e);
                    else
                        logger.warn(String.format("failed to process tasks %s for backend %s", flushes, backendId));

                }
            }
        } catch (ZepException e) {
            if (logger.isDebugEnabled())
                logger.warn(String.format("failed to find events for UUIDs %s for backend %s", indexTasks.keySet(),
                        backendId), e);
            else
                logger.warn(String.format("failed to find events for UUIDs %s for backend %s", indexTasks.keySet(),
                        backendId));
        }
    }

    /**
     * The thread will exit cleanly once its backend has no registered work queue (in workQueues).
     */
    private class WorkerThread extends Thread {
        private final String backendId;

        public WorkerThread(String backendId) {
            this.backendId = backendId;
            this.setDaemon(true);
            this.setName(
                    MultiBackendEventIndexDao.this + " backend " + backendId + " event indexing worker thread");
        }

        @Override
        public void run() {
            logger.info("Started processing queue for {}", backendId);
            WorkQueue q = workQueues.get(backendId);
            EventIndexBackendConfiguration config = getBackendConfiguration(backendId);

            EventIndexBackend backend = config.getBackend();
            if (backend == null) {
                logger.error("Stopping worker for unknown backend: {}", backendId);
            } else {
                List<EventIndexBackendTask> tasks;
                while (q != null && config != null && workers.get(backendId) == this) {
                    boolean sleep_and_continue = false;

                    if (!backend.isReady()) {
                        logger.info("Waiting for backend {} to be ready", backendId);
                        sleep_and_continue = true;
                    } else if (!backend.ping()) {
                        logger.warn("Backend {} cannot be pinged", backendId);
                        sleep_and_continue = true;
                    } else if (enableAsyncProcessing && config.isAsyncUpdates() && !q.isReady()) {
                        logger.warn("Backend {}: Worker queue is not ready.", backendId);
                        sleep_and_continue = true;
                    }

                    if (sleep_and_continue) {
                        try {
                            sleep(1000);
                        } catch (InterruptedException e) {
                            this.interrupt();
                        }
                        continue;
                    }

                    try {
                        int batchSize = config.getBatchSize();
                        //logger.debug("Polling for tasks to process");
                        tasks = q.poll(batchSize, 500, TimeUnit.MILLISECONDS);
                        if (tasks == null || tasks.isEmpty())
                            continue;
                        backendsUse.lock();
                        try {
                            logger.debug(getName() + " fetched {} tasks to process", tasks.size());
                            processTasks(backendId, tasks, q);
                        } catch (ZepException e) {
                            logger.warn(
                                    String.format("failed to process tasks %s for backend %s", tasks, backendId),
                                    e);
                        } finally {
                            backendsUse.unlock();
                        }
                    } catch (InterruptedException e) {
                        // continue
                    } catch (RuntimeException e) {
                        logger.warn(String.format("failed to fetch tasks for backend %s", backendId), e);
                        try {
                            sleep(1000);
                        } catch (InterruptedException ie) {
                            /* ignore */ }
                    } finally {
                        q = workQueues.get(backendId);
                        config = getBackendConfiguration(backendId);
                    }
                }
            }
            logger.info("Stopped processing queue for {}", backendId);
        }
    }

    /**
     * The thread will exit cleanly once its queue is no longer found in workQueues.
     */
    private class RequeueThread extends Thread {
        private final String backendId;
        private final WorkQueue q;

        public RequeueThread(String backendId, WorkQueue q) {
            this.backendId = backendId;
            this.q = q;
            this.setDaemon(true);
            this.setName(MultiBackendEventIndexDao.this + " backend " + backendId + " requeue thread");
        }

        @Override
        public void run() {
            logger.debug(getName() + " started");
            while (workQueues.get(backendId) == q) {
                long requeued = q.requeueOldTasks();
                if (requeued > 0) {
                    logger.warn(getName() + " requeued " + requeued + " old tasks");
                }
                try {
                    sleep(1000);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                }
            }
            logger.debug(getName() + " exited");
        }

    }

    protected EventIndexBackendConfiguration getBackendConfiguration(String backendId) {
        backendsUse.lock();
        try {
            EventIndexBackendConfiguration result = backends.get(backendId);
            if (result == null)
                return null;
            return result.clone();
        } finally {
            backendsUse.unlock();
        }
    }

    /**
     * Start a background thread to pull tasks off a work queue and pass them to the backend.
     *
     * Pre-condition: The backend must have an entry in workQueues (the worker thread will exit if not).
     */
    protected void startBackendWorker(String backendId) {
        backendsModify.lock();
        try {
            stopBackendWorker(backendId);
            if (!enableAsyncProcessing)
                return;
            WorkQueue q = queueBuilder.build(backendId);
            workQueues.put(backendId, q);
            WorkerThread worker = new WorkerThread(backendId);
            worker.start();
            if (worker.isAlive())
                workers.put(backendId, worker);
            else
                logger.error("Failed to start worker thread for event indexing backend {}", backendId);
            new RequeueThread(backendId, q).start();
        } finally {
            backendsModify.unlock();
        }
    }

    protected void stopBackendWorker(String backendId) {
        WorkerThread worker;
        backendsModify.lock();
        try {
            worker = workers.remove(backendId);
            workQueues.remove(backendId);
        } finally {
            backendsModify.unlock();
        }
        if (worker != null) {
            logger.info("Stopping backend worker for " + getName() + " backend " + backendId);
            try {
                worker.join();
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
        }
    }

    public void close() throws IOException {
        backendsModify.lock();
        try {
            for (EventIndexBackendConfiguration b : backends.values()) {
                closeBackendSavedSearches(b.getName());
                try {
                    b.getBackend().close();
                } catch (IOException e) {
                    logger.warn("exception while closing backend: " + b.getName(), e);
                }
            }
            workers.clear(); // This also causes each of the worker threads to exit cleanly.
            workQueues.clear();
        } finally {
            backendsModify.unlock();
        }
    }

    /**
     * Prevents the use of redis for configuration rebuilder
     */
    public void setUseRedis(boolean useRedis) {
        this.useRedis = useRedis;
        if (!useRedis) {
            this.disableRebuilders();
        }
    }

    /**
     * Set the initial list of EventIndexBackendConfiguration
     */
    public void setInitialBackendConfigurations(List<EventIndexBackendConfiguration> backends) {
        initialBackendConfigurations.addAll(Collections2.filter(backends, Predicates.notNull()));
    }

    public List<EventIndexBackendConfiguration> getInitialBackendConfigurations() {
        return initialBackendConfigurations;
    }

    public void init() throws ZepException {
        setBackends(initialBackendConfigurations);
    }

    protected void setBackends(List<EventIndexBackendConfiguration> backends) throws ZepException {
        int readers = 0;
        backendsModify.lock();
        try {
            final Set<String> toDisable = enabledBackends();
            for (EventIndexBackendConfiguration newConf : backends) {
                reconfigureBackend(newConf);
                if (newConf.getStatus() == null) {
                    toDisable.remove(newConf.getName());
                } else
                    switch (newConf.getStatus()) {
                    case READER:
                        if (++readers > 1)
                            logger.error("Multi-backend event indexer " + getName()
                                    + " configured with multiple readers! (Only one will be selected)");
                        // Intentional fall-through...
                    case WRITER:
                        toDisable.remove(newConf.getName());
                    }
            }
            for (String backendId : toDisable) {
                EventIndexBackendConfiguration config = this.backends.get(backendId);
                stopRebuilder(backendId);
                stopBackendWorker(backendId);
                if (config.isWriter()) {
                    logger.info("disabled backend (" + backendId + ") for " + getName());
                    config.setStatus(BackendStatus.REGISTERED);
                }
            }
            Set<String> enabled = enabledBackends();
            if (!enabled.contains(readerBackendId)) {
                logger.warn("reader backend (" + readerBackendId + ") was disabled");
                if (enabled.isEmpty()) {
                    readerBackendId = null;
                    logger.warn("all event indexer backends for " + getName() + " have been disabled!");
                } else {
                    String newReaderId = enabled.iterator().next();
                    this.backends.get(newReaderId).setStatus(BackendStatus.READER);
                    readerBackendId = newReaderId;
                    logger.warn("auto-selected backend (" + newReaderId + ") as reader for " + getName());
                }
            }
        } finally {
            backendsModify.unlock();
        }
    }

    private void reconfigureBackend(EventIndexBackendConfiguration input) throws ZepException {
        backendsModify.lock();
        try {
            final String backendId = input.getName();
            for (EventIndexBackendConfiguration conf : backends.values()) {
                if (conf.getBackend().equals(input.getBackend()) && !conf.getName().equals(backendId))
                    throw new ZepException("the backend (" + backendId
                            + ") has already been registered with a different ID: " + conf.getName());
            }
            EventIndexBackendConfiguration conf = backends.get(backendId);
            if (conf == null) {
                if (input.getBackend() == null)
                    throw new ZepException("unknown backend (" + backendId + ") for " + getName());
                conf = input.clone();
                backends.put(backendId, conf);
                switch (conf.getStatus()) {
                case READER:
                    if (backendId != null && !backendId.equals(readerBackendId)) {
                        EventIndexBackend reader = getReader();
                        if (reader != null)
                            reader.closeSavedSearches();
                        readerBackendId = backendId;
                        logger.info("selected backend (" + backendId + ") as reader for " + getName());
                    }
                    // Intentional fall-through...
                case WRITER:
                    logger.info("enabled backend (" + backendId + ") for " + getName());
                    if (enableAsyncProcessing && conf.isAsyncUpdates()) {
                        startBackendWorker(backendId);
                    }
                    startRebuilder(backendId);
                }
            } else {
                if (input.getBackend() != null)
                    conf.setBackend(input.getBackend()); //throws ZepException if it changed.
                switch (input.getStatus()) {
                case REGISTERED:
                    stopRebuilder(backendId);
                    stopBackendWorker(backendId);
                    if (conf.isWriter())
                        logger.info("disabled backend (" + backendId + ") for " + getName());
                    if (readerBackendId.equals(backendId) && conf.getBackend() != null)
                        conf.getBackend().closeSavedSearches();
                    break;
                case READER:
                    if (backendId != null && !backendId.equals(readerBackendId)) {
                        EventIndexBackend reader = getReader();
                        if (reader != null)
                            reader.closeSavedSearches();
                        readerBackendId = backendId;
                        logger.info("selected backend (" + backendId + ") as reader for " + getName());
                    }
                    // Intentional fall-through...
                case WRITER:
                    if (!conf.isWriter())
                        logger.info("enabled backend (" + backendId + ") for " + getName());
                    if (enableAsyncProcessing && input.isAsyncUpdates()) {
                        if (!workers.containsKey(backendId)) {
                            startBackendWorker(backendId);
                        }
                    }
                    if (!rebuilders.containsKey(backendId))
                        startRebuilder(backendId);
                    break;
                }
                conf.merge(input);
            }
        } finally {
            backendsModify.unlock();
        }
    }

    protected Set<String> enabledBackends() {
        backendsUse.lock();
        try {
            Set<String> result = Sets.newLinkedHashSet();
            for (EventIndexBackendConfiguration config : backends.values())
                if (config.isWriter())
                    result.add(config.getName());
            return result;
        } finally {
            backendsUse.unlock();
        }
    }

    public static enum BackendStatus {
        REGISTERED, WRITER, READER
    }

    /** Do not use this outside of a {@link #backendsUse} lock-block. */
    private EventIndexBackend getReader() {
        if (readerBackendId == null)
            return null;
        final EventIndexBackendConfiguration c = backends.get(readerBackendId);
        return (c == null) ? null : c.getBackend();
    }

    @Override
    public void index(EventSummary event) throws ZepException {
        backendsUse.lock();
        try {
            for (EventIndexBackendConfiguration config : backends.values()) {
                if (config.isWriter()) {
                    if (enableAsyncProcessing && config.isAsyncUpdates()) {
                        workQueues.get(config.getName())
                                .add(EventIndexBackendTask.Index(event.getUuid(), event.getLastSeenTime()));
                    } else {
                        config.getBackend().index(event);
                    }
                }
            }
        } finally {
            backendsUse.unlock();
        }
    }

    @Override
    public void indexMany(List<EventSummary> events) throws ZepException {
        if (events == null || events.isEmpty())
            return;
        backendsUse.lock();
        try {
            List<EventIndexBackendTask> tasks = null;
            for (EventIndexBackendConfiguration config : backends.values()) {
                if (config.isWriter()) {
                    if (enableAsyncProcessing && config.isAsyncUpdates()) {
                        if (tasks == null) {
                            tasks = Lists.newArrayListWithExpectedSize(events.size());
                            for (EventSummary event : events)
                                tasks.add(EventIndexBackendTask.Index(event.getUuid(), event.getLastSeenTime()));
                        }
                        workQueues.get(config.getName()).addAll(tasks);
                    } else {
                        config.getBackend().index(events);
                    }
                }
            }
        } finally {
            backendsUse.unlock();
        }
    }

    @Override
    public void delete(String uuid) throws ZepException {
        backendsUse.lock();
        try {
            for (EventIndexBackendConfiguration config : backends.values()) {
                if (config.isWriter() && config.isHonorDeletes()) {
                    if (enableAsyncProcessing && config.isAsyncUpdates()) {
                        workQueues.get(config.getName()).add(EventIndexBackendTask.Index(uuid, null));
                    } else {
                        config.getBackend().delete(uuid);
                    }
                }
            }
        } finally {
            backendsUse.unlock();
        }
    }

    @Override
    public void delete(List<String> uuids) throws ZepException {
        if (uuids == null || uuids.isEmpty())
            return;
        backendsUse.lock();
        try {
            List<EventIndexBackendTask> tasks = null;
            for (EventIndexBackendConfiguration config : backends.values()) {
                if (config.isWriter() && config.isHonorDeletes()) {
                    if (enableAsyncProcessing && config.isAsyncUpdates()) {
                        if (tasks == null) {
                            tasks = Lists.newArrayListWithExpectedSize(uuids.size());
                            for (String uuid : uuids)
                                tasks.add(EventIndexBackendTask.Index(uuid, null));
                        }
                        workQueues.get(config.getName()).addAll(tasks);
                    } else {
                        config.getBackend().delete(uuids);
                    }
                }
            }
        } finally {
            backendsUse.unlock();
        }
    }

    @Override
    public void clear() throws ZepException {
        backendsUse.lock();
        try {
            for (EventIndexBackendConfiguration config : backends.values()) {
                if (config.isWriter() && config.isHonorDeletes()) {
                    config.getBackend().clear();
                }
            }
        } finally {
            backendsUse.unlock();
        }
    }

    protected void clear(String backendId) throws ZepException {
        backendsUse.lock();
        try {
            EventIndexBackendConfiguration config = backends.get(backendId);
            if (config != null && config.isWriter() && config.isHonorDeletes()) {
                config.getBackend().clear();
            }
        } finally {
            backendsUse.unlock();
        }
    }

    @Override
    public void purge(Date threshold) throws ZepException {
        backendsUse.lock();
        try {
            for (EventIndexBackendConfiguration config : backends.values()) {
                if (config.isWriter() && config.isHonorDeletes()) {
                    config.getBackend().purge(threshold);
                }
            }
        } finally {
            backendsUse.unlock();
        }
    }

    @Override
    public void commit() throws ZepException {
        backendsUse.lock();
        try {
            for (EventIndexBackendConfiguration config : backends.values()) {
                if (config.isWriter()) {
                    if (enableAsyncProcessing && config.isAsyncUpdates()) {
                        workQueues.get(config.getName()).add(EventIndexBackendTask.Flush());
                    } else {
                        config.getBackend().flush();
                    }
                }
            }
        } finally {
            backendsUse.unlock();
        }
    }

    @Override
    public int getNumDocs() throws ZepException {
        backendsUse.lock();
        try {
            return (int) getReader().count();
        } finally {
            backendsUse.unlock();
        }
    }

    @Override
    public long getSize() {
        backendsUse.lock();
        try {
            return getReader().sizeInBytes();
        } catch (UnsupportedOperationException e) {
            return -1; //TODO: figure out the right thing to do
        } finally {
            backendsUse.unlock();
        }
    }

    @Override
    public EventSummary findByUuid(String uuid) throws ZepException {
        backendsUse.lock();
        try {
            return getReader().findByUuid(uuid);
        } finally {
            backendsUse.unlock();
        }
    }

    @Override
    public EventSummaryResult list(EventSummaryRequest request) throws ZepException {
        backendsUse.lock();
        try {
            return getReader().list(request);
        } finally {
            backendsUse.unlock();
        }
    }

    @Override
    public EventSummaryResult listUuids(EventSummaryRequest request) throws ZepException {
        backendsUse.lock();
        try {
            return getReader().listUuids(request);
        } finally {
            backendsUse.unlock();
        }
    }

    @Override
    public EventTagSeveritiesSet getEventTagSeverities(EventFilter filter) throws ZepException {
        backendsUse.lock();
        try {
            return getReader().getEventTagSeverities(filter);
        } finally {
            backendsUse.unlock();
        }
    }

    @Override
    public MultiBackendSavedSearch buildSavedSearch(String uuid, EventQuery eventQuery) throws ZepException {
        backendsUse.lock();
        try {
            final String savedSearchId = getReader().createSavedSearch(eventQuery);
            return new MultiBackendSavedSearch(generateUuid(), eventQuery.getTimeout(), readerBackendId,
                    savedSearchId, this);
        } finally {
            backendsUse.unlock();
        }
    }

    @Override
    public SavedSearchProcessor<MultiBackendSavedSearch> savedSearchProcessor() {
        return new SavedSearchProcessor<MultiBackendSavedSearch>() {
            @Override
            public EventSummaryResult result(MultiBackendSavedSearch search, int offset, int limit)
                    throws ZepException {
                return backends.get(search.backendId).getBackend().savedSearch(search.savedSearchId, offset, limit);
            }
        };
    }

    @Override
    public SavedSearchProcessor<MultiBackendSavedSearch> savedSearchUuidsProcessor() {
        return new SavedSearchProcessor<MultiBackendSavedSearch>() {
            @Override
            public EventSummaryResult result(MultiBackendSavedSearch search, int offset, int limit)
                    throws ZepException {
                return backends.get(search.backendId).getBackend().savedSearchUuids(search.savedSearchId, offset,
                        limit);
            }
        };
    }

    public void closeBackendSavedSearch(String backendId, String savedSearchId) {
        EventIndexBackendConfiguration config = backends.get(backendId);
        if (config == null)
            return;
        EventIndexBackend backend = config.getBackend();
        if (backend == null)
            return;
        backend.closeSavedSearch(savedSearchId);
    }

    private void closeBackendSavedSearches(String backendId) {
        EventIndexBackendConfiguration config = backends.get(backendId);
        if (config == null)
            return;
        EventIndexBackend backend = config.getBackend();
        if (backend == null)
            return;
        backend.closeSavedSearches();
    }

    public final synchronized void disableRebuilders() {
        this.disableRebuilders = true;
        backendsUse.lock();
        try {
            for (String backendId : Lists.newArrayList(rebuilders.keySet())) {
                stopRebuilder(backendId);
            }
        } finally {
            backendsUse.unlock();
        }
    }

    public final synchronized void startRebuilder(String backendId) {
        stopRebuilder(backendId);
        if (disableRebuilders)
            return;
        RebuilderThread rebuilder = new RebuilderThread(this, backendId);
        rebuilders.put(backendId, rebuilder);
        rebuilder.start();
        if (!rebuilder.isAlive())
            logger.error("failed to start event index rebuilder thread for " + getName() + " backend " + backendId);
    }

    public final synchronized void stopRebuilder(String backendId) {
        RebuilderThread rebuilder = rebuilders.remove(backendId);
        if (rebuilder != null) {
            logger.info("shutting down event index rebuilder thread for " + getName() + " backend " + backendId);
            try {
                rebuilder.join();
            } catch (InterruptedException e) {
                /* no problem */ }
        }
    }

    public final synchronized void forceRebuild(String backendId) {
        RebuilderThread rebuilder = rebuilders.get(backendId);
        if (rebuilder == null)
            logger.error(
                    "unable to force rebuild with rebuilder stopped for " + getName() + " backend " + backendId);
        else {
            rebuilder.forceRebuild = true;
        }
    }

    private static class RebuilderProgress implements Serializable {
        public final long throughTime;
        public final EventBatchParams nextBatch;
        public final boolean done;

        public static RebuilderProgress begin(long throughTime) {
            return new RebuilderProgress(throughTime, null, false);
        }

        public static RebuilderProgress done(RebuilderProgress progress) {
            return new RebuilderProgress(progress.throughTime, null, true);
        }

        public static RebuilderProgress next(RebuilderProgress progress, EventBatchParams nextBatch) {
            return new RebuilderProgress(progress.throughTime, nextBatch, false);
        }

        private RebuilderProgress(long throughTime, EventBatchParams nextBatch, boolean done) {
            this.throughTime = throughTime;
            this.nextBatch = nextBatch;
            this.done = done;
        }

        public String toString() {
            StringBuilder sb = new StringBuilder();
            sb.append("throughTime:");
            sb.append(throughTime);
            sb.append(" ");
            if (nextBatch != null) {
                sb.append("nextLastSeen:");
                sb.append(nextBatch.nextLastSeen);
                sb.append(" ");
                if (nextBatch.nextUuid != null) {
                    sb.append("nextUuid:");
                    sb.append(nextBatch.nextUuid);
                    sb.append(" ");
                }
            }
            sb.append("done:");
            sb.append(done);
            return sb.toString();
        }

        public static RebuilderProgress parse(String s) {
            try {
                Map<String, String> pairs = new HashMap<String, String>();
                for (String pair : s.split(" ")) {
                    String[] splitPair = pair.split(":", 2);
                    if (splitPair.length == 2)
                        pairs.put(splitPair[0], splitPair[1]);
                }
                long throughTime = Long.parseLong(pairs.get("throughTime"));
                if ("true".equalsIgnoreCase(pairs.get("done")))
                    return new RebuilderProgress(throughTime, null, true);
                String lastSeenStr = pairs.get("nextLastSeen");
                if (lastSeenStr == null)
                    return new RebuilderProgress(throughTime, null, false);
                long nextLastSeen = Long.parseLong(lastSeenStr);
                String nextUuid = pairs.get("nextUuid");
                EventBatchParams nextBatch = new EventBatchParams(nextLastSeen, nextUuid);
                return new RebuilderProgress(throughTime, nextBatch, false);
            } catch (RuntimeException e) {
                throw new IllegalArgumentException("Unable to parse: " + s, e);
            }
        }
    }

    private static final SimpleDateFormat UTC;
    static {
        UTC = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S 'UTC'");
        UTC.setTimeZone(TimeZone.getTimeZone("UTC"));
    }

    private static String formatUTC(long time) {
        synchronized (UTC) {
            return UTC.format(new Date(time));
        }
    }

    private class RebuilderThread extends Thread {

        private final String backendId;
        private final byte[] storeKey;
        private long nextStatusLog = 0;
        public transient boolean forceRebuild;

        public RebuilderThread(MultiBackendEventIndexDao dao, String backendId) {
            this.backendId = backendId;
            this.forceRebuild = false;
            this.storeKey = ("org.zenoss.zep.index.rebuilder:" + dao.getName() + ":" + backendId).getBytes();
            this.setDaemon(true);
            this.setName(dao.getName() + " backend " + backendId + " event index rebuilder thread");
        }

        private void saveRebuildProgress(RebuilderProgress progress) throws IOException {
            store.store(storeKey, progress.toString().getBytes());
        }

        private RebuilderProgress loadRebuildProgress() throws IOException {
            final byte[] data = store.load(storeKey);
            if (data == null || data.length == 0)
                return null;
            try {
                return RebuilderProgress.parse(new String(data));
            } catch (IllegalArgumentException e) {
                logger.error("exception parsing index rebuilder progress", e);
                return null;
            }
        }

        private void logStatus(String msg) {
            long now = System.currentTimeMillis();
            if (now > nextStatusLog) {
                long count = -1;
                EventIndexBackendConfiguration configuration = backends.get(backendId);
                if (configuration != null) {
                    EventIndexBackend backend = configuration.getBackend();
                    if (backend != null) {
                        try {
                            count = backend.count();
                        } catch (ZepException e) {
                            // ignore it
                        } catch (RuntimeException e) {
                            // ignore it
                        }
                    }
                }
                WorkQueue q = workQueues.get(backendId);
                logger.info(getName() + " " + msg + " [current queue size: " + (q == null ? "n/a" : q.size())
                        + ", index size: " + ((count >= 0) ? count : "unknown") + "]");
                nextStatusLog = now + 60000; // no more than once a minute
            }
        }

        @Override
        public void run() {
            logger.info(getName() + " has started its run");
            EventIndexBackendConfiguration configuration = backends.get(backendId);
            if (configuration == null) {
                logger.error(getName() + " running with a missing backend configuration! Exiting!");
                return;
            }
            EventIndexBackend backend = configuration.getBackend();
            if (backend == null) {
                logger.error(getName() + " running with a missing backend! Exiting!");
                return;
            }

            while (rebuilders.get(backendId) == this) {
                try {
                    if (!backend.isReady()) {
                        logStatus("is waiting for backend to be ready");
                        sleep(1000);
                        continue;
                    }
                    if (enableAsyncProcessing && configuration.isAsyncUpdates()
                            && !workQueues.get(backendId).isReady()) {
                        logStatus("is waiting for work queue to be ready");
                        sleep(1000);
                        continue;
                    }

                    RebuilderProgress progress = loadRebuildProgress();
                    if (forceRebuild) {
                        logStatus("is starting a new rebuild");
                        saveRebuildProgress(RebuilderProgress.begin(System.currentTimeMillis()));
                        forceRebuild = false;
                        progress = loadRebuildProgress();
                    }

                    if (progress == null || progress.done) {
                        logStatus("is dormant");
                        sleep(1000);
                        continue;
                    }

                    int batchSize = configuration.getBatchSize();
                    EventBatch batch = null;
                    backendsUse.lock();
                    try {
                        if (configuration.isWriter()) {
                            List<EventSummary> events = null;
                            try {
                                batch = eventDao.listBatch(progress.nextBatch, progress.throughTime, batchSize);
                                events = batch.events;
                            } catch (RuntimeException e) {
                                logger.debug("Unable to listBatch due to exception: " + e.getMessage(), e);
                            }

                            if (enableAsyncProcessing && configuration.isAsyncUpdates()) {
                                if (events.isEmpty()) {
                                    // Finished!
                                    workQueues.get(backendId).add(EventIndexBackendTask.Flush());
                                } else {
                                    logger.debug("Converting {} events into tasks.", events.size());
                                    List<EventIndexBackendTask> tasks = Lists
                                            .newArrayListWithExpectedSize(events.size());
                                    for (EventSummary event : events)
                                        tasks.add(EventIndexBackendTask.Index(event.getUuid(),
                                                event.getLastSeenTime()));
                                    logger.debug("Queuing up another {} events.", events.size());
                                    workQueues.get(backendId).addAll(tasks);
                                    logger.debug("Done queuing up {} events.", events.size());
                                }
                                logStatus("queued events up to:" + batch);
                            } else {
                                backend.index(events);
                                logStatus("indexed events up to:" + batch);
                                if (events.isEmpty()) {
                                    // Finished!
                                    backend.flush();
                                }
                            }
                        }
                    } finally {
                        backendsUse.unlock();
                    }
                    if (batch.events.isEmpty())
                        progress = RebuilderProgress.done(progress);
                    else
                        progress = RebuilderProgress.next(progress, batch.nextParams);
                    saveRebuildProgress(progress);
                } catch (InterruptedException e) {
                    // ignore it
                } catch (ZepException e) {
                    logger.warn("error while rebuilding for " + getName(), e);
                } catch (IOException e) {
                    logger.warn("error while rebuilding for " + getName(), e);
                } catch (RuntimeException e) {
                    logger.warn("error while rebuilding for " + getName(), e);
                }
            }
            logger.info(getName() + " has ended its run");
        }
    }

}