Java tutorial
/***************************************************************************** * * Copyright (C) Zenoss, Inc. 2014, all rights reserved. * * This content is made available according to terms specified in * License.zenoss under the directory where your Zenoss product is installed. * ****************************************************************************/ package org.zenoss.zep.index.impl; import com.google.common.base.Predicates; import com.google.common.collect.Collections2; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.springframework.scheduling.TaskScheduler; import org.zenoss.protobufs.zep.Zep.*; import org.zenoss.zep.Messages; import org.zenoss.zep.UUIDGenerator; import org.zenoss.zep.ZepException; import org.zenoss.zep.dao.EventBatch; import org.zenoss.zep.dao.EventBatchParams; import org.zenoss.zep.dao.EventSummaryBaseDao; import org.zenoss.zep.index.EventIndexBackend; import org.zenoss.zep.index.SavedSearchProcessor; import org.zenoss.zep.index.WorkQueue; import org.zenoss.zep.index.WorkQueueBuilder; import org.zenoss.zep.utils.KeyValueStore; import java.io.IOException; import java.io.Serializable; import java.text.SimpleDateFormat; import java.util.*; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; public class MultiBackendEventIndexDao extends BaseEventIndexDaoImpl<MultiBackendSavedSearch> { protected boolean useRedis; private String readerBackendId; private boolean disableRebuilders = false; private boolean enableAsyncProcessing = true; private final WorkQueueBuilder queueBuilder; private final List<EventIndexBackendConfiguration> initialBackendConfigurations; private final Map<String, EventIndexBackendConfiguration> backends; private final Map<String, WorkQueue> workQueues; private final Map<String, WorkerThread> workers; private final Map<String, RebuilderThread> rebuilders; private final KeyValueStore store; private final EventSummaryBaseDao eventDao; private final ReadWriteLock backendsLock = new ReentrantReadWriteLock(); private final Lock backendsUse = backendsLock.readLock(); private final Lock backendsModify = backendsLock.writeLock(); public MultiBackendEventIndexDao(String name, EventSummaryBaseDao eventDao, WorkQueueBuilder queueBuilder, KeyValueStore store, Messages messages, TaskScheduler scheduler, UUIDGenerator uuidGenerator) { super(name, messages, scheduler, uuidGenerator); this.store = store; this.eventDao = eventDao; this.queueBuilder = queueBuilder; backends = Maps.newLinkedHashMap(); workQueues = Maps.newConcurrentMap(); workers = Maps.newConcurrentMap(); rebuilders = Maps.newConcurrentMap(); initialBackendConfigurations = Lists.newArrayList(); } public final synchronized void disableAsyncProcessing() { this.enableAsyncProcessing = false; backendsUse.lock(); try { for (String backendId : Lists.newArrayList(workers.keySet())) stopBackendWorker(backendId); } finally { backendsUse.unlock(); } } /** Only call this within a backendsUse.lock() block. */ private void processTasks(String backendId, List<EventIndexBackendTask> tasks, WorkQueue q) throws ZepException { final EventIndexBackendConfiguration configuration = backends.get(backendId); if (configuration == null) throw new ZepException("Tried to process tasks for unknown backend: " + backendId); final EventIndexBackend backend = configuration.getBackend(); if (backend == null) throw new ZepException("Tried to process tasks for unknown backend: " + backendId); logger.debug("Processing {} tasks for backend {}", tasks.size(), backendId); final Set<EventIndexBackendTask> flushes = Sets.newHashSet(); final Map<String, EventIndexBackendTask> indexTasks = Maps.newHashMapWithExpectedSize(tasks.size()); final Set<EventSummary> toIndex = Sets.newHashSetWithExpectedSize(tasks.size()); for (EventIndexBackendTask task : tasks) { switch (task.op) { case FLUSH: flushes.add(task); break; case INDEX_EVENT: indexTasks.put(task.uuid, task); toIndex.add(EventSummary.newBuilder().setUuid(task.uuid).setLastSeenTime(task.lastSeen).build()); break; default: logger.error("UNEXPECTED TASK OPERATION: {}", task.op); q.complete(task); } } try { if (!toIndex.isEmpty()) { logger.debug(String.format("Looking up %d events by primary key", toIndex.size())); List<EventSummary> events = eventDao.findByKey(toIndex); if (events.size() != toIndex.size()) logger.info("Found {} of {} events by primary key", events.size(), toIndex.size()); else logger.debug("Found {} of {} events by primary key", events.size(), toIndex.size()); try { backend.index(events); logger.debug("Indexed {} events", events.size()); } catch (ZepException e) { if (logger.isDebugEnabled()) logger.warn(String.format("failed to process task to index events (%d) for backend %s", events.size(), backendId), e); else logger.warn(String.format("failed to process task to index events (%d) for backend %s", events.size(), backendId)); } List<EventIndexBackendTask> completedTasks = Lists.newArrayListWithExpectedSize(events.size()); for (EventSummary event : events) { EventIndexBackendTask task = indexTasks.remove(event.getUuid()); if (task != null) // should always be true completedTasks.add(task); } q.completeAll(completedTasks); if (!indexTasks.isEmpty()) { try { if (configuration.isHonorDeletes()) { logger.debug( "Removing {} events from the index since they weren't found by primary key in the database", indexTasks.size()); backend.delete(indexTasks.keySet()); } q.completeAll(indexTasks.values()); } catch (ZepException e) { if (logger.isDebugEnabled()) logger.warn(String.format("failed to delete %d events from backend %s", toIndex.size(), backendId), e); else logger.warn(String.format("failed to delete %d events from backend %s", toIndex.size(), backendId)); } } } if (!flushes.isEmpty()) { try { logger.debug("flushing backend"); backend.flush(); q.completeAll(flushes); } catch (ZepException e) { if (logger.isDebugEnabled()) logger.warn(String.format("failed to process tasks %s for backend %s", flushes, backendId), e); else logger.warn(String.format("failed to process tasks %s for backend %s", flushes, backendId)); } } } catch (ZepException e) { if (logger.isDebugEnabled()) logger.warn(String.format("failed to find events for UUIDs %s for backend %s", indexTasks.keySet(), backendId), e); else logger.warn(String.format("failed to find events for UUIDs %s for backend %s", indexTasks.keySet(), backendId)); } } /** * The thread will exit cleanly once its backend has no registered work queue (in workQueues). */ private class WorkerThread extends Thread { private final String backendId; public WorkerThread(String backendId) { this.backendId = backendId; this.setDaemon(true); this.setName( MultiBackendEventIndexDao.this + " backend " + backendId + " event indexing worker thread"); } @Override public void run() { logger.info("Started processing queue for {}", backendId); WorkQueue q = workQueues.get(backendId); EventIndexBackendConfiguration config = getBackendConfiguration(backendId); EventIndexBackend backend = config.getBackend(); if (backend == null) { logger.error("Stopping worker for unknown backend: {}", backendId); } else { List<EventIndexBackendTask> tasks; while (q != null && config != null && workers.get(backendId) == this) { boolean sleep_and_continue = false; if (!backend.isReady()) { logger.info("Waiting for backend {} to be ready", backendId); sleep_and_continue = true; } else if (!backend.ping()) { logger.warn("Backend {} cannot be pinged", backendId); sleep_and_continue = true; } else if (enableAsyncProcessing && config.isAsyncUpdates() && !q.isReady()) { logger.warn("Backend {}: Worker queue is not ready.", backendId); sleep_and_continue = true; } if (sleep_and_continue) { try { sleep(1000); } catch (InterruptedException e) { this.interrupt(); } continue; } try { int batchSize = config.getBatchSize(); //logger.debug("Polling for tasks to process"); tasks = q.poll(batchSize, 500, TimeUnit.MILLISECONDS); if (tasks == null || tasks.isEmpty()) continue; backendsUse.lock(); try { logger.debug(getName() + " fetched {} tasks to process", tasks.size()); processTasks(backendId, tasks, q); } catch (ZepException e) { logger.warn( String.format("failed to process tasks %s for backend %s", tasks, backendId), e); } finally { backendsUse.unlock(); } } catch (InterruptedException e) { // continue } catch (RuntimeException e) { logger.warn(String.format("failed to fetch tasks for backend %s", backendId), e); try { sleep(1000); } catch (InterruptedException ie) { /* ignore */ } } finally { q = workQueues.get(backendId); config = getBackendConfiguration(backendId); } } } logger.info("Stopped processing queue for {}", backendId); } } /** * The thread will exit cleanly once its queue is no longer found in workQueues. */ private class RequeueThread extends Thread { private final String backendId; private final WorkQueue q; public RequeueThread(String backendId, WorkQueue q) { this.backendId = backendId; this.q = q; this.setDaemon(true); this.setName(MultiBackendEventIndexDao.this + " backend " + backendId + " requeue thread"); } @Override public void run() { logger.debug(getName() + " started"); while (workQueues.get(backendId) == q) { long requeued = q.requeueOldTasks(); if (requeued > 0) { logger.warn(getName() + " requeued " + requeued + " old tasks"); } try { sleep(1000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } logger.debug(getName() + " exited"); } } protected EventIndexBackendConfiguration getBackendConfiguration(String backendId) { backendsUse.lock(); try { EventIndexBackendConfiguration result = backends.get(backendId); if (result == null) return null; return result.clone(); } finally { backendsUse.unlock(); } } /** * Start a background thread to pull tasks off a work queue and pass them to the backend. * * Pre-condition: The backend must have an entry in workQueues (the worker thread will exit if not). */ protected void startBackendWorker(String backendId) { backendsModify.lock(); try { stopBackendWorker(backendId); if (!enableAsyncProcessing) return; WorkQueue q = queueBuilder.build(backendId); workQueues.put(backendId, q); WorkerThread worker = new WorkerThread(backendId); worker.start(); if (worker.isAlive()) workers.put(backendId, worker); else logger.error("Failed to start worker thread for event indexing backend {}", backendId); new RequeueThread(backendId, q).start(); } finally { backendsModify.unlock(); } } protected void stopBackendWorker(String backendId) { WorkerThread worker; backendsModify.lock(); try { worker = workers.remove(backendId); workQueues.remove(backendId); } finally { backendsModify.unlock(); } if (worker != null) { logger.info("Stopping backend worker for " + getName() + " backend " + backendId); try { worker.join(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } } public void close() throws IOException { backendsModify.lock(); try { for (EventIndexBackendConfiguration b : backends.values()) { closeBackendSavedSearches(b.getName()); try { b.getBackend().close(); } catch (IOException e) { logger.warn("exception while closing backend: " + b.getName(), e); } } workers.clear(); // This also causes each of the worker threads to exit cleanly. workQueues.clear(); } finally { backendsModify.unlock(); } } /** * Prevents the use of redis for configuration rebuilder */ public void setUseRedis(boolean useRedis) { this.useRedis = useRedis; if (!useRedis) { this.disableRebuilders(); } } /** * Set the initial list of EventIndexBackendConfiguration */ public void setInitialBackendConfigurations(List<EventIndexBackendConfiguration> backends) { initialBackendConfigurations.addAll(Collections2.filter(backends, Predicates.notNull())); } public List<EventIndexBackendConfiguration> getInitialBackendConfigurations() { return initialBackendConfigurations; } public void init() throws ZepException { setBackends(initialBackendConfigurations); } protected void setBackends(List<EventIndexBackendConfiguration> backends) throws ZepException { int readers = 0; backendsModify.lock(); try { final Set<String> toDisable = enabledBackends(); for (EventIndexBackendConfiguration newConf : backends) { reconfigureBackend(newConf); if (newConf.getStatus() == null) { toDisable.remove(newConf.getName()); } else switch (newConf.getStatus()) { case READER: if (++readers > 1) logger.error("Multi-backend event indexer " + getName() + " configured with multiple readers! (Only one will be selected)"); // Intentional fall-through... case WRITER: toDisable.remove(newConf.getName()); } } for (String backendId : toDisable) { EventIndexBackendConfiguration config = this.backends.get(backendId); stopRebuilder(backendId); stopBackendWorker(backendId); if (config.isWriter()) { logger.info("disabled backend (" + backendId + ") for " + getName()); config.setStatus(BackendStatus.REGISTERED); } } Set<String> enabled = enabledBackends(); if (!enabled.contains(readerBackendId)) { logger.warn("reader backend (" + readerBackendId + ") was disabled"); if (enabled.isEmpty()) { readerBackendId = null; logger.warn("all event indexer backends for " + getName() + " have been disabled!"); } else { String newReaderId = enabled.iterator().next(); this.backends.get(newReaderId).setStatus(BackendStatus.READER); readerBackendId = newReaderId; logger.warn("auto-selected backend (" + newReaderId + ") as reader for " + getName()); } } } finally { backendsModify.unlock(); } } private void reconfigureBackend(EventIndexBackendConfiguration input) throws ZepException { backendsModify.lock(); try { final String backendId = input.getName(); for (EventIndexBackendConfiguration conf : backends.values()) { if (conf.getBackend().equals(input.getBackend()) && !conf.getName().equals(backendId)) throw new ZepException("the backend (" + backendId + ") has already been registered with a different ID: " + conf.getName()); } EventIndexBackendConfiguration conf = backends.get(backendId); if (conf == null) { if (input.getBackend() == null) throw new ZepException("unknown backend (" + backendId + ") for " + getName()); conf = input.clone(); backends.put(backendId, conf); switch (conf.getStatus()) { case READER: if (backendId != null && !backendId.equals(readerBackendId)) { EventIndexBackend reader = getReader(); if (reader != null) reader.closeSavedSearches(); readerBackendId = backendId; logger.info("selected backend (" + backendId + ") as reader for " + getName()); } // Intentional fall-through... case WRITER: logger.info("enabled backend (" + backendId + ") for " + getName()); if (enableAsyncProcessing && conf.isAsyncUpdates()) { startBackendWorker(backendId); } startRebuilder(backendId); } } else { if (input.getBackend() != null) conf.setBackend(input.getBackend()); //throws ZepException if it changed. switch (input.getStatus()) { case REGISTERED: stopRebuilder(backendId); stopBackendWorker(backendId); if (conf.isWriter()) logger.info("disabled backend (" + backendId + ") for " + getName()); if (readerBackendId.equals(backendId) && conf.getBackend() != null) conf.getBackend().closeSavedSearches(); break; case READER: if (backendId != null && !backendId.equals(readerBackendId)) { EventIndexBackend reader = getReader(); if (reader != null) reader.closeSavedSearches(); readerBackendId = backendId; logger.info("selected backend (" + backendId + ") as reader for " + getName()); } // Intentional fall-through... case WRITER: if (!conf.isWriter()) logger.info("enabled backend (" + backendId + ") for " + getName()); if (enableAsyncProcessing && input.isAsyncUpdates()) { if (!workers.containsKey(backendId)) { startBackendWorker(backendId); } } if (!rebuilders.containsKey(backendId)) startRebuilder(backendId); break; } conf.merge(input); } } finally { backendsModify.unlock(); } } protected Set<String> enabledBackends() { backendsUse.lock(); try { Set<String> result = Sets.newLinkedHashSet(); for (EventIndexBackendConfiguration config : backends.values()) if (config.isWriter()) result.add(config.getName()); return result; } finally { backendsUse.unlock(); } } public static enum BackendStatus { REGISTERED, WRITER, READER } /** Do not use this outside of a {@link #backendsUse} lock-block. */ private EventIndexBackend getReader() { if (readerBackendId == null) return null; final EventIndexBackendConfiguration c = backends.get(readerBackendId); return (c == null) ? null : c.getBackend(); } @Override public void index(EventSummary event) throws ZepException { backendsUse.lock(); try { for (EventIndexBackendConfiguration config : backends.values()) { if (config.isWriter()) { if (enableAsyncProcessing && config.isAsyncUpdates()) { workQueues.get(config.getName()) .add(EventIndexBackendTask.Index(event.getUuid(), event.getLastSeenTime())); } else { config.getBackend().index(event); } } } } finally { backendsUse.unlock(); } } @Override public void indexMany(List<EventSummary> events) throws ZepException { if (events == null || events.isEmpty()) return; backendsUse.lock(); try { List<EventIndexBackendTask> tasks = null; for (EventIndexBackendConfiguration config : backends.values()) { if (config.isWriter()) { if (enableAsyncProcessing && config.isAsyncUpdates()) { if (tasks == null) { tasks = Lists.newArrayListWithExpectedSize(events.size()); for (EventSummary event : events) tasks.add(EventIndexBackendTask.Index(event.getUuid(), event.getLastSeenTime())); } workQueues.get(config.getName()).addAll(tasks); } else { config.getBackend().index(events); } } } } finally { backendsUse.unlock(); } } @Override public void delete(String uuid) throws ZepException { backendsUse.lock(); try { for (EventIndexBackendConfiguration config : backends.values()) { if (config.isWriter() && config.isHonorDeletes()) { if (enableAsyncProcessing && config.isAsyncUpdates()) { workQueues.get(config.getName()).add(EventIndexBackendTask.Index(uuid, null)); } else { config.getBackend().delete(uuid); } } } } finally { backendsUse.unlock(); } } @Override public void delete(List<String> uuids) throws ZepException { if (uuids == null || uuids.isEmpty()) return; backendsUse.lock(); try { List<EventIndexBackendTask> tasks = null; for (EventIndexBackendConfiguration config : backends.values()) { if (config.isWriter() && config.isHonorDeletes()) { if (enableAsyncProcessing && config.isAsyncUpdates()) { if (tasks == null) { tasks = Lists.newArrayListWithExpectedSize(uuids.size()); for (String uuid : uuids) tasks.add(EventIndexBackendTask.Index(uuid, null)); } workQueues.get(config.getName()).addAll(tasks); } else { config.getBackend().delete(uuids); } } } } finally { backendsUse.unlock(); } } @Override public void clear() throws ZepException { backendsUse.lock(); try { for (EventIndexBackendConfiguration config : backends.values()) { if (config.isWriter() && config.isHonorDeletes()) { config.getBackend().clear(); } } } finally { backendsUse.unlock(); } } protected void clear(String backendId) throws ZepException { backendsUse.lock(); try { EventIndexBackendConfiguration config = backends.get(backendId); if (config != null && config.isWriter() && config.isHonorDeletes()) { config.getBackend().clear(); } } finally { backendsUse.unlock(); } } @Override public void purge(Date threshold) throws ZepException { backendsUse.lock(); try { for (EventIndexBackendConfiguration config : backends.values()) { if (config.isWriter() && config.isHonorDeletes()) { config.getBackend().purge(threshold); } } } finally { backendsUse.unlock(); } } @Override public void commit() throws ZepException { backendsUse.lock(); try { for (EventIndexBackendConfiguration config : backends.values()) { if (config.isWriter()) { if (enableAsyncProcessing && config.isAsyncUpdates()) { workQueues.get(config.getName()).add(EventIndexBackendTask.Flush()); } else { config.getBackend().flush(); } } } } finally { backendsUse.unlock(); } } @Override public int getNumDocs() throws ZepException { backendsUse.lock(); try { return (int) getReader().count(); } finally { backendsUse.unlock(); } } @Override public long getSize() { backendsUse.lock(); try { return getReader().sizeInBytes(); } catch (UnsupportedOperationException e) { return -1; //TODO: figure out the right thing to do } finally { backendsUse.unlock(); } } @Override public EventSummary findByUuid(String uuid) throws ZepException { backendsUse.lock(); try { return getReader().findByUuid(uuid); } finally { backendsUse.unlock(); } } @Override public EventSummaryResult list(EventSummaryRequest request) throws ZepException { backendsUse.lock(); try { return getReader().list(request); } finally { backendsUse.unlock(); } } @Override public EventSummaryResult listUuids(EventSummaryRequest request) throws ZepException { backendsUse.lock(); try { return getReader().listUuids(request); } finally { backendsUse.unlock(); } } @Override public EventTagSeveritiesSet getEventTagSeverities(EventFilter filter) throws ZepException { backendsUse.lock(); try { return getReader().getEventTagSeverities(filter); } finally { backendsUse.unlock(); } } @Override public MultiBackendSavedSearch buildSavedSearch(String uuid, EventQuery eventQuery) throws ZepException { backendsUse.lock(); try { final String savedSearchId = getReader().createSavedSearch(eventQuery); return new MultiBackendSavedSearch(generateUuid(), eventQuery.getTimeout(), readerBackendId, savedSearchId, this); } finally { backendsUse.unlock(); } } @Override public SavedSearchProcessor<MultiBackendSavedSearch> savedSearchProcessor() { return new SavedSearchProcessor<MultiBackendSavedSearch>() { @Override public EventSummaryResult result(MultiBackendSavedSearch search, int offset, int limit) throws ZepException { return backends.get(search.backendId).getBackend().savedSearch(search.savedSearchId, offset, limit); } }; } @Override public SavedSearchProcessor<MultiBackendSavedSearch> savedSearchUuidsProcessor() { return new SavedSearchProcessor<MultiBackendSavedSearch>() { @Override public EventSummaryResult result(MultiBackendSavedSearch search, int offset, int limit) throws ZepException { return backends.get(search.backendId).getBackend().savedSearchUuids(search.savedSearchId, offset, limit); } }; } public void closeBackendSavedSearch(String backendId, String savedSearchId) { EventIndexBackendConfiguration config = backends.get(backendId); if (config == null) return; EventIndexBackend backend = config.getBackend(); if (backend == null) return; backend.closeSavedSearch(savedSearchId); } private void closeBackendSavedSearches(String backendId) { EventIndexBackendConfiguration config = backends.get(backendId); if (config == null) return; EventIndexBackend backend = config.getBackend(); if (backend == null) return; backend.closeSavedSearches(); } public final synchronized void disableRebuilders() { this.disableRebuilders = true; backendsUse.lock(); try { for (String backendId : Lists.newArrayList(rebuilders.keySet())) { stopRebuilder(backendId); } } finally { backendsUse.unlock(); } } public final synchronized void startRebuilder(String backendId) { stopRebuilder(backendId); if (disableRebuilders) return; RebuilderThread rebuilder = new RebuilderThread(this, backendId); rebuilders.put(backendId, rebuilder); rebuilder.start(); if (!rebuilder.isAlive()) logger.error("failed to start event index rebuilder thread for " + getName() + " backend " + backendId); } public final synchronized void stopRebuilder(String backendId) { RebuilderThread rebuilder = rebuilders.remove(backendId); if (rebuilder != null) { logger.info("shutting down event index rebuilder thread for " + getName() + " backend " + backendId); try { rebuilder.join(); } catch (InterruptedException e) { /* no problem */ } } } public final synchronized void forceRebuild(String backendId) { RebuilderThread rebuilder = rebuilders.get(backendId); if (rebuilder == null) logger.error( "unable to force rebuild with rebuilder stopped for " + getName() + " backend " + backendId); else { rebuilder.forceRebuild = true; } } private static class RebuilderProgress implements Serializable { public final long throughTime; public final EventBatchParams nextBatch; public final boolean done; public static RebuilderProgress begin(long throughTime) { return new RebuilderProgress(throughTime, null, false); } public static RebuilderProgress done(RebuilderProgress progress) { return new RebuilderProgress(progress.throughTime, null, true); } public static RebuilderProgress next(RebuilderProgress progress, EventBatchParams nextBatch) { return new RebuilderProgress(progress.throughTime, nextBatch, false); } private RebuilderProgress(long throughTime, EventBatchParams nextBatch, boolean done) { this.throughTime = throughTime; this.nextBatch = nextBatch; this.done = done; } public String toString() { StringBuilder sb = new StringBuilder(); sb.append("throughTime:"); sb.append(throughTime); sb.append(" "); if (nextBatch != null) { sb.append("nextLastSeen:"); sb.append(nextBatch.nextLastSeen); sb.append(" "); if (nextBatch.nextUuid != null) { sb.append("nextUuid:"); sb.append(nextBatch.nextUuid); sb.append(" "); } } sb.append("done:"); sb.append(done); return sb.toString(); } public static RebuilderProgress parse(String s) { try { Map<String, String> pairs = new HashMap<String, String>(); for (String pair : s.split(" ")) { String[] splitPair = pair.split(":", 2); if (splitPair.length == 2) pairs.put(splitPair[0], splitPair[1]); } long throughTime = Long.parseLong(pairs.get("throughTime")); if ("true".equalsIgnoreCase(pairs.get("done"))) return new RebuilderProgress(throughTime, null, true); String lastSeenStr = pairs.get("nextLastSeen"); if (lastSeenStr == null) return new RebuilderProgress(throughTime, null, false); long nextLastSeen = Long.parseLong(lastSeenStr); String nextUuid = pairs.get("nextUuid"); EventBatchParams nextBatch = new EventBatchParams(nextLastSeen, nextUuid); return new RebuilderProgress(throughTime, nextBatch, false); } catch (RuntimeException e) { throw new IllegalArgumentException("Unable to parse: " + s, e); } } } private static final SimpleDateFormat UTC; static { UTC = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S 'UTC'"); UTC.setTimeZone(TimeZone.getTimeZone("UTC")); } private static String formatUTC(long time) { synchronized (UTC) { return UTC.format(new Date(time)); } } private class RebuilderThread extends Thread { private final String backendId; private final byte[] storeKey; private long nextStatusLog = 0; public transient boolean forceRebuild; public RebuilderThread(MultiBackendEventIndexDao dao, String backendId) { this.backendId = backendId; this.forceRebuild = false; this.storeKey = ("org.zenoss.zep.index.rebuilder:" + dao.getName() + ":" + backendId).getBytes(); this.setDaemon(true); this.setName(dao.getName() + " backend " + backendId + " event index rebuilder thread"); } private void saveRebuildProgress(RebuilderProgress progress) throws IOException { store.store(storeKey, progress.toString().getBytes()); } private RebuilderProgress loadRebuildProgress() throws IOException { final byte[] data = store.load(storeKey); if (data == null || data.length == 0) return null; try { return RebuilderProgress.parse(new String(data)); } catch (IllegalArgumentException e) { logger.error("exception parsing index rebuilder progress", e); return null; } } private void logStatus(String msg) { long now = System.currentTimeMillis(); if (now > nextStatusLog) { long count = -1; EventIndexBackendConfiguration configuration = backends.get(backendId); if (configuration != null) { EventIndexBackend backend = configuration.getBackend(); if (backend != null) { try { count = backend.count(); } catch (ZepException e) { // ignore it } catch (RuntimeException e) { // ignore it } } } WorkQueue q = workQueues.get(backendId); logger.info(getName() + " " + msg + " [current queue size: " + (q == null ? "n/a" : q.size()) + ", index size: " + ((count >= 0) ? count : "unknown") + "]"); nextStatusLog = now + 60000; // no more than once a minute } } @Override public void run() { logger.info(getName() + " has started its run"); EventIndexBackendConfiguration configuration = backends.get(backendId); if (configuration == null) { logger.error(getName() + " running with a missing backend configuration! Exiting!"); return; } EventIndexBackend backend = configuration.getBackend(); if (backend == null) { logger.error(getName() + " running with a missing backend! Exiting!"); return; } while (rebuilders.get(backendId) == this) { try { if (!backend.isReady()) { logStatus("is waiting for backend to be ready"); sleep(1000); continue; } if (enableAsyncProcessing && configuration.isAsyncUpdates() && !workQueues.get(backendId).isReady()) { logStatus("is waiting for work queue to be ready"); sleep(1000); continue; } RebuilderProgress progress = loadRebuildProgress(); if (forceRebuild) { logStatus("is starting a new rebuild"); saveRebuildProgress(RebuilderProgress.begin(System.currentTimeMillis())); forceRebuild = false; progress = loadRebuildProgress(); } if (progress == null || progress.done) { logStatus("is dormant"); sleep(1000); continue; } int batchSize = configuration.getBatchSize(); EventBatch batch = null; backendsUse.lock(); try { if (configuration.isWriter()) { List<EventSummary> events = null; try { batch = eventDao.listBatch(progress.nextBatch, progress.throughTime, batchSize); events = batch.events; } catch (RuntimeException e) { logger.debug("Unable to listBatch due to exception: " + e.getMessage(), e); } if (enableAsyncProcessing && configuration.isAsyncUpdates()) { if (events.isEmpty()) { // Finished! workQueues.get(backendId).add(EventIndexBackendTask.Flush()); } else { logger.debug("Converting {} events into tasks.", events.size()); List<EventIndexBackendTask> tasks = Lists .newArrayListWithExpectedSize(events.size()); for (EventSummary event : events) tasks.add(EventIndexBackendTask.Index(event.getUuid(), event.getLastSeenTime())); logger.debug("Queuing up another {} events.", events.size()); workQueues.get(backendId).addAll(tasks); logger.debug("Done queuing up {} events.", events.size()); } logStatus("queued events up to:" + batch); } else { backend.index(events); logStatus("indexed events up to:" + batch); if (events.isEmpty()) { // Finished! backend.flush(); } } } } finally { backendsUse.unlock(); } if (batch.events.isEmpty()) progress = RebuilderProgress.done(progress); else progress = RebuilderProgress.next(progress, batch.nextParams); saveRebuildProgress(progress); } catch (InterruptedException e) { // ignore it } catch (ZepException e) { logger.warn("error while rebuilding for " + getName(), e); } catch (IOException e) { logger.warn("error while rebuilding for " + getName(), e); } catch (RuntimeException e) { logger.warn("error while rebuilding for " + getName(), e); } } logger.info(getName() + " has ended its run"); } } }