Java tutorial
/* * Copyright 2011 Greg Haines * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.greghaines.jesque.worker; import static net.greghaines.jesque.utils.ResqueConstants.*; import static net.greghaines.jesque.worker.JobExecutor.State.*; import static net.greghaines.jesque.worker.WorkerEvent.*; import java.io.IOException; import java.lang.Throwable; import java.lang.management.ManagementFactory; import java.net.InetAddress; import java.net.UnknownHostException; import java.text.SimpleDateFormat; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.Set; import java.util.concurrent.BlockingDeque; import java.util.concurrent.Callable; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import net.greghaines.jesque.Config; import net.greghaines.jesque.Job; import net.greghaines.jesque.JobFailure; import net.greghaines.jesque.WorkerStatus; import net.greghaines.jesque.json.ObjectMapperFactory; import net.greghaines.jesque.utils.JedisUtils; import net.greghaines.jesque.utils.JesqueUtils; import net.greghaines.jesque.utils.VersionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.Transaction; import redis.clients.jedis.exceptions.JedisException; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; /** * Basic implementation of the Worker interface. Obeys the contract of a Resque * worker in Redis. * * @author Greg Haines * @author Animesh Kumar */ public class WorkerImpl implements Worker { private static final Logger LOG = LoggerFactory.getLogger(WorkerImpl.class); private static final AtomicLong WORKER_COUNTER = new AtomicLong(0); protected static final long EMPTY_QUEUE_SLEEP_TIME = 500; // 500 ms protected static final long RECONNECT_SLEEP_TIME = 5000; // 5 sec protected static final int RECONNECT_ATTEMPTS = 120; // Total time: 10 min // Set the thread name to the message for debugging private static volatile boolean threadNameChangingEnabled = false; /** * @return true if worker threads names will change during normal operation */ public static boolean isThreadNameChangingEnabled() { return threadNameChangingEnabled; } /** * Enable/disable worker thread renaming during normal operation. (Disabled * by default) * <p> * <strong>Warning: Enabling this feature is very expensive * CPU-wise!</strong><br> * This feature is designed to assist in debugging worker state but should * be disabled in production environments for performance reasons. * * @param enabled * whether threads' names should change during normal operation */ public static void setThreadNameChangingEnabled(final boolean enabled) { threadNameChangingEnabled = enabled; } /** * Verify that the given queues are all valid. * * @param queues * the given queues */ protected static void checkQueues(final Iterable<String> queues) { if (queues == null) { throw new IllegalArgumentException("queues must not be null"); } for (final String queue : queues) { if (queue == null || "".equals(queue)) { throw new IllegalArgumentException("queues' members must not be null: " + queues); } } } protected final Config config; protected final Jedis jedis; protected final String namespace; protected final BlockingDeque<String> queueNames = new LinkedBlockingDeque<String>(); private final String name; protected final WorkerListenerDelegate listenerDelegate = new WorkerListenerDelegate(); protected final AtomicReference<State> state = new AtomicReference<State>(NEW); private final AtomicBoolean paused = new AtomicBoolean(false); private final AtomicBoolean processingJob = new AtomicBoolean(false); private final long workerId = WORKER_COUNTER.getAndIncrement(); private final String threadNameBase = "Worker-" + this.workerId + " Jesque-" + VersionUtils.getVersion() + ": "; private final AtomicReference<Thread> threadRef = new AtomicReference<Thread>(null); private final AtomicReference<ExceptionHandler> exceptionHandlerRef = new AtomicReference<ExceptionHandler>( new DefaultExceptionHandler()); private final JobFactory jobFactory; /** * Creates a new WorkerImpl, which creates it's own connection to Redis * using values from the config. The worker will only listen to the supplied * queues and execute jobs that are provided by the given job factory. * * @param config * used to create a connection to Redis and the package prefix * for incoming jobs * @param queues * the list of queues to poll * @param jobFactory * the job factory that materializes the jobs * @throws IllegalArgumentException * if either config, queues or jobFactory is null */ public WorkerImpl(final Config config, final Collection<String> queues, final JobFactory jobFactory) { this(config, queues, jobFactory, new Jedis(config.getHost(), config.getPort(), config.getTimeout())); } /** * Creates a new WorkerImpl, with the given connection to Redis. * The worker will only listen to the supplied queues and execute jobs * that are provided by the given job factory. * * @param config * used to create a connection to Redis and the package prefix * for incoming jobs * @param queues * the list of queues to poll * @param jobFactory * the job factory that materializes the jobs * @param jedis * the connection to Redis * @throws IllegalArgumentException * if either config, queues, jobFactory or jedis is null */ public WorkerImpl(final Config config, final Collection<String> queues, final JobFactory jobFactory, final Jedis jedis) { if (config == null) { throw new IllegalArgumentException("config must not be null"); } if (jobFactory == null) { throw new IllegalArgumentException("jobFactory must not be null"); } if (jedis == null) { throw new IllegalArgumentException("jedis must not be null"); } checkQueues(queues); this.config = config; this.jobFactory = jobFactory; this.namespace = config.getNamespace(); this.jedis = jedis; authenticateAndSelectDB(); setQueues(queues); this.name = createName(); } /** * @return this worker's identifier */ public long getWorkerId() { return this.workerId; } /** * Starts this worker. Registers the worker in Redis and begins polling the * queues for jobs. Stop this worker by calling end() on any thread. */ @Override public void run() { if (this.state.compareAndSet(NEW, RUNNING)) { try { renameThread("RUNNING"); this.threadRef.set(Thread.currentThread()); this.jedis.sadd(key(WORKERS), this.name); this.jedis.set(key(WORKER, this.name, STARTED), new SimpleDateFormat(DATE_FORMAT).format(new Date())); this.listenerDelegate.fireEvent(WORKER_START, this, null, null, null, null, null); poll(); } finally { renameThread("STOPPING"); this.listenerDelegate.fireEvent(WORKER_STOP, this, null, null, null, null, null); this.jedis.srem(key(WORKERS), this.name); this.jedis.del(key(WORKER, this.name), key(WORKER, this.name, STARTED), key(STAT, FAILED, this.name), key(STAT, PROCESSED, this.name)); this.jedis.quit(); this.threadRef.set(null); } } else { if (RUNNING.equals(this.state.get())) { throw new IllegalStateException("This WorkerImpl is already running"); } else { throw new IllegalStateException("This WorkerImpl is shutdown"); } } } /** * Shutdown this Worker.<br> * <b>The worker cannot be started again; create a new worker in this * case.</b> * * @param now * if true, an effort will be made to stop any job in progress */ @Override public void end(final boolean now) { if (now) { this.state.set(SHUTDOWN_IMMEDIATE); final Thread workerThread = this.threadRef.get(); if (workerThread != null) { workerThread.interrupt(); } } else { this.state.set(SHUTDOWN); } togglePause(false); // Release any threads waiting in checkPaused() } /** * {@inheritDoc} */ @Override public boolean isShutdown() { return SHUTDOWN.equals(this.state.get()) || SHUTDOWN_IMMEDIATE.equals(this.state.get()); } /** * {@inheritDoc} */ @Override public boolean isPaused() { return this.paused.get(); } /** * {@inheritDoc} */ @Override public boolean isProcessingJob() { return this.processingJob.get(); } /** * {@inheritDoc} */ @Override public void togglePause(final boolean paused) { this.paused.set(paused); synchronized (this.paused) { this.paused.notifyAll(); } } /** * {@inheritDoc} */ @Override public String getName() { return this.name; } /** * {@inheritDoc} */ @Override public WorkerEventEmitter getWorkerEventEmitter() { return this.listenerDelegate; } /** * {@inheritDoc} */ @Override public Collection<String> getQueues() { return Collections.unmodifiableCollection(this.queueNames); } /** * {@inheritDoc} */ @Override public void addQueue(final String queueName) { if (queueName == null || "".equals(queueName)) { throw new IllegalArgumentException("queueName must not be null or empty: " + queueName); } this.queueNames.add(queueName); } /** * {@inheritDoc} */ @Override public void removeQueue(final String queueName, final boolean all) { if (queueName == null || "".equals(queueName)) { throw new IllegalArgumentException("queueName must not be null or empty: " + queueName); } if (all) { // Remove all instances boolean tryAgain = true; while (tryAgain) { tryAgain = this.queueNames.remove(queueName); } } else { // Only remove one instance this.queueNames.remove(queueName); } } /** * {@inheritDoc} */ @Override public void removeAllQueues() { this.queueNames.clear(); } /** * {@inheritDoc} */ @Override public void setQueues(final Collection<String> queues) { checkQueues(queues); this.queueNames.clear(); this.queueNames.addAll((queues == ALL_QUEUES) // Using object equality on purpose ? this.jedis.smembers(key(QUEUES)) // Like '*' in other clients : queues); } /** * {@inheritDoc} */ @Override public JobFactory getJobFactory() { return this.jobFactory; } /** * {@inheritDoc} */ @Override public ExceptionHandler getExceptionHandler() { return this.exceptionHandlerRef.get(); } /** * {@inheritDoc} */ @Override public void setExceptionHandler(final ExceptionHandler exceptionHandler) { if (exceptionHandler == null) { throw new IllegalArgumentException("exceptionHandler must not be null"); } this.exceptionHandlerRef.set(exceptionHandler); } /** * {@inheritDoc} */ @Override public void join(final long millis) throws InterruptedException { final Thread workerThread = this.threadRef.get(); if (workerThread != null && workerThread.isAlive()) { workerThread.join(millis); } } /** * @return the number of times this Worker will attempt to reconnect to * Redis before giving up */ protected int getReconnectAttempts() { return RECONNECT_ATTEMPTS; } /** * Polls the queues for jobs and executes them. */ protected void poll() { int missCount = 0; String curQueue = null; while (RUNNING.equals(this.state.get())) { try { if (threadNameChangingEnabled) { renameThread("Waiting for " + JesqueUtils.join(",", this.queueNames)); } curQueue = this.queueNames.poll(EMPTY_QUEUE_SLEEP_TIME, TimeUnit.MILLISECONDS); if (curQueue != null) { this.queueNames.add(curQueue); // Rotate the queues checkPaused(); // Might have been waiting in poll()/checkPaused() for a while if (RUNNING.equals(this.state.get())) { this.listenerDelegate.fireEvent(WORKER_POLL, this, curQueue, null, null, null, null); final String payload = pop(curQueue); if (payload != null) { final Job job = ObjectMapperFactory.get().readValue(payload, Job.class); process(job, curQueue); missCount = 0; } else if (++missCount >= this.queueNames.size() && RUNNING.equals(this.state.get())) { // Keeps worker from busy-spinning on empty queues missCount = 0; Thread.sleep(EMPTY_QUEUE_SLEEP_TIME); } } } } catch (InterruptedException ie) { if (!isShutdown()) { recoverFromException(curQueue, ie); } } catch (JsonParseException | JsonMappingException e) { // If the job JSON is not deserializable, we never want to submit it again... removeInFlight(curQueue); recoverFromException(curQueue, e); } catch (Exception e) { recoverFromException(curQueue, e); } } } /** * Remove a job from the given queue. * @param curQueue the queue to remove a job from * @return a JSON string of a job or null if there was nothing to de-queue */ protected String pop(final String curQueue) { final String key = key(QUEUE, curQueue); String payload = null; // If a delayed queue, peek and remove from ZSET if (JedisUtils.isDelayedQueue(this.jedis, key)) { final long now = System.currentTimeMillis(); // Peek ==> is there any item scheduled to run between -INF and now? final Set<String> payloadSet = this.jedis.zrangeByScore(key, -1, now, 0, 1); if (payloadSet != null && !payloadSet.isEmpty()) { final String tmp = payloadSet.iterator().next(); // Try to acquire this job if (this.jedis.zrem(key, tmp) == 1) { payload = tmp; } } } else if (JedisUtils.isRegularQueue(this.jedis, key)) { // If a regular queue, pop from it payload = lpoplpush(key, key(INFLIGHT, this.name, curQueue)); } return payload; } /** * Handle an exception that was thrown from inside {@link #poll()} * * @param curQueue * the name of the queue that was being processed when the * exception was thrown * @param e * the exception that was thrown */ protected void recoverFromException(final String curQueue, final Exception e) { final RecoveryStrategy recoveryStrategy = this.exceptionHandlerRef.get().onException(this, e, curQueue); switch (recoveryStrategy) { case RECONNECT: LOG.info("Reconnecting to Redis in response to exception", e); final int reconAttempts = getReconnectAttempts(); if (!JedisUtils.reconnect(this.jedis, reconAttempts, RECONNECT_SLEEP_TIME)) { LOG.warn("Terminating in response to exception after " + reconAttempts + " to reconnect", e); end(false); } else { authenticateAndSelectDB(); LOG.info("Reconnected to Redis"); } break; case TERMINATE: LOG.warn("Terminating in response to exception", e); end(false); break; case PROCEED: this.listenerDelegate.fireEvent(WORKER_ERROR, this, curQueue, null, null, null, e); break; default: LOG.error("Unknown RecoveryStrategy: " + recoveryStrategy + " while attempting to recover from the following exception; worker proceeding...", e); break; } } private void authenticateAndSelectDB() { if (this.config.getPassword() != null) { this.jedis.auth(this.config.getPassword()); } this.jedis.select(this.config.getDatabase()); } /** * Checks to see if worker is paused. If so, wait until unpaused. * * @throws IOException * if there was an error creating the pause message */ protected void checkPaused() throws IOException { if (this.paused.get()) { synchronized (this.paused) { if (this.paused.get()) { this.jedis.set(key(WORKER, this.name), pauseMsg()); } while (this.paused.get()) { try { this.paused.wait(); } catch (InterruptedException ie) { LOG.warn("Worker interrupted", ie); } } this.jedis.del(key(WORKER, this.name)); } } } /** * Materializes and executes the given job. * * @param job * the Job to process * @param curQueue * the queue the payload came from */ protected void process(final Job job, final String curQueue) { try { this.processingJob.set(true); if (threadNameChangingEnabled) { renameThread("Processing " + curQueue + " since " + System.currentTimeMillis()); } this.listenerDelegate.fireEvent(JOB_PROCESS, this, curQueue, job, null, null, null); this.jedis.set(key(WORKER, this.name), statusMsg(curQueue, job)); final Object instance = this.jobFactory.materializeJob(job); final Object result = execute(job, curQueue, instance); success(job, instance, result, curQueue); } catch (Throwable t) { failure(t, job, curQueue); } finally { removeInFlight(curQueue); this.jedis.del(key(WORKER, this.name)); this.processingJob.set(false); } } private void removeInFlight(String curQueue) { if (SHUTDOWN_IMMEDIATE.equals(this.state.get())) { lpoplpush(key(INFLIGHT, this.name, curQueue), key(QUEUE, curQueue)); } else { this.jedis.lpop(key(INFLIGHT, this.name, curQueue)); } } /** * Executes the given job. * * @param job * the job to execute * @param curQueue * the queue the job came from * @param instance * the materialized job * @throws Exception * if the instance is a {@link Callable} and throws an exception * @return result of the execution */ protected Object execute(final Job job, final String curQueue, final Object instance) throws Exception { if (instance instanceof WorkerAware) { ((WorkerAware) instance).setWorker(this); } this.listenerDelegate.fireEvent(JOB_EXECUTE, this, curQueue, job, instance, null, null); final Object result; if (instance instanceof Callable) { result = ((Callable<?>) instance).call(); // The job is executing! } else if (instance instanceof Runnable) { ((Runnable) instance).run(); // The job is executing! result = null; } else { // Should never happen since we're testing the class earlier throw new ClassCastException("Instance must be a Runnable or a Callable: " + instance.getClass().getName() + " - " + instance); } return result; } /** * Update the status in Redis on success. * * @param job * the Job that succeeded * @param runner * the materialized Job * @param result * the result of the successful execution of the Job * @param curQueue * the queue the Job came from */ protected void success(final Job job, final Object runner, final Object result, final String curQueue) { // The job may have taken a long time; make an effort to ensure the // connection is OK JedisUtils.ensureJedisConnection(this.jedis); try { this.jedis.incr(key(STAT, PROCESSED)); this.jedis.incr(key(STAT, PROCESSED, this.name)); } catch (JedisException je) { LOG.warn("Error updating success stats for job=" + job, je); } this.listenerDelegate.fireEvent(JOB_SUCCESS, this, curQueue, job, runner, result, null); } /** * Update the status in Redis on failure * * @param t * the Throwable that occurred * @param job * the Job that failed * @param curQueue * the queue the Job came from */ protected void failure(final Throwable t, final Job job, final String curQueue) { // The job may have taken a long time; make an effort to ensure the // connection is OK JedisUtils.ensureJedisConnection(this.jedis); try { this.jedis.incr(key(STAT, FAILED)); this.jedis.incr(key(STAT, FAILED, this.name)); this.jedis.rpush(key(FAILED), failMsg(t, curQueue, job)); } catch (JedisException je) { LOG.warn("Error updating failure stats for throwable=" + t + " job=" + job, je); } catch (IOException ioe) { LOG.warn("Error serializing failure payload for throwable=" + t + " job=" + job, ioe); } this.listenerDelegate.fireEvent(JOB_FAILURE, this, curQueue, job, null, null, t); } /** * Create and serialize a JobFailure. * * @param t * the Throwable that occurred * @param queue * the queue the job came from * @param job * the Job that failed * @return the JSON representation of a new JobFailure * @throws IOException * if there was an error serializing the JobFailure */ protected String failMsg(final Throwable t, final String queue, final Job job) throws IOException { final JobFailure failure = new JobFailure(); failure.setFailedAt(new Date()); failure.setWorker(this.name); failure.setQueue(queue); failure.setPayload(job); failure.setThrowable(t); return ObjectMapperFactory.get().writeValueAsString(failure); } /** * Create and serialize a WorkerStatus. * * @param queue * the queue the Job came from * @param job * the Job currently being processed * @return the JSON representation of a new WorkerStatus * @throws IOException * if there was an error serializing the WorkerStatus */ protected String statusMsg(final String queue, final Job job) throws IOException { final WorkerStatus status = new WorkerStatus(); status.setRunAt(new Date()); status.setQueue(queue); status.setPayload(job); return ObjectMapperFactory.get().writeValueAsString(status); } /** * Create and serialize a WorkerStatus for a pause event. * * @return the JSON representation of a new WorkerStatus * @throws IOException * if there was an error serializing the WorkerStatus */ protected String pauseMsg() throws IOException { final WorkerStatus status = new WorkerStatus(); status.setRunAt(new Date()); status.setPaused(isPaused()); return ObjectMapperFactory.get().writeValueAsString(status); } /** * Creates a unique name, suitable for use with Resque. * * @return a unique name for this worker */ protected String createName() { final StringBuilder buf = new StringBuilder(128); try { buf.append(InetAddress.getLocalHost().getHostName()).append(COLON) .append(ManagementFactory.getRuntimeMXBean().getName().split("@")[0]) // PID .append('-').append(this.workerId).append(COLON).append(JAVA_DYNAMIC_QUEUES); for (final String queueName : this.queueNames) { buf.append(',').append(queueName); } } catch (UnknownHostException uhe) { throw new RuntimeException(uhe); } return buf.toString(); } /** * Builds a namespaced Redis key with the given arguments. * * @param parts * the key parts to be joined * @return an assembled String key */ protected String key(final String... parts) { return JesqueUtils.createKey(this.namespace, parts); } /** * Rename the current thread with the given message. * * @param msg * the message to add to the thread name */ protected void renameThread(final String msg) { Thread.currentThread().setName(this.threadNameBase + msg); } protected String lpoplpush(final String from, final String to) { String result = null; while (JedisUtils.isRegularQueue(this.jedis, from)) { this.jedis.watch(from); // Get the leftmost value of the 'from' list. If it does not exist, there is nothing to pop. String val = null; if (JedisUtils.isRegularQueue(this.jedis, from)) { val = this.jedis.lindex(from, 0); } if (val == null) { this.jedis.unwatch(); result = val; break; } final Transaction tx = this.jedis.multi(); tx.lpop(from); tx.lpush(to, val); if (tx.exec() != null) { result = val; break; } // If execution of the transaction failed, this means that 'from' // was modified while we were watching it and the transaction was // not executed. We simply retry the operation. } return result; } /** * {@inheritDoc} */ @Override public String toString() { return this.namespace + COLON + WORKER + COLON + this.name; } }