com.opengamma.engine.view.worker.ParallelRecompilationViewProcessWorker.java Source code

Java tutorial

Introduction

Here is the source code for com.opengamma.engine.view.worker.ParallelRecompilationViewProcessWorker.java

Source

/**
 * Copyright (C) 2013 - present by OpenGamma Inc. and the OpenGamma group of companies
 *
 * Please see distribution for license.
 */
package com.opengamma.engine.view.worker;

import java.util.Collection;
import java.util.EnumSet;
import java.util.Map;
import java.util.Set;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.threeten.bp.Instant;

import com.google.common.collect.Sets;
import com.opengamma.engine.target.ComputationTargetReference;
import com.opengamma.engine.view.ViewComputationResultModel;
import com.opengamma.engine.view.ViewDefinition;
import com.opengamma.engine.view.compilation.CompiledViewDefinitionWithGraphs;
import com.opengamma.engine.view.cycle.ViewCycle;
import com.opengamma.engine.view.cycle.ViewCycleMetadata;
import com.opengamma.engine.view.execution.ExecutionOptions;
import com.opengamma.engine.view.execution.ViewCycleExecutionOptions;
import com.opengamma.engine.view.execution.ViewCycleExecutionSequence;
import com.opengamma.engine.view.execution.ViewExecutionFlags;
import com.opengamma.engine.view.execution.ViewExecutionOptions;
import com.opengamma.engine.view.impl.ViewProcessContext;
import com.opengamma.id.ObjectId;
import com.opengamma.id.UniqueId;
import com.opengamma.id.VersionCorrection;
import com.opengamma.util.ArgumentChecker;

/**
 * Implementation of {@link ViewProcessWorker} that rolls work between two delegate workers to allow the secondary one to recompile a view while the first is still calculating values from the old
 * compilation.
 */
public class ParallelRecompilationViewProcessWorker implements ViewProcessWorker {

    private static final Logger s_logger = LoggerFactory.getLogger(ParallelRecompilationViewProcessWorker.class);

    private static enum WorkerAction {

        TERMINATE,

        PROCEED,

        DEFER,

        BLOCK

    };

    /**
     * Base class of the context used by the delegate workers.
     */
    protected abstract class AbstractViewProcessWorkerContext implements ViewProcessWorkerContext {

        private final int _id;
        private final ViewCycleExecutionSequence _sequence;
        private CompiledViewDefinitionWithGraphs _lastCompiled;
        private ViewProcessWorker _worker;
        private WorkerAction _action;
        private ViewExecutionDataProvider _deferredCompilation;
        private ViewCycleMetadata _deferredCycleStarted;

        // caller must hold the outer class monitor
        public AbstractViewProcessWorkerContext(final ViewExecutionOptions options) {
            _id = _nextWorkerId++;
            _sequence = options.getExecutionSequence().copy();
            _worker = getDelegate().createWorker(this, options, getViewDefinition());
            // Discard the first item from the sequence
            _sequence.poll(options.getDefaultExecutionOptions());
        }

        protected ViewCycleExecutionSequence getSequence() {
            return _sequence;
        }

        protected abstract AbstractViewProcessWorkerContext newInstance(ViewExecutionOptions options);

        protected AbstractViewProcessWorkerContext createSecondaryWorker() {
            return createSecondaryWorker(getSequence());
        }

        protected AbstractViewProcessWorkerContext createSecondaryWorker(
                final ViewCycleExecutionSequence sequence) {
            return newInstance(getOptions(sequence));
        }

        protected void setCompiled(final CompiledViewDefinitionWithGraphs compiled) {
            _lastCompiled = compiled;
        }

        protected boolean isCompiled() {
            return _lastCompiled != null;
        }

        /**
         * Called on a secondary context to indicate a primary is about to start a cycle.
         * 
         * @return {@code TERMINATE} to terminate the primary (must unblock this one), or {@code PROCEED} to allow the primary to continue (and start calculating)
         */
        protected abstract WorkerAction primaryCycleStarted();

        /**
         * Called on a primary context to indicate a secondary is about to start a cycle.
         * 
         * @return {@code PROCEED} to allow the secondary to continue (may terminate this one), {@code IGNORE} to discard/defer the notification, or {@code BLOCK} to block the secondary
         */
        protected abstract WorkerAction secondaryCycleStarted();

        /**
         * Called on a secondary context to indicate a primary has completed a fragment.
         * 
         * @return {@code TERMINATE} to terminate the primary (must unblock this one), or {@code PROCEED} to allow the primary to continue (and post its result)
         */
        protected abstract WorkerAction primaryCycleFragmentCompleted();

        /**
         * Called on a secondary context to indicate a primary has completed a cycle.
         * 
         * @return {@code TERMINATE} to terminate the primary (must unblock this one), or {@code PROCEED} to allow the primary to continue (and post its result)
         */
        protected abstract WorkerAction primaryCycleCompleted();

        protected synchronized WorkerAction block() {
            s_logger.debug("Blocking worker {}", this);
            WorkerAction action = _action;
            while (action == null) {
                try {
                    wait();
                    action = _action;
                } catch (InterruptedException e) {
                    s_logger.debug("Interrupted", e);
                    action = WorkerAction.TERMINATE;
                }
            }
            _action = null;
            s_logger.debug("Unblocked {} with action {}", this, action);
            return action;
        }

        protected synchronized void unblock(final WorkerAction action) {
            s_logger.debug("Unblocking worker {} with {}", this, action);
            _action = action;
            notifyAll();
        }

        protected void terminate() {
            s_logger.debug("Terminating delegate");
            if (!ParallelRecompilationViewProcessWorker.this.terminate(this)) {
                s_logger.debug("Worker for {} already terminated", this);
            }
        }

        protected void deferredActions() {
            if (_deferredCompilation != null) {
                try {
                    s_logger.debug("Notifying context of deferred compilation by {}", this);
                    getContext().viewDefinitionCompiled(_deferredCompilation, _lastCompiled);
                } finally {
                    _deferredCompilation = null;
                }
            }
            if (_deferredCycleStarted != null) {
                try {
                    s_logger.debug("Notifying context of deferred cycle start by {}", this);
                    getContext().cycleStarted(_deferredCycleStarted);
                } finally {
                    _deferredCycleStarted = null;
                }
            }
        }

        protected void notifyCycleStarted(ViewCycleMetadata cycleMetadata) {
            deferredActions();
            s_logger.debug("Notifying context of cycle started by {}", this);
            getContext().cycleStarted(cycleMetadata);
        }

        protected void notifyCycleFragmentCompleted(ViewComputationResultModel result,
                ViewDefinition viewDefinition) {
            deferredActions();
            s_logger.debug("Notifying context of cycle fragment completed by {}", this);
            getContext().cycleFragmentCompleted(result, viewDefinition);
        }

        protected void notifyCycleCompleted(ViewCycle cycle) {
            s_logger.debug("Notifying context of cycle completed by {}", this);
            getContext().cycleCompleted(cycle);
        }

        protected void notifyCycleExecutionFailed(ViewCycleExecutionOptions options, Exception exception) {
            s_logger.debug("Notifying context of cycle execution failed by {}", this);
            getContext().cycleExecutionFailed(options, exception);
        }

        // ViewProcessWorkerContext

        @Override
        public final ViewProcessContext getProcessContext() {
            return getContext().getProcessContext();
        }

        @Override
        public final void viewDefinitionCompiled(ViewExecutionDataProvider dataProvider,
                CompiledViewDefinitionWithGraphs compiled) {
            if (ParallelRecompilationViewProcessWorker.this.viewDefinitionCompiled(this, compiled)) {
                _deferredCompilation = dataProvider;
            } else {
                terminate();
            }
        }

        @Override
        public final void viewDefinitionCompilationFailed(Instant compilationTime, Exception exception) {
            s_logger.info("View definition compilation failure");
            try {
                getContext().viewDefinitionCompilationFailed(compilationTime, exception);
            } finally {
                terminate();
            }
        }

        @Override
        public final void cycleStarted(ViewCycleMetadata cycleMetadata) {
            WorkerAction action = ParallelRecompilationViewProcessWorker.this.cycleStarted(this);
            while (action == WorkerAction.BLOCK) {
                action = block();
            }
            if (action == WorkerAction.TERMINATE) {
                terminate();
                return;
            }
            if (action != WorkerAction.DEFER) {
                notifyCycleStarted(cycleMetadata);
            } else {
                _deferredCycleStarted = cycleMetadata;
            }
        }

        @Override
        public final void cycleFragmentCompleted(ViewComputationResultModel result, ViewDefinition viewDefinition) {
            WorkerAction action = ParallelRecompilationViewProcessWorker.this.cycleFragmentCompleted(this);
            while (action == WorkerAction.BLOCK) {
                action = block();
            }
            if (action == WorkerAction.TERMINATE) {
                terminate();
                return;
            }
            if (action != WorkerAction.DEFER) {
                notifyCycleFragmentCompleted(result, viewDefinition);
            }
        }

        @Override
        public final void cycleCompleted(ViewCycle cycle) {
            WorkerAction action = ParallelRecompilationViewProcessWorker.this.cycleCompleted(this);
            while (action == WorkerAction.BLOCK) {
                action = block();
            }
            if (action == WorkerAction.TERMINATE) {
                terminate();
                return;
            }
            deferredActions();
            notifyCycleCompleted(cycle);
        }

        @Override
        public final void cycleExecutionFailed(ViewCycleExecutionOptions options, Exception exception) {
            WorkerAction action = ParallelRecompilationViewProcessWorker.this.cycleExecutionFailed(this);
            while (action == WorkerAction.BLOCK) {
                action = block();
            }
            if (action == WorkerAction.TERMINATE) {
                terminate();
                return;
            }
            deferredActions();
            try {
                notifyCycleExecutionFailed(options, exception);
            } finally {
                workerCompleted();
            }
        }

        @Override
        public final void workerCompleted() {
            if (ParallelRecompilationViewProcessWorker.this.workerCompleted(this)) {
                getContext().workerCompleted();
            }
        }

        // Object

        @Override
        public String toString() {
            return _id + "[" + getContext() + "]";
        }

    }

    /**
     * Context used by workers that are participating in a parallel execution strategy.
     */
    protected class ParallelExecutionContext extends AbstractViewProcessWorkerContext {

        private boolean _resultsAvailable;

        public ParallelExecutionContext(final ViewExecutionOptions options) {
            super(options);
        }

        // AbstractViewProcessWorkerContext

        @Override
        protected AbstractViewProcessWorkerContext newInstance(final ViewExecutionOptions options) {
            return new ParallelExecutionContext(options);
        }

        @Override
        protected void notifyCycleFragmentCompleted(ViewComputationResultModel result,
                ViewDefinition viewDefinition) {
            _resultsAvailable = true;
            super.notifyCycleFragmentCompleted(result, viewDefinition);
        }

        @Override
        protected void notifyCycleCompleted(ViewCycle cycle) {
            _resultsAvailable = true;
            super.notifyCycleCompleted(cycle);
        }

        @Override
        protected WorkerAction primaryCycleStarted() {
            // Allow primary to continue until we've got a result
            if (_resultsAvailable) {
                return WorkerAction.TERMINATE;
            } else {
                return WorkerAction.PROCEED;
            }
        }

        @Override
        protected WorkerAction secondaryCycleStarted() {
            // Secondary can always run, eventually
            return WorkerAction.DEFER;
        }

        @Override
        protected WorkerAction primaryCycleFragmentCompleted() {
            // Allow primary to continue until we've got a result
            if (_resultsAvailable) {
                return WorkerAction.TERMINATE;
            } else {
                return WorkerAction.PROCEED;
            }
        }

        @Override
        protected WorkerAction primaryCycleCompleted() {
            // Allow primary to continue until we've got a result
            if (_resultsAvailable) {
                return WorkerAction.TERMINATE;
            } else {
                return WorkerAction.PROCEED;
            }
        }

        // Object

        @Override
        public String toString() {
            return "Parallel/" + super.toString();
        }

    }

    /**
     * Context used by workers that are participating in a deferred execution strategy.
     */
    protected class DeferredExecutionContext extends AbstractViewProcessWorkerContext {

        public DeferredExecutionContext(final ViewExecutionOptions options) {
            super(options);
        }

        // AbstractViewProcessWorkerContext

        @Override
        protected AbstractViewProcessWorkerContext newInstance(final ViewExecutionOptions options) {
            return new DeferredExecutionContext(options);
        }

        @Override
        protected WorkerAction primaryCycleStarted() {
            if (isCompiled()) {
                // Terminate the primary worker, and unblock this one
                unblock(WorkerAction.PROCEED);
                return WorkerAction.TERMINATE;
            } else {
                // Allow the primary to continue until we've compiled ourselves
                return WorkerAction.PROCEED;
            }
        }

        @Override
        protected WorkerAction secondaryCycleStarted() {
            // Block the secondary worker until the primary has finished a cycle
            return WorkerAction.BLOCK;
        }

        @Override
        protected WorkerAction primaryCycleFragmentCompleted() {
            // Allow the primary to continue
            return WorkerAction.PROCEED;
        }

        @Override
        protected WorkerAction primaryCycleCompleted() {
            if (isCompiled()) {
                // Compiled and ready to run - unblock
                unblock(WorkerAction.PROCEED);
            }
            // Allow the primary worker to post its result; we'll kill it when it starts its next cycle
            return WorkerAction.PROCEED;
        }

        // Object

        @Override
        public String toString() {
            return "Deferred/" + super.toString();
        }

    }

    /**
     * Context used by workers that are participating in an immediate execution strategy.
     */
    protected class ImmediateExecutionContext extends AbstractViewProcessWorkerContext {

        public ImmediateExecutionContext(final ViewExecutionOptions options) {
            super(options);
        }

        // AbstractViewProcessWorkerContext

        @Override
        protected AbstractViewProcessWorkerContext newInstance(final ViewExecutionOptions options) {
            return new ImmediateExecutionContext(options);
        }

        @Override
        protected WorkerAction primaryCycleStarted() {
            if (isCompiled()) {
                // Terminate the primary worker
                return WorkerAction.TERMINATE;
            } else {
                // Allow the primary to continue until we've compiled ourselves
                return WorkerAction.PROCEED;
            }
        }

        @Override
        protected WorkerAction secondaryCycleStarted() {
            // Always continue - we'll kill the primary at the first opportunity
            return WorkerAction.PROCEED;
        }

        @Override
        protected WorkerAction primaryCycleFragmentCompleted() {
            if (isCompiled()) {
                // Terminate the primary worker
                return WorkerAction.TERMINATE;
            } else {
                return WorkerAction.PROCEED;
            }
        }

        @Override
        protected WorkerAction primaryCycleCompleted() {
            if (isCompiled()) {
                // Terminate the primary worker
                return WorkerAction.TERMINATE;
            } else {
                return WorkerAction.PROCEED;
            }
        }

        // Object

        @Override
        public String toString() {
            return "Immediate/" + super.toString();
        }

    }

    private final ViewProcessWorkerFactory _delegate;
    private final ViewProcessWorkerContext _context;
    private final EnumSet<ViewExecutionFlags> _flags;
    private final Integer _maxSuccessiveDeltaCycles;
    private final ViewCycleExecutionOptions _defaultExecutionOptions;

    private TargetResolverChangeListener _resolverChanges;
    private int _nextWorkerId;
    private ViewDefinition _viewDefinition;
    private AbstractViewProcessWorkerContext _primary;
    private AbstractViewProcessWorkerContext _secondary;
    private boolean _terminated;

    /**
     * Creates a new worker. The worker will not do anything; the caller must spawn a primary delegate.
     * 
     * @param delegate the factory for spawning delegate workers, not null
     * @param context the context controlling this worker, not null
     * @param options the options for this worker (and its spawned workers), not null
     * @param viewDefinition the initial view definition, not null
     */
    public ParallelRecompilationViewProcessWorker(final ViewProcessWorkerFactory delegate,
            final ViewProcessWorkerContext context, final ViewExecutionOptions options,
            final ViewDefinition viewDefinition) {
        ArgumentChecker.notNull(delegate, "delegate");
        ArgumentChecker.notNull(context, "context");
        ArgumentChecker.notNull(options, "options");
        ArgumentChecker.notNull(viewDefinition, "viewDefinition");
        _delegate = delegate;
        _context = context;
        _flags = options.getFlags();
        _maxSuccessiveDeltaCycles = options.getMaxSuccessiveDeltaCycles();
        _defaultExecutionOptions = options.getDefaultExecutionOptions();
        _viewDefinition = viewDefinition;
    }

    protected ViewProcessWorkerFactory getDelegate() {
        return _delegate;
    }

    protected ViewProcessWorkerContext getContext() {
        return _context;
    }

    protected EnumSet<ViewExecutionFlags> getFlags() {
        return _flags;
    }

    protected Integer getMaxSuccessiveDeltaCycles() {
        return _maxSuccessiveDeltaCycles;
    }

    protected ViewCycleExecutionOptions getDefaultExecutionOptions() {
        return _defaultExecutionOptions;
    }

    /* package */AbstractViewProcessWorkerContext getPrimary() {
        return _primary;
    }

    /* package */AbstractViewProcessWorkerContext getSecondary() {
        return _secondary;
    }

    protected ViewExecutionOptions getOptions(final ViewCycleExecutionSequence sequence) {
        return new ExecutionOptions(sequence, getFlags(), getMaxSuccessiveDeltaCycles(),
                getDefaultExecutionOptions());
    }

    protected synchronized ViewDefinition getViewDefinition() {
        return _viewDefinition;
    }

    protected AbstractViewProcessWorkerContext createParallel(final ViewExecutionOptions options) {
        return new ParallelExecutionContext(options);
    }

    public synchronized void startParallel(final ViewExecutionOptions options) {
        setPrimary(createParallel(options));
    }

    protected AbstractViewProcessWorkerContext createDeferred(final ViewExecutionOptions options) {
        return new DeferredExecutionContext(options);
    }

    public synchronized void startDeferred(final ViewExecutionOptions options) {
        setPrimary(createDeferred(options));
    }

    protected AbstractViewProcessWorkerContext createImmediate(final ViewExecutionOptions options) {
        return new ImmediateExecutionContext(options);
    }

    public synchronized void startImmediate(final ViewExecutionOptions options) {
        setPrimary(createImmediate(options));
    }

    private void setPrimary(final AbstractViewProcessWorkerContext primary) {
        if (_terminated || (_primary != null)) {
            throw new IllegalStateException();
        }
        _primary = primary;
    }

    // caller must hold the monitor
    /* package */void startSecondaryWorker(final AbstractViewProcessWorkerContext primary,
            final ViewCycleExecutionSequence tailSequence) {
        s_logger.info("Starting secondary worker");
        _secondary = primary.createSecondaryWorker(tailSequence);
    }

    /* package */void promoteSecondaryWorker() {
        s_logger.info("Promoting secondary worker");
        _primary = _secondary;
        _secondary = null;
    }

    // caller must hold the monitor
    protected void checkForRecompilation(final AbstractViewProcessWorkerContext primary,
            CompiledViewDefinitionWithGraphs compiled) {
        final ViewCycleExecutionSequence tailSequence = (getSecondary() == null) ? primary.getSequence().copy()
                : null;
        final ViewCycleExecutionOptions nextCycle = primary.getSequence().poll(getDefaultExecutionOptions());
        if (nextCycle != null) {
            final VersionCorrection vc = nextCycle.getResolverVersionCorrection();
            boolean changes = false;
            if ((vc == null) || VersionCorrection.LATEST.equals(vc)) {
                if (_resolverChanges == null) {
                    _resolverChanges = new TargetResolverChangeListener() {
                        @Override
                        protected void onChanged() {
                            // Something has changed; request a cycle on the primary and that may then do the necessary
                            ViewProcessWorker worker = null;
                            synchronized (this) {
                                if (!_terminated && (getPrimary() != null)) {
                                    worker = getPrimary()._worker;
                                }
                            }
                            if (worker != null) {
                                worker.requestCycle();
                            }
                        }
                    };
                    getContext().getProcessContext().getFunctionCompilationService().getFunctionCompilationContext()
                            .getRawComputationTargetResolver().changeManager().addChangeListener(_resolverChanges);
                }
                final Collection<UniqueId> uids = compiled.getResolvedIdentifiers().values();
                final Set<ObjectId> oids = Sets.newHashSetWithExpectedSize(uids.size());
                for (UniqueId uid : uids) {
                    final ObjectId oid = uid.getObjectId();
                    if (tailSequence != null) {
                        changes |= _resolverChanges.isChanged(oid);
                    }
                    oids.add(oid);
                }
                _resolverChanges.watchOnly(oids);
            } else {
                if (_resolverChanges != null) {
                    getContext().getProcessContext().getFunctionCompilationService().getFunctionCompilationContext()
                            .getRawComputationTargetResolver().changeManager()
                            .removeChangeListener(_resolverChanges);
                    _resolverChanges = null;
                }
            }
            if (tailSequence == null) {
                // Already got a secondary worker; just went this far to update any change listeners
                s_logger.debug("Secondary worker already active");
                return;
            }
            if ((_resolverChanges == null) || changes) {
                startSecondaryWorker(primary, tailSequence);
            }
        }
    }

    protected synchronized boolean viewDefinitionCompiled(final AbstractViewProcessWorkerContext context,
            final CompiledViewDefinitionWithGraphs compiled) {
        if (!_terminated) {
            if (getPrimary() == context) {
                s_logger.info("View definition compiled by primary worker");
                getPrimary().setCompiled(compiled);
                checkForRecompilation(context, compiled);
                return true;
            }
            if (getSecondary() == context) {
                s_logger.info("View definition compiled by secondary worker");
                CompiledViewDefinitionWithGraphs primaryCompile = getPrimary()._lastCompiled;
                final Map<ComputationTargetReference, UniqueId> primaryResolutions = primaryCompile
                        .getResolvedIdentifiers();
                final Map<ComputationTargetReference, UniqueId> secondaryResolutions = compiled
                        .getResolvedIdentifiers();
                if (primaryResolutions.equals(secondaryResolutions)) {
                    // Nothing has changed, the primary is still valid
                    s_logger.debug("Rejecting compilation from secondary worker");
                    _secondary = null;
                    return false;
                } else {
                    s_logger.debug("Secondary compilation valid");
                    getSecondary().setCompiled(compiled);
                    return true;
                }
            }
        }
        return false;
    }

    protected synchronized WorkerAction cycleStarted(final AbstractViewProcessWorkerContext context) {
        if (!_terminated) {
            if (getPrimary() == context) {
                s_logger.info("Cycle started from primary worker");
                checkForRecompilation(context, context._lastCompiled);
                if (getSecondary() != null) {
                    final WorkerAction action = getSecondary().primaryCycleStarted();
                    if (action == WorkerAction.TERMINATE) {
                        promoteSecondaryWorker();
                    }
                    return action;
                } else {
                    return WorkerAction.PROCEED;
                }
            }
            if (getSecondary() == context) {
                s_logger.info("Cycle started from secondary worker");
                final WorkerAction action = getPrimary().secondaryCycleStarted();
                if (action == WorkerAction.TERMINATE) {
                    s_logger.info("Terminating secondary worker");
                    _secondary = null;
                }
                return action;
            }
        }
        return WorkerAction.TERMINATE;
    }

    protected synchronized WorkerAction cycleFragmentCompleted(final AbstractViewProcessWorkerContext context) {
        if (!_terminated) {
            if (getPrimary() == context) {
                s_logger.debug("Cycle fragment completed from primary worker");
                if (getSecondary() != null) {
                    WorkerAction action = getSecondary().primaryCycleFragmentCompleted();
                    if (action == WorkerAction.TERMINATE) {
                        promoteSecondaryWorker();
                    }
                    return action;
                } else {
                    return WorkerAction.PROCEED;
                }
            }
            if (getSecondary() == context) {
                s_logger.debug("Cycle fragment completed from secondary worker");
                return WorkerAction.PROCEED;
            }
        }
        return WorkerAction.TERMINATE;
    }

    protected synchronized WorkerAction cycleCompleted(final AbstractViewProcessWorkerContext context) {
        if (!_terminated) {
            if (getPrimary() == context) {
                s_logger.info("Cycle completed from primary worker");
                if (getSecondary() != null) {
                    WorkerAction action = getSecondary().primaryCycleCompleted();
                    if (action == WorkerAction.TERMINATE) {
                        promoteSecondaryWorker();
                    }
                    return action;
                } else {
                    return WorkerAction.PROCEED;
                }
            }
            if (getSecondary() == context) {
                s_logger.info("Cycle completed from secondary worker");
                return WorkerAction.PROCEED;
            }
        }
        return WorkerAction.TERMINATE;
    }

    protected synchronized WorkerAction cycleExecutionFailed(final AbstractViewProcessWorkerContext context) {
        if (!_terminated) {
            if (getPrimary() == context) {
                s_logger.info("Cycle execution failed from primary worker");
                if (getSecondary() != null) {
                    promoteSecondaryWorker();
                    return WorkerAction.TERMINATE;
                } else {
                    return WorkerAction.PROCEED;
                }
            }
            if (getSecondary() == context) {
                s_logger.info("Cycle execution failed from secondary worker");
                s_logger.info("Terminating secondary worker");
                _secondary = null;
                return WorkerAction.TERMINATE;
            }
        }
        return WorkerAction.TERMINATE;
    }

    protected boolean workerCompleted(final AbstractViewProcessWorkerContext context) {
        if (!terminate(context)) {
            s_logger.info("Worker for {} already terminated", context);
            return false;
        }
        synchronized (this) {
            if (getPrimary() == context) {
                final AbstractViewProcessWorkerContext secondary = getSecondary();
                if (secondary != null) {
                    promoteSecondaryWorker();
                    secondary.unblock(WorkerAction.PROCEED);
                    return false;
                } else {
                    s_logger.info("Primary worker completed - no secondary worker");
                    _primary = null;
                    if (_resolverChanges != null) {
                        getContext().getProcessContext().getFunctionCompilationService()
                                .getFunctionCompilationContext().getRawComputationTargetResolver().changeManager()
                                .removeChangeListener(_resolverChanges);
                        _resolverChanges = null;
                    }
                    return true;
                }
            } else if (getSecondary() == context) {
                assert getPrimary() != null;
                s_logger.info("Secondary worker completed");
                _secondary = null;
                return false;
            } else {
                // E.g. a late or incorrect notification from a worker 
                throw new IllegalStateException();
            }
        }
    }

    protected boolean terminate(final AbstractViewProcessWorkerContext context) {
        ViewProcessWorker worker;
        synchronized (this) {
            worker = context._worker;
            if (worker == null) {
                return false;
            }
            context._worker = null;
        }
        worker.terminate();
        return true;
    }

    // ViewProcessWorker

    @Override
    public boolean triggerCycle() {
        do {
            ViewProcessWorker primary = null;
            synchronized (this) {
                if (!_terminated && (getPrimary() != null)) {
                    primary = getPrimary()._worker;
                }
            }
            if (primary != null) {
                s_logger.debug("Triggering cycle on primary worker {}", primary);
                if (primary.triggerCycle()) {
                    return true;
                } else {
                    synchronized (this) {
                        if (!_terminated && (getPrimary() != null) && (primary == getPrimary()._worker)) {
                            s_logger.debug("Primary worker unable to handle request");
                            return false;
                        }
                    }
                    s_logger.debug("Primary worker has terminated; repeating request");
                    continue;
                }
            } else {
                s_logger.debug("Ignoring triggerCycle on terminated worker");
                return false;
            }
        } while (true);
    }

    @Override
    public boolean requestCycle() {
        do {
            ViewProcessWorker primary = null;
            synchronized (this) {
                if (!_terminated && (getPrimary() != null)) {
                    primary = getPrimary()._worker;
                }
            }
            if (primary != null) {
                s_logger.debug("Requesting cycle from primary worker {}", primary);
                if (primary.requestCycle()) {
                    return true;
                } else {
                    synchronized (this) {
                        if (!_terminated && (getPrimary() != null) && (primary == getPrimary()._worker)) {
                            s_logger.debug("Primary worker unable to handle request");
                            return false;
                        }
                    }
                    s_logger.debug("Primary worker has terminated; repeating request");
                    continue;
                }
            } else {
                s_logger.debug("Ignoring requestCycle on terminated worker");
                return false;
            }
        } while (true);
    }

    @Override
    public void updateViewDefinition(ViewDefinition viewDefinition) {
        s_logger.info("Updating view definition");
        ViewProcessWorker worker = null;
        synchronized (this) {
            _viewDefinition = viewDefinition;
            if (getSecondary() != null) {
                worker = getSecondary()._worker;
                _secondary = getSecondary().createSecondaryWorker();
            } else if (getPrimary() != null) {
                _secondary = getPrimary().createSecondaryWorker();
            }
        }
        if (worker != null) {
            s_logger.info("Terminating previous secondary worker {}", worker);
            worker.terminate();
        }
    }

    @Override
    public void terminate() {
        s_logger.info("Terminating worker(s)");
        ViewProcessWorker primary, secondary;
        synchronized (this) {
            if (_terminated) {
                s_logger.warn("Already terminated");
                return;
            }
            primary = (getPrimary() != null) ? getPrimary()._worker : null;
            secondary = (getSecondary() != null) ? getSecondary()._worker : null;
            _terminated = true;
        }
        if (primary != null) {
            s_logger.debug("Terminating primary worker {}", primary);
            primary.terminate();
        }
        if (secondary != null) {
            s_logger.debug("Terminating secondary worker {}", secondary);
            secondary.terminate();
        }
    }

    @Override
    public void join() throws InterruptedException {
        s_logger.info("Joining worker(s)");
        do {
            ViewProcessWorker primary;
            synchronized (this) {
                if (getPrimary() == null) {
                    break;
                }
                primary = getPrimary()._worker;
            }
            s_logger.debug("Joining primary worker {}", primary);
            primary.join();
            synchronized (this) {
                if (getPrimary() == null) {
                    break;
                } else {
                    if (getPrimary()._worker == primary) {
                        if (getSecondary() != null) {
                            promoteSecondaryWorker();
                        } else {
                            _primary = null;
                            break;
                        }
                    } else {
                        s_logger.debug("Primary worker {} changed to {} during wait", primary, getPrimary());
                    }
                }
            }
        } while (true);
        s_logger.debug("Primary worker joined");
    }

    @Override
    public boolean join(final long timeout) throws InterruptedException {
        s_logger.info("Joining worker(s)");
        final long waitUntil = System.currentTimeMillis() + timeout;
        do {
            ViewProcessWorker primary;
            synchronized (this) {
                if (getPrimary() == null) {
                    break;
                }
                primary = getPrimary()._worker;
            }
            s_logger.debug("Joining primary worker {}", primary);
            final long waitDuration = waitUntil - System.currentTimeMillis();
            if (waitDuration > 0) {
                s_logger.debug("Waiting for {}ms for primary worker {}", waitDuration, primary);
                if (!primary.join(waitDuration)) {
                    return false;
                }
            } else {
                s_logger.debug("Timeout elapsed joining {}", primary);
                return false;
            }
            synchronized (this) {
                if (getPrimary() == null) {
                    break;
                } else {
                    if (getPrimary()._worker == primary) {
                        if (getSecondary() != null) {
                            promoteSecondaryWorker();
                        } else {
                            _primary = null;
                            break;
                        }
                    } else {
                        s_logger.debug("Primary worker {} changed to {} during wait", primary, getPrimary());
                    }
                }
            }
        } while (true);
        s_logger.debug("Primary worker joined");
        return true;
    }

    @Override
    public synchronized boolean isTerminated() {
        return getPrimary() == null;
    }

    @Override
    public void forceGraphRebuild() {
        ViewProcessWorker primary;
        ViewProcessWorker secondary;
        synchronized (this) {
            if (_terminated) {
                s_logger.warn("Already terminated");
                return;
            }
            primary = (getPrimary() != null) ? getPrimary()._worker : null;
            secondary = (getSecondary() != null) ? getSecondary()._worker : null;
            _terminated = true;
        }
        if (primary != null) {
            primary.forceGraphRebuild();
        }
        if (secondary != null) {
            secondary.forceGraphRebuild();
        }
    }

}