rapture.kernel.DecisionApiImpl.java Source code

Java tutorial

Introduction

Here is the source code for rapture.kernel.DecisionApiImpl.java

Source

/**
 * The MIT License (MIT)
 *
 * Copyright (c) 2011-2016 Incapture Technologies LLC
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
package rapture.kernel;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.http.HttpStatus;
import org.apache.log4j.Logger;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.LocalDate;
import org.joda.time.Period;

import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Optional;
import com.google.common.collect.Maps;

import rapture.common.AppStatus;
import rapture.common.AppStatusGroup;
import rapture.common.AppStatusGroupStorage;
import rapture.common.AppStatusStorage;
import rapture.common.CallingContext;
import rapture.common.CreateResponse;
import rapture.common.ErrorWrapper;
import rapture.common.LogQueryResponse;
import rapture.common.RaptureFolderInfo;
import rapture.common.RaptureJobExec;
import rapture.common.RaptureJobExecStorage;
import rapture.common.RaptureURI;
import rapture.common.Scheme;
import rapture.common.SemaphoreAcquireResponse;
import rapture.common.WorkOrderExecutionState;
import rapture.common.WorkflowJobDetails;
import rapture.common.api.DecisionApi;
import rapture.common.dp.AppStatusDetails;
import rapture.common.dp.ContextValueType;
import rapture.common.dp.ContextVariables;
import rapture.common.dp.ExecutionContext;
import rapture.common.dp.ExecutionContextField;
import rapture.common.dp.ExecutionContextFieldStorage;
import rapture.common.dp.ExpectedArgument;
import rapture.common.dp.PassedArgument;
import rapture.common.dp.Step;
import rapture.common.dp.StepHelper;
import rapture.common.dp.StepRecord;
import rapture.common.dp.Transition;
import rapture.common.dp.WorkOrder;
import rapture.common.dp.WorkOrderArguments;
import rapture.common.dp.WorkOrderArgumentsStorage;
import rapture.common.dp.WorkOrderCancellation;
import rapture.common.dp.WorkOrderCancellationStorage;
import rapture.common.dp.WorkOrderDebug;
import rapture.common.dp.WorkOrderInitialArgsHash;
import rapture.common.dp.WorkOrderInitialArgsHashStorage;
import rapture.common.dp.WorkOrderPathBuilder;
import rapture.common.dp.WorkOrderStatus;
import rapture.common.dp.WorkOrderStorage;
import rapture.common.dp.Worker;
import rapture.common.dp.WorkerExecutionState;
import rapture.common.dp.WorkerStorage;
import rapture.common.dp.Workflow;
import rapture.common.dp.WorkflowHistoricalMetrics;
import rapture.common.dp.WorkflowStorage;
import rapture.common.exception.ExceptionToString;
import rapture.common.exception.RaptureException;
import rapture.common.exception.RaptureExceptionFactory;
import rapture.common.impl.jackson.JacksonUtil;
import rapture.dp.ArgsHashFactory;
import rapture.dp.DecisionProcessExecutorFactory;
import rapture.dp.InvocableUtils;
import rapture.dp.WorkOrderFactory;
import rapture.dp.metrics.WorkflowMetricsFactory;
import rapture.dp.semaphore.LockKeyFactory;
import rapture.dp.semaphore.WorkOrderSemaphore;
import rapture.dp.semaphore.WorkOrderSemaphoreFactory;
import rapture.kernel.dp.DpDebugReader;
import rapture.kernel.dp.ExecutionContextUtil;
import rapture.kernel.dp.StepRecordUtil;
import rapture.kernel.dp.WorkflowValidator;
import rapture.log.management.LogManagerConnection;
import rapture.log.management.LogReadException;
import rapture.log.management.SessionExpiredException;

public class DecisionApiImpl extends KernelBase implements DecisionApi {
    private static final String ERROR_LIST_CONSTANT = "errorList";
    private static final Logger logger = Logger.getLogger(DecisionApiImpl.class);
    private static final String FIRST_WORKER_ID = "0";
    private static final String LOCK_TIMEOUT_SECONDS = "LOCK_TIMEOUT_SECONDS";

    public DecisionApiImpl(Kernel raptureKernel) {
        super(raptureKernel);
    }

    @Override
    public void putWorkflow(CallingContext context, Workflow workflow) {
        WorkflowValidator.validate(workflow);
        WorkflowStorage.add(new RaptureURI(workflow.getWorkflowURI(), Scheme.WORKFLOW), workflow, context.getUser(),
                "Define workflow");
    }

    @Override
    public Workflow getWorkflow(CallingContext context, String workflowURI) {
        RaptureURI addressURI = new RaptureURI(workflowURI, Scheme.WORKFLOW);
        return getWorkflow(addressURI);
    }

    private Workflow getWorkflow(RaptureURI uri) {
        return WorkflowStorage.readByAddress(uri.withoutDecoration());
    }

    @Override
    public List<Workflow> getAllWorkflows(CallingContext context) {
        return WorkflowStorage.readAll();
    }

    @Override
    public void addStep(CallingContext context, String workflowURI, Step step) {
        Workflow workflow = getWorkflowNotNull(context, workflowURI);
        List<Step> steps = workflow.getSteps();
        for (Step currStep : steps) {
            if (currStep.getName().equals(step.getName())) {
                throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_BAD_REQUEST,
                        String.format("Trying to add step that already exists: Step name '%s'", step.getName()));
            }
        }
        steps.add(step);
        WorkflowStorage.add(new RaptureURI(workflow.getWorkflowURI(), Scheme.WORKFLOW), workflow, context.getUser(),
                "Add step");
    }

    /**
     * Returns a workflow, or throws a {@link RaptureException} if no workflow exists at the given URI. This should only be used internally by methods that are
     * intended to modify an existing {@link Workflow} and require it to already be defined
     */
    private Workflow getWorkflowNotNull(CallingContext context, String workflowURI) {
        Workflow workflow = getWorkflow(context, workflowURI);
        if (workflow == null) {
            throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_BAD_REQUEST,
                    String.format("Workflow does not exist: '%s'", workflowURI));
        } else {
            return workflow;
        }
    }

    private Workflow getWorkflowNotNull(CallingContext context, RaptureURI uri) {
        Workflow workflow = getWorkflow(uri);
        if (workflow == null) {
            throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_BAD_REQUEST,
                    String.format("Workflow does not exist: '%s'", uri.toString()));
        } else {
            return workflow;
        }
    }

    @Override
    public void removeStep(CallingContext context, String workflowURI, String stepName) {
        Workflow workflow = getWorkflowNotNull(context, workflowURI);
        List<Step> steps = workflow.getSteps();
        int index = 0;
        boolean found = false;
        for (Step step : steps) {
            if (stepName.equals(step.getName())) {
                found = true;
                break;
            }
            index++;
        }
        if (found) {
            steps.remove(index);
            WorkflowStorage.add(new RaptureURI(workflow.getWorkflowURI(), Scheme.WORKFLOW), workflow,
                    context.getUser(), "Remove step");
        }
    }

    @Override
    public void addTransition(CallingContext context, String workflowURI, String stepName, Transition transition) {
        Workflow workflow = getWorkflowNotNull(context, workflowURI);
        Step step = getStep(workflow, stepName);
        if (step != null) {
            step.getTransitions().add(transition);
        } else {
            throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_BAD_REQUEST,
                    String.format("Attempting to add a transition from non-existent step: '%s'", stepName));
        }

        WorkflowStorage.add(new RaptureURI(workflow.getWorkflowURI(), Scheme.WORKFLOW), workflow, context.getUser(),
                "Add transition");
    }

    public static Step getStep(Workflow workflow, String stepName) {
        String decodedName = StepHelper.decode(stepName);
        if (StepHelper.isImpliedStep(decodedName)) {
            Step impliedStep = new Step();
            impliedStep.setName(decodedName);
            impliedStep.setExecutable(decodedName);
            return impliedStep;
        }
        for (Step step : workflow.getSteps()) {
            if (decodedName.equals(step.getName())) {
                return step;
            }
        }
        return null;
    }

    @Override
    public void removeTransition(CallingContext context, String workflowURI, String stepName,
            String transitionName) {
        Workflow workflow = getWorkflowNotNull(context, workflowURI);
        Step step = getStep(workflow, stepName);
        if (step != null) {
            int index = 0;
            boolean found = false;
            List<Transition> transitions = step.getTransitions();
            for (Transition transition : transitions) {
                if (transitionName.equals(transition.getName())) {
                    found = true;
                    break;
                }
                index++;
            }

            if (found) {
                transitions.remove(index);
                WorkflowStorage.add(new RaptureURI(workflow.getWorkflowURI(), Scheme.WORKFLOW), workflow,
                        context.getUser(), "Remove transition");
            }
        }
    }

    @Override
    public void deleteWorkflow(CallingContext context, String workflowURI) {
        RaptureURI addressURI = new RaptureURI(workflowURI, Scheme.WORKFLOW);
        WorkflowStorage.deleteByAddress(addressURI, context.getUser(), "Delete workflow");
    }

    private static final String WORK_ORDER_IDGEN_URI = new RaptureURI("//sys/dp/workOrder", Scheme.IDGEN)
            .toString();

    @Override
    public String createWorkOrder(CallingContext context, String workflowURI, Map<String, String> contextMap) {
        return createWorkOrderP(context, workflowURI, contextMap, null).getUri();
    }

    @Override
    public CreateResponse createWorkOrderP(CallingContext context, String workflowURI, Map<String, String> argsMap,
            String appStatusNamePattern) {
        /*
         * Check that we can create the Work Order. If at capacity, return null
         */
        if (logger.isTraceEnabled())
            logger.trace("createWorkOrder URI = " + workflowURI + "argsMap = " + argsMap);

        RaptureURI workflowUri = new RaptureURI(workflowURI, Scheme.WORKFLOW);
        workflowURI = workflowUri.toString();
        Workflow workflow = getWorkflowNotNull(context, workflowUri);
        String startStep = workflowUri.hasElement() ? workflowUri.getElement() : workflow.getStartStep();

        Kernel.getStackContainer().pushStack(context, workflowUri.toString());

        Map<String, String> contextMap = setupContextMap(workflow.getExpectedArguments(), argsMap);

        WorkOrderSemaphore semaphore = WorkOrderSemaphoreFactory.create(context, workflow.getSemaphoreType(),
                workflow.getSemaphoreConfig());
        Long startInstant = System.currentTimeMillis();
        String lockKey = null;
        try {
            lockKey = LockKeyFactory.createLockKey(workflow.getSemaphoreType(), workflow.getSemaphoreConfig(),
                    workflow.getWorkflowURI(), contextMap);
            /**
             * Save the lock key so we can use it when releasing later
             */
            if (lockKey != null) {
                contextMap.put(ContextVariables.LOCK_KEY, lockKey);
            }
        } catch (RaptureException e) {
            CreateResponse ret = new CreateResponse();
            ret.setIsCreated(false);
            ret.setMessage(e.getMessage());
            return ret;
        }

        long timeout = semaphore.getTimeout();
        if (contextMap.containsKey(LOCK_TIMEOUT_SECONDS)) {
            timeout = Long.parseLong(contextMap.get(LOCK_TIMEOUT_SECONDS));
        }
        SemaphoreAcquireResponse response;
        if (timeout > 0) {
            response = semaphore.acquirePermit(workflow.getWorkflowURI(), startInstant, lockKey, timeout);
        } else {
            response = semaphore.tryAcquirePermit(workflow.getWorkflowURI(), startInstant, lockKey);
        }

        if (!response.getIsAcquired()) {
            List<String> existingWoDesc = new LinkedList<>();
            Set<String> existingStakeholderURIs = response.getExistingStakeholderURIs();
            for (String stakeHolderURI : existingStakeholderURIs) {
                String jobURI = getContextValue(context, stakeHolderURI, ContextVariables.PARENT_JOB_URI);
                if (jobURI != null) {
                    existingWoDesc
                            .add(String.format("{workOrderURI=%s, created by jobURI=%s}", stakeHolderURI, jobURI));
                } else {
                    existingWoDesc.add(String.format("{workOrderURI=%s}", stakeHolderURI));
                }
            }
            String error = String.format(
                    "Unable to acquire a permit for a new WorkOrder for Workflow %s, with lockKey %s. The lock is already being held by the following"
                            + " WorkOrder(s): %s",
                    workflowURI, lockKey, StringUtils.join(existingWoDesc, ", "));
            logger.warn(error);
            CreateResponse ret = new CreateResponse();
            ret.setIsCreated(false);
            ret.setMessage(error);
            return ret;
        } else {
            RaptureURI workOrderURI = new RaptureURI(response.getAcquiredURI());
            Kernel.getStackContainer().pushStack(context, workOrderURI.toString());

            // Register the separate pieces
            // 1. The WorkOrder object
            WorkOrder workOrder = new WorkOrder();

            workOrder.setWorkOrderURI(workOrderURI.toString());
            workOrder.setWorkflowURI(workflowURI);
            workOrder.setPriority(0);
            workOrder.setStartTime(startInstant);
            List<String> workerIds = Arrays.asList(FIRST_WORKER_ID);
            workOrder.setWorkerIds(workerIds);
            workOrder.setPendingIds(workerIds);
            workOrder.setSemaphoreConfig(workflow.getSemaphoreConfig());
            workOrder.setSemaphoreType(workflow.getSemaphoreType());
            workOrder.setStatus(WorkOrderExecutionState.NEW);

            WorkOrderStorage.add(workOrderURI, workOrder, context.getUser(), "Create work order");
            if (logger.isTraceEnabled())
                logger.trace("workOrder = " + workOrder.debug());

            // 2. The ExecutionContext
            ExecutionContext executionContext = new ExecutionContext();
            Map<String, String> data = new HashMap<>();
            for (Entry<String, String> entry : contextMap.entrySet()) {
                data.put(entry.getKey(), ContextValueType.LITERAL.marker + entry.getValue());
            }

            executionContext.setWorkOrderURI(workOrder.getWorkOrderURI());
            executionContext.setData(data);
            ExecutionContextUtil.saveContext(executionContext, context.getUser(), "Create work order");
            if (logger.isTraceEnabled())
                logger.trace("Execution Context is " + executionContext.debug() + " user = " + context.getUser());

            // 3. The WorkOrderArguments
            writeWorkOrderArguments(context, argsMap, workOrder);

            // 4. The Worker
            Worker worker = new Worker();
            worker.setWorkOrderURI(workOrder.getWorkOrderURI());
            worker.setId("" + FIRST_WORKER_ID);
            worker.setParent("");
            worker.setSiblingCount(1);
            worker.setSiblingPosition(0);
            worker.setWaitCount(0);
            worker.setStatus(WorkerExecutionState.READY);
            worker.setPriority(workOrder.getPriority());
            worker.setCallingContext(context);
            worker.getLocalView().add(workflow.getView());
            RaptureURI qualifiedURI = new RaptureURI(workflowURI, Scheme.WORKFLOW);
            String activityId = Kernel.getActivity().createActivity(context, qualifiedURI.toString(), "Started", 0L,
                    100L);
            worker.setActivityId(activityId);

            String appStatusURI = InvocableUtils.createAppStatusName(context, workflow, worker,
                    appStatusNamePattern);
            worker.getAppStatusNameStack().add(0, appStatusURI);

            List<String> stack = worker.getStack();
            String encodedName = StepHelper.encode(startStep);
            stack.add(RaptureURI.builder(new RaptureURI(workflowURI, Scheme.WORKFLOW)).element(encodedName)
                    .asString());
            WorkerStorage.add(worker, context.getUser(), "Create work order");
            if (logger.isTraceEnabled())
                logger.trace("worker = " + worker.debug());

            // 4. The workorder initial args hash
            String hashValue = ArgsHashFactory.createHashValue(contextMap);
            WorkOrderInitialArgsHash argsHash = new WorkOrderInitialArgsHash();
            argsHash.setWorkOrderURI(workOrderURI.toString());
            argsHash.setHashValue(hashValue);
            WorkOrderInitialArgsHashStorage.add(argsHash, context.getUser(), "Created work order");

            // start the work order
            DecisionProcessExecutorFactory.getDefault().start(context, worker);
            Kernel.getStackContainer().popStack(context);
            Kernel.getStackContainer().popStack(context);
            CreateResponse ret = new CreateResponse();
            String order = workOrderURI.toString();
            ret.setIsCreated(true);
            ret.setUri(order);

            return ret;
        }
    }

    private static void writeWorkOrderArguments(CallingContext context, Map<String, String> argsMap,
            WorkOrder workOrder) {
        WorkOrderArguments workOrderArguments = new WorkOrderArguments();
        workOrderArguments.setWorkOrderURI(workOrder.getWorkOrderURI());
        if (argsMap != null) {
            for (Entry<String, String> entry : argsMap.entrySet()) {
                PassedArgument arg = new PassedArgument();
                arg.setName(entry.getKey());
                arg.setValue(entry.getValue());
                workOrderArguments.getArguments().add(arg);
            }
        }
        WorkOrderArgumentsStorage.add(workOrderArguments, context.getUser(), "Create work order");
    }

    private Map<String, String> setupContextMap(List<ExpectedArgument> expectedArguments,
            Map<String, String> argsMap) {
        HashMap<String, String> contextMap;
        if (argsMap == null) {
            contextMap = new HashMap<>();
        } else {
            contextMap = Maps.newHashMap(argsMap);
        }

        long timestamp = System.currentTimeMillis();
        if (!contextMap.containsKey(ContextVariables.TIMESTAMP)) {
            contextMap.put(ContextVariables.TIMESTAMP, timestamp + "");
        }
        if (!contextMap.containsKey(ContextVariables.LOCAL_DATE)) {
            DateTimeZone timezone = DateTimeZone.UTC;
            LocalDate ld = new LocalDate(timestamp, timezone);
            contextMap.put(ContextVariables.LOCAL_DATE, ContextVariables.FORMATTER.print(ld));
        }

        for (ExpectedArgument expectedArgument : expectedArguments) {
            if (!contextMap.containsKey(expectedArgument.getName())
                    && !StringUtils.isBlank(expectedArgument.getDefaultValue())) {
                contextMap.put(expectedArgument.getName(), expectedArgument.getDefaultValue());
            }
        }

        return contextMap;
    }

    @Override
    public void releaseWorkOrderLock(CallingContext context, String workOrderURI) {
        releaseWorkOrderLock(context, WorkOrderFactory.getWorkOrderNotNull(context, workOrderURI));
    }

    public void releaseWorkOrderLock(CallingContext ctx, WorkOrder workOrder) {
        WorkOrderSemaphore semaphore = WorkOrderSemaphoreFactory.create(ctx, workOrder.getSemaphoreType(),
                workOrder.getSemaphoreConfig());
        String lockKey = getContextValue(ctx, workOrder.getWorkOrderURI(), ContextVariables.LOCK_KEY);
        semaphore.releasePermit(workOrder.getWorkOrderURI(), lockKey);
    }

    public RaptureURI generateWorkOrderURI(CallingContext context, String workflowURI, Long startInstant) {
        // use epoch at start of day as authority
        String authority = getStartOfDayEpoch(startInstant);
        String id = Kernel.getIdGen().nextIds(context, WORK_ORDER_IDGEN_URI, 1L);
        String docPath = new RaptureURI(workflowURI, Scheme.WORKFLOW).getFullPath() + "/" + id;
        return RaptureURI.builder(Scheme.WORKORDER, authority).docPath(docPath).build();
    }

    private String getStartOfDayEpoch(Long startInstant) {
        DateTime dateTime = new DateTime(startInstant, DateTimeZone.UTC);
        return "" + dateTime.withTimeAtStartOfDay().toInstant().getMillis() / 1000;
    }

    @Override
    public WorkOrderStatus getWorkOrderStatus(CallingContext context, String workOrderURI) {

        if (workOrderURI == null)
            return null;
        WorkOrder workOrder = WorkOrderFactory.getWorkOrderNotNull(context, workOrderURI);
        WorkOrderStatus retVal = new WorkOrderStatus();
        retVal.setStatus(workOrder.getStatus());
        List<String> workerIds = workOrder.getWorkerIds();
        Map<String, String> workerOutput = workOrder.getOutputs();
        if (workerOutput == null) {
            workerOutput = new HashMap<>();
        }
        retVal.setWorkerOutput(workerOutput);

        DocApiImpl docApi = Kernel.getDoc().getTrusted();
        String outputUri = RaptureURI.newScheme(workOrderURI, Scheme.DOCUMENT).toShortString();

        for (String workerId : workerIds) {
            String id = outputUri + "#" + workerId;
            if (!workerOutput.containsKey(id)) {
                String str = docApi.getDocEphemeral(context, outputUri);
                if (str != null) {
                    Map<String, Object> map = JacksonUtil.getMapFromJson(str);
                    Object out = map.get(id);
                    if (out != null)
                        workerOutput.put(id, out.toString());
                }
            }
        }
        return retVal;
    }

    @Override
    public void cancelWorkOrder(CallingContext context, String workOrderURI) {
        if (this.wasCancelCalled(context, workOrderURI))
            return;
        String nakedUri = null;
        try {
            RaptureURI uri = new RaptureURI(workOrderURI, Scheme.WORKORDER);
            nakedUri = uri.withoutDecoration().toString();
        } catch (Exception ex) {
            throw RaptureExceptionFactory.create("Unparseable URI");
        }
        WorkOrderCancellation woc = new WorkOrderCancellation();
        woc.setTime(System.currentTimeMillis());
        woc.setWorkOrderURI(nakedUri);
        WorkOrderCancellationStorage.add(woc, context.getUser(), "Try to Cancel");
    }

    /**
     * resumeWorkOrder acts like createWorkOrder except that we must get the context from the previous order
     *
     * @param context
     * @param workOrderURI
     * @param workflowStepURI
     * @return CreateResponse object - same as Create Work Order
     */
    @Override
    public CreateResponse resumeWorkOrder(CallingContext context, String workOrderURI, String workflowStepURI) {
        if (workflowStepURI == null) {
            // Error?
            throw RaptureExceptionFactory.create("Workflow Step to resume from must be specified - cannot be null");
        }
        try {
            // Sanity check. Don't try to resume a flow at a step from a
            // different work order - that would be silly.
            // Is this valid in all cases?
            RaptureURI stepURI = new RaptureURI(workflowStepURI, Scheme.WORKFLOW);
            String stepName = stepURI.getElement();
            if ((stepName == null) || stepName.isEmpty()) {
                throw RaptureExceptionFactory
                        .create("Workflow Step to resume from must be specified - element missing");
            }
            RaptureURI orderURI = new RaptureURI(workOrderURI, Scheme.WORKORDER);
            if (!orderURI.getDocPath().startsWith(stepURI.getShortPath())) {
                throw RaptureExceptionFactory
                        .create("Step " + workflowStepURI + " does not match work order " + workOrderURI);
            }
        } catch (Exception ex) {
            throw RaptureExceptionFactory.create("Not a valid Workflow URI " + workflowStepURI);
        }

        Map<String, String> contextMap = new HashMap<>();
        RaptureURI uri = new RaptureURI(workOrderURI, Scheme.WORKORDER);
        List<ExecutionContextField> ecfs = ExecutionContextFieldStorage.readAll(uri.getFullPath());
        for (ExecutionContextField ecf : ecfs) {
            String value = ecf.getValue();
            contextMap.put(ecf.getVarName(), (value.charAt(0) == '#') ? value.substring(1) : value);
        }

        // How do we recover the original appStatus?
        // I don't see a way - so let's try this:
        String appStatusNamePattern = null;

        // Get the appStatusNameStack from the old work order
        Worker worker = WorkerStorage.readByFields(workOrderURI, "0");
        if (worker != null) {
            List<String> appStatusNameStack = worker.getAppStatusNameStack();
            if ((appStatusNameStack != null) && !appStatusNameStack.isEmpty()) {
                appStatusNamePattern = appStatusNameStack.get(0);
                if ((appStatusNamePattern != null) && (!appStatusNamePattern.startsWith("%")))
                    appStatusNamePattern = "%" + appStatusNamePattern;
            }
        }
        return createWorkOrderP(context, workflowStepURI, contextMap, appStatusNamePattern);

    }

    @Override
    public WorkOrderDebug getWorkOrderDebug(CallingContext context, String workOrderURI) {
        DpDebugReader reader = new DpDebugReader();
        return reader.getWorkOrderDebug(context, workOrderURI);
    }

    public List<Worker> getWorkers(WorkOrder workOrder) {
        List<String> workerIds = workOrder.getWorkerIds();
        List<Worker> workers = new ArrayList<>(workerIds.size());
        for (String workerId : workerIds) {
            workers.add(getWorkerNotNull(workOrder.getWorkOrderURI(), workerId));
        }
        return workers;
    }

    @Override
    public void setWorkOrderIdGenConfig(CallingContext context, String config, Boolean force) {
        if (force || !Kernel.getIdGen().idGenExists(context, WORK_ORDER_IDGEN_URI)) {
            Kernel.getIdGen().createIdGen(context, WORK_ORDER_IDGEN_URI, config);
        }
    }

    @Override
    public Step getWorkflowStep(CallingContext context, String stepURI) {
        RaptureURI uri = new RaptureURI(stepURI, Scheme.WORKFLOW);
        String stepName = uri.getElement();
        if (stepName == null) {
            throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_BAD_REQUEST, String.format(
                    "The Step URI passed in does not contain an 'element' indicating the step, but it requires one: [%s]",
                    uri.toString()));
        }
        Workflow workflow = getWorkflowNotNull(context, stepURI);
        return getStep(workflow, stepName);
    }

    /**
     * Trusted method to get the workflow and the selected step as a pair.
     */
    public Pair<Workflow, Step> getWorkflowWithStep(CallingContext context, String stepURI) {
        RaptureURI uri = new RaptureURI(stepURI, Scheme.WORKFLOW);
        String stepName = uri.getElement();
        if (stepName == null) {
            throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_BAD_REQUEST, String.format(
                    "The Step URI passed in does not contain an 'element' indicating the step, but it requires one: [%s]",
                    uri.toString()));
        }
        Workflow workflow = getWorkflowNotNull(context, stepURI);
        return ImmutablePair.of(workflow, getStep(workflow, stepName));
    }

    @Override
    public String getStepCategory(CallingContext context, String stepURI) {
        RaptureURI uri = new RaptureURI(stepURI, Scheme.WORKFLOW);
        Workflow workflow = getWorkflowNotNull(context, stepURI);
        if (uri.getElement() == null) {
            throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_BAD_REQUEST, String.format(
                    "The Step URI passed in does not contain an 'element' indicating the step, but it requires one: '%s'",
                    uri.toString()));
        }
        Step step = getStep(workflow, uri.getElement());
        if (step == null) {
            throw RaptureExceptionFactory.create(String.format("Error! No step exists for URI %s", stepURI));
        } else if (step.getCategoryOverride() == null || "".equals(step.getCategoryOverride())) {
            return workflow.getCategory();
        } else {
            return step.getCategoryOverride();
        }
    }

    @Override
    public void setContextTemplate(CallingContext context, String workerURI, String varAlias, String literalValue) {
        setContextValue(context, workerURI, varAlias, ContextValueType.TEMPLATE, literalValue);
    }

    @Override
    public void setContextLiteral(CallingContext context, String workerURI, String varAlias, String literalValue) {
        setContextValue(context, workerURI, varAlias, ContextValueType.LITERAL, literalValue);
    }

    private void setContextValue(CallingContext context, String workerURI, String varAlias, ContextValueType type,
            String literalValue) {
        RaptureURI uri = new RaptureURI(workerURI, Scheme.WORKORDER);
        String workOrderUri = uri.toShortString();
        Map<String, String> view = getView(uri, workOrderUri);
        ExecutionContextUtil.setValueECF(context, workOrderUri, view, varAlias, type, literalValue);
    }

    private Map<String, String> getView(RaptureURI uri, String workOrderURI) {
        String workerId = uri.getElement();
        if (workerId == null) {
            workerId = FIRST_WORKER_ID;
        }
        Worker worker = WorkerStorage.readByFields(workOrderURI, workerId);
        Map<String, String> view;
        if (worker != null) {
            view = InvocableUtils.getLocalViewOverlay(worker);
        } else {
            view = new HashMap<>();
        }
        return view;
    }

    @Override
    public void setContextLink(CallingContext context, String workerURI, String varAlias, String expressionValue) {
        setContextValue(context, workerURI, varAlias, ContextValueType.LINK, expressionValue);
    }

    @Override
    public String getContextValue(CallingContext callingContext, String workerURI, String varAlias) {
        RaptureURI uri = new RaptureURI(workerURI, Scheme.WORKORDER);
        String workOrderURI = uri.toShortString();
        Map<String, String> view = getView(uri, workOrderURI);
        return ExecutionContextUtil.getValueECF(callingContext, workOrderURI, varAlias, view);
    }

    @Override
    public void addErrorToContext(CallingContext context, String workerURI, ErrorWrapper errorWrapper) {
        List<ErrorWrapper> errorList = getErrorsFromContext(context, workerURI);
        errorList.add(errorWrapper);
        String json = JacksonUtil.jsonFromObject(errorList);
        setContextLiteral(ContextFactory.getKernelUser(), workerURI, ERROR_LIST_CONSTANT, json);
    }

    @Override
    public List<ErrorWrapper> getErrorsFromContext(CallingContext context, String workerURI) {
        String json = getContextValue(ContextFactory.getKernelUser(), workerURI, ERROR_LIST_CONSTANT);
        List<ErrorWrapper> errorList;
        if (json == null) {
            errorList = new LinkedList<>();
        } else {
            errorList = JacksonUtil.objectFromJson(json, new TypeReference<List<ErrorWrapper>>() {
            });
        }
        return errorList;
    }

    @Override
    public List<ErrorWrapper> getExceptionInfo(CallingContext context, String workOrderURI) {
        WorkOrder order = getWorkOrder(context, workOrderURI);
        List<ErrorWrapper> list = new LinkedList<>();
        if (order != null) {
            List<Worker> workers = getWorkers(order);
            for (Worker worker : workers) {
                ErrorWrapper workerEx = worker.getExceptionInfo();
                if (workerEx != null) {
                    list.add(workerEx);
                }
                for (StepRecord stepRecord : StepRecordUtil.getStepRecords(worker)) {
                    ErrorWrapper stepEx = stepRecord.getExceptionInfo();
                    if (stepEx != null) {
                        list.add(stepEx);
                    }
                }
            }
        } else {
            throw RaptureExceptionFactory.create(HttpStatus.SC_BAD_REQUEST,
                    String.format("WorkOrder not found for URI: [%s]", workOrderURI));
        }
        return list;
    }

    @Override
    public void reportStepProgress(CallingContext context, String workerURI, Long stepStartTime, String message,
            Long progress, Long max) {
        RaptureURI uri = new RaptureURI(workerURI, Scheme.WORKORDER);
        String workOrderURI = uri.toShortString();
        String workerId = uri.getElement();
        if (workerId == null) {
            throw RaptureExceptionFactory.create(HttpStatus.SC_BAD_REQUEST,
                    String.format("Bad uri passed, a workerURI must contain the worker id: [%s]", workerURI));
        }
        Optional<StepRecord> stepRecordOptional = StepRecordUtil.getRecord(workOrderURI, workerId, stepStartTime);
        if (stepRecordOptional.isPresent()) {
            StepRecord stepRecord = stepRecordOptional.get();
            String activityId = stepRecord.getActivityId();
            boolean isFinished = progress.longValue() == max.longValue();
            if (activityId == null) {
                activityId = Kernel.getActivity().createActivity(context, stepRecord.getStepURI(), message,
                        progress, max);
                if (isFinished) {
                    Kernel.getActivity().finishActivity(context, activityId, message);
                }
                stepRecord.setActivityId(activityId);
                StepRecordUtil.writeStepRecord(context, workOrderURI, workerId, stepRecord,
                        "reportStepProgress(): adding activity id");
            } else {
                if (isFinished) {
                    Kernel.getActivity().finishActivity(context, activityId, message);
                } else {
                    Kernel.getActivity().updateActivity(context, activityId, message, progress, max);
                }
            }
            InvocableUtils.writeWorkflowAuditEntry(context, workerURI,
                    message + String.format("; progress is %s of %s", progress, max), false);
        } else {
            log.error(String.format("Step record not found; workOrderURI [%s], workerId [%s], stepStartTime [%s]",
                    workOrderURI, workerId, stepStartTime));
        }
    }

    private Worker getWorkerNotNull(String workOrderURI, String workerId) {
        Worker found = WorkerStorage.readByFields(workOrderURI, workerId);
        if (found == null) {
            throw RaptureExceptionFactory.create(String.format("No worker found for id: '%s'", workerId));
        } else {
            return found;
        }
    }

    @Override
    public Worker getWorker(CallingContext context, String workOrderURI, String workerId) {
        return WorkerStorage.readByFields(workOrderURI, workerId);
    }

    @Override
    public List<RaptureFolderInfo> getWorkflowChildren(CallingContext context, String parentFolderPath) {
        return WorkflowStorage.getChildren(parentFolderPath);
    }

    @Override
    public List<RaptureFolderInfo> getWorkOrderChildren(CallingContext context, String parentFolderPath) {
        return WorkOrderStorage.getChildren(parentFolderPath);
    }

    @Override
    public WorkOrder getWorkOrder(CallingContext context, String workOrderURI) {
        if (StringUtils.isBlank(workOrderURI)) {
            throw RaptureExceptionFactory.create(HttpStatus.SC_BAD_REQUEST,
                    String.format("Invalid workOrderURI: [%s]", workOrderURI));
        }
        RaptureURI addressURI = new RaptureURI(workOrderURI, Scheme.WORKORDER).withoutDecoration();
        return WorkOrderStorage.readByAddress(addressURI);
    }

    @Override
    public List<WorkOrder> getWorkOrdersByDay(CallingContext context, Long startTimeInstant) {
        String startDate = getStartOfDayEpoch(startTimeInstant);
        return WorkOrderStorage.readAll(startDate);
    }

    @Override
    public Map<String, List<String>> getWorkOrderStatusesByWorkflow(CallingContext context, Long startTimeInstant,
            String workflowUri) {
        Map<String, List<String>> ret = new HashMap<>();
        List<WorkOrder> wos = getWorkOrderObjectsByWorkflow(context, startTimeInstant, workflowUri);
        for (WorkOrder wo : wos) {
            String status = wo.getStatus().toString();
            List<String> workorders = ret.get(status);
            if (workorders == null) {
                ret.put(status, new ArrayList<>(Arrays.asList(wo.getWorkOrderURI())));
            } else {
                workorders.add(wo.getWorkOrderURI());
            }
        }
        return ret;
    }

    @Override
    public List<String> getWorkOrdersByWorkflow(CallingContext context, Long startTimeInstant, String workflowUri) {
        List<String> ret = new ArrayList<>();
        List<WorkOrder> wos = getWorkOrderObjectsByWorkflow(context, startTimeInstant, workflowUri);
        for (WorkOrder wo : wos) {
            ret.add(wo.getWorkOrderURI());
        }
        return ret;
    }

    private List<WorkOrder> getWorkOrderObjectsByWorkflow(CallingContext context, Long startTimeInstant,
            String workflowUri) {
        List<WorkOrder> ret = new ArrayList<>();
        // if the document://WorkOrder repo doesn't exist, aka no workflows have been run yet, just return empty
        if (!Kernel.getDoc().docRepoExists(context, WorkOrderPathBuilder.getRepoName())) {
            return ret;
        }
        if (startTimeInstant == null) {
            startTimeInstant = 0L;
        }
        DateTime startDate = new DateTime(startTimeInstant, DateTimeZone.UTC).withTimeAtStartOfDay();
        DateTime nowDate = new DateTime(System.currentTimeMillis(), DateTimeZone.UTC).withTimeAtStartOfDay();
        RaptureURI uri = new RaptureURI(workflowUri, Scheme.WORKFLOW);
        log.info(String.format("Requested startDate is [%s] and current date is [%s]", startDate.toString(),
                nowDate.toString()));

        final String workOrderPrefix = new WorkOrderPathBuilder().buildStorageLocation().toString();
        Map<String, RaptureFolderInfo> existingTimes = Kernel.getDoc().listDocsByUriPrefix(context, workOrderPrefix,
                1);
        Map<String, RaptureFolderInfo> existingTimesWithAuthority = Kernel.getDoc().listDocsByUriPrefix(context,
                workOrderPrefix, 2);

        for (Map.Entry<String, RaptureFolderInfo> entry : existingTimes.entrySet()) {
            DateTime potentialTimestamp = new DateTime(Long.parseLong(entry.getValue().getName()) * 1000,
                    DateTimeZone.UTC);
            // check if the timestamp is within range and also if there is an matching workflow authority
            if (!startDate.isAfter(potentialTimestamp) && existingTimesWithAuthority
                    .containsKey(String.format("%s%s/", entry.getKey(), uri.getAuthority()))) {
                String prefix = String.format("%s/%s/%s", entry.getValue().getName(), uri.getAuthority(),
                        uri.getDocPath());
                ret.addAll(WorkOrderStorage.readAll(prefix));
            }
        }
        return ret;
    }

    @Override
    public WorkOrder getWorkOrderByJobExec(CallingContext context, RaptureJobExec raptureJobExec) {
        if (raptureJobExec == null) {
            throw RaptureExceptionFactory.create("Invalid jobexec");
        }
        String execDetails = raptureJobExec.getExecDetails();
        if (StringUtils.isNotBlank(execDetails)) {
            return getWorkOrder(context,
                    JacksonUtil.objectFromJson(execDetails, WorkflowJobDetails.class).getWorkOrderURI());
        }
        return null;
    }

    @Override
    public Map<RaptureJobExec, WorkOrder> getJobExecsAndWorkOrdersByDay(CallingContext context,
            Long startTimeInstant) {
        Map<RaptureJobExec, WorkOrder> ret = new HashMap<>();
        List<WorkOrder> workOrders = getWorkOrdersByDay(context, startTimeInstant);
        for (WorkOrder workOrder : workOrders) {
            // we find the associated RaptureJobExec using the built-in execution context params
            String jobUri = ExecutionContextUtil.getValueECF(context, workOrder.getWorkOrderURI(),
                    ContextVariables.PARENT_JOB_URI, null);
            // this will be blank if its a workorder that was not initiated by a job, so we skip those
            if (!StringUtils.isBlank(jobUri)) {
                String execTime = ExecutionContextUtil.getValueECF(context, workOrder.getWorkOrderURI(),
                        ContextVariables.TIMESTAMP, null);
                log.info(String.format("Retrieving job exec for job uri [%s] and time [%s]", jobUri, execTime));
                ret.put(RaptureJobExecStorage.readByFields(jobUri, Long.valueOf(execTime)), workOrder);
            }
        }
        return ret;
    }

    @Override
    public Boolean wasCancelCalled(CallingContext context, String workOrderURI) {
        String nakedUri;
        try {
            RaptureURI uri = new RaptureURI(workOrderURI, Scheme.WORKORDER);
            nakedUri = uri.withoutDecoration().toString();
        } catch (Exception ex) {
            throw RaptureExceptionFactory.create("Unparseable URI");
        }
        try {
            WorkOrderCancellation woc = WorkOrderCancellationStorage.readByFields(nakedUri);
            return woc != null;
        } catch (Exception ex) {
            return false;
        }
    }

    @Override
    public WorkOrderCancellation getCancellationDetails(CallingContext context, String workOrderURI) {
        try {
            return WorkOrderCancellationStorage.readByFields(workOrderURI);
        } catch (Exception ex) {
            return null;
        }
    }

    @Override
    public List<AppStatus> getAppStatuses(CallingContext context, String prefix) {
        AppStatusGroup sample = new AppStatusGroup();
        sample.setName(prefix);
        String encodedPrefix = sample.getStoragePath();
        List<AppStatusGroup> groups = AppStatusGroupStorage.readAll(encodedPrefix);
        List<AppStatus> ret = new LinkedList<>();
        for (AppStatusGroup group : groups) {
            ret.addAll(group.getIdToStatus().values());
        }
        // TODO remove line below, it's just here so we read data stored prior
        // to migration
        ret.addAll(AppStatusStorage.readAll(encodedPrefix));
        return ret;

    }

    @Override
    public List<AppStatusDetails> getAppStatusDetails(CallingContext context, String prefix,
            List<String> extraContextValues) {
        List<AppStatus> statuses = getAppStatuses(context, prefix);
        List<AppStatusDetails> detailedList = new ArrayList<>(statuses.size());
        for (AppStatus status : statuses) {
            AppStatusDetails asd = new AppStatusDetails();
            asd.setAppStatus(status);
            String workOrderURI = status.getWorkOrderURI();
            String logURI = InvocableUtils.getWorkflowAuditLog(status.getName(), workOrderURI, null);
            asd.setLogURI(logURI);
            WorkOrder workOrder = WorkOrderFactory.getWorkOrderNotNull(context, workOrderURI);
            List<Worker> workers = getWorkers(workOrder);
            Map<String, List<StepRecord>> workerIdToSteps = new HashMap<>();
            for (String varAlias : extraContextValues) {
                String value = getContextValue(context, status.getWorkOrderURI(), varAlias);
                asd.getExtraContextValues().put(varAlias, value);
            }
            for (Worker worker : workers) {
                List<StepRecord> steps = StepRecordUtil.getStepRecords(worker);
                workerIdToSteps.put(worker.getId(), steps);
            }
            asd.setWorkerIdToSteps(workerIdToSteps);
            detailedList.add(asd);
        }
        return detailedList;
    }

    @Override
    public WorkflowHistoricalMetrics getMonthlyMetrics(CallingContext context, String workflowURIIn,
            String jobURIIn, String argsHashValue, String stateIn) {
        WorkOrderExecutionState state = WorkOrderExecutionState.valueOf(stateIn);

        if (StringUtils.isEmpty(workflowURIIn)) {
            throw RaptureExceptionFactory.create(HttpStatus.SC_BAD_REQUEST, "Workflow URI must be defined!");
        }

        RaptureURI workflowURI = new RaptureURI(workflowURIIn, Scheme.WORKFLOW);

        WorkflowHistoricalMetrics result = new WorkflowHistoricalMetrics();

        try {
            String workflowMetric = WorkflowMetricsFactory.createWorkflowMetricName(workflowURI, state);
            result.setWorkflowAverage(
                    Kernel.getMetricsService().getMetricAverage(workflowMetric, new Period().withMonths(1)));
            Long workflowAverageCount = Kernel.getMetricsService().getMetricCount(workflowMetric,
                    new Period().withMonths(1));

            result.setWorkflowMetricName(workflowMetric);

            if (!StringUtils.isEmpty(jobURIIn)) {
                RaptureURI jobURI = new RaptureURI(jobURIIn, Scheme.JOB);
                String jobMetric = WorkflowMetricsFactory.createJobMetricName(jobURI, state);
                Long averageCount = Kernel.getMetricsService().getMetricCount(jobMetric,
                        new Period().withMonths(1));
                if (averageCount > 10 && workflowAverageCount > averageCount) {
                    // if this is too small, it's a negligible stat and we should use the workflow average instead
                    Double average = Kernel.getMetricsService().getMetricAverage(jobMetric,
                            new Period().withMonths(1));
                    result.setJobAverage(average);
                }
                result.setJobMetricName(jobMetric);
            }

            if (!StringUtils.isEmpty(argsHashValue)) {
                String argsMetric = WorkflowMetricsFactory.createWorkflowWithArgsMetric(workflowURI, state,
                        argsHashValue);
                Long averageCount = Kernel.getMetricsService().getMetricCount(argsMetric,
                        new Period().withMonths(1));
                if (averageCount > 10 && workflowAverageCount > averageCount) {
                    // if this is too small, it's a negligible stat and we should use the workflow average instead
                    Double metricAverage = Kernel.getMetricsService().getMetricAverage(argsMetric,
                            new Period().withMonths(1));
                    result.setWorkflowWithArgsAverage(metricAverage);
                }
                result.setArgsHashMetricName(argsMetric);
            }
        } catch (IOException e) {
            throw RaptureExceptionFactory.create("Error while getting average: " + e.getMessage(), e);
        }

        return result;
    }

    @Override
    public LogQueryResponse queryLogs(CallingContext context, String workOrderURI, Long startTime, Long endTime,
            Long keepAlive, Long bufferSize, String nextBatchId, String stepName, String stepStartTimeStr) {

        try {
            LogManagerConnection connection = Kernel.getLogManagerConnection();
            if (StringUtils.isBlank(nextBatchId)) {
                Long stepStartTimeLong = null;
                try {
                    if (!StringUtils.isBlank(stepStartTimeStr)) {
                        stepStartTimeLong = Long.parseLong(stepStartTimeStr);
                    }
                } catch (NumberFormatException e) {
                    log.error(ExceptionToString.format(e));
                }
                Optional<Long> stepStartTime = Optional.fromNullable(stepStartTimeLong);
                nextBatchId = connection.searchByWorkOrder(workOrderURI,
                        Optional.fromNullable(StringUtils.trimToNull(stepName)), stepStartTime, startTime, endTime,
                        keepAlive.intValue(), bufferSize.intValue());
            }
            return connection.getScrollingResults(nextBatchId, keepAlive.intValue());
        } catch (LogReadException e) {
            log.error(ExceptionToString.format(e));
            throw RaptureExceptionFactory.create(HttpStatus.SC_INTERNAL_SERVER_ERROR,
                    "Error while reading from the log server.", e);
        } catch (SessionExpiredException e) {
            log.error(ExceptionToString.format(e));
            throw RaptureExceptionFactory.create(HttpStatus.SC_BAD_REQUEST,
                    String.format("Error reading logs, passed in expired batch id [%s].", nextBatchId), e);
        }
    }

    @Override
    public void writeWorkflowAuditEntry(CallingContext context, String workOrderURI, String message,
            Boolean error) {
        InvocableUtils.writeWorkflowAuditEntry(context, workOrderURI, message, error);
    }

}