co.cask.cdap.gateway.handlers.AppFabricHttpHandler.java Source code

Java tutorial

Introduction

Here is the source code for co.cask.cdap.gateway.handlers.AppFabricHttpHandler.java

Source

/*
 * Copyright  2014 Cask Data, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package co.cask.cdap.gateway.handlers;

import co.cask.cdap.api.ProgramSpecification;
import co.cask.cdap.api.data.DataSetInstantiationException;
import co.cask.cdap.api.data.stream.StreamSpecification;
import co.cask.cdap.api.dataset.DatasetSpecification;
import co.cask.cdap.api.flow.FlowSpecification;
import co.cask.cdap.api.flow.FlowletConnection;
import co.cask.cdap.api.flow.FlowletDefinition;
import co.cask.cdap.api.mapreduce.MapReduceSpecification;
import co.cask.cdap.api.procedure.ProcedureSpecification;
import co.cask.cdap.api.service.ServiceSpecification;
import co.cask.cdap.api.spark.SparkSpecification;
import co.cask.cdap.api.workflow.WorkflowSpecification;
import co.cask.cdap.app.ApplicationSpecification;
import co.cask.cdap.app.deploy.Manager;
import co.cask.cdap.app.deploy.ManagerFactory;
import co.cask.cdap.app.program.Program;
import co.cask.cdap.app.program.Programs;
import co.cask.cdap.app.runtime.ProgramController;
import co.cask.cdap.app.runtime.ProgramRuntimeService;
import co.cask.cdap.app.services.Data;
import co.cask.cdap.app.services.DeployStatus;
import co.cask.cdap.app.store.Store;
import co.cask.cdap.app.store.StoreFactory;
import co.cask.cdap.common.conf.CConfiguration;
import co.cask.cdap.common.conf.Constants;
import co.cask.cdap.common.discovery.RandomEndpointStrategy;
import co.cask.cdap.common.discovery.TimeLimitEndpointStrategy;
import co.cask.cdap.common.metrics.MetricsScope;
import co.cask.cdap.common.queue.QueueName;
import co.cask.cdap.data.Namespace;
import co.cask.cdap.data2.OperationException;
import co.cask.cdap.data2.datafabric.DefaultDatasetNamespace;
import co.cask.cdap.data2.dataset2.DatasetFramework;
import co.cask.cdap.data2.dataset2.NamespacedDatasetFramework;
import co.cask.cdap.data2.transaction.queue.QueueAdmin;
import co.cask.cdap.data2.transaction.stream.StreamAdmin;
import co.cask.cdap.data2.transaction.stream.StreamConsumerFactory;
import co.cask.cdap.gateway.auth.Authenticator;
import co.cask.cdap.gateway.handlers.util.AbstractAppFabricHttpHandler;
import co.cask.cdap.internal.UserErrors;
import co.cask.cdap.internal.UserMessages;
import co.cask.cdap.internal.app.deploy.ProgramTerminator;
import co.cask.cdap.internal.app.deploy.SessionInfo;
import co.cask.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms;
import co.cask.cdap.internal.app.runtime.AbstractListener;
import co.cask.cdap.internal.app.runtime.BasicArguments;
import co.cask.cdap.internal.app.runtime.ProgramOptionConstants;
import co.cask.cdap.internal.app.runtime.SimpleProgramOptions;
import co.cask.cdap.internal.app.runtime.flow.FlowUtils;
import co.cask.cdap.internal.app.runtime.schedule.ScheduledRuntime;
import co.cask.cdap.internal.app.runtime.schedule.Scheduler;
import co.cask.cdap.internal.filesystem.LocationCodec;
import co.cask.cdap.proto.ApplicationRecord;
import co.cask.cdap.proto.Containers;
import co.cask.cdap.proto.DatasetRecord;
import co.cask.cdap.proto.Id;
import co.cask.cdap.proto.Instances;
import co.cask.cdap.proto.NotRunningProgramLiveInfo;
import co.cask.cdap.proto.ProgramLiveInfo;
import co.cask.cdap.proto.ProgramRecord;
import co.cask.cdap.proto.ProgramStatus;
import co.cask.cdap.proto.ProgramType;
import co.cask.cdap.proto.ProgramTypes;
import co.cask.cdap.proto.StreamRecord;
import co.cask.http.BodyConsumer;
import co.cask.http.HttpResponder;
import co.cask.tephra.TransactionSystemClient;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.base.Throwables;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import com.google.common.io.Closeables;
import com.google.common.io.InputSupplier;
import com.google.common.io.OutputSupplier;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonSyntaxException;
import com.google.gson.reflect.TypeToken;
import com.google.inject.Inject;
import com.ning.http.client.Body;
import com.ning.http.client.BodyGenerator;
import com.ning.http.client.Response;
import com.ning.http.client.SimpleAsyncHttpClient;
import org.apache.commons.io.IOUtils;
import org.apache.twill.api.RunId;
import org.apache.twill.api.RuntimeSpecification;
import org.apache.twill.common.Threads;
import org.apache.twill.discovery.Discoverable;
import org.apache.twill.discovery.DiscoveryServiceClient;
import org.apache.twill.filesystem.Location;
import org.apache.twill.filesystem.LocationFactory;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBufferInputStream;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.handler.codec.http.HttpHeaders;
import org.jboss.netty.handler.codec.http.HttpRequest;
import org.jboss.netty.handler.codec.http.HttpResponseStatus;
import org.jboss.netty.handler.codec.http.QueryStringDecoder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;

/**
 *  HttpHandler class for app-fabric requests.
 */
@Path(Constants.Gateway.GATEWAY_VERSION) //this will be removed/changed when gateway goes.
public class AppFabricHttpHandler extends AbstractAppFabricHttpHandler {
    private static final Logger LOG = LoggerFactory.getLogger(AppFabricHttpHandler.class);

    private static final java.lang.reflect.Type MAP_STRING_STRING_TYPE = new TypeToken<Map<String, String>>() {
    }.getType();

    /**
     * Json serializer.
     */
    private static final Gson GSON = new Gson();

    /**
     * Timeout to get response from metrics system.
     */
    private static final long METRICS_SERVER_RESPONSE_TIMEOUT = TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES);

    private static final String ARCHIVE_NAME_HEADER = "X-Archive-Name";

    /**
     * Timeout to upload to remote app fabric.
     */
    private static final long UPLOAD_TIMEOUT = TimeUnit.MILLISECONDS.convert(10, TimeUnit.MINUTES);

    private static final Map<String, ProgramType> RUNNABLE_TYPE_MAP = new ImmutableMap.Builder<String, ProgramType>()
            .put("mapreduce", ProgramType.MAPREDUCE).put("spark", ProgramType.SPARK).put("flows", ProgramType.FLOW)
            .put("procedures", ProgramType.PROCEDURE).put("workflows", ProgramType.WORKFLOW)
            .put("webapp", ProgramType.WEBAPP).put("services", ProgramType.SERVICE).build();

    /**
     * Configuration object passed from higher up.
     */
    private final CConfiguration configuration;

    /**
     * Factory for handling the location - can do both in either Distributed or Local mode.
     */
    private final LocationFactory locationFactory;

    /**
     * Runtime program service for running and managing programs.
     */
    private final ProgramRuntimeService runtimeService;

    /**
     * Client talking to transaction system.
     */
    private TransactionSystemClient txClient;

    /**
     * Access Dataset Service
     */
    private final DatasetFramework dsFramework;

    /**
     * App fabric output directory.
     */
    private final String appFabricDir;

    /**
     * Maintains a mapping of transient session state. The state is stored in memory,
     * in case of failure, all the current running sessions will be terminated. As
     * per the current implementation only connection per account is allowed to upload.
     */
    private final Map<String, SessionInfo> sessions = Maps.newConcurrentMap();

    /**
     * Store manages non-runtime lifecycle.
     */
    private final Store store;

    private final WorkflowClient workflowClient;

    private final DiscoveryServiceClient discoveryServiceClient;

    private final QueueAdmin queueAdmin;

    private final StreamAdmin streamAdmin;

    private final StreamConsumerFactory streamConsumerFactory;

    /**
     * Number of seconds for timing out a service endpoint discovery.
     */
    private static final long DISCOVERY_TIMEOUT_SECONDS = 3;

    /**
     * The directory where the uploaded files would be placed.
     */
    private final String archiveDir;

    private final ManagerFactory<Location, ApplicationWithPrograms> managerFactory;
    private final Scheduler scheduler;

    private static final class AppFabricServiceStatus {

        private static final AppFabricServiceStatus OK = new AppFabricServiceStatus(HttpResponseStatus.OK, "");

        private static final AppFabricServiceStatus PROGRAM_STILL_RUNNING = new AppFabricServiceStatus(
                HttpResponseStatus.FORBIDDEN, "Program is still running");

        private static final AppFabricServiceStatus PROGRAM_ALREADY_RUNNING = new AppFabricServiceStatus(
                HttpResponseStatus.CONFLICT, "Program is already running");

        private static final AppFabricServiceStatus PROGRAM_ALREADY_STOPPED = new AppFabricServiceStatus(
                HttpResponseStatus.CONFLICT, "Program already stopped");

        private static final AppFabricServiceStatus RUNTIME_INFO_NOT_FOUND = new AppFabricServiceStatus(
                HttpResponseStatus.CONFLICT, UserMessages.getMessage(UserErrors.RUNTIME_INFO_NOT_FOUND));

        private static final AppFabricServiceStatus PROGRAM_NOT_FOUND = new AppFabricServiceStatus(
                HttpResponseStatus.NOT_FOUND, "Program not found");

        private static final AppFabricServiceStatus INTERNAL_ERROR = new AppFabricServiceStatus(
                HttpResponseStatus.INTERNAL_SERVER_ERROR, "Internal server error");

        private final HttpResponseStatus code;
        private final String message;

        /**
         * Describes the output status of app fabric operations.
         */
        private AppFabricServiceStatus(HttpResponseStatus code, String message) {
            this.code = code;
            this.message = message;
        }

        public HttpResponseStatus getCode() {
            return code;
        }

        public String getMessage() {
            return message;
        }
    }

    /**
     * Constructs an new instance. Parameters are binded by Guice.
     */
    @Inject
    public AppFabricHttpHandler(Authenticator authenticator, CConfiguration configuration,
            LocationFactory locationFactory, ManagerFactory<Location, ApplicationWithPrograms> managerFactory,
            StoreFactory storeFactory, ProgramRuntimeService runtimeService, StreamAdmin streamAdmin,
            StreamConsumerFactory streamConsumerFactory, WorkflowClient workflowClient, Scheduler service,
            QueueAdmin queueAdmin, DiscoveryServiceClient discoveryServiceClient, TransactionSystemClient txClient,
            DatasetFramework dsFramework) {

        super(authenticator);
        this.locationFactory = locationFactory;
        this.managerFactory = managerFactory;
        this.streamAdmin = streamAdmin;
        this.streamConsumerFactory = streamConsumerFactory;
        this.configuration = configuration;
        this.runtimeService = runtimeService;
        this.appFabricDir = configuration.get(Constants.AppFabric.OUTPUT_DIR, System.getProperty("java.io.tmpdir"));
        this.archiveDir = this.appFabricDir + "/archive";
        this.store = storeFactory.create();
        this.workflowClient = workflowClient;
        this.scheduler = service;
        this.discoveryServiceClient = discoveryServiceClient;
        this.queueAdmin = queueAdmin;
        this.txClient = txClient;
        this.dsFramework = new NamespacedDatasetFramework(dsFramework,
                new DefaultDatasetNamespace(configuration, Namespace.USER));
    }

    /**
     * Ping to check handler status.
     */
    @Path("/ping")
    @GET
    public void ping(HttpRequest request, HttpResponder responder) {
        responder.sendStatus(HttpResponseStatus.OK);
    }

    /**
     * Retrieve the state of the transaction manager.
     */
    @Path("/transactions/state")
    @GET
    public void getTxManagerSnapshot(HttpRequest request, HttpResponder responder) {
        try {
            LOG.trace("Taking transaction manager snapshot at time {}", System.currentTimeMillis());
            InputStream in = txClient.getSnapshotInputStream();
            LOG.trace("Took and retrieved transaction manager snapshot successfully.");
            try {
                responder.sendChunkStart(HttpResponseStatus.OK, ImmutableMultimap.<String, String>of());
                while (true) {
                    // netty doesn't copy the readBytes buffer, so we have to reallocate a new buffer
                    byte[] readBytes = new byte[4096];
                    int res = in.read(readBytes, 0, 4096);
                    if (res == -1) {
                        break;
                    }
                    responder.sendChunk(ChannelBuffers.wrappedBuffer(readBytes, 0, res));
                }
                responder.sendChunkEnd();
            } finally {
                in.close();
            }
        } catch (Exception e) {
            LOG.error("Could not take transaction manager snapshot", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Invalidate a transaction.
     * @param txId transaction ID.
     */
    @Path("/transactions/{tx-id}/invalidate")
    @POST
    public void invalidateTx(HttpRequest request, HttpResponder responder, @PathParam("tx-id") final String txId) {
        try {
            long txIdLong = Long.parseLong(txId);
            boolean success = txClient.invalidate(txIdLong);
            if (success) {
                LOG.info("Transaction {} successfully invalidated", txId);
                responder.sendStatus(HttpResponseStatus.OK);
            } else {
                LOG.info("Transaction {} could not be invalidated: not in progress.", txId);
                responder.sendStatus(HttpResponseStatus.CONFLICT);
            }
        } catch (NumberFormatException e) {
            LOG.info("Could not invalidate transaction: {} is not a valid tx id", txId);
            responder.sendStatus(HttpResponseStatus.BAD_REQUEST);
        }
    }

    /**
     * Reset the state of the transaction manager.
     */
    @Path("/transactions/state")
    @POST
    public void resetTxManagerState(HttpRequest request, HttpResponder responder) {
        txClient.resetState();
        responder.sendStatus(HttpResponseStatus.OK);
    }

    /**
     * Returns status of a runnable specified by the type{flows,workflows,mapreduce,spark,procedures,services}.
     */
    @GET
    @Path("/apps/{app-id}/{runnable-type}/{runnable-id}/status")
    public void getStatus(final HttpRequest request, final HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("runnable-type") final String runnableType,
            @PathParam("runnable-id") final String runnableId) {

        try {
            String accountId = getAuthenticatedAccountId(request);
            final Id.Program id = Id.Program.from(accountId, appId, runnableId);
            final ProgramType type = RUNNABLE_TYPE_MAP.get(runnableType);
            StatusMap statusMap = getStatus(id, type);
            // If status is null, then there was an error
            if (statusMap.getStatus() == null) {
                responder.sendString(HttpResponseStatus.valueOf(statusMap.getStatusCode()), statusMap.getError());
                return;
            }
            Map<String, String> status = ImmutableMap.of("status", statusMap.getStatus());
            responder.sendJson(HttpResponseStatus.OK, status);
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Returns a map where the pairs map from status to program status (e.g. {"status" : "RUNNING"}) or
     * in case of an error in the input (e.g. invalid id, program not found), a map from statusCode to integer and
     * error to error message (e.g. {"statusCode": 404, "error": "Program not found"})
     *
     * @param id The Program Id to get the status of
     * @param type The Type of the Program to get the status of
     * @throws Throwable
     */
    private StatusMap getStatus(final Id.Program id, final ProgramType type) throws Throwable {
        // check that app exists
        final StatusMap statusMap = new StatusMap();
        ApplicationSpecification appSpec = store.getApplication(id.getApplication());
        if (appSpec == null) {
            return new StatusMap(null, "App: " + id.getApplicationId() + " not found",
                    HttpResponseStatus.NOT_FOUND.getCode());
        }
        // must do it this way to allow anon function in workflow to modify status
        if (type == ProgramType.MAPREDUCE) {
            // check that mapreduce exists
            if (!appSpec.getMapReduce().containsKey(id.getId())) {
                return new StatusMap(null, "Program: " + id.getId() + " not found",
                        HttpResponseStatus.NOT_FOUND.getCode());
            }
            String workflowName = getWorkflowName(id.getId());
            if (workflowName != null) {
                //mapreduce is part of a workflow
                workflowClient.getWorkflowStatus(id.getAccountId(), id.getApplicationId(), workflowName,
                        new WorkflowClient.Callback() {
                            @Override
                            public void handle(WorkflowClient.Status status) {
                                if (status.getCode().equals(WorkflowClient.Status.Code.OK)) {
                                    statusMap.setStatus("RUNNING");
                                    statusMap.setStatusCode(HttpResponseStatus.OK.getCode());
                                } else {
                                    //mapreduce name might follow the same format even when its not part of the workflow.
                                    try {
                                        // getProgramStatus returns program status or http response status NOT_FOUND
                                        storeProgramStatus(id, type, statusMap);
                                    } catch (Exception e) {
                                        LOG.error("Got exception: ", e);
                                        // error occurred so say internal server error
                                        statusMap.setStatusCode(HttpResponseStatus.INTERNAL_SERVER_ERROR.getCode());
                                        statusMap.setError(e.getMessage());
                                    }
                                }
                            }
                        });
                // wait for status to come back in case we are polling mapreduce status in workflow
                // status map contains either a status or an error
                while (statusMap.getStatus() == null
                        || (statusMap.getStatus().isEmpty() && statusMap.getError().isEmpty())) {
                    Thread.sleep(1);
                }
            } else {
                //mapreduce is not part of a workflow
                storeProgramStatus(id, type, statusMap);
            }
        } else if (type == null) {
            // invalid type does not exist
            return new StatusMap(null, "Invalid program type provided", HttpResponseStatus.BAD_REQUEST.getCode());
        } else {
            // all other programs
            storeProgramStatus(id, type, statusMap);
        }
        return statusMap;
    }

    private void storeProgramStatus(final Id.Program id, final ProgramType type, final StatusMap statusMap)
            throws Exception {
        // getProgramStatus returns program status or http response status NOT_FOUND
        String progStatus = getProgramStatus(id, type).getStatus();
        if (progStatus.equals(HttpResponseStatus.NOT_FOUND.toString())) {
            statusMap.setStatusCode(HttpResponseStatus.NOT_FOUND.getCode());
            statusMap.setError("Program not found");
        } else {
            statusMap.setStatus(progStatus);
            statusMap.setStatusCode(HttpResponseStatus.OK.getCode());
        }
    }

    /**
     * starts a webapp.
     */
    @POST
    @Path("/apps/{app-id}/webapp/start")
    public void webappStart(final HttpRequest request, final HttpResponder responder,
            @PathParam("app-id") final String appId) {
        runnableStartStop(request, responder, appId, ProgramType.WEBAPP.getPrettyName().toLowerCase(),
                ProgramType.WEBAPP, "start");
    }

    /**
     * stops a webapp.
     */
    @POST
    @Path("/apps/{app-id}/webapp/stop")
    public void webappStop(final HttpRequest request, final HttpResponder responder,
            @PathParam("app-id") final String appId) {
        runnableStartStop(request, responder, appId, ProgramType.WEBAPP.getPrettyName().toLowerCase(),
                ProgramType.WEBAPP, "stop");
    }

    /**
     * Returns status of a webapp.
     */
    @GET
    @Path("/apps/{app-id}/webapp/status")
    public void webappStatus(final HttpRequest request, final HttpResponder responder,
            @PathParam("app-id") final String appId) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            Id.Program id = Id.Program.from(accountId, appId, ProgramType.WEBAPP.getPrettyName().toLowerCase());
            runnableStatus(responder, id, ProgramType.WEBAPP);
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable t) {
            LOG.error("Got exception:", t);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Get workflow name from mapreduceId.
     * Format of mapreduceId: WorkflowName_mapreduceName, if the mapreduce is a part of workflow.
     *
     * @param mapreduceId id of the mapreduce job in CDAP
     * @return workflow name if exists null otherwise
     */
    private String getWorkflowName(String mapreduceId) {
        String[] splits = mapreduceId.split("_");
        if (splits.length > 1) {
            return splits[0];
        } else {
            return null;
        }
    }

    private void runnableStatus(HttpResponder responder, Id.Program id, ProgramType type) {
        try {
            ProgramStatus status = getProgramStatus(id, type);
            if (status.getStatus().equals(HttpResponseStatus.NOT_FOUND.toString())) {
                responder.sendStatus(HttpResponseStatus.NOT_FOUND);
            } else {
                JsonObject reply = new JsonObject();
                reply.addProperty("status", status.getStatus());
                responder.sendJson(HttpResponseStatus.OK, reply);
            }
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Starts a program.
     */
    @POST
    @Path("/apps/{app-id}/{runnable-type}/{runnable-id}/start")
    public void startProgram(HttpRequest request, HttpResponder responder, @PathParam("app-id") final String appId,
            @PathParam("runnable-type") final String runnableType,
            @PathParam("runnable-id") final String runnableId) {
        startStopProgram(request, responder, appId, runnableType, runnableId, "start");
    }

    /**
     * Starts a program with debugging enabled.
     */
    @POST
    @Path("/apps/{app-id}/{runnable-type}/{runnable-id}/debug")
    public void debugProgram(HttpRequest request, HttpResponder responder, @PathParam("app-id") final String appId,
            @PathParam("runnable-type") final String runnableType,
            @PathParam("runnable-id") final String runnableId) {
        if (!("flows".equals(runnableType) || "procedures".equals(runnableType)
                || "services".equals(runnableType))) {
            responder.sendStatus(HttpResponseStatus.NOT_IMPLEMENTED);
            return;
        }
        startStopProgram(request, responder, appId, runnableType, runnableId, "debug");
    }

    /**
     * Stops a program.
     */
    @POST
    @Path("/apps/{app-id}/{runnable-type}/{runnable-id}/stop")
    public void stopProgram(HttpRequest request, HttpResponder responder, @PathParam("app-id") final String appId,
            @PathParam("runnable-type") final String runnableType,
            @PathParam("runnable-id") final String runnableId) {
        startStopProgram(request, responder, appId, runnableType, runnableId, "stop");
    }

    /**
     * Returns program run history.
     */
    @GET
    @Path("/apps/{app-id}/{runnable-type}/{runnable-id}/history")
    public void runnableHistory(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("runnable-type") final String runnableType,
            @PathParam("runnable-id") final String runnableId) {
        ProgramType type = RUNNABLE_TYPE_MAP.get(runnableType);
        if (type == null || type == ProgramType.WEBAPP) {
            responder.sendStatus(HttpResponseStatus.NOT_FOUND);
            return;
        }

        QueryStringDecoder decoder = new QueryStringDecoder(request.getUri());
        String startTs = getQueryParameter(decoder.getParameters(), Constants.AppFabric.QUERY_PARAM_START_TIME);
        String endTs = getQueryParameter(decoder.getParameters(), Constants.AppFabric.QUERY_PARAM_END_TIME);
        String resultLimit = getQueryParameter(decoder.getParameters(), Constants.AppFabric.QUERY_PARAM_LIMIT);

        long start = startTs == null ? Long.MIN_VALUE : Long.parseLong(startTs);
        long end = endTs == null ? Long.MAX_VALUE : Long.parseLong(endTs);
        int limit = resultLimit == null ? Constants.AppFabric.DEFAULT_HISTORY_RESULTS_LIMIT
                : Integer.parseInt(resultLimit);
        getHistory(request, responder, appId, runnableId, start, end, limit);
    }

    /**
     * Get runnable runtime args.
     */
    @GET
    @Path("/apps/{app-id}/{runnable-type}/{runnable-id}/runtimeargs")
    public void getRunnableRuntimeArgs(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("runnable-type") final String runnableType,
            @PathParam("runnable-id") final String runnableId) {
        ProgramType type = RUNNABLE_TYPE_MAP.get(runnableType);
        if (type == null || type == ProgramType.WEBAPP) {
            responder.sendStatus(HttpResponseStatus.NOT_FOUND);
            return;
        }

        String accountId = getAuthenticatedAccountId(request);
        Id.Program id = Id.Program.from(accountId, appId, runnableId);

        try {
            if (!store.programExists(id, type)) {
                responder.sendString(HttpResponseStatus.NOT_FOUND, "Runnable not found");
                return;
            }
            Map<String, String> runtimeArgs = store.getRunArguments(id);
            responder.sendJson(HttpResponseStatus.OK, runtimeArgs);
        } catch (Throwable e) {
            LOG.error("Error getting runtime args {}", e.getMessage(), e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Save runnable runtime args.
     */
    @PUT
    @Path("/apps/{app-id}/{runnable-type}/{runnable-id}/runtimeargs")
    public void saveRunnableRuntimeArgs(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("runnable-type") final String runnableType,
            @PathParam("runnable-id") final String runnableId) {
        ProgramType type = RUNNABLE_TYPE_MAP.get(runnableType);
        if (type == null || type == ProgramType.WEBAPP) {
            responder.sendStatus(HttpResponseStatus.NOT_FOUND);
            return;
        }

        String accountId = getAuthenticatedAccountId(request);
        Id.Program id = Id.Program.from(accountId, appId, runnableId);

        try {
            if (!store.programExists(id, type)) {
                responder.sendString(HttpResponseStatus.NOT_FOUND, "Runnable not found");
                return;
            }
            Map<String, String> args = decodeArguments(request);
            store.storeRunArguments(id, args);
            responder.sendStatus(HttpResponseStatus.OK);
        } catch (Throwable e) {
            LOG.error("Error getting runtime args {}", e.getMessage(), e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    private String getQueryParameter(Map<String, List<String>> parameters, String parameterName) {
        if (parameters == null || parameters.isEmpty()) {
            return null;
        } else {
            List<String> matchedParams = parameters.get(parameterName);
            return matchedParams == null || matchedParams.isEmpty() ? null : matchedParams.get(0);
        }
    }

    private void getHistory(HttpRequest request, HttpResponder responder, String appId, String runnableId,
            long start, long end, int limit) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            Id.Program programId = Id.Program.from(accountId, appId, runnableId);
            try {
                responder.sendJson(HttpResponseStatus.OK, store.getRunHistory(programId, start, end, limit));
            } catch (OperationException e) {
                LOG.warn(String.format(UserMessages.getMessage(UserErrors.PROGRAM_NOT_FOUND), programId.toString(),
                        e.getMessage()), e);
                responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
            }
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    private synchronized void startStopProgram(HttpRequest request, HttpResponder responder, final String appId,
            final String runnableType, final String runnableId, final String action) {
        ProgramType type = RUNNABLE_TYPE_MAP.get(runnableType);

        if (type == null || (type == ProgramType.WORKFLOW && "stop".equals(action))) {
            responder.sendStatus(HttpResponseStatus.NOT_FOUND);
        } else {
            LOG.trace("{} call from AppFabricHttpHandler for app {}, flow type {} id {}", action, appId,
                    runnableType, runnableId);
            runnableStartStop(request, responder, appId, runnableId, type, action);
        }
    }

    private void runnableStartStop(HttpRequest request, HttpResponder responder, String appId, String runnableId,
            ProgramType type, String action) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            Id.Program id = Id.Program.from(accountId, appId, runnableId);
            AppFabricServiceStatus status = null;
            if ("start".equals(action)) {
                status = start(id, type, decodeArguments(request), false);
            } else if ("debug".equals(action)) {
                status = start(id, type, decodeArguments(request), true);
            } else if ("stop".equals(action)) {
                status = stop(id, type);
            }
            if (status == AppFabricServiceStatus.INTERNAL_ERROR) {
                responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
                return;
            }

            responder.sendString(status.getCode(), status.getMessage());
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Starts a Program.
     */
    private AppFabricServiceStatus start(final Id.Program id, ProgramType type, Map<String, String> overrides,
            boolean debug) {

        try {
            ProgramRuntimeService.RuntimeInfo existingRuntimeInfo = findRuntimeInfo(id, type);
            if (existingRuntimeInfo != null) {
                return AppFabricServiceStatus.PROGRAM_ALREADY_RUNNING;
            }

            Program program = store.loadProgram(id, type);
            if (program == null) {
                return AppFabricServiceStatus.PROGRAM_NOT_FOUND;
            }

            Map<String, String> userArgs = store.getRunArguments(id);
            if (overrides != null) {
                for (Map.Entry<String, String> entry : overrides.entrySet()) {
                    userArgs.put(entry.getKey(), entry.getValue());
                }
            }

            BasicArguments userArguments = new BasicArguments(userArgs);
            ProgramRuntimeService.RuntimeInfo runtimeInfo = runtimeService.run(program,
                    new SimpleProgramOptions(id.getId(), new BasicArguments(), userArguments, debug));

            ProgramController controller = runtimeInfo.getController();
            final String runId = controller.getRunId().getId();

            controller.addListener(new AbstractListener() {
                @Override
                public void stopped() {
                    store.setStop(id, runId,
                            TimeUnit.SECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS),
                            ProgramController.State.STOPPED.toString());
                }

                @Override
                public void error(Throwable cause) {
                    LOG.info("Program stopped with error {}, {}", id, runId, cause);
                    store.setStop(id, runId,
                            TimeUnit.SECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS),
                            ProgramController.State.ERROR.toString());
                }
            }, Threads.SAME_THREAD_EXECUTOR);

            store.setStart(id, runId, TimeUnit.SECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS));
            return AppFabricServiceStatus.OK;
        } catch (DataSetInstantiationException e) {
            return new AppFabricServiceStatus(HttpResponseStatus.UNPROCESSABLE_ENTITY, e.getMessage());
        } catch (Throwable throwable) {
            LOG.error(throwable.getMessage(), throwable);
            if (throwable instanceof FileNotFoundException) {
                return AppFabricServiceStatus.PROGRAM_NOT_FOUND;
            }
            return AppFabricServiceStatus.INTERNAL_ERROR;
        }
    }

    /**
     * Stops a Program.
     */
    private AppFabricServiceStatus stop(Id.Program identifier, ProgramType type) {
        ProgramRuntimeService.RuntimeInfo runtimeInfo = findRuntimeInfo(identifier, type);
        if (runtimeInfo == null) {
            try {
                ProgramStatus status = getProgramStatus(identifier, type);
                if (status.getStatus().equals(HttpResponseStatus.NOT_FOUND.toString())) {
                    return AppFabricServiceStatus.PROGRAM_NOT_FOUND;
                } else if (ProgramController.State.STOPPED.toString().equals(status.getStatus())) {
                    return AppFabricServiceStatus.PROGRAM_ALREADY_STOPPED;
                } else {
                    return AppFabricServiceStatus.RUNTIME_INFO_NOT_FOUND;
                }
            } catch (Exception e) {
                if (e instanceof FileNotFoundException) {
                    return AppFabricServiceStatus.PROGRAM_NOT_FOUND;
                }
                return AppFabricServiceStatus.INTERNAL_ERROR;
            }
        }

        try {
            Preconditions.checkNotNull(runtimeInfo, UserMessages.getMessage(UserErrors.RUNTIME_INFO_NOT_FOUND));
            ProgramController controller = runtimeInfo.getController();
            controller.stop().get();
            return AppFabricServiceStatus.OK;
        } catch (Throwable throwable) {
            LOG.warn(throwable.getMessage(), throwable);
            return AppFabricServiceStatus.INTERNAL_ERROR;
        }
    }

    /**
     * Returns number of instances for a procedure.
     */
    @GET
    @Path("/apps/{app-id}/procedures/{procedure-id}/instances")
    public void getProcedureInstances(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("procedure-id") final String procedureId) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            Id.Program programId = Id.Program.from(accountId, appId, procedureId);

            if (!store.programExists(programId, ProgramType.PROCEDURE)) {
                responder.sendString(HttpResponseStatus.NOT_FOUND, "Runnable not found");
                return;
            }

            int count = getProgramInstances(programId);
            responder.sendJson(HttpResponseStatus.OK, new Instances(count));
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable throwable) {
            LOG.error("Got exception : ", throwable);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Sets number of instances for a procedure.
     */
    @PUT
    @Path("/apps/{app-id}/procedures/{procedure-id}/instances")
    public void setProcedureInstances(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("procedure-id") final String procedureId) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            Id.Program programId = Id.Program.from(accountId, appId, procedureId);

            if (!store.programExists(programId, ProgramType.PROCEDURE)) {
                responder.sendString(HttpResponseStatus.NOT_FOUND, "Runnable not found");
                return;
            }

            int instances = getInstances(request);
            if (instances < 1) {
                responder.sendString(HttpResponseStatus.BAD_REQUEST, "Instance count should be greater than 0");
                return;
            }

            setProgramInstances(programId, instances);
            responder.sendStatus(HttpResponseStatus.OK);
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable throwable) {
            LOG.error("Got exception : ", throwable);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    private void setProgramInstances(Id.Program programId, int instances) throws Exception {
        try {
            store.setProcedureInstances(programId, instances);
            ProgramRuntimeService.RuntimeInfo runtimeInfo = findRuntimeInfo(programId, ProgramType.PROCEDURE);
            if (runtimeInfo != null) {
                runtimeInfo.getController()
                        .command(ProgramOptionConstants.INSTANCES, ImmutableMap.of(programId.getId(), instances))
                        .get();
            }
        } catch (Throwable throwable) {
            LOG.warn("Exception when getting instances for {}.{} to {}. {}", programId.getId(),
                    ProgramType.PROCEDURE.getPrettyName(), throwable.getMessage(), throwable);
            throw new Exception(throwable.getMessage());
        }
    }

    private int getProgramInstances(Id.Program programId) throws Exception {
        try {
            return store.getProcedureInstances(programId);
        } catch (Throwable throwable) {
            LOG.warn("Exception when getting instances for {}.{} to {}.{}", programId.getId(),
                    ProgramType.PROCEDURE.getPrettyName(), throwable.getMessage(), throwable);
            throw new Exception(throwable.getMessage());
        }
    }

    /**
     * Returns number of instances for a flowlet within a flow.
     */
    @GET
    @Path("/apps/{app-id}/flows/{flow-id}/flowlets/{flowlet-id}/instances")
    public void getFlowletInstances(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("flow-id") final String flowId,
            @PathParam("flowlet-id") final String flowletId) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            int count = store.getFlowletInstances(Id.Program.from(accountId, appId, flowId), flowletId);
            responder.sendJson(HttpResponseStatus.OK, new Instances(count));
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            if (respondIfElementNotFound(e, responder)) {
                return;
            }
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Returns the number of instances for all program runnables that are passed into the data. The data is an array of
     * Json objects where each object must contain the following three elements: appId, programType, and programId
     * (flow name, service name, or procedure name). Retrieving instances only applies to flows, procedures, and user
     * services. For flows and procedures, another parameter, "runnableId", must be provided. This corresponds to the
     * flowlet/runnable for which to retrieve the instances. This does not apply to procedures.
     *
     * Example input:
     * [{"appId": "App1", "programType": "Service", "programId": "Service1", "runnableId": "Runnable1"},
     *  {"appId": "App1", "programType": "Procedure", "programId": "Proc2"},
     *  {"appId": "App2", "programType": "Flow", "programId": "Flow1", "runnableId": "Flowlet1"}]
     *
     * The response will be an array of JsonObjects each of which will contain the three input parameters
     * as well as 3 fields:
     * "provisioned" which maps to the number of instances actually provided for the input runnable,
     * "requested" which maps to the number of instances the user has requested for the input runnable,
     * "statusCode" which maps to the http status code for the data in that JsonObjects. (200, 400, 404)
     * If an error occurs in the input (i.e. in the example above, Flowlet1 does not exist), then all JsonObjects for
     * which the parameters have a valid instances will have the provisioned and requested fields status code fields
     * but all JsonObjects for which the parameters are not valid will have an error message and statusCode.
     *
     * E.g. given the above data, if there is no Flowlet1, then the response would be 200 OK with following possible data:
     * [{"appId": "App1", "programType": "Service", "programId": "Service1", "runnableId": "Runnable1",
     *   "statusCode": 200, "provisioned": 2, "requested": 2},
     *  {"appId": "App1", "programType": "Procedure", "programId": "Proc2", "statusCode": 200, "provisioned": 1,
     *   "requested": 3},
     *  {"appId": "App2", "programType": "Flow", "programId": "Flow1", "runnableId": "Flowlet1", "statusCode": 404,
     *   "error": "Runnable": Flowlet1 not found"}]
     *
     * @param request
     * @param responder
     */
    @POST
    @Path("/instances")
    public void getInstances(HttpRequest request, HttpResponder responder) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            List<BatchEndpointInstances> args = instancesFromBatchArgs(decodeArrayArguments(request, responder));
            // if args is null then the response has already been sent
            if (args == null) {
                return;
            }
            for (int i = 0; i < args.size(); ++i) {
                BatchEndpointInstances requestedObj = (BatchEndpointInstances) args.get(i);
                String appId = requestedObj.getAppId();
                String programTypeStr = requestedObj.getProgramType();
                String programId = requestedObj.getProgramId();
                // these values will be overwritten later
                int requested, provisioned;
                ApplicationSpecification spec = store.getApplication(Id.Application.from(accountId, appId));
                if (spec == null) {
                    addCodeError(requestedObj, HttpResponseStatus.NOT_FOUND.getCode(),
                            "App: " + appId + " not found");
                    continue;
                }
                ProgramType programType = ProgramType.valueOfPrettyName(programTypeStr);
                String runnableId;
                if (programType == ProgramType.PROCEDURE) {
                    // the "runnable" for procedures has the same id as the procedure name
                    runnableId = programId;
                    if (spec.getProcedures().containsKey(programId)) {
                        requested = store.getProcedureInstances(Id.Program.from(accountId, appId, programId));
                    } else {
                        addCodeError(requestedObj, HttpResponseStatus.NOT_FOUND.getCode(),
                                "Procedure: " + programId + " not found");
                        continue;
                    }
                } else {
                    // cant get instances for things that are not flows, services, or procedures
                    if (programType != ProgramType.FLOW && programType != ProgramType.SERVICE) {
                        addCodeError(requestedObj, HttpResponseStatus.BAD_REQUEST.getCode(),
                                "Program type: " + programType + " is not a valid program type to get instances");
                        continue;
                    }
                    // services and flows must have runnable id
                    if (requestedObj.getRunnableId() == null) {
                        responder.sendJson(HttpResponseStatus.BAD_REQUEST,
                                "Must provide a string runnableId for flows/services");
                        return;
                    }
                    runnableId = requestedObj.getRunnableId();
                    if (programType == ProgramType.FLOW) {
                        FlowSpecification flowSpec = spec.getFlows().get(programId);
                        if (flowSpec != null) {
                            Map<String, FlowletDefinition> flowletSpecs = flowSpec.getFlowlets();
                            if (flowletSpecs != null && flowletSpecs.containsKey(runnableId)) {
                                requested = flowletSpecs.get(runnableId).getInstances();
                            } else {
                                addCodeError(requestedObj, HttpResponseStatus.NOT_FOUND.getCode(),
                                        "Flowlet: " + runnableId + " not found");
                                continue;
                            }
                        } else {
                            addCodeError(requestedObj, HttpResponseStatus.NOT_FOUND.getCode(),
                                    "Flow: " + programId + " not found");
                            continue;
                        }
                    } else {
                        // Services
                        ServiceSpecification serviceSpec = spec.getServices().get(programId);
                        if (serviceSpec != null) {
                            Map<String, RuntimeSpecification> runtimeSpecs = serviceSpec.getRunnables();
                            if (runtimeSpecs != null && runtimeSpecs.containsKey(runnableId)) {
                                requested = runtimeSpecs.get(runnableId).getResourceSpecification().getInstances();
                            } else {
                                addCodeError(requestedObj, HttpResponseStatus.NOT_FOUND.getCode(),
                                        "Runnable: " + runnableId + " not found");
                                continue;
                            }
                        } else {
                            addCodeError(requestedObj, HttpResponseStatus.NOT_FOUND.getCode(),
                                    "Service: " + programId + " not found");
                            continue;
                        }
                    }
                }
                // use the pretty name of program types to be consistent
                requestedObj.setProgramType(programType.getPrettyName());
                provisioned = getRunnableCount(accountId, appId, programType, programId, runnableId);
                requestedObj.setStatusCode(HttpResponseStatus.OK.getCode());
                requestedObj.setRequested(requested);
                requestedObj.setProvisioned(provisioned);
            }
            responder.sendJson(HttpResponseStatus.OK, args);
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (JsonSyntaxException e) {
            responder.sendStatus(HttpResponseStatus.BAD_REQUEST);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Returns the status for all programs that are passed into the data. The data is an array of Json objects
     * where each object must contain the following three elements: appId, programType, and programId
     * (flow name, service name, etc.).
     * <p/>
     * Example input:
     * [{"appId": "App1", "programType": "Service", "programId": "Service1"},
     * {"appId": "App1", "programType": "Procedure", "programId": "Proc2"},
     * {"appId": "App2", "programType": "Flow", "programId": "Flow1"}]
     * <p/>
     * The response will be an array of JsonObjects each of which will contain the three input parameters
     * as well as 2 fields, "status" which maps to the status of the program and "statusCode" which maps to the
     * status code for the data in that JsonObjects. If an error occurs in the
     * input (i.e. in the example above, App2 does not exist), then all JsonObjects for which the parameters
     * have a valid status will have the status field but all JsonObjects for which the parameters do not have a valid
     * status will have an error message and statusCode.
     * <p/>
     * For example, if there is no App2 in the data above, then the response would be 200 OK with following possible data:
     * [{"appId": "App1", "programType": "Service", "programId": "Service1", "statusCode": 200, "status": "RUNNING"},
     * {"appId": "App1", "programType": "Procedure", "programId": "Proc2"}, "statusCode": 200, "status": "STOPPED"},
     * {"appId":"App2", "programType":"Flow", "programId":"Flow1", "statusCode":404, "error": "App: App2 not found"}]
     *
     * @param request
     * @param responder
     */
    @POST
    @Path("/status")
    public void getStatuses(HttpRequest request, HttpResponder responder) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            List<BatchEndpointStatus> args = statusFromBatchArgs(decodeArrayArguments(request, responder));
            // if args is null, then there was an error in decoding args and response was already sent
            if (args == null) {
                return;
            }
            for (int i = 0; i < args.size(); ++i) {
                BatchEndpointStatus requestedObj = args.get(i);
                Id.Program progId = Id.Program.from(accountId, requestedObj.getAppId(),
                        requestedObj.getProgramId());
                ProgramType programType = ProgramType.valueOfPrettyName(requestedObj.getProgramType());
                // get th statuses
                StatusMap statusMap = getStatus(progId, programType);
                if (statusMap.getStatus() != null) {
                    requestedObj.setStatusCode(HttpResponseStatus.OK.getCode());
                    requestedObj.setStatus(statusMap.getStatus());
                } else {
                    requestedObj.setStatusCode(statusMap.getStatusCode());
                    requestedObj.setError(statusMap.getError());
                }
                // set the program type to the pretty name in case the request originally didn't have pretty name
                requestedObj.setProgramType(programType.getPrettyName());
            }
            responder.sendJson(HttpResponseStatus.OK, args);
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Adds the status code and error to the JsonObject. The JsonObject will have 2 new properties:
     * 'statusCode': code, 'error': error
     *
     * @param object The JsonObject to add the code and error to
     * @param code The status code to add
     * @param error The error message to add
     */
    private void addCodeError(BatchEndpointArgs object, int code, String error) {
        object.setStatusCode(code);
        object.setError(error);
    }

    /**
     * Returns the number of instances currently running for different runnables for different programs
     *
     * @param accountId
     * @param appId
     * @param programType
     * @param programId
     * @param runnableId
     * @return
     */
    private int getRunnableCount(String accountId, String appId, ProgramType programType, String programId,
            String runnableId) {
        ProgramLiveInfo info = runtimeService.getLiveInfo(Id.Program.from(accountId, appId, programId),
                programType);
        int count = 0;
        if (info instanceof NotRunningProgramLiveInfo) {
            return count;
        } else if (info instanceof Containers) {
            Containers containers = (Containers) info;
            for (Containers.ContainerInfo container : containers.getContainers()) {
                if (container.getName().equals(runnableId)) {
                    count++;
                }
            }
            return count;
        } else {
            // Not running on YARN default 1
            return 1;
        }
    }

    /**
     * Deserializes and parses the HttpRequest data into a list of JsonObjects. Checks the HttpRequest data to see that
     * the input has valid fields corresponding to the /instances and /status endpoints. If the input data is empty or
     * the data is not of the form of an array of json objects, it sends an appropriate response through the responder
     * and returns null.
     *
     * @param request The HttpRequest to parse
     * @param responder The HttpResponder used to send responses in case of errors
     * @return List of JsonObjects from the request data
     * @throws IOException Thrown in case of Exceptions when reading the http request data
     */
    @Nullable
    private List<BatchEndpointArgs> decodeArrayArguments(HttpRequest request, HttpResponder responder)
            throws IOException {
        ChannelBuffer content = request.getContent();
        if (!content.readable()) {
            responder.sendString(HttpResponseStatus.BAD_REQUEST, "Cannot read request");
            return null;
        }
        Reader reader = new InputStreamReader(new ChannelBufferInputStream(content), Charsets.UTF_8);
        try {
            List<BatchEndpointArgs> input = GSON.fromJson(reader, new TypeToken<List<BatchEndpointArgs>>() {
            }.getType());
            for (int i = 0; i < input.size(); ++i) {
                BatchEndpointArgs requestedObj;
                try {
                    requestedObj = input.get(i);
                } catch (ClassCastException e) {
                    responder.sendString(HttpResponseStatus.BAD_REQUEST,
                            "All elements in array must be valid JSON Objects");
                    return null;
                }
                // make sure the following args exist
                if (requestedObj.getAppId() == null || requestedObj.getProgramId() == null
                        || requestedObj.getProgramType() == null) {
                    responder.sendJson(HttpResponseStatus.BAD_REQUEST,
                            "Must provide appId, programType, and programId as strings for each object");
                    return null;
                }
                // invalid type
                try {
                    if (ProgramType.valueOfPrettyName(requestedObj.getProgramType()) == null) {
                        responder.sendJson(HttpResponseStatus.BAD_REQUEST,
                                "Invalid program type provided: " + requestedObj.getProgramType());
                        return null;
                    }
                } catch (IllegalArgumentException e) {
                    responder.sendJson(HttpResponseStatus.BAD_REQUEST,
                            "Invalid program type provided: " + requestedObj.getProgramType());
                    return null;
                }

            }
            return input;
        } catch (JsonSyntaxException e) {
            responder.sendJson(HttpResponseStatus.BAD_REQUEST, "Invalid Json object provided");
            return null;
        } finally {
            reader.close();
        }
    }

    /**
     * Increases number of instance for a flowlet within a flow.
     */
    @PUT
    @Path("/apps/{app-id}/flows/{flow-id}/flowlets/{flowlet-id}/instances")
    public void setFlowletInstances(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("flow-id") final String flowId,
            @PathParam("flowlet-id") final String flowletId) {
        int instances = 0;
        try {
            instances = getInstances(request);
            if (instances < 1) {
                responder.sendString(HttpResponseStatus.BAD_REQUEST, "Instance count should be greater than 0");
                return;
            }
        } catch (Throwable th) {
            responder.sendString(HttpResponseStatus.BAD_REQUEST, "Invalid instance count.");
            return;
        }

        try {
            String accountId = getAuthenticatedAccountId(request);
            Id.Program programID = Id.Program.from(accountId, appId, flowId);
            int oldInstances = store.getFlowletInstances(programID, flowletId);
            if (oldInstances != instances) {
                store.setFlowletInstances(programID, flowletId, instances);
                ProgramRuntimeService.RuntimeInfo runtimeInfo = findRuntimeInfo(accountId, appId, flowId,
                        ProgramType.FLOW);
                if (runtimeInfo != null) {
                    runtimeInfo.getController()
                            .command(ProgramOptionConstants.FLOWLET_INSTANCES,
                                    ImmutableMap.of("flowlet", flowletId, "newInstances", String.valueOf(instances),
                                            "oldInstances", String.valueOf(oldInstances)))
                            .get();
                }
            }
            responder.sendStatus(HttpResponseStatus.OK);
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            if (respondIfElementNotFound(e, responder)) {
                return;
            }
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Changes input stream for a flowlet connection.
     */
    @PUT
    @Path("/apps/{app-id}/flows/{flow-id}/flowlets/{flowlet-id}/connections/{stream-id}")
    public void changeFlowletStreamConnection(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("flow-id") final String flowId,
            @PathParam("flowlet-id") final String flowletId, @PathParam("stream-id") final String streamId)
            throws IOException {

        try {
            Map<String, String> arguments = decodeArguments(request);
            String oldStreamId = arguments.get("oldStreamId");
            if (oldStreamId == null) {
                responder.sendString(HttpResponseStatus.BAD_REQUEST, "oldStreamId param is required");
                return;
            }

            String accountId = getAuthenticatedAccountId(request);
            StreamSpecification stream = store.getStream(Id.Account.from(accountId), streamId);
            if (stream == null) {
                responder.sendString(HttpResponseStatus.BAD_REQUEST,
                        "Stream specified with streamId param does not exist");
                return;
            }

            Id.Program programID = Id.Program.from(accountId, appId, flowId);
            store.changeFlowletSteamConnection(programID, flowletId, oldStreamId, streamId);
            responder.sendStatus(HttpResponseStatus.OK);
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            if (respondIfElementNotFound(e, responder)) {
                return;
            }
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    private ProgramStatus getProgramStatus(Id.Program id, ProgramType type) throws Exception {

        try {
            ProgramRuntimeService.RuntimeInfo runtimeInfo = findRuntimeInfo(id, type);

            if (runtimeInfo == null) {
                if (type != ProgramType.WEBAPP) {
                    //Runtime info not found. Check to see if the program exists.
                    String spec = getProgramSpecification(id, type);
                    if (spec == null || spec.isEmpty()) {
                        // program doesn't exist
                        return new ProgramStatus(id.getApplicationId(), id.getId(),
                                HttpResponseStatus.NOT_FOUND.toString());
                    } else {
                        // program exists and not running. so return stopped.
                        return new ProgramStatus(id.getApplicationId(), id.getId(),
                                ProgramController.State.STOPPED.toString());
                    }
                } else {
                    // TODO: Fetching webapp status is a hack. This will be fixed when webapp spec is added.
                    Location webappLoc = null;
                    try {
                        webappLoc = Programs.programLocation(locationFactory, appFabricDir, id, ProgramType.WEBAPP);
                    } catch (FileNotFoundException e) {
                        // No location found for webapp, no need to log this exception
                    }

                    if (webappLoc != null && webappLoc.exists()) {
                        // webapp exists and not running. so return stopped.
                        return new ProgramStatus(id.getApplicationId(), id.getId(),
                                ProgramController.State.STOPPED.toString());
                    } else {
                        // webapp doesn't exist
                        return new ProgramStatus(id.getApplicationId(), id.getId(),
                                HttpResponseStatus.NOT_FOUND.toString());
                    }
                }
            }

            String status = controllerStateToString(runtimeInfo.getController().getState());
            return new ProgramStatus(id.getApplicationId(), id.getId(), status);
        } catch (Throwable throwable) {
            LOG.warn(throwable.getMessage(), throwable);
            throw new Exception(throwable.getMessage());
        }
    }

    /**
     * Deploys an application with the specified name.
     */
    @PUT
    @Path("/apps/{app-id}")
    public BodyConsumer deploy(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId) {
        try {
            return deployAppStream(request, responder, appId);
        } catch (Exception ex) {
            responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Deploy failed: {}" + ex.getMessage());
            return null;
        }

    }

    /**
     * Deploys an application.
     */
    @POST
    @Path("/apps")
    public BodyConsumer deploy(HttpRequest request, HttpResponder responder) {
        // null means use name provided by app spec
        try {
            return deployAppStream(request, responder, null);
        } catch (Exception ex) {
            responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, "Deploy failed: {}" + ex.getMessage());
            return null;
        }
    }

    /**
     * Returns next scheduled runtime of a workflow.
     */
    @GET
    @Path("/apps/{app-id}/workflows/{workflow-id}/nextruntime")
    public void getScheduledRunTime(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("workflow-id") final String workflowId) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            Id.Program id = Id.Program.from(accountId, appId, workflowId);
            List<ScheduledRuntime> runtimes = scheduler.nextScheduledRuntime(id, ProgramType.WORKFLOW);

            JsonArray array = new JsonArray();
            for (ScheduledRuntime runtime : runtimes) {
                JsonObject object = new JsonObject();
                object.addProperty("id", runtime.getScheduleId());
                object.addProperty("time", runtime.getTime());
                array.add(object);
            }
            responder.sendJson(HttpResponseStatus.OK, array);
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Returns the schedule ids for a given workflow.
     */
    @GET
    @Path("/apps/{app-id}/workflows/{workflow-id}/schedules")
    public void workflowSchedules(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("workflow-id") final String workflowId) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            Id.Program id = Id.Program.from(accountId, appId, workflowId);
            responder.sendJson(HttpResponseStatus.OK, scheduler.getScheduleIds(id, ProgramType.WORKFLOW));
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Get schedule state.
     */
    @GET
    @Path("/apps/{app-id}/workflows/{workflow-id}/schedules/{schedule-id}/status")
    public void getScheuleState(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("workflow-id") final String workflowId,
            @PathParam("schedule-id") final String scheduleId) {
        try {
            // get the accountId to catch if there is a security exception
            String accountId = getAuthenticatedAccountId(request);
            JsonObject json = new JsonObject();
            json.addProperty("status", scheduler.scheduleState(scheduleId).toString());
            responder.sendJson(HttpResponseStatus.OK, json);
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Suspend a workflow schedule.
     */
    @POST
    @Path("/apps/{app-id}/workflows/{workflow-id}/schedules/{schedule-id}/suspend")
    public void workflowScheduleSuspend(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("workflow-id") final String workflowId,
            @PathParam("schedule-id") final String scheduleId) {
        try {
            // get the accountId to catch if there is a security exception
            String accountId = getAuthenticatedAccountId(request);
            Scheduler.ScheduleState state = scheduler.scheduleState(scheduleId);
            switch (state) {
            case NOT_FOUND:
                responder.sendStatus(HttpResponseStatus.NOT_FOUND);
                break;
            case SCHEDULED:
                scheduler.suspendSchedule(scheduleId);
                responder.sendJson(HttpResponseStatus.OK, "OK");
                break;
            case SUSPENDED:
                responder.sendJson(HttpResponseStatus.CONFLICT, "Schedule already suspended");
                break;
            }
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Resume a workflow schedule.
     */
    @POST
    @Path("/apps/{app-id}/workflows/{workflow-id}/schedules/{schedule-id}/resume")
    public void workflowScheduleResume(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("workflow-id") final String workflowId,
            @PathParam("schedule-id") final String scheduleId) {

        try {
            // get the accountId to catch if there is a security exception
            String accountId = getAuthenticatedAccountId(request);
            Scheduler.ScheduleState state = scheduler.scheduleState(scheduleId);
            switch (state) {
            case NOT_FOUND:
                responder.sendStatus(HttpResponseStatus.NOT_FOUND);
                break;
            case SCHEDULED:
                responder.sendJson(HttpResponseStatus.CONFLICT, "Already resumed");
                break;
            case SUSPENDED:
                scheduler.resumeSchedule(scheduleId);
                responder.sendJson(HttpResponseStatus.OK, "OK");
                break;
            }
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    @GET
    @Path("/apps/{app-id}/procedures/{procedure-id}/live-info")
    @SuppressWarnings("unused")
    public void procedureLiveInfo(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("procedure-id") final String procedureId) {
        getLiveInfo(request, responder, appId, procedureId, ProgramType.PROCEDURE);
    }

    @GET
    @Path("/apps/{app-id}/flows/{flow-id}/live-info")
    @SuppressWarnings("unused")
    public void flowLiveInfo(HttpRequest request, HttpResponder responder, @PathParam("app-id") final String appId,
            @PathParam("flow-id") final String flowId) {
        getLiveInfo(request, responder, appId, flowId, ProgramType.FLOW);
    }

    /**
     * Returns specification of a runnable - flow.
     */
    @GET
    @Path("/apps/{app-id}/flows/{flow-id}")
    public void flowSpecification(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("flow-id") final String flowId) {
        runnableSpecification(request, responder, appId, ProgramType.FLOW, flowId);
    }

    /**
     * Returns specification of procedure.
     */
    @GET
    @Path("/apps/{app-id}/procedures/{procedure-id}")
    public void procedureSpecification(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("procedure-id") final String procId) {
        runnableSpecification(request, responder, appId, ProgramType.PROCEDURE, procId);
    }

    /**
     * Returns specification of mapreduce.
     */
    @GET
    @Path("/apps/{app-id}/mapreduce/{mapreduce-id}")
    public void mapreduceSpecification(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("mapreduce-id") final String mapreduceId) {
        runnableSpecification(request, responder, appId, ProgramType.MAPREDUCE, mapreduceId);
    }

    /**
     * Returns specification of spark program.
     */
    @GET
    @Path("/apps/{app-id}/spark/{spark-id}")
    public void sparkSpecification(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("spark-id") final String sparkId) {
        runnableSpecification(request, responder, appId, ProgramType.SPARK, sparkId);
    }

    /**
     * Returns specification of workflow.
     */
    @GET
    @Path("/apps/{app-id}/workflows/{workflow-id}")
    public void workflowSpecification(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("workflow-id") final String workflowId) {
        runnableSpecification(request, responder, appId, ProgramType.WORKFLOW, workflowId);
    }

    private void runnableSpecification(HttpRequest request, HttpResponder responder, final String appId,
            ProgramType runnableType, final String runnableId) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            Id.Program id = Id.Program.from(accountId, appId, runnableId);
            String specification = getProgramSpecification(id, runnableType);
            if (specification == null || specification.isEmpty()) {
                responder.sendStatus(HttpResponseStatus.NOT_FOUND);
            } else {
                responder.sendByteArray(HttpResponseStatus.OK, specification.getBytes(Charsets.UTF_8),
                        ImmutableMultimap.of(HttpHeaders.Names.CONTENT_TYPE, "application/json"));
            }
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    private BodyConsumer deployAppStream(final HttpRequest request, final HttpResponder responder,
            final String appId) throws IOException {
        final String archiveName = request.getHeader(ARCHIVE_NAME_HEADER);
        final String accountId = getAuthenticatedAccountId(request);
        final Location uploadDir = locationFactory.create(archiveDir + "/" + accountId);
        final Location archive = uploadDir.append(archiveName);
        final OutputStream os = archive.getOutputStream();

        if (archiveName == null || archiveName.isEmpty()) {
            responder.sendString(HttpResponseStatus.BAD_REQUEST, ARCHIVE_NAME_HEADER + " header not present");
        }

        final SessionInfo sessionInfo = new SessionInfo(accountId, appId, archiveName, archive,
                DeployStatus.UPLOADING);
        sessions.put(accountId, sessionInfo);

        return new BodyConsumer() {
            @Override
            public void chunk(ChannelBuffer request, HttpResponder responder) {
                try {
                    request.readBytes(os, request.readableBytes());
                } catch (IOException e) {
                    sessionInfo.setStatus(DeployStatus.FAILED);
                    LOG.error("Failed to write deploy jar", e);
                    responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, e.getMessage());
                }
            }

            @Override
            public void finished(HttpResponder responder) {
                try {
                    os.close();
                    sessionInfo.setStatus(DeployStatus.VERIFYING);
                    deploy(accountId, appId, archive);
                    sessionInfo.setStatus(DeployStatus.DEPLOYED);
                    responder.sendString(HttpResponseStatus.OK, "Deploy Complete");
                } catch (Exception e) {
                    sessionInfo.setStatus(DeployStatus.FAILED);
                    LOG.error("Deploy failure", e);
                    responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage());
                } finally {
                    save(sessionInfo.setStatus(sessionInfo.getStatus()), accountId);
                    sessions.remove(accountId);
                }
            }

            @Override
            public void handleError(Throwable t) {
                try {
                    os.close();
                    sessionInfo.setStatus(DeployStatus.FAILED);
                    responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, t.getCause().getMessage());
                } catch (IOException e) {
                    LOG.error("Error while saving deploy jar.", e);
                } finally {
                    save(sessionInfo.setStatus(sessionInfo.getStatus()), accountId);
                    sessions.remove(accountId);
                }
            }
        };

    }

    // deploy helper
    private void deploy(final String accountId, final String appId, Location archive) throws Exception {

        try {
            Id.Account id = Id.Account.from(accountId);
            Location archiveLocation = archive;
            Manager<Location, ApplicationWithPrograms> manager = managerFactory.create(new ProgramTerminator() {
                @Override
                public void stop(Id.Account id, Id.Program programId, ProgramType type) throws ExecutionException {
                    deleteHandler(programId, type);
                }
            });

            ApplicationWithPrograms applicationWithPrograms = manager.deploy(id, appId, archiveLocation).get();
            ApplicationSpecification specification = applicationWithPrograms.getAppSpecLoc().getSpecification();
            setupSchedules(accountId, specification);
        } catch (Throwable e) {
            LOG.warn(e.getMessage(), e);
            throw new Exception(e.getMessage());
        }
    }

    private void setupSchedules(String accountId, ApplicationSpecification specification) throws IOException {

        for (Map.Entry<String, WorkflowSpecification> entry : specification.getWorkflows().entrySet()) {
            Id.Program programId = Id.Program.from(accountId, specification.getName(), entry.getKey());
            List<String> existingSchedules = scheduler.getScheduleIds(programId, ProgramType.WORKFLOW);
            //Delete the existing schedules and add new ones.
            if (!existingSchedules.isEmpty()) {
                scheduler.deleteSchedules(programId, ProgramType.WORKFLOW, existingSchedules);
            }
            // Add new schedules.
            if (!entry.getValue().getSchedules().isEmpty()) {
                scheduler.schedule(programId, ProgramType.WORKFLOW, entry.getValue().getSchedules());
            }
        }
    }

    /**
     * Defines the class for sending deploy status to client.
     */
    private static class Status {
        private final int code;
        private final String status;
        private final String message;

        public Status(int code, String message) {
            this.code = code;
            this.status = DeployStatus.getMessage(code);
            this.message = message;
        }
    }

    /**
     * Gets application deployment status.
     */
    @GET
    @Path("/deploy/status")
    public void getDeployStatus(HttpRequest request, HttpResponder responder) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            DeployStatus status = dstatus(accountId);
            LOG.trace("Deployment status call at AppFabricHttpHandler , Status: {}", status);
            responder.sendJson(HttpResponseStatus.OK, new Status(status.getCode(), status.getMessage()));
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Promote an application to another CDAP instance.
     */
    @POST
    @Path("/apps/{app-id}/promote")
    public void promoteApp(HttpRequest request, HttpResponder responder, @PathParam("app-id") final String appId) {
        try {
            String postBody = null;

            try {
                postBody = IOUtils.toString(new ChannelBufferInputStream(request.getContent()));
            } catch (IOException e) {
                responder.sendError(HttpResponseStatus.BAD_REQUEST, e.getMessage());
                return;
            }

            Map<String, String> content = null;
            try {
                content = GSON.fromJson(postBody, MAP_STRING_STRING_TYPE);
            } catch (JsonSyntaxException e) {
                responder.sendError(HttpResponseStatus.BAD_REQUEST, "Not a valid body specified.");
                return;
            }

            if (!content.containsKey("hostname")) {
                responder.sendError(HttpResponseStatus.BAD_REQUEST, "Hostname not specified.");
                return;
            }

            // Checks DNS, Ipv4, Ipv6 address in one go.
            String hostname = content.get("hostname");
            Preconditions.checkArgument(!hostname.isEmpty(), "Empty hostname passed.");

            String accountId = getAuthenticatedAccountId(request);
            String token = request.getHeader(Constants.Gateway.API_KEY);

            final Location appArchive = store.getApplicationArchiveLocation(Id.Application.from(accountId, appId));
            if (appArchive == null || !appArchive.exists()) {
                throw new IOException("Unable to locate the application.");
            }

            if (!promote(token, accountId, appId, hostname)) {
                responder.sendError(HttpResponseStatus.INTERNAL_SERVER_ERROR,
                        "Failed to promote application " + appId);
            } else {
                responder.sendStatus(HttpResponseStatus.OK);
            }
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    public boolean promote(String authToken, String accountId, String appId, String hostname) throws Exception {

        try {
            final Location appArchive = store.getApplicationArchiveLocation(Id.Application.from(accountId, appId));
            if (appArchive == null || !appArchive.exists()) {
                throw new Exception("Unable to locate the application.");
            }

            String schema = "https";
            if ("localhost".equals(hostname)) {
                schema = "http";
            }

            // Construct URL for promotion of application to remote cluster
            int gatewayPort;
            if (configuration.getBoolean(Constants.Security.SSL_ENABLED)) {
                gatewayPort = Integer.parseInt(configuration.get(Constants.Router.ROUTER_SSL_PORT,
                        Constants.Router.DEFAULT_ROUTER_SSL_PORT));
            } else {
                gatewayPort = Integer.parseInt(
                        configuration.get(Constants.Router.ROUTER_PORT, Constants.Router.DEFAULT_ROUTER_PORT));
            }

            String url = String.format("%s://%s:%s/v2/apps/%s", schema, hostname, gatewayPort, appId);

            SimpleAsyncHttpClient client = new SimpleAsyncHttpClient.Builder().setUrl(url)
                    .setRequestTimeoutInMs((int) UPLOAD_TIMEOUT).setHeader("X-Archive-Name", appArchive.getName())
                    .setHeader(Constants.Gateway.API_KEY, authToken).build();

            try {
                Future<Response> future = client.put(new LocationBodyGenerator(appArchive));
                Response response = future.get(UPLOAD_TIMEOUT, TimeUnit.MILLISECONDS);
                if (response.getStatusCode() != 200) {
                    throw new RuntimeException(response.getResponseBody());
                }
                return true;
            } finally {
                client.close();
            }
        } catch (Exception ex) {
            LOG.warn(ex.getMessage(), ex);
            throw ex;
        }
    }

    private static final class LocationBodyGenerator implements BodyGenerator {

        private final Location location;

        private LocationBodyGenerator(Location location) {
            this.location = location;
        }

        @Override
        public Body createBody() throws IOException {
            final InputStream input = location.getInputStream();

            return new Body() {
                @Override
                public long getContentLength() {
                    try {
                        return location.length();
                    } catch (IOException e) {
                        throw Throwables.propagate(e);
                    }
                }

                @Override
                public long read(ByteBuffer buffer) throws IOException {
                    // Fast path
                    if (buffer.hasArray()) {
                        int len = input.read(buffer.array(), buffer.arrayOffset() + buffer.position(),
                                buffer.remaining());
                        if (len > 0) {
                            buffer.position(buffer.position() + len);
                        }
                        return len;
                    }

                    byte[] bytes = new byte[buffer.remaining()];
                    int len = input.read(bytes);
                    if (len < 0) {
                        return len;
                    }
                    buffer.put(bytes, 0, len);
                    return len;
                }

                @Override
                public void close() throws IOException {
                    input.close();
                }
            };
        }
    }

    /**
     * Delete an application specified by appId.
     */
    @DELETE
    @Path("/apps/{app-id}")
    public void deleteApp(HttpRequest request, HttpResponder responder, @PathParam("app-id") final String appId) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            Id.Program id = Id.Program.from(accountId, appId, "");
            AppFabricServiceStatus appStatus = removeApplication(id);
            LOG.trace("Delete call for Application {} at AppFabricHttpHandler", appId);
            responder.sendString(appStatus.getCode(), appStatus.getMessage());
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception: ", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Deletes all applications in CDAP.
     */
    @DELETE
    @Path("/apps")
    public void deleteAllApps(HttpRequest request, HttpResponder responder) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            Id.Account id = Id.Account.from(accountId);
            AppFabricServiceStatus status = removeAll(id);
            LOG.trace("Delete All call at AppFabricHttpHandler");
            responder.sendString(status.getCode(), status.getMessage());
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception: ", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Deletes queues.
     */
    @DELETE
    @Path("/apps/{app-id}/flows/{flow-id}/queues")
    public void deleteFlowQueues(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId, @PathParam("flow-id") final String flowId) {
        String accountId = getAuthenticatedAccountId(request);
        Id.Program programId = Id.Program.from(accountId, appId, flowId);
        try {
            ProgramStatus status = getProgramStatus(programId, ProgramType.FLOW);
            if (status.getStatus().equals(HttpResponseStatus.NOT_FOUND.toString())) {
                responder.sendStatus(HttpResponseStatus.NOT_FOUND);
            } else if (status.getStatus().equals("RUNNING")) {
                responder.sendString(HttpResponseStatus.FORBIDDEN, "Flow is running, please stop it first.");
            } else {
                queueAdmin.dropAllForFlow(appId, flowId);
                // delete process metrics that are used to calculate the queue size (process.events.pending metric name)
                deleteProcessMetricsForFlow(appId, flowId);
                responder.sendStatus(HttpResponseStatus.OK);
            }
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    @DELETE
    @Path("/queues")
    public void clearQueues(HttpRequest request, final HttpResponder responder) {
        clear(request, responder, ToClear.QUEUES);
    }

    @DELETE
    @Path("/streams")
    public void clearStreams(HttpRequest request, final HttpResponder responder) {
        clear(request, responder, ToClear.STREAMS);
    }

    private static enum ToClear {
        QUEUES, STREAMS
    }

    private void clear(HttpRequest request, final HttpResponder responder, ToClear toClear) {
        try {
            getAuthenticatedAccountId(request);
            try {
                if (toClear == ToClear.QUEUES) {
                    queueAdmin.dropAll();
                } else if (toClear == ToClear.STREAMS) {
                    streamAdmin.dropAll();
                }
                responder.sendStatus(HttpResponseStatus.OK);
            } catch (Exception e) {
                LOG.error("Exception clearing data fabric: ", e);
                responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
            }
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (IllegalArgumentException e) {
            responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage());
        } catch (Throwable e) {
            LOG.error("Caught exception", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /*
     * Retrieves a {@link SessionInfo} from the file system.
     */
    @Nullable
    private SessionInfo retrieve(String accountId) {
        try {
            final Location outputDir = locationFactory.create(archiveDir + "/" + accountId);
            if (!outputDir.exists()) {
                return null;
            }
            final Location sessionInfoFile = outputDir.append("session.json");
            InputSupplier<Reader> reader = new InputSupplier<Reader>() {
                @Override
                public Reader getInput() throws IOException {
                    return new InputStreamReader(sessionInfoFile.getInputStream(), "UTF-8");
                }
            };

            Gson gson = new GsonBuilder().registerTypeAdapter(Location.class, new LocationCodec(locationFactory))
                    .create();
            Reader r = reader.getInput();
            try {
                return gson.fromJson(r, SessionInfo.class);
            } finally {
                Closeables.closeQuietly(r);
            }
        } catch (IOException e) {
            LOG.warn("Failed to retrieve session info for account.");
        }
        return null;
    }

    private AppFabricServiceStatus removeAll(Id.Account identifier) throws Exception {
        List<ApplicationSpecification> allSpecs = new ArrayList<ApplicationSpecification>(
                store.getAllApplications(identifier));

        //Check if any App associated with this account is running
        final Id.Account accId = Id.Account.from(identifier.getId());
        boolean appRunning = checkAnyRunning(new Predicate<Id.Program>() {
            @Override
            public boolean apply(Id.Program programId) {
                return programId.getApplication().getAccount().equals(accId);
            }
        }, ProgramType.values());

        if (appRunning) {
            return AppFabricServiceStatus.PROGRAM_STILL_RUNNING;
        }

        //All Apps are STOPPED, delete them
        for (ApplicationSpecification appSpec : allSpecs) {
            Id.Program id = Id.Program.from(identifier.getId(), appSpec.getName(), "");
            removeApplication(id);
        }
        return AppFabricServiceStatus.OK;
    }

    private AppFabricServiceStatus removeApplication(Id.Program identifier) throws Exception {
        Id.Account accountId = Id.Account.from(identifier.getAccountId());
        final Id.Application appId = Id.Application.from(accountId, identifier.getApplicationId());

        //Check if all are stopped.
        boolean appRunning = checkAnyRunning(new Predicate<Id.Program>() {
            @Override
            public boolean apply(Id.Program programId) {
                return programId.getApplication().equals(appId);
            }
        }, ProgramType.values());

        if (appRunning) {
            return AppFabricServiceStatus.PROGRAM_STILL_RUNNING;
        }

        ApplicationSpecification spec = store.getApplication(appId);
        if (spec == null) {
            return AppFabricServiceStatus.PROGRAM_NOT_FOUND;
        }

        //Delete the schedules
        for (WorkflowSpecification workflowSpec : spec.getWorkflows().values()) {
            Id.Program workflowProgramId = Id.Program.from(appId, workflowSpec.getName());
            List<String> schedules = scheduler.getScheduleIds(workflowProgramId, ProgramType.WORKFLOW);
            if (!schedules.isEmpty()) {
                scheduler.deleteSchedules(workflowProgramId, ProgramType.WORKFLOW, schedules);
            }
        }

        deleteMetrics(identifier.getAccountId(), identifier.getApplicationId());

        // Delete all streams and queues state of each flow
        // TODO: This should be unified with the DeletedProgramHandlerStage
        for (FlowSpecification flowSpecification : spec.getFlows().values()) {
            Id.Program flowProgramId = Id.Program.from(appId, flowSpecification.getName());

            // Collects stream name to all group ids consuming that stream
            Multimap<String, Long> streamGroups = HashMultimap.create();
            for (FlowletConnection connection : flowSpecification.getConnections()) {
                if (connection.getSourceType() == FlowletConnection.Type.STREAM) {
                    long groupId = FlowUtils.generateConsumerGroupId(flowProgramId, connection.getTargetName());
                    streamGroups.put(connection.getSourceName(), groupId);
                }
            }
            // Remove all process states and group states for each stream
            String namespace = String.format("%s.%s", flowProgramId.getApplicationId(), flowProgramId.getId());
            for (Map.Entry<String, Collection<Long>> entry : streamGroups.asMap().entrySet()) {
                streamConsumerFactory.dropAll(QueueName.fromStream(entry.getKey()), namespace, entry.getValue());
            }

            queueAdmin.dropAllForFlow(identifier.getApplicationId(), flowSpecification.getName());
        }
        deleteProgramLocations(appId);

        Location appArchive = store.getApplicationArchiveLocation(appId);
        Preconditions.checkNotNull(appArchive, "Could not find the location of application", appId.getId());
        appArchive.delete();
        store.removeApplication(appId);
        return AppFabricServiceStatus.OK;
    }

    private void deleteMetrics(String accountId, String applicationId) throws IOException, OperationException {
        Collection<ApplicationSpecification> applications = Lists.newArrayList();
        if (applicationId == null) {
            applications = this.store.getAllApplications(new Id.Account(accountId));
        } else {
            ApplicationSpecification spec = this.store
                    .getApplication(new Id.Application(new Id.Account(accountId), applicationId));
            applications.add(spec);
        }
        Iterable<Discoverable> discoverables = this.discoveryServiceClient.discover(Constants.Service.METRICS);
        Discoverable discoverable = new TimeLimitEndpointStrategy(new RandomEndpointStrategy(discoverables),
                DISCOVERY_TIMEOUT_SECONDS, TimeUnit.SECONDS).pick();

        if (discoverable == null) {
            LOG.error("Fail to get any metrics endpoint for deleting metrics.");
            throw new IOException("Can't find Metrics endpoint");
        }

        for (MetricsScope scope : MetricsScope.values()) {
            for (ApplicationSpecification application : applications) {
                String url = String.format("http://%s:%d%s/metrics/%s/apps/%s",
                        discoverable.getSocketAddress().getHostName(), discoverable.getSocketAddress().getPort(),
                        Constants.Gateway.GATEWAY_VERSION, scope.name().toLowerCase(), application.getName());
                sendMetricsDelete(url);
            }
        }

        if (applicationId == null) {
            String url = String.format("http://%s:%d%s/metrics", discoverable.getSocketAddress().getHostName(),
                    discoverable.getSocketAddress().getPort(), Constants.Gateway.GATEWAY_VERSION);
            sendMetricsDelete(url);
        }
    }

    // deletes the process metrics for a flow
    private void deleteProcessMetricsForFlow(String application, String flow) throws IOException {
        Iterable<Discoverable> discoverables = this.discoveryServiceClient.discover(Constants.Service.METRICS);
        Discoverable discoverable = new TimeLimitEndpointStrategy(new RandomEndpointStrategy(discoverables), 3L,
                TimeUnit.SECONDS).pick();

        if (discoverable == null) {
            LOG.error("Fail to get any metrics endpoint for deleting metrics.");
            throw new IOException("Can't find Metrics endpoint");
        }

        LOG.debug("Deleting metrics for flow {}.{}", application, flow);
        String url = String.format("http://%s:%d%s/metrics/system/apps/%s/flows/%s?prefixEntity=process",
                discoverable.getSocketAddress().getHostName(), discoverable.getSocketAddress().getPort(),
                Constants.Gateway.GATEWAY_VERSION, application, flow);

        long timeout = TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES);

        SimpleAsyncHttpClient client = new SimpleAsyncHttpClient.Builder().setUrl(url)
                .setRequestTimeoutInMs((int) timeout).build();

        try {
            client.delete().get(timeout, TimeUnit.MILLISECONDS);
        } catch (Exception e) {
            LOG.error("exception making metrics delete call", e);
            Throwables.propagate(e);
        } finally {
            client.close();
        }
    }

    private void sendMetricsDelete(String url) {
        SimpleAsyncHttpClient client = new SimpleAsyncHttpClient.Builder().setUrl(url)
                .setRequestTimeoutInMs((int) METRICS_SERVER_RESPONSE_TIMEOUT).build();

        try {
            client.delete().get(METRICS_SERVER_RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS);
        } catch (Exception e) {
            LOG.error("exception making metrics delete call", e);
            Throwables.propagate(e);
        } finally {
            client.close();
        }
    }

    /**
     * Check if any program that satisfy the given {@link Predicate} is running.
     *
     * @param predicate Get call on each running {@link Id.Program}.
     * @param types Types of program to check
     * returns True if a program is running as defined by the predicate.
     */
    private boolean checkAnyRunning(Predicate<Id.Program> predicate, ProgramType... types) {
        for (ProgramType type : types) {
            for (Map.Entry<RunId, ProgramRuntimeService.RuntimeInfo> entry : runtimeService.list(type).entrySet()) {
                Id.Program programId = entry.getValue().getProgramId();
                if (predicate.apply(programId)) {
                    LOG.trace("Program still running in checkAnyRunning: {} {} {} {}", programId.getApplicationId(),
                            type, programId.getId(), entry.getValue().getController().getRunId());
                    return true;
                }
            }
        }
        return false;
    }

    /**
     * Delete the jar location of the program.
     *
     * @param appId        applicationId.
     * @throws IOException if there are errors with location IO
     */
    private void deleteProgramLocations(Id.Application appId) throws IOException, OperationException {
        ApplicationSpecification specification = store.getApplication(appId);

        Iterable<ProgramSpecification> programSpecs = Iterables.concat(specification.getFlows().values(),
                specification.getMapReduce().values(), specification.getProcedures().values(),
                specification.getWorkflows().values());

        for (ProgramSpecification spec : programSpecs) {
            ProgramType type = ProgramTypes.fromSpecification(spec);
            Id.Program programId = Id.Program.from(appId, spec.getName());
            try {
                Location location = Programs.programLocation(locationFactory, appFabricDir, programId, type);
                location.delete();
            } catch (FileNotFoundException e) {
                LOG.warn("Program jar for program {} not found.", programId.toString(), e);
            }
        }

        // Delete webapp
        // TODO: this will go away once webapp gets a spec
        try {
            Id.Program programId = Id.Program.from(appId.getAccountId(), appId.getId(),
                    ProgramType.WEBAPP.name().toLowerCase());
            Location location = Programs.programLocation(locationFactory, appFabricDir, programId,
                    ProgramType.WEBAPP);
            location.delete();
        } catch (FileNotFoundException e) {
            // expected exception when webapp is not present.
        }
    }

    /*
     * Returns DeploymentStatus
     */
    private DeployStatus dstatus(String accountId) {
        if (!sessions.containsKey(accountId)) {
            SessionInfo info = retrieve(accountId);
            if (info == null) {
                return DeployStatus.NOT_FOUND;
            }
            return info.getStatus();
        } else {
            SessionInfo info = sessions.get(accountId);
            return info.getStatus();
        }
    }

    private void deleteHandler(Id.Program programId, ProgramType type) throws ExecutionException {
        try {
            switch (type) {
            case FLOW:
                //Stop the flow if it not running
                ProgramRuntimeService.RuntimeInfo flowRunInfo = findRuntimeInfo(programId.getAccountId(),
                        programId.getApplicationId(), programId.getId(), type);
                if (flowRunInfo != null) {
                    doStop(flowRunInfo);
                }
                break;
            case PROCEDURE:
                //Stop the procedure if it not running
                ProgramRuntimeService.RuntimeInfo procedureRunInfo = findRuntimeInfo(programId.getAccountId(),
                        programId.getApplicationId(), programId.getId(), type);
                if (procedureRunInfo != null) {
                    doStop(procedureRunInfo);
                }
                break;
            case WORKFLOW:
                List<String> scheduleIds = scheduler.getScheduleIds(programId, type);
                scheduler.deleteSchedules(programId, ProgramType.WORKFLOW, scheduleIds);
                break;
            case MAPREDUCE:
                //no-op
                break;
            }
        } catch (InterruptedException e) {
            throw new ExecutionException(e);
        }
    }

    /**
     * Saves the {@link SessionInfo} to the filesystem.
     *
     * @param info to be saved.
     * @return true if and only if successful; false otherwise.
     */
    private boolean save(SessionInfo info, String accountId) {
        try {
            Gson gson = new GsonBuilder().registerTypeAdapter(Location.class, new LocationCodec(locationFactory))
                    .create();
            Location outputDir = locationFactory.create(archiveDir + "/" + accountId);
            if (!outputDir.exists()) {
                return false;
            }
            final Location sessionInfoFile = outputDir.append("session.json");
            OutputSupplier<Writer> writer = new OutputSupplier<Writer>() {
                @Override
                public Writer getOutput() throws IOException {
                    return new OutputStreamWriter(sessionInfoFile.getOutputStream(), "UTF-8");
                }
            };

            Writer w = writer.getOutput();
            try {
                gson.toJson(info, w);
            } finally {
                Closeables.closeQuietly(w);
            }
        } catch (IOException e) {
            LOG.warn(e.getMessage(), e);
            return false;
        }
        return true;
    }

    private void doStop(ProgramRuntimeService.RuntimeInfo runtimeInfo)
            throws ExecutionException, InterruptedException {
        Preconditions.checkNotNull(runtimeInfo, UserMessages.getMessage(UserErrors.RUNTIME_INFO_NOT_FOUND));
        ProgramController controller = runtimeInfo.getController();
        controller.stop().get();
    }

    /** NOTE: This was a temporary hack done to map the status to something that is
     * UI friendly. Internal states of program controller are reasonable and hence
     * no point in changing them.
     */
    private String controllerStateToString(ProgramController.State state) {
        if (state == ProgramController.State.ALIVE) {
            return "RUNNING";
        }
        if (state == ProgramController.State.ERROR) {
            return "FAILED";
        }
        return state.toString();
    }

    private String getProgramSpecification(Id.Program id, ProgramType type) throws Exception {

        ApplicationSpecification appSpec;
        try {
            appSpec = store.getApplication(id.getApplication());
            if (appSpec == null) {
                return "";
            }
            String runnableId = id.getId();
            if (type == ProgramType.FLOW && appSpec.getFlows().containsKey(runnableId)) {
                return GSON.toJson(appSpec.getFlows().get(id.getId()));
            } else if (type == ProgramType.PROCEDURE && appSpec.getProcedures().containsKey(runnableId)) {
                return GSON.toJson(appSpec.getProcedures().get(id.getId()));
            } else if (type == ProgramType.MAPREDUCE && appSpec.getMapReduce().containsKey(runnableId)) {
                return GSON.toJson(appSpec.getMapReduce().get(id.getId()));
            } else if (type == ProgramType.SPARK && appSpec.getSpark().containsKey(runnableId)) {
                return GSON.toJson(appSpec.getSpark().get(id.getId()));
            } else if (type == ProgramType.WORKFLOW && appSpec.getWorkflows().containsKey(runnableId)) {
                return GSON.toJson(appSpec.getWorkflows().get(id.getId()));
            } else if (type == ProgramType.SERVICE && appSpec.getServices().containsKey(runnableId)) {
                return GSON.toJson(appSpec.getServices().get(id.getId()));
            }
        } catch (Throwable throwable) {
            LOG.warn(throwable.getMessage(), throwable);
            throw new Exception(throwable.getMessage());
        }
        return "";
    }

    private ProgramRuntimeService.RuntimeInfo findRuntimeInfo(Id.Program identifier, ProgramType type) {
        Collection<ProgramRuntimeService.RuntimeInfo> runtimeInfos = runtimeService.list(type).values();
        Preconditions.checkNotNull(runtimeInfos, UserMessages.getMessage(UserErrors.RUNTIME_INFO_NOT_FOUND),
                identifier.getAccountId(), identifier.getApplicationId());
        for (ProgramRuntimeService.RuntimeInfo info : runtimeInfos) {
            if (identifier.equals(info.getProgramId())) {
                return info;
            }
        }
        return null;
    }

    @GET
    @Path("/apps/{app-id}/workflows/{workflow-name}/current")
    public void workflowStatus(HttpRequest request, final HttpResponder responder,
            @PathParam("app-id") String appId, @PathParam("workflow-name") String workflowName) {

        try {
            String accountId = getAuthenticatedAccountId(request);
            workflowClient.getWorkflowStatus(accountId, appId, workflowName, new WorkflowClient.Callback() {
                @Override
                public void handle(WorkflowClient.Status status) {
                    if (status.getCode() == WorkflowClient.Status.Code.NOT_FOUND) {
                        responder.sendStatus(HttpResponseStatus.NOT_FOUND);
                    } else if (status.getCode() == WorkflowClient.Status.Code.OK) {
                        responder.sendByteArray(HttpResponseStatus.OK, status.getResult().getBytes(),
                                ImmutableMultimap.of(HttpHeaders.Names.CONTENT_TYPE,
                                        "application/json; charset=utf-8"));

                    } else {
                        responder.sendError(HttpResponseStatus.INTERNAL_SERVER_ERROR, status.getResult());
                    }
                }
            });
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Caught exception", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Returns a list of flows associated with account.
     */
    @GET
    @Path("/flows")
    public void getAllFlows(HttpRequest request, HttpResponder responder) {
        programList(request, responder, ProgramType.FLOW, null);
    }

    /**
     * Returns a list of procedures associated with account.
     */
    @GET
    @Path("/procedures")
    public void getAllProcedures(HttpRequest request, HttpResponder responder) {
        programList(request, responder, ProgramType.PROCEDURE, null);
    }

    /**
     * Returns a list of map/reduces associated with account.
     */
    @GET
    @Path("/mapreduce")
    public void getAllMapReduce(HttpRequest request, HttpResponder responder) {
        programList(request, responder, ProgramType.MAPREDUCE, null);
    }

    /**
     * Returns a list of spark jobs associated with account.
     */
    @GET
    @Path("/spark")
    public void getAllSpark(HttpRequest request, HttpResponder responder) {
        programList(request, responder, ProgramType.SPARK, null);
    }

    /**
     * Returns a list of workflows associated with account.
     */
    @GET
    @Path("/workflows")
    public void getAllWorkflows(HttpRequest request, HttpResponder responder) {
        programList(request, responder, ProgramType.WORKFLOW, null);
    }

    /**
     * Returns a list of applications associated with account.
     */
    @GET
    @Path("/apps")
    public void getAllApps(HttpRequest request, HttpResponder responder) {
        getAppDetails(request, responder, null);
    }

    /**
     * Returns the info associated with the application.
     */
    @GET
    @Path("/apps/{app-id}")
    public void getAppInfo(HttpRequest request, HttpResponder responder, @PathParam("app-id") final String appId) {
        getAppDetails(request, responder, appId);
    }

    /**
     * Returns a list of procedure associated with account & application.
     */
    @GET
    @Path("/apps/{app-id}/flows")
    public void getFlowsByApp(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId) {
        programList(request, responder, ProgramType.FLOW, appId);
    }

    /**
     * Returns a list of procedure associated with account & application.
     */
    @GET
    @Path("/apps/{app-id}/procedures")
    public void getProceduresByApp(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId) {
        programList(request, responder, ProgramType.PROCEDURE, appId);
    }

    /**
     * Returns a list of procedure associated with account & application.
     */
    @GET
    @Path("/apps/{app-id}/mapreduce")
    public void getMapreduceByApp(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId) {
        programList(request, responder, ProgramType.MAPREDUCE, appId);
    }

    /**
     * Returns a list of spark jobs associated with account & application.
     */
    @GET
    @Path("/apps/{app-id}/spark")
    public void getSparkByApp(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId) {
        programList(request, responder, ProgramType.SPARK, appId);
    }

    /**
     * Returns a list of procedure associated with account & application.
     */
    @GET
    @Path("/apps/{app-id}/workflows")
    public void getWorkflowssByApp(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId) {
        programList(request, responder, ProgramType.WORKFLOW, appId);
    }

    private void getAppDetails(HttpRequest request, HttpResponder responder, String appid) {
        if (appid != null && appid.isEmpty()) {
            responder.sendString(HttpResponseStatus.BAD_REQUEST, "app-id is empty");
            return;
        }

        try {
            String accountId = getAuthenticatedAccountId(request);
            Id.Account accId = Id.Account.from(accountId);
            List<ApplicationRecord> result = Lists.newArrayList();
            List<ApplicationSpecification> specList;
            if (appid == null) {
                specList = new ArrayList<ApplicationSpecification>(store.getAllApplications(accId));
            } else {
                ApplicationSpecification appSpec = store.getApplication(new Id.Application(accId, appid));
                if (appSpec == null) {
                    responder.sendStatus(HttpResponseStatus.NOT_FOUND);
                    return;
                }
                specList = Collections.singletonList(store.getApplication(new Id.Application(accId, appid)));
            }

            for (ApplicationSpecification appSpec : specList) {
                result.add(makeAppRecord(appSpec));
            }

            String json;
            if (appid == null) {
                json = GSON.toJson(result);
            } else {
                json = GSON.toJson(result.get(0));
            }

            responder.sendByteArray(HttpResponseStatus.OK, json.getBytes(Charsets.UTF_8),
                    ImmutableMultimap.of(HttpHeaders.Names.CONTENT_TYPE, "application/json"));
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception : ", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    private void programList(HttpRequest request, HttpResponder responder, ProgramType type, String appid) {
        if (appid != null && appid.isEmpty()) {
            responder.sendString(HttpResponseStatus.BAD_REQUEST, "app-id is null or empty");
            return;
        }

        try {
            String accountId = getAuthenticatedAccountId(request);
            String list;
            if (appid == null) {
                Id.Account accId = Id.Account.from(accountId);
                list = listPrograms(accId, type);
            } else {
                Id.Application appId = Id.Application.from(accountId, appid);
                list = listProgramsByApp(appId, type);
            }

            if (list.isEmpty()) {
                responder.sendStatus(HttpResponseStatus.NOT_FOUND);
            } else {
                responder.sendByteArray(HttpResponseStatus.OK, list.getBytes(Charsets.UTF_8),
                        ImmutableMultimap.of(HttpHeaders.Names.CONTENT_TYPE, "application/json"));
            }
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception: ", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    private String listProgramsByApp(Id.Application appId, ProgramType type) throws Exception {
        ApplicationSpecification appSpec;
        try {
            appSpec = store.getApplication(appId);
            if (appSpec == null) {
                return "";
            } else {
                return listPrograms(Collections.singletonList(appSpec), type);
            }
        } catch (Throwable throwable) {
            LOG.warn(throwable.getMessage(), throwable);
            throw new Exception("Could not retrieve application spec for " + appId.toString() + ", reason: "
                    + throwable.getMessage());
        }
    }

    private String listPrograms(Id.Account accId, ProgramType type) throws Exception {
        try {
            Collection<ApplicationSpecification> appSpecs = store.getAllApplications(accId);
            if (appSpecs == null) {
                return "";
            } else {
                return listPrograms(appSpecs, type);
            }
        } catch (Throwable throwable) {
            LOG.warn(throwable.getMessage(), throwable);
            throw new Exception("Could not retrieve application spec for " + accId.toString() + ", reason: "
                    + throwable.getMessage());
        }
    }

    private String listPrograms(Collection<ApplicationSpecification> appSpecs, ProgramType type) throws Exception {
        List<ProgramRecord> result = Lists.newArrayList();
        for (ApplicationSpecification appSpec : appSpecs) {
            if (type == ProgramType.FLOW) {
                for (FlowSpecification flowSpec : appSpec.getFlows().values()) {
                    result.add(makeProgramRecord(appSpec.getName(), flowSpec, ProgramType.FLOW));
                }
            } else if (type == ProgramType.PROCEDURE) {
                for (ProcedureSpecification procedureSpec : appSpec.getProcedures().values()) {
                    result.add(makeProgramRecord(appSpec.getName(), procedureSpec, ProgramType.PROCEDURE));
                }
            } else if (type == ProgramType.MAPREDUCE) {
                for (MapReduceSpecification mrSpec : appSpec.getMapReduce().values()) {
                    result.add(makeProgramRecord(appSpec.getName(), mrSpec, ProgramType.MAPREDUCE));
                }
            } else if (type == ProgramType.SPARK) {
                for (SparkSpecification sparkSpec : appSpec.getSpark().values()) {
                    result.add(makeProgramRecord(appSpec.getName(), sparkSpec, ProgramType.SPARK));
                }
            } else if (type == ProgramType.WORKFLOW) {
                for (WorkflowSpecification wfSpec : appSpec.getWorkflows().values()) {
                    result.add(makeProgramRecord(appSpec.getName(), wfSpec, ProgramType.WORKFLOW));
                }
            } else {
                throw new Exception("Unknown program type: " + type.name());
            }
        }
        return GSON.toJson(result);
    }

    private ProgramRuntimeService.RuntimeInfo findRuntimeInfo(String accountId, String appId, String flowId,
            ProgramType typeId) {
        ProgramType type = ProgramType.valueOf(typeId.name());
        Collection<ProgramRuntimeService.RuntimeInfo> runtimeInfos = runtimeService.list(type).values();
        Preconditions.checkNotNull(runtimeInfos, UserMessages.getMessage(UserErrors.RUNTIME_INFO_NOT_FOUND),
                accountId, flowId);

        Id.Program programId = Id.Program.from(accountId, appId, flowId);

        for (ProgramRuntimeService.RuntimeInfo info : runtimeInfos) {
            if (programId.equals(info.getProgramId())) {
                return info;
            }
        }
        return null;
    }

    private void getLiveInfo(HttpRequest request, HttpResponder responder, final String appId,
            final String programId, ProgramType type) {
        try {
            String accountId = getAuthenticatedAccountId(request);
            responder.sendJson(HttpResponseStatus.OK,
                    runtimeService.getLiveInfo(Id.Program.from(accountId, appId, programId), type));
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    /**
     * Returns a list of streams associated with account.
     */
    @GET
    @Path("/streams")
    public void getStreams(HttpRequest request, HttpResponder responder) {
        dataList(request, responder, Data.STREAM, null, null);
    }

    /**
     * Returns a stream associated with account.
     */
    @GET
    @Path("/streams/{stream-id}")
    public void getStreamSpecification(HttpRequest request, HttpResponder responder,
            @PathParam("stream-id") final String streamId) {
        dataList(request, responder, Data.STREAM, streamId, null);
    }

    /**
     * Returns a list of streams associated with application.
     */
    @GET
    @Path("/apps/{app-id}/streams")
    public void getStreamsByApp(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId) {
        dataList(request, responder, Data.STREAM, null, appId);
    }

    /**
     * Returns a list of dataset associated with account.
     */
    @GET
    @Path("/datasets")
    public void getDatasets(HttpRequest request, HttpResponder responder) {
        dataList(request, responder, Data.DATASET, null, null);
    }

    /**
     * Returns a dataset associated with account.
     */
    @GET
    @Path("/datasets/{dataset-id}")
    public void getDatasetSpecification(HttpRequest request, HttpResponder responder,
            @PathParam("dataset-id") final String datasetId) {
        dataList(request, responder, Data.DATASET, datasetId, null);
    }

    /**
     * Returns a list of dataset associated with application.
     */
    @GET
    @Path("/apps/{app-id}/datasets")
    public void getDatasetsByApp(HttpRequest request, HttpResponder responder,
            @PathParam("app-id") final String appId) {
        dataList(request, responder, Data.DATASET, null, appId);
    }

    private void dataList(HttpRequest request, HttpResponder responder, Data type, String name, String appId) {
        try {
            if ((name != null && name.isEmpty()) || (appId != null && appId.isEmpty())) {
                responder.sendString(HttpResponseStatus.BAD_REQUEST, "Empty name provided");
                return;
            }

            String accountId = getAuthenticatedAccountId(request);
            Id.Program program = Id.Program.from(accountId, appId == null ? "" : appId, "");
            String json = name != null ? getDataEntity(program, type, name)
                    : appId != null ? listDataEntitiesByApp(program, type) : listDataEntities(program, type);
            if (json.isEmpty()) {
                responder.sendStatus(HttpResponseStatus.NOT_FOUND);
            } else {
                responder.sendByteArray(HttpResponseStatus.OK, json.getBytes(Charsets.UTF_8),
                        ImmutableMultimap.of(HttpHeaders.Names.CONTENT_TYPE, "application/json"));
            }
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception : ", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    private String getDataEntity(Id.Program programId, Data type, String name) throws Exception {
        try {
            Id.Account account = new Id.Account(programId.getAccountId());
            if (type == Data.DATASET) {
                DatasetSpecification dsSpec = getDatasetSpec(name);
                String typeName = null;
                if (dsSpec != null) {
                    typeName = dsSpec.getType();
                }
                return GSON.toJson(makeDataSetRecord(name, typeName));
            } else if (type == Data.STREAM) {
                StreamSpecification spec = store.getStream(account, name);
                return spec == null ? "" : GSON.toJson(makeStreamRecord(spec.getName(), spec));
            }
            return "";
        } catch (OperationException e) {
            LOG.warn(e.getMessage(), e);
            throw new Exception(
                    "Could not retrieve data specs for " + programId.toString() + ", reason: " + e.getMessage());
        }
    }

    private String listDataEntities(Id.Program programId, Data type) throws Exception {
        try {
            if (type == Data.DATASET) {
                Collection<DatasetSpecification> instances = dsFramework.getInstances();
                List<DatasetRecord> result = Lists.newArrayListWithExpectedSize(instances.size());
                for (DatasetSpecification instance : instances) {
                    result.add(makeDataSetRecord(instance.getName(), instance.getType()));
                }
                return GSON.toJson(result);
            } else if (type == Data.STREAM) {
                Collection<StreamSpecification> specs = store
                        .getAllStreams(new Id.Account(programId.getAccountId()));
                List<StreamRecord> result = Lists.newArrayListWithExpectedSize(specs.size());
                for (StreamSpecification spec : specs) {
                    result.add(makeStreamRecord(spec.getName(), null));
                }
                return GSON.toJson(result);
            }
            return "";
        } catch (OperationException e) {
            LOG.warn(e.getMessage(), e);
            throw new Exception(
                    "Could not retrieve data specs for " + programId.toString() + ", reason: " + e.getMessage());
        }
    }

    private String listDataEntitiesByApp(Id.Program programId, Data type) throws Exception {
        try {
            Id.Account account = new Id.Account(programId.getAccountId());
            ApplicationSpecification appSpec = store
                    .getApplication(new Id.Application(account, programId.getApplicationId()));
            if (type == Data.DATASET) {
                Set<String> dataSetsUsed = dataSetsUsedBy(appSpec);
                List<DatasetRecord> result = Lists.newArrayListWithExpectedSize(dataSetsUsed.size());
                for (String dsName : dataSetsUsed) {
                    String typeName = null;
                    DatasetSpecification dsSpec = getDatasetSpec(dsName);
                    if (dsSpec != null) {
                        typeName = dsSpec.getType();
                    }
                    result.add(makeDataSetRecord(dsName, typeName));
                }
                return GSON.toJson(result);
            }
            if (type == Data.STREAM) {
                Set<String> streamsUsed = streamsUsedBy(appSpec);
                List<StreamRecord> result = Lists.newArrayListWithExpectedSize(streamsUsed.size());
                for (String streamName : streamsUsed) {
                    result.add(makeStreamRecord(streamName, null));
                }
                return GSON.toJson(result);
            }
            return "";
        } catch (OperationException e) {
            LOG.warn(e.getMessage(), e);
            throw new Exception(
                    "Could not retrieve data specs for " + programId.toString() + ", reason: " + e.getMessage());
        }
    }

    @Nullable
    private DatasetSpecification getDatasetSpec(String dsName) {
        try {
            return dsFramework.getDatasetSpec(dsName);
        } catch (Exception e) {
            LOG.warn("Couldn't get spec for dataset: " + dsName);
            return null;
        }
    }

    private Set<String> dataSetsUsedBy(FlowSpecification flowSpec) {
        Set<String> result = Sets.newHashSet();
        for (FlowletDefinition flowlet : flowSpec.getFlowlets().values()) {
            result.addAll(flowlet.getDatasets());
        }
        return result;
    }

    private Set<String> dataSetsUsedBy(ApplicationSpecification appSpec) {
        Set<String> result = Sets.newHashSet();
        for (FlowSpecification flowSpec : appSpec.getFlows().values()) {
            result.addAll(dataSetsUsedBy(flowSpec));
        }
        for (ProcedureSpecification procSpec : appSpec.getProcedures().values()) {
            result.addAll(procSpec.getDataSets());
        }
        for (MapReduceSpecification mrSpec : appSpec.getMapReduce().values()) {
            result.addAll(mrSpec.getDataSets());
        }
        return result;
    }

    private Set<String> streamsUsedBy(FlowSpecification flowSpec) {
        Set<String> result = Sets.newHashSet();
        for (FlowletConnection con : flowSpec.getConnections()) {
            if (FlowletConnection.Type.STREAM == con.getSourceType()) {
                result.add(con.getSourceName());
            }
        }
        return result;
    }

    private Set<String> streamsUsedBy(ApplicationSpecification appSpec) {
        Set<String> result = Sets.newHashSet();
        for (FlowSpecification flowSpec : appSpec.getFlows().values()) {
            result.addAll(streamsUsedBy(flowSpec));
        }
        result.addAll(appSpec.getStreams().keySet());
        return result;
    }

    /**
     * Returns all flows associated with a stream.
     */
    @GET
    @Path("/streams/{stream-id}/flows")
    public void getFlowsByStream(HttpRequest request, HttpResponder responder,
            @PathParam("stream-id") final String streamId) {
        programListByDataAccess(request, responder, ProgramType.FLOW, Data.STREAM, streamId);
    }

    /**
     * Returns all flows associated with a dataset.
     */
    @GET
    @Path("/datasets/{dataset-id}/flows")
    public void getFlowsByDataset(HttpRequest request, HttpResponder responder,
            @PathParam("dataset-id") final String datasetId) {
        programListByDataAccess(request, responder, ProgramType.FLOW, Data.DATASET, datasetId);
    }

    private void programListByDataAccess(HttpRequest request, HttpResponder responder, ProgramType type, Data data,
            String name) {
        try {
            if (name.isEmpty()) {
                responder.sendString(HttpResponseStatus.BAD_REQUEST,
                        data.prettyName().toLowerCase() + " name is empty");
                return;
            }
            String accountId = getAuthenticatedAccountId(request);
            Id.Program programId = Id.Program.from(accountId, "", "");
            String list = listProgramsByDataAccess(programId, type, data, name);
            if (list.isEmpty()) {
                responder.sendStatus(HttpResponseStatus.NOT_FOUND);
            } else {
                responder.sendByteArray(HttpResponseStatus.OK, list.getBytes(Charsets.UTF_8),
                        ImmutableMultimap.of(HttpHeaders.Names.CONTENT_TYPE, "application/json"));
            }
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.error("Got exception:", e);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    }

    private String listProgramsByDataAccess(Id.Program programId, ProgramType type, Data data, String name)
            throws Exception {
        try {
            List<ProgramRecord> result = Lists.newArrayList();
            Collection<ApplicationSpecification> appSpecs = store
                    .getAllApplications(new Id.Account(programId.getAccountId()));
            if (appSpecs != null) {
                for (ApplicationSpecification appSpec : appSpecs) {
                    if (type == ProgramType.FLOW) {
                        for (FlowSpecification flowSpec : appSpec.getFlows().values()) {
                            if ((data == Data.DATASET && usesDataSet(flowSpec, name))
                                    || (data == Data.STREAM && usesStream(flowSpec, name))) {
                                result.add(makeProgramRecord(appSpec.getName(), flowSpec, ProgramType.FLOW));
                            }
                        }
                    } else if (type == ProgramType.PROCEDURE) {
                        for (ProcedureSpecification procedureSpec : appSpec.getProcedures().values()) {
                            if (data == Data.DATASET && procedureSpec.getDataSets().contains(name)) {
                                result.add(
                                        makeProgramRecord(appSpec.getName(), procedureSpec, ProgramType.PROCEDURE));
                            }
                        }
                    } else if (type == ProgramType.MAPREDUCE) {
                        for (MapReduceSpecification mrSpec : appSpec.getMapReduce().values()) {
                            if (data == Data.DATASET && mrSpec.getDataSets().contains(name)) {
                                result.add(makeProgramRecord(appSpec.getName(), mrSpec, ProgramType.MAPREDUCE));
                            }
                        }
                    }
                }
            }
            return GSON.toJson(result);
        } catch (OperationException e) {
            LOG.warn(e.getMessage(), e);
            throw new Exception("Could not retrieve application specs for " + programId.toString() + ", reason: "
                    + e.getMessage());
        }
    }

    private static boolean usesDataSet(FlowSpecification flowSpec, String dataset) {
        for (FlowletDefinition flowlet : flowSpec.getFlowlets().values()) {
            if (flowlet.getDatasets().contains(dataset)) {
                return true;
            }
        }
        return false;
    }

    private static boolean usesStream(FlowSpecification flowSpec, String stream) {
        for (FlowletConnection con : flowSpec.getConnections()) {
            if (FlowletConnection.Type.STREAM == con.getSourceType() && stream.equals(con.getSourceName())) {
                return true;
            }
        }
        return false;
    }

    /* -----------------  helpers to return Json consistently -------------- */

    private static ApplicationRecord makeAppRecord(ApplicationSpecification appSpec) {
        return new ApplicationRecord("App", appSpec.getName(), appSpec.getName(), appSpec.getDescription());
    }

    private static ProgramRecord makeProgramRecord(String appId, ProgramSpecification spec, ProgramType type) {
        return new ProgramRecord(type, appId, spec.getName(), spec.getName(), spec.getDescription());
    }

    private static DatasetRecord makeDataSetRecord(String name, String classname) {
        return new DatasetRecord("Dataset", name, name, classname);
    }

    private static StreamRecord makeStreamRecord(String name, StreamSpecification specification) {
        return new StreamRecord("Stream", name, name, GSON.toJson(specification));
    }

    /**
     * DO NOT DOCUMENT THIS API.
     */
    @POST
    @Path("/unrecoverable/reset")
    public void resetCDAP(HttpRequest request, HttpResponder responder) {

        try {
            if (!configuration.getBoolean(Constants.Dangerous.UNRECOVERABLE_RESET,
                    Constants.Dangerous.DEFAULT_UNRECOVERABLE_RESET)) {
                responder.sendStatus(HttpResponseStatus.FORBIDDEN);
                return;
            }
            String account = getAuthenticatedAccountId(request);
            final Id.Account accountId = Id.Account.from(account);

            // Check if any program is still running
            boolean appRunning = checkAnyRunning(new Predicate<Id.Program>() {
                @Override
                public boolean apply(Id.Program programId) {
                    return programId.getAccountId().equals(accountId.getId());
                }
            }, ProgramType.values());

            if (appRunning) {
                throw new Exception("App Still Running");
            }

            LOG.info("Deleting all data for account '" + account + "'.");

            // NOTE: deleting new datasets stuff first because old datasets system deletes all blindly by prefix
            //       which may damage metadata
            for (DatasetSpecification spec : dsFramework.getInstances()) {
                dsFramework.deleteInstance(spec.getName());
            }
            dsFramework.deleteAllModules();

            deleteMetrics(account, null);
            // delete all meta data
            store.removeAll(accountId);
            // todo: delete only for specified account
            // delete queues and streams data
            queueAdmin.dropAll();
            streamAdmin.dropAll();

            LOG.info("All data for account '" + account + "' deleted.");
            responder.sendStatus(HttpResponseStatus.OK);
        } catch (SecurityException e) {
            responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
        } catch (Throwable e) {
            LOG.warn(e.getMessage(), e);
            responder.sendString(HttpResponseStatus.BAD_REQUEST,
                    String.format(UserMessages.getMessage(UserErrors.RESET_FAIL), e.getMessage()));
        }
    }

    /**
     * Convenience class for representing the necessary components in the batch endpoint.
     */
    private class BatchEndpointArgs {
        private String appId = null;
        private String programType = null;
        private String programId = null;
        private String runnableId = null;
        private String error = null;
        private Integer statusCode = null;

        private BatchEndpointArgs(String appId, String programType, String programId, String runnableId,
                String error, Integer statusCode) {
            this.appId = appId;
            this.programType = programType;
            this.programId = programId;
            this.runnableId = runnableId;
            this.error = error;
            this.statusCode = statusCode;
        }

        public BatchEndpointArgs(BatchEndpointArgs arg) {
            this(arg.appId, arg.programType, arg.programId, arg.runnableId, arg.error, arg.statusCode);
        }

        public String getRunnableId() {
            return runnableId;
        }

        public void setRunnableId(String runnableId) {
            this.runnableId = runnableId;
        }

        public void setError(String error) {
            this.error = error;
        }

        public void setStatusCode(Integer statusCode) {
            this.statusCode = statusCode;
        }

        public int getStatusCode() {
            return statusCode;
        }

        public String getError() {
            return error;
        }

        public String getProgramId() {
            return programId;
        }

        public String getProgramType() {
            return programType;
        }

        public String getAppId() {
            return appId;
        }

        public void setProgramType(String programType) {
            this.programType = programType;
        }
    }

    private class BatchEndpointInstances extends BatchEndpointArgs {
        private Integer requested = null;
        private Integer provisioned = null;

        public BatchEndpointInstances(BatchEndpointArgs arg) {
            super(arg);
        }

        public Integer getProvisioned() {
            return provisioned;
        }

        public void setProvisioned(Integer provisioned) {
            this.provisioned = provisioned;
        }

        public Integer getRequested() {
            return requested;
        }

        public void setRequested(Integer requested) {
            this.requested = requested;
        }
    }

    private class BatchEndpointStatus extends BatchEndpointArgs {
        private String status = null;

        public BatchEndpointStatus(BatchEndpointArgs arg) {
            super(arg);
        }

        public String getStatus() {
            return status;
        }

        public void setStatus(String status) {
            this.status = status;
        }
    }

    private List<BatchEndpointInstances> instancesFromBatchArgs(List<BatchEndpointArgs> args) {
        if (args == null) {
            return null;
        }
        List<BatchEndpointInstances> retVal = new ArrayList<BatchEndpointInstances>(args.size());
        for (BatchEndpointArgs arg : args) {
            retVal.add(new BatchEndpointInstances(arg));
        }
        return retVal;
    }

    private List<BatchEndpointStatus> statusFromBatchArgs(List<BatchEndpointArgs> args) {
        if (args == null) {
            return null;
        }
        List<BatchEndpointStatus> retVal = new ArrayList<BatchEndpointStatus>(args.size());
        for (BatchEndpointArgs arg : args) {
            retVal.add(new BatchEndpointStatus(arg));
        }
        return retVal;
    }

    /**
     * Convenience class for representing the necessary components for retrieving status
     */
    private class StatusMap {
        private String status = null;
        private String error = null;
        private Integer statusCode = null;

        private StatusMap(String status, String error, int statusCode) {
            this.status = status;
            this.error = error;
            this.statusCode = statusCode;
        }

        public StatusMap() {
        }

        public int getStatusCode() {
            return statusCode;
        }

        public String getError() {
            return error;
        }

        public String getStatus() {
            return status;
        }

        public void setStatusCode(int statusCode) {
            this.statusCode = statusCode;
        }

        public void setError(String error) {
            this.error = error;
        }

        public void setStatus(String status) {
            this.status = status;
        }
    }
}