co.cask.cdap.internal.app.services.http.handlers.ProgramLifecycleHttpHandlerTest.java Source code

Java tutorial

Introduction

Here is the source code for co.cask.cdap.internal.app.services.http.handlers.ProgramLifecycleHttpHandlerTest.java

Source

/*
 * Copyright  2014-2015 Cask Data, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package co.cask.cdap.internal.app.services.http.handlers;

import co.cask.cdap.AppWithMultipleScheduledWorkflows;
import co.cask.cdap.AppWithServices;
import co.cask.cdap.AppWithWorker;
import co.cask.cdap.AppWithWorkflow;
import co.cask.cdap.DummyAppWithTrackingTable;
import co.cask.cdap.SleepingWorkflowApp;
import co.cask.cdap.WordCountApp;
import co.cask.cdap.api.schedule.ScheduleSpecification;
import co.cask.cdap.api.service.ServiceSpecification;
import co.cask.cdap.api.service.http.HttpServiceHandlerSpecification;
import co.cask.cdap.api.service.http.ServiceHttpEndpoint;
import co.cask.cdap.api.workflow.WorkflowActionSpecification;
import co.cask.cdap.common.NotFoundException;
import co.cask.cdap.common.conf.Constants;
import co.cask.cdap.common.queue.QueueName;
import co.cask.cdap.common.utils.Tasks;
import co.cask.cdap.data2.queue.ConsumerConfig;
import co.cask.cdap.data2.queue.DequeueStrategy;
import co.cask.cdap.data2.queue.QueueClientFactory;
import co.cask.cdap.data2.queue.QueueConsumer;
import co.cask.cdap.data2.queue.QueueEntry;
import co.cask.cdap.data2.queue.QueueProducer;
import co.cask.cdap.gateway.handlers.ProgramLifecycleHttpHandler;
import co.cask.cdap.internal.app.ServiceSpecificationCodec;
import co.cask.cdap.internal.app.services.http.AppFabricTestBase;
import co.cask.cdap.proto.ApplicationDetail;
import co.cask.cdap.proto.Id;
import co.cask.cdap.proto.Instances;
import co.cask.cdap.proto.ProgramRecord;
import co.cask.cdap.proto.ProgramRunStatus;
import co.cask.cdap.proto.ProgramType;
import co.cask.cdap.proto.RunRecord;
import co.cask.cdap.proto.ServiceInstances;
import co.cask.cdap.proto.codec.ScheduleSpecificationCodec;
import co.cask.cdap.proto.codec.WorkflowActionSpecificationCodec;
import co.cask.cdap.test.SlowTests;
import co.cask.cdap.test.XSlowTests;
import co.cask.common.http.HttpMethod;
import co.cask.tephra.TransactionAware;
import co.cask.tephra.TransactionExecutor;
import co.cask.tephra.TransactionExecutorFactory;
import com.google.common.base.Charsets;
import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonObject;
import com.google.gson.reflect.TypeToken;
import org.apache.http.HttpResponse;
import org.apache.http.util.EntityUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.lang.reflect.Type;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;

/**
 * Tests for {@link ProgramLifecycleHttpHandler}
 */
public class ProgramLifecycleHttpHandlerTest extends AppFabricTestBase {
    private static final Logger LOG = LoggerFactory.getLogger(ProgramLifecycleHttpHandlerTest.class);

    private static final Gson GSON = new GsonBuilder()
            .registerTypeAdapter(ScheduleSpecification.class, new ScheduleSpecificationCodec())
            .registerTypeAdapter(WorkflowActionSpecification.class, new WorkflowActionSpecificationCodec())
            .create();
    private static final Type LIST_OF_JSONOBJECT_TYPE = new TypeToken<List<JsonObject>>() {
    }.getType();
    private static final Type LIST_OF_RUN_RECORD = new TypeToken<List<RunRecord>>() {
    }.getType();

    private static final String WORDCOUNT_APP_NAME = "WordCountApp";
    private static final String WORDCOUNT_FLOW_NAME = "WordCountFlow";
    private static final String WORDCOUNT_MAPREDUCE_NAME = "VoidMapReduceJob";
    private static final String WORDCOUNT_FLOWLET_NAME = "StreamSource";
    private static final String DUMMY_APP_ID = "dummy";
    private static final String DUMMY_MR_NAME = "dummy-batch";
    private static final String SLEEP_WORKFLOW_APP_ID = "SleepWorkflowApp";
    private static final String SLEEP_WORKFLOW_NAME = "SleepWorkflow";
    private static final String APP_WITH_SERVICES_APP_ID = "AppWithServices";
    private static final String APP_WITH_SERVICES_SERVICE_NAME = "NoOpService";
    private static final String APP_WITH_WORKFLOW_APP_ID = "AppWithWorkflow";
    private static final String APP_WITH_WORKFLOW_WORKFLOW_NAME = "SampleWorkflow";
    private static final String APP_WITH_MULTIPLE_WORKFLOWS_APP_NAME = "AppWithMultipleScheduledWorkflows";
    private static final String APP_WITH_MULTIPLE_WORKFLOWS_SOMEWORKFLOW = "SomeWorkflow";
    private static final String APP_WITH_MULTIPLE_WORKFLOWS_ANOTHERWORKFLOW = "AnotherWorkflow";

    private static final String EMPTY_ARRAY_JSON = "[]";
    private static final String STOPPED = "STOPPED";

    @Category(XSlowTests.class)
    @Test
    public void testProgramStartStopStatus() throws Exception {
        // deploy, check the status
        HttpResponse response = deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        Id.Flow wordcountFlow1 = Id.Flow.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME);
        Id.Flow wordcountFlow2 = Id.Flow.from(TEST_NAMESPACE2, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME);

        // flow is stopped initially
        Assert.assertEquals(STOPPED, getProgramStatus(wordcountFlow1));

        // start flow in the wrong namespace and verify that it does not start
        startProgram(wordcountFlow2, 404);
        Assert.assertEquals(STOPPED, getProgramStatus(wordcountFlow1));

        // start a flow and check the status
        startProgram(wordcountFlow1);
        waitState(wordcountFlow1, ProgramRunStatus.RUNNING.toString());

        // stop the flow and check the status
        stopProgram(wordcountFlow1);
        waitState(wordcountFlow1, STOPPED);

        // deploy another app in a different namespace and verify
        response = deploy(DummyAppWithTrackingTable.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        Id.Program dummyMR1 = Id.Program.from(TEST_NAMESPACE1, DUMMY_APP_ID, ProgramType.MAPREDUCE, DUMMY_MR_NAME);
        Id.Program dummyMR2 = Id.Program.from(TEST_NAMESPACE2, DUMMY_APP_ID, ProgramType.MAPREDUCE, DUMMY_MR_NAME);

        // mapreduce is stopped initially
        Assert.assertEquals(STOPPED, getProgramStatus(dummyMR2));

        // start mapreduce in the wrong namespace and verify it does not start
        startProgram(dummyMR1, 404);
        Assert.assertEquals(STOPPED, getProgramStatus(dummyMR2));

        // start map-reduce and verify status
        startProgram(dummyMR2);
        waitState(dummyMR2, ProgramRunStatus.RUNNING.toString());

        // stop the mapreduce program and check the status
        stopProgram(dummyMR2);
        waitState(dummyMR2, STOPPED);

        // start multiple runs of the map-reduce program
        startProgram(dummyMR2);
        startProgram(dummyMR2);

        // verify that more than one map-reduce program runs are running
        verifyProgramRuns(dummyMR2, "running", 1);

        // get run records corresponding to the program runs
        List<RunRecord> historyRuns = getProgramRuns(dummyMR2, "running");
        Assert.assertTrue(2 == historyRuns.size());

        // stop individual runs of the map-reduce program
        String runId = historyRuns.get(0).getPid();
        stopProgram(dummyMR2, runId, 200);

        runId = historyRuns.get(1).getPid();
        stopProgram(dummyMR2, runId, 200);

        waitState(dummyMR2, STOPPED);

        // deploy an app containing a workflow
        response = deploy(SleepingWorkflowApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        Id.Program sleepWorkflow1 = Id.Program.from(TEST_NAMESPACE1, SLEEP_WORKFLOW_APP_ID, ProgramType.WORKFLOW,
                SLEEP_WORKFLOW_NAME);
        Id.Program sleepWorkflow2 = Id.Program.from(TEST_NAMESPACE2, SLEEP_WORKFLOW_APP_ID, ProgramType.WORKFLOW,
                SLEEP_WORKFLOW_NAME);

        // workflow is stopped initially
        Assert.assertEquals(STOPPED, getProgramStatus(sleepWorkflow2));

        // start workflow in the wrong namespace and verify that it does not start
        startProgram(sleepWorkflow1, 404);
        Assert.assertEquals(STOPPED, getProgramStatus(sleepWorkflow2));

        // start workflow and check status
        startProgram(sleepWorkflow2);
        waitState(sleepWorkflow2, ProgramRunStatus.RUNNING.toString());

        // workflow will stop itself
        waitState(sleepWorkflow2, STOPPED);

        // cleanup
        response = doDelete(getVersionedAPIPath("apps/", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1));
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        response = doDelete(getVersionedAPIPath("apps/", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2));
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    }

    @Category(XSlowTests.class)
    @Test
    public void testProgramStartStopStatusErrors() throws Exception {
        // deploy, check the status
        HttpResponse response = deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        // start unknown program
        startProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, "noexist"), 404);
        // start program in unknonw app
        startProgram(Id.Program.from(TEST_NAMESPACE1, "noexist", ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 404);
        // start program in unknown namespace
        startProgram(Id.Program.from("noexist", WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 404);

        // debug unknown program
        debugProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, "noexist"), 404);
        // debug a program that does not support it
        debugProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.MAPREDUCE,
                WORDCOUNT_MAPREDUCE_NAME), 501); // not implemented

        // status for unknown program
        programStatus(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, "noexist"), 404);
        // status for program in unknonw app
        programStatus(Id.Program.from(TEST_NAMESPACE1, "noexist", ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 404);
        // status for program in unknown namespace
        programStatus(Id.Program.from("noexist", WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 404);

        // stop unknown program
        stopProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, "noexist"), 404);
        // stop program in unknonw app
        stopProgram(Id.Program.from(TEST_NAMESPACE1, "noexist", ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 404);
        // stop program in unknown namespace
        stopProgram(Id.Program.from("noexist", WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 404);
        // stop program that is not running
        stopProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME),
                400);
        // stop run of a program with ill-formed run id
        stopProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME),
                "norunid", 400);

        // start program twice
        startProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME));
        waitState(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME),
                "RUNNING");

        startProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME),
                409); // conflict

        // get run records for later use
        List<RunRecord> runs = getProgramRuns(
                Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME),
                "running");
        Assert.assertEquals(1, runs.size());
        String runId = runs.get(0).getPid();

        // stop program
        stopProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME),
                200);
        waitState(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME),
                "STOPPED");

        // get run records again, should be empty now
        Tasks.waitFor(true, new Callable<Boolean>() {
            @Override
            public Boolean call() throws Exception {
                Id.Program id = Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW,
                        WORDCOUNT_FLOW_NAME);
                return getProgramRuns(id, "running").isEmpty();
            }
        }, 10, TimeUnit.SECONDS);

        // stop run of the program that is not running
        stopProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME),
                runId, 404); // active run not found

        // cleanup
        response = doDelete(getVersionedAPIPath("apps/", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1));
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    }

    /**
     * Tests history of a flow.
     */
    @Test
    public void testFlowHistory() throws Exception {
        testHistory(WordCountApp.class,
                Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME));
    }

    /**
     * Tests history of a mapreduce.
     */
    @Category(XSlowTests.class)
    @Test
    public void testMapreduceHistory() throws Exception {
        testHistory(DummyAppWithTrackingTable.class,
                Id.Program.from(TEST_NAMESPACE2, DUMMY_APP_ID, ProgramType.MAPREDUCE, DUMMY_MR_NAME));
    }

    /**
     * Tests history of a workflow.
     */
    @Category(SlowTests.class)
    @Test
    public void testWorkflowHistory() throws Exception {
        try {
            deploy(SleepingWorkflowApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
            Id.Program sleepWorkflow1 = Id.Program.from(TEST_NAMESPACE1, SLEEP_WORKFLOW_APP_ID,
                    ProgramType.WORKFLOW, SLEEP_WORKFLOW_NAME);

            // first run
            startProgram(sleepWorkflow1);
            waitState(sleepWorkflow1, ProgramRunStatus.RUNNING.toString());
            // workflow stops by itself after actions are done
            waitState(sleepWorkflow1, STOPPED);

            // second run
            startProgram(sleepWorkflow1);
            waitState(sleepWorkflow1, ProgramRunStatus.RUNNING.toString());
            // workflow stops by itself after actions are done
            waitState(sleepWorkflow1, STOPPED);

            String url = String.format("apps/%s/%s/%s/runs?status=completed", SLEEP_WORKFLOW_APP_ID,
                    ProgramType.WORKFLOW.getCategoryName(), SLEEP_WORKFLOW_NAME);
            historyStatusWithRetry(getVersionedAPIPath(url, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1),
                    2);

        } finally {
            Assert.assertEquals(200,
                    doDelete(getVersionedAPIPath("apps/" + SLEEP_WORKFLOW_APP_ID,
                            Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1)).getStatusLine()
                                    .getStatusCode());
        }
    }

    @Test
    public void testFlowRuntimeArgs() throws Exception {
        testRuntimeArgs(WordCountApp.class, TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW.getCategoryName(),
                WORDCOUNT_FLOW_NAME);
    }

    @Test
    public void testWorkflowRuntimeArgs() throws Exception {
        testRuntimeArgs(SleepingWorkflowApp.class, TEST_NAMESPACE2, SLEEP_WORKFLOW_APP_ID,
                ProgramType.WORKFLOW.getCategoryName(), SLEEP_WORKFLOW_NAME);
    }

    @Test
    public void testMapreduceRuntimeArgs() throws Exception {
        testRuntimeArgs(DummyAppWithTrackingTable.class, TEST_NAMESPACE1, DUMMY_APP_ID,
                ProgramType.MAPREDUCE.getCategoryName(), DUMMY_MR_NAME);
    }

    @Test
    public void testBatchStatus() throws Exception {
        final String statusUrl1 = getVersionedAPIPath("status", Constants.Gateway.API_VERSION_3_TOKEN,
                TEST_NAMESPACE1);
        final String statusUrl2 = getVersionedAPIPath("status", Constants.Gateway.API_VERSION_3_TOKEN,
                TEST_NAMESPACE2);

        // invalid json must return 400
        Assert.assertEquals(400, doPost(statusUrl1, "").getStatusLine().getStatusCode());
        Assert.assertEquals(400, doPost(statusUrl2, "").getStatusLine().getStatusCode());
        // empty array is valid args
        Assert.assertEquals(200, doPost(statusUrl1, EMPTY_ARRAY_JSON).getStatusLine().getStatusCode());
        Assert.assertEquals(200, doPost(statusUrl2, EMPTY_ARRAY_JSON).getStatusLine().getStatusCode());

        // deploy an app in namespace1
        deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
        // deploy another app in namespace2
        deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);

        // data requires appId, programId, and programType. Test missing fields/invalid programType
        Assert.assertEquals(400, doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'Flow'}]")
                .getStatusLine().getStatusCode());
        Assert.assertEquals(400, doPost(statusUrl1, "[{'appId':'WordCountApp', 'programId':'WordCountFlow'}]")
                .getStatusLine().getStatusCode());
        Assert.assertEquals(400,
                doPost(statusUrl1,
                        "[{'programType':'Flow', 'programId':'WordCountFlow'}, {'appId':"
                                + "'AppWithServices', 'programType': 'service', 'programId': 'NoOpService'}]")
                                        .getStatusLine().getStatusCode());
        Assert.assertEquals(400,
                doPost(statusUrl1,
                        "[{'appId':'WordCountApp', 'programType':'Flow' " + "'programId':'WordCountFlow'}]")
                                .getStatusLine().getStatusCode());
        // Test missing app, programType, etc
        List<JsonObject> returnedBody = readResponse(
                doPost(statusUrl1,
                        "[{'appId':'NotExist', 'programType':'Flow', " + "'programId':'WordCountFlow'}]"),
                LIST_OF_JSONOBJECT_TYPE);
        Assert.assertEquals(new NotFoundException(Id.Application.from("testnamespace1", "NotExist")).getMessage(),
                returnedBody.get(0).get("error").getAsString());
        returnedBody = readResponse(
                doPost(statusUrl1,
                        "[{'appId':'WordCountApp', 'programType':'flow', 'programId':'NotExist'},"
                                + "{'appId':'WordCountApp', 'programType':'flow', 'programId':'WordCountFlow'}]"),
                LIST_OF_JSONOBJECT_TYPE);
        Assert.assertEquals(new NotFoundException(
                Id.Program.from("testnamespace1", "WordCountApp", ProgramType.FLOW, "NotExist")).getMessage(),
                returnedBody.get(0).get("error").getAsString());
        Assert.assertEquals(new NotFoundException(
                Id.Program.from("testnamespace1", "WordCountApp", ProgramType.FLOW, "NotExist")).getMessage(),
                returnedBody.get(0).get("error").getAsString());
        // The programType should be consistent. Second object should have proper status
        Assert.assertEquals("Flow", returnedBody.get(1).get("programType").getAsString());
        Assert.assertEquals(STOPPED, returnedBody.get(1).get("status").getAsString());

        // test valid cases for namespace1
        HttpResponse response = doPost(statusUrl1,
                "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow'},"
                        + "{'appId': 'WordCountApp', 'programType': 'Service', 'programId': "
                        + "'WordFrequencyService'}]");
        verifyInitialBatchStatusOutput(response);

        // test valid cases for namespace2
        response = doPost(statusUrl2,
                "[{'appId': 'AppWithServices', 'programType': 'Service', 'programId': " + "'NoOpService'}]");
        verifyInitialBatchStatusOutput(response);

        // start the flow
        Id.Program wordcountFlow1 = Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW,
                WORDCOUNT_FLOW_NAME);
        Id.Program service2 = Id.Program.from(TEST_NAMESPACE2, APP_WITH_SERVICES_APP_ID, ProgramType.SERVICE,
                APP_WITH_SERVICES_SERVICE_NAME);
        startProgram(wordcountFlow1);

        // test status API after starting the flow
        response = doPost(statusUrl1,
                "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow'},"
                        + "{'appId': 'WordCountApp', 'programType': 'Mapreduce', 'programId': 'VoidMapReduceJob'}]");
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE);
        Assert.assertEquals(ProgramRunStatus.RUNNING.toString(), returnedBody.get(0).get("status").getAsString());
        Assert.assertEquals(STOPPED, returnedBody.get(1).get("status").getAsString());

        // start the service
        startProgram(service2);
        // test status API after starting the service
        response = doPost(statusUrl2,
                "[{'appId': 'AppWithServices', 'programType': 'Service', 'programId': " + "'NoOpService'}]");
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE);
        Assert.assertEquals(ProgramRunStatus.RUNNING.toString(), returnedBody.get(0).get("status").getAsString());

        // stop the flow
        stopProgram(wordcountFlow1);
        waitState(wordcountFlow1, STOPPED);

        // stop the service
        stopProgram(service2);
        waitState(service2, STOPPED);

        // try posting a status request with namespace2 for apps in namespace1
        response = doPost(statusUrl2,
                "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow'},"
                        + "{'appId': 'WordCountApp', 'programType': 'Service', 'programId': 'WordFrequencyService'},"
                        + "{'appId': 'WordCountApp', 'programType': 'Mapreduce', 'programId': 'VoidMapReduceJob'}]");
        returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE);
        Assert.assertEquals(
                new NotFoundException(Id.Application.from("testnamespace2", "WordCountApp")).getMessage(),
                returnedBody.get(0).get("error").getAsString());
        Assert.assertEquals(
                new NotFoundException(Id.Application.from("testnamespace2", "WordCountApp")).getMessage(),
                returnedBody.get(1).get("error").getAsString());
        Assert.assertEquals(
                new NotFoundException(Id.Application.from("testnamespace2", "WordCountApp")).getMessage(),
                returnedBody.get(2).get("error").getAsString());
    }

    @Test
    public void testBatchInstances() throws Exception {
        final String instancesUrl1 = getVersionedAPIPath("instances", Constants.Gateway.API_VERSION_3_TOKEN,
                TEST_NAMESPACE1);
        final String instancesUrl2 = getVersionedAPIPath("instances", Constants.Gateway.API_VERSION_3_TOKEN,
                TEST_NAMESPACE2);

        Assert.assertEquals(400, doPost(instancesUrl1, "").getStatusLine().getStatusCode());
        Assert.assertEquals(400, doPost(instancesUrl2, "").getStatusLine().getStatusCode());

        // empty array is valid args
        Assert.assertEquals(200, doPost(instancesUrl1, "[]").getStatusLine().getStatusCode());
        Assert.assertEquals(200, doPost(instancesUrl2, "[]").getStatusLine().getStatusCode());

        deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
        deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);

        // data requires appId, programId, and programType. Test missing fields/invalid programType
        // TODO: These json strings should be replaced with JsonObjects so it becomes easier to refactor in future
        Assert.assertEquals(400, doPost(instancesUrl1, "[{'appId':'WordCountApp', 'programType':'Flow'}]")
                .getStatusLine().getStatusCode());
        Assert.assertEquals(400, doPost(instancesUrl1, "[{'appId':'WordCountApp', 'programId':'WordCountFlow'}]")
                .getStatusLine().getStatusCode());
        Assert.assertEquals(400,
                doPost(instancesUrl1, "[{'programType':'Flow', 'programId':'WordCountFlow'},"
                        + "{'appId': 'WordCountApp', 'programType': 'Mapreduce', 'programId': 'WordFrequency'}]")
                                .getStatusLine().getStatusCode());
        Assert.assertEquals(400,
                doPost(instancesUrl1,
                        "[{'appId':'WordCountApp', 'programType':'NotExist', " + "'programId':'WordCountFlow'}]")
                                .getStatusLine().getStatusCode());

        // Test malformed json
        Assert.assertEquals(400,
                doPost(instancesUrl1,
                        "[{'appId':'WordCountApp', 'programType':'Flow' 'programId':'WordCountFlow'}]")
                                .getStatusLine().getStatusCode());

        // Test missing app, programType, etc
        List<JsonObject> returnedBody = readResponse(
                doPost(instancesUrl1, "[{'appId':'NotExist', 'programType':'Flow', 'programId':'WordCountFlow'}]"),
                LIST_OF_JSONOBJECT_TYPE);
        Assert.assertEquals(404, returnedBody.get(0).get("statusCode").getAsInt());
        returnedBody = readResponse(doPost(instancesUrl1,
                "[{'appId':'WordCountApp', 'programType':'flow', 'programId':'WordCountFlow', " + "'runnableId': "
                        + "NotExist'}]"),
                LIST_OF_JSONOBJECT_TYPE);
        Assert.assertEquals(404, returnedBody.get(0).get("statusCode").getAsInt());

        // valid test in namespace1
        HttpResponse response = doPost(instancesUrl1,
                "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow', "
                        + "'runnableId': 'StreamSource'},"
                        + "{'appId': 'WordCountApp', 'programType': 'Service', 'programId': "
                        + "'WordFrequencyService', 'runnableId': 'WordFrequencyService'}]");

        verifyInitialBatchInstanceOutput(response);

        // valid test in namespace2
        response = doPost(instancesUrl2,
                "[{'appId': 'AppWithServices', 'programType':'Service', 'programId':'NoOpService', "
                        + "'runnableId':'NoOpService'}]");
        verifyInitialBatchInstanceOutput(response);

        // start the flow
        Id.Program wordcountFlow1 = Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW,
                WORDCOUNT_FLOW_NAME);
        startProgram(wordcountFlow1);

        response = doPost(instancesUrl1,
                "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow',"
                        + "'runnableId': 'StreamSource'}]");
        returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE);
        Assert.assertEquals(1, returnedBody.get(0).get("provisioned").getAsInt());

        // start the service
        Id.Program service2 = Id.Program.from(TEST_NAMESPACE2, APP_WITH_SERVICES_APP_ID, ProgramType.SERVICE,
                APP_WITH_SERVICES_SERVICE_NAME);
        startProgram(service2);

        response = doPost(instancesUrl2,
                "[{'appId':'AppWithServices', 'programType':'Service','programId':'NoOpService',"
                        + " 'runnableId':'NoOpService'}]");
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE);
        Assert.assertEquals(1, returnedBody.get(0).get("provisioned").getAsInt());

        // request for 2 more instances of the flowlet
        Assert.assertEquals(200, requestFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME,
                WORDCOUNT_FLOWLET_NAME, 2));
        returnedBody = readResponse(
                doPost(instancesUrl1,
                        "[{'appId':'WordCountApp', 'programType':'Flow',"
                                + "'programId':'WordCountFlow', 'runnableId': 'StreamSource'}]"),
                LIST_OF_JSONOBJECT_TYPE);
        // verify that 2 more instances were requested
        Assert.assertEquals(2, returnedBody.get(0).get("requested").getAsInt());

        stopProgram(wordcountFlow1);
        stopProgram(service2);
        waitState(wordcountFlow1, STOPPED);
        waitState(service2, STOPPED);
    }

    /**
     * Tests for program list calls
     */
    @Test
    public void testProgramList() throws Exception {
        // test initial state
        testListInitialState(TEST_NAMESPACE1, ProgramType.FLOW);
        testListInitialState(TEST_NAMESPACE2, ProgramType.MAPREDUCE);
        testListInitialState(TEST_NAMESPACE1, ProgramType.WORKFLOW);
        testListInitialState(TEST_NAMESPACE2, ProgramType.SPARK);
        testListInitialState(TEST_NAMESPACE1, ProgramType.SERVICE);

        // deploy WordCountApp in namespace1 and verify
        HttpResponse response = deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        // deploy AppWithServices in namespace2 and verify
        response = deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        // verify list by namespace
        verifyProgramList(TEST_NAMESPACE1, ProgramType.FLOW, 1);
        verifyProgramList(TEST_NAMESPACE1, ProgramType.MAPREDUCE, 1);
        verifyProgramList(TEST_NAMESPACE2, ProgramType.SERVICE, 1);

        // verify list by app
        verifyProgramList(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, 1);
        verifyProgramList(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.MAPREDUCE, 1);
        verifyProgramList(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.WORKFLOW, 0);
        verifyProgramList(TEST_NAMESPACE2, APP_WITH_SERVICES_APP_ID, ProgramType.SERVICE, 1);

        // verify invalid namespace
        Assert.assertEquals(404, getAppFDetailResponseCode(TEST_NAMESPACE1, APP_WITH_SERVICES_APP_ID,
                ProgramType.SERVICE.getCategoryName()));
        // verify invalid app
        Assert.assertEquals(404,
                getAppFDetailResponseCode(TEST_NAMESPACE1, "random", ProgramType.FLOW.getCategoryName()));
    }

    /**
     * Worker Specification tests
     */
    @Test
    public void testWorkerSpecification() throws Exception {
        // deploy AppWithWorker in namespace1 and verify
        HttpResponse response = deploy(AppWithWorker.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        verifyProgramSpecification(TEST_NAMESPACE1, AppWithWorker.NAME, ProgramType.WORKER.getCategoryName(),
                AppWithWorker.WORKER);
        Assert.assertEquals(404, getProgramSpecificationResponseCode(TEST_NAMESPACE2, AppWithWorker.NAME,
                ProgramType.WORKER.getCategoryName(), AppWithWorker.WORKER));
    }

    @Test
    public void testServiceSpecification() throws Exception {
        deploy(AppWithServices.class);
        HttpResponse response = doGet("/v3/namespaces/default/apps/AppWithServices/services/NoOpService");
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        Set<ServiceHttpEndpoint> expectedEndpoints = ImmutableSet.of(new ServiceHttpEndpoint("GET", "/ping"),
                new ServiceHttpEndpoint("POST", "/multi"), new ServiceHttpEndpoint("GET", "/multi"),
                new ServiceHttpEndpoint("GET", "/multi/ping"));

        GsonBuilder gsonBuilder = new GsonBuilder();
        gsonBuilder.registerTypeAdapter(ServiceSpecification.class, new ServiceSpecificationCodec());
        Gson gson = gsonBuilder.create();
        ServiceSpecification specification = readResponse(response, ServiceSpecification.class, gson);

        Set<ServiceHttpEndpoint> returnedEndpoints = new HashSet<>();
        for (HttpServiceHandlerSpecification httpServiceHandlerSpecification : specification.getHandlers()
                .values()) {
            returnedEndpoints.addAll(httpServiceHandlerSpecification.getEndpoints());
        }

        Assert.assertEquals("NoOpService", specification.getName());
        Assert.assertTrue(returnedEndpoints.equals(expectedEndpoints));
    }

    /**
     * Program specification tests through appfabric apis.
     */
    @Test
    public void testProgramSpecification() throws Exception {
        // deploy WordCountApp in namespace1 and verify
        HttpResponse response = deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        // deploy AppWithServices in namespace2 and verify
        response = deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        // deploy AppWithWorkflow in namespace2 and verify
        response = deploy(AppWithWorkflow.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        // deploy AppWithWorker in namespace1 and verify
        response = deploy(AppWithWorker.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        // verify program specification
        verifyProgramSpecification(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW.getCategoryName(),
                WORDCOUNT_FLOW_NAME);
        verifyProgramSpecification(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.MAPREDUCE.getCategoryName(),
                WORDCOUNT_MAPREDUCE_NAME);
        verifyProgramSpecification(TEST_NAMESPACE2, APP_WITH_SERVICES_APP_ID, ProgramType.SERVICE.getCategoryName(),
                APP_WITH_SERVICES_SERVICE_NAME);
        verifyProgramSpecification(TEST_NAMESPACE2, APP_WITH_WORKFLOW_APP_ID,
                ProgramType.WORKFLOW.getCategoryName(), APP_WITH_WORKFLOW_WORKFLOW_NAME);
        verifyProgramSpecification(TEST_NAMESPACE1, AppWithWorker.NAME, ProgramType.WORKER.getCategoryName(),
                AppWithWorker.WORKER);

        // verify invalid namespace
        Assert.assertEquals(404, getProgramSpecificationResponseCode(TEST_NAMESPACE1, APP_WITH_SERVICES_APP_ID,
                ProgramType.SERVICE.getCategoryName(), APP_WITH_SERVICES_SERVICE_NAME));
        // verify invalid app
        Assert.assertEquals(404, getProgramSpecificationResponseCode(TEST_NAMESPACE2, APP_WITH_SERVICES_APP_ID,
                ProgramType.WORKFLOW.getCategoryName(), APP_WITH_WORKFLOW_WORKFLOW_NAME));
        // verify invalid program type
        Assert.assertEquals(405, getProgramSpecificationResponseCode(TEST_NAMESPACE2, APP_WITH_SERVICES_APP_ID,
                "random", APP_WITH_WORKFLOW_WORKFLOW_NAME));

        // verify invalid program type
        Assert.assertEquals(404, getProgramSpecificationResponseCode(TEST_NAMESPACE2, AppWithWorker.NAME,
                ProgramType.WORKER.getCategoryName(), AppWithWorker.WORKER));
    }

    @Test
    public void testFlows() throws Exception {
        // deploy WordCountApp in namespace1 and verify
        HttpResponse response = deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        // check initial flowlet instances
        int initial = getFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME,
                WORDCOUNT_FLOWLET_NAME);
        Assert.assertEquals(1, initial);

        // request two more instances
        Assert.assertEquals(200, requestFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME,
                WORDCOUNT_FLOWLET_NAME, 3));
        // verify
        int after = getFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME,
                WORDCOUNT_FLOWLET_NAME);
        Assert.assertEquals(3, after);

        // invalid namespace
        Assert.assertEquals(404, requestFlowletInstances(TEST_NAMESPACE2, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME,
                WORDCOUNT_FLOWLET_NAME, 3));
        // invalid app
        Assert.assertEquals(404, requestFlowletInstances(TEST_NAMESPACE1, APP_WITH_SERVICES_APP_ID,
                WORDCOUNT_FLOW_NAME, WORDCOUNT_FLOWLET_NAME, 3));
        // invalid flow
        Assert.assertEquals(404,
                requestFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, "random", WORDCOUNT_FLOWLET_NAME, 3));
        // invalid flowlet
        Assert.assertEquals(404,
                requestFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME, "random", 3));

        // test live info
        // send invalid program type to live info
        response = sendLiveInfoRequest(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, "random", WORDCOUNT_FLOW_NAME);
        Assert.assertEquals(405, response.getStatusLine().getStatusCode());

        // test valid live info
        JsonObject liveInfo = getLiveInfo(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW.getCategoryName(),
                WORDCOUNT_FLOW_NAME);
        Assert.assertEquals(WORDCOUNT_APP_NAME, liveInfo.get("app").getAsString());
        Assert.assertEquals(ProgramType.FLOW.getPrettyName(), liveInfo.get("type").getAsString());
        Assert.assertEquals(WORDCOUNT_FLOW_NAME, liveInfo.get("id").getAsString());

        // start flow
        Id.Program wordcountFlow1 = Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW,
                WORDCOUNT_FLOW_NAME);
        startProgram(wordcountFlow1);

        liveInfo = getLiveInfo(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW.getCategoryName(),
                WORDCOUNT_FLOW_NAME);
        Assert.assertEquals(WORDCOUNT_APP_NAME, liveInfo.get("app").getAsString());
        Assert.assertEquals(ProgramType.FLOW.getPrettyName(), liveInfo.get("type").getAsString());
        Assert.assertEquals(WORDCOUNT_FLOW_NAME, liveInfo.get("id").getAsString());
        Assert.assertEquals("in-memory", liveInfo.get("runtime").getAsString());

        // should not delete queues while running
        Assert.assertEquals(403, deleteQueues(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME));
        Assert.assertEquals(403, deleteQueues(TEST_NAMESPACE1));

        // stop
        stopProgram(wordcountFlow1);

        // delete queues
        Assert.assertEquals(200, deleteQueues(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME));
    }

    @Test
    public void testMultipleWorkflowSchedules() throws Exception {
        // Deploy the app
        HttpResponse response = deploy(AppWithMultipleScheduledWorkflows.class,
                Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        List<ScheduleSpecification> someSchedules = getSchedules(TEST_NAMESPACE2,
                APP_WITH_MULTIPLE_WORKFLOWS_APP_NAME, APP_WITH_MULTIPLE_WORKFLOWS_SOMEWORKFLOW);
        Assert.assertEquals(2, someSchedules.size());
        Assert.assertEquals(APP_WITH_MULTIPLE_WORKFLOWS_SOMEWORKFLOW,
                someSchedules.get(0).getProgram().getProgramName());
        Assert.assertEquals(APP_WITH_MULTIPLE_WORKFLOWS_SOMEWORKFLOW,
                someSchedules.get(1).getProgram().getProgramName());

        List<ScheduleSpecification> anotherSchedules = getSchedules(TEST_NAMESPACE2,
                APP_WITH_MULTIPLE_WORKFLOWS_APP_NAME, APP_WITH_MULTIPLE_WORKFLOWS_ANOTHERWORKFLOW);
        Assert.assertEquals(3, anotherSchedules.size());
        Assert.assertEquals(APP_WITH_MULTIPLE_WORKFLOWS_ANOTHERWORKFLOW,
                anotherSchedules.get(0).getProgram().getProgramName());
        Assert.assertEquals(APP_WITH_MULTIPLE_WORKFLOWS_ANOTHERWORKFLOW,
                anotherSchedules.get(1).getProgram().getProgramName());
        Assert.assertEquals(APP_WITH_MULTIPLE_WORKFLOWS_ANOTHERWORKFLOW,
                anotherSchedules.get(2).getProgram().getProgramName());
    }

    @Test
    public void testServices() throws Exception {
        HttpResponse response = deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN,
                TEST_NAMESPACE2);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        Id.Service service1 = Id.Service.from(Id.Namespace.from(TEST_NAMESPACE1), APP_WITH_SERVICES_APP_ID,
                APP_WITH_SERVICES_SERVICE_NAME);
        Id.Service service2 = Id.Service.from(Id.Namespace.from(TEST_NAMESPACE2), APP_WITH_SERVICES_APP_ID,
                APP_WITH_SERVICES_SERVICE_NAME);

        // start service in wrong namespace
        startProgram(service1, 404);
        startProgram(service2);

        // verify instances
        try {
            getServiceInstances(service1);
            Assert.fail("Should not find service in " + TEST_NAMESPACE1);
        } catch (AssertionError expected) {
            // expected
        }
        ServiceInstances instances = getServiceInstances(service2);
        Assert.assertEquals(1, instances.getRequested());
        Assert.assertEquals(1, instances.getProvisioned());

        // request 2 additional instances
        int code = setServiceInstances(service1, 3);
        Assert.assertEquals(404, code);
        code = setServiceInstances(service2, 3);
        Assert.assertEquals(200, code);

        // verify that additional instances were provisioned
        instances = getServiceInstances(service2);
        Assert.assertEquals(3, instances.getRequested());
        Assert.assertEquals(3, instances.getProvisioned());

        // verify that endpoints are not available in the wrong namespace
        response = callService(service1, HttpMethod.POST, "multi");
        code = response.getStatusLine().getStatusCode();
        Assert.assertEquals(404, code);

        response = callService(service1, HttpMethod.GET, "multi/ping");
        code = response.getStatusLine().getStatusCode();
        Assert.assertEquals(404, code);

        // stop service
        stopProgram(service1, 404);
        stopProgram(service2);
    }

    @Test
    public void testDeleteQueues() throws Exception {
        QueueName queueName = QueueName.fromFlowlet(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME,
                WORDCOUNT_FLOWLET_NAME, "out");

        // enqueue some data
        enqueue(queueName, new QueueEntry("x".getBytes(Charsets.UTF_8)));

        // verify it exists
        Assert.assertTrue(dequeueOne(queueName));

        // clear queue in wrong namespace
        Assert.assertEquals(200,
                doDelete("/v3/namespaces/" + TEST_NAMESPACE2 + "/queues").getStatusLine().getStatusCode());
        // verify queue is still here
        Assert.assertTrue(dequeueOne(queueName));

        // clear queue in the right namespace
        Assert.assertEquals(200,
                doDelete("/v3/namespaces/" + TEST_NAMESPACE1 + "/queues").getStatusLine().getStatusCode());

        // verify queue is gone
        Assert.assertFalse(dequeueOne(queueName));
    }

    @After
    public void cleanup() throws Exception {
        doDelete(getVersionedAPIPath("apps/", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1));
        doDelete(getVersionedAPIPath("apps/", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2));
    }

    // TODO: Duplicate from AppFabricHttpHandlerTest, remove the AppFabricHttpHandlerTest method after deprecating v2 APIs
    private void enqueue(QueueName queueName, final QueueEntry queueEntry) throws Exception {
        QueueClientFactory queueClientFactory = AppFabricTestBase.getInjector()
                .getInstance(QueueClientFactory.class);
        final QueueProducer producer = queueClientFactory.createProducer(queueName);
        // doing inside tx
        TransactionExecutorFactory txExecutorFactory = AppFabricTestBase.getInjector()
                .getInstance(TransactionExecutorFactory.class);
        txExecutorFactory.createExecutor(ImmutableList.of((TransactionAware) producer))
                .execute(new TransactionExecutor.Subroutine() {
                    @Override
                    public void apply() throws Exception {
                        // write more than one so that we can dequeue multiple times for multiple checks
                        // we only dequeue twice, but ensure that the drop queues call drops the rest of the entries as well
                        int numEntries = 0;
                        while (numEntries++ < 5) {
                            producer.enqueue(queueEntry);
                        }
                    }
                });
    }

    private boolean dequeueOne(QueueName queueName) throws Exception {
        QueueClientFactory queueClientFactory = AppFabricTestBase.getInjector()
                .getInstance(QueueClientFactory.class);
        final QueueConsumer consumer = queueClientFactory.createConsumer(queueName,
                new ConsumerConfig(1L, 0, 1, DequeueStrategy.ROUND_ROBIN, null), 1);
        // doing inside tx
        TransactionExecutorFactory txExecutorFactory = AppFabricTestBase.getInjector()
                .getInstance(TransactionExecutorFactory.class);
        return txExecutorFactory.createExecutor(ImmutableList.of((TransactionAware) consumer))
                .execute(new Callable<Boolean>() {
                    @Override
                    public Boolean call() throws Exception {
                        return !consumer.dequeue(1).isEmpty();
                    }
                });
    }

    private ServiceInstances getServiceInstances(Id.Service serviceId) throws Exception {
        String instanceUrl = String.format("apps/%s/services/%s/instances", serviceId.getApplicationId(),
                serviceId.getId());
        String versionedInstanceUrl = getVersionedAPIPath(instanceUrl, Constants.Gateway.API_VERSION_3_TOKEN,
                serviceId.getNamespaceId());
        HttpResponse response = doGet(versionedInstanceUrl);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        return readResponse(response, ServiceInstances.class);
    }

    private int setServiceInstances(Id.Service serviceId, int instances) throws Exception {
        String instanceUrl = String.format("apps/%s/services/%s/instances", serviceId.getApplicationId(),
                serviceId.getId());
        String versionedInstanceUrl = getVersionedAPIPath(instanceUrl, Constants.Gateway.API_VERSION_3_TOKEN,
                serviceId.getNamespaceId());
        String instancesBody = GSON.toJson(new Instances(instances));
        return doPut(versionedInstanceUrl, instancesBody).getStatusLine().getStatusCode();
    }

    private HttpResponse callService(Id.Service serviceId, HttpMethod method, String endpoint) throws Exception {
        String serviceUrl = String.format("apps/%s/service/%s/methods/%s", serviceId.getApplicationId(),
                serviceId.getId(), endpoint);
        String versionedServiceUrl = getVersionedAPIPath(serviceUrl, Constants.Gateway.API_VERSION_3_TOKEN,
                serviceId.getNamespaceId());
        if (HttpMethod.GET.equals(method)) {
            return doGet(versionedServiceUrl);
        } else if (HttpMethod.POST.equals(method)) {
            return doPost(versionedServiceUrl);
        }
        throw new IllegalArgumentException("Only GET and POST supported right now.");
    }

    private int deleteQueues(String namespace) throws Exception {
        String versionedDeleteUrl = getVersionedAPIPath("queues", Constants.Gateway.API_VERSION_3_TOKEN, namespace);
        HttpResponse response = doDelete(versionedDeleteUrl);
        return response.getStatusLine().getStatusCode();
    }

    private int deleteQueues(String namespace, String appId, String flow) throws Exception {
        String deleteQueuesUrl = String.format("apps/%s/flows/%s/queues", appId, flow);
        String versionedDeleteUrl = getVersionedAPIPath(deleteQueuesUrl, Constants.Gateway.API_VERSION_3_TOKEN,
                namespace);
        HttpResponse response = doDelete(versionedDeleteUrl);
        return response.getStatusLine().getStatusCode();
    }

    private JsonObject getLiveInfo(String namespace, String appId, String programType, String programId)
            throws Exception {
        HttpResponse response = sendLiveInfoRequest(namespace, appId, programType, programId);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        return readResponse(response, JsonObject.class);
    }

    private HttpResponse sendLiveInfoRequest(String namespace, String appId, String programType, String programId)
            throws Exception {
        String liveInfoUrl = String.format("apps/%s/%s/%s/live-info", appId, programType, programId);
        String versionedUrl = getVersionedAPIPath(liveInfoUrl, Constants.Gateway.API_VERSION_3_TOKEN, namespace);
        return doGet(versionedUrl);
    }

    private int requestFlowletInstances(String namespace, String appId, String flow, String flowlet,
            int noRequested) throws Exception {
        String flowletInstancesVersionedUrl = getFlowletInstancesVersionedUrl(namespace, appId, flow, flowlet);
        JsonObject instances = new JsonObject();
        instances.addProperty("instances", noRequested);
        String body = GSON.toJson(instances);
        return doPut(flowletInstancesVersionedUrl, body).getStatusLine().getStatusCode();
    }

    private int getFlowletInstances(String namespace, String appId, String flow, String flowlet) throws Exception {
        String flowletInstancesUrl = getFlowletInstancesVersionedUrl(namespace, appId, flow, flowlet);
        String response = readResponse(doGet(flowletInstancesUrl));
        Assert.assertNotNull(response);
        JsonObject instances = GSON.fromJson(response, JsonObject.class);
        Assert.assertTrue(instances.has("instances"));
        return instances.get("instances").getAsInt();
    }

    private String getFlowletInstancesVersionedUrl(String namespace, String appId, String flow, String flowlet) {
        String flowletInstancesUrl = String.format("apps/%s/%s/%s/flowlets/%s/instances", appId,
                ProgramType.FLOW.getCategoryName(), flow, flowlet);
        return getVersionedAPIPath(flowletInstancesUrl, Constants.Gateway.API_VERSION_3_TOKEN, namespace);
    }

    private void verifyProgramSpecification(String namespace, String appId, String programType, String programId)
            throws Exception {
        JsonObject programSpec = getProgramSpecification(namespace, appId, programType, programId);
        Assert.assertTrue(
                programSpec.has("className") && programSpec.has("name") && programSpec.has("description"));
        Assert.assertEquals(programId, programSpec.get("name").getAsString());
    }

    private JsonObject getProgramSpecification(String namespace, String appId, String programType, String programId)
            throws Exception {
        HttpResponse response = requestProgramSpecification(namespace, appId, programType, programId);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        String result = EntityUtils.toString(response.getEntity());
        Assert.assertNotNull(result);
        return GSON.fromJson(result, JsonObject.class);
    }

    private int getProgramSpecificationResponseCode(String namespace, String appId, String programType,
            String programId) throws Exception {
        HttpResponse response = requestProgramSpecification(namespace, appId, programType, programId);
        return response.getStatusLine().getStatusCode();
    }

    private HttpResponse requestProgramSpecification(String namespace, String appId, String programType,
            String programId) throws Exception {
        String uri = getVersionedAPIPath(String.format("apps/%s/%s/%s", appId, programType, programId),
                Constants.Gateway.API_VERSION_3_TOKEN, namespace);
        return doGet(uri);
    }

    private void testListInitialState(String namespace, ProgramType programType) throws Exception {
        HttpResponse response = doGet(getVersionedAPIPath(programType.getCategoryName(),
                Constants.Gateway.API_VERSION_3_TOKEN, namespace));
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        Assert.assertEquals(EMPTY_ARRAY_JSON, readResponse(response));
    }

    private void verifyProgramList(String namespace, ProgramType programType, int expected) throws Exception {
        HttpResponse response = requestProgramList(namespace, programType.getCategoryName());
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        String json = EntityUtils.toString(response.getEntity());
        List<Map<String, String>> programs = GSON.fromJson(json, LIST_MAP_STRING_STRING_TYPE);
        Assert.assertEquals(expected, programs.size());
    }

    private void verifyProgramList(String namespace, String appName, final ProgramType programType, int expected)
            throws Exception {
        HttpResponse response = requestAppDetail(namespace, appName);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        String json = EntityUtils.toString(response.getEntity());
        ApplicationDetail appDetail = GSON.fromJson(json, ApplicationDetail.class);
        Collection<ProgramRecord> programs = Collections2.filter(appDetail.getPrograms(),
                new Predicate<ProgramRecord>() {
                    @Override
                    public boolean apply(@Nullable ProgramRecord record) {
                        return programType.getCategoryName().equals(record.getType().getCategoryName());
                    }
                });
        Assert.assertEquals(expected, programs.size());
    }

    private int getAppFDetailResponseCode(String namespace, @Nullable String appName, String programType)
            throws Exception {
        HttpResponse response = requestAppDetail(namespace, appName);
        return response.getStatusLine().getStatusCode();
    }

    private HttpResponse requestProgramList(String namespace, String programType) throws Exception {
        return doGet(getVersionedAPIPath(programType, Constants.Gateway.API_VERSION_3_TOKEN, namespace));
    }

    private HttpResponse requestAppDetail(String namespace, String appName) throws Exception {
        String uri = getVersionedAPIPath(String.format("apps/%s", appName), Constants.Gateway.API_VERSION_3_TOKEN,
                namespace);
        return doGet(uri);
    }

    private void verifyInitialBatchStatusOutput(HttpResponse response) throws IOException {
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        List<JsonObject> returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE);
        for (JsonObject obj : returnedBody) {
            Assert.assertEquals(200, obj.get("statusCode").getAsInt());
            Assert.assertEquals(STOPPED, obj.get("status").getAsString());
        }
    }

    private void verifyInitialBatchInstanceOutput(HttpResponse response) throws IOException {
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        List<JsonObject> returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE);
        for (JsonObject obj : returnedBody) {
            Assert.assertEquals(200, obj.get("statusCode").getAsInt());
            Assert.assertEquals(1, obj.get("requested").getAsInt());
            Assert.assertEquals(0, obj.get("provisioned").getAsInt());
        }
    }

    private void testHistory(Class<?> app, Id.Program program) throws Exception {
        try {
            String namespace = program.getNamespaceId();
            deploy(app, Constants.Gateway.API_VERSION_3_TOKEN, namespace);
            // first run
            startProgram(program);
            waitState(program, ProgramRunStatus.RUNNING.toString());
            stopProgram(program);
            waitState(program, STOPPED);

            // second run
            startProgram(program);
            waitState(program, ProgramRunStatus.RUNNING.toString());
            String url = String.format("apps/%s/%s/%s/runs?status=running", program.getApplicationId(),
                    program.getType().getCategoryName(), program.getId());

            //active size should be 1
            historyStatusWithRetry(getVersionedAPIPath(url, Constants.Gateway.API_VERSION_3_TOKEN, namespace), 1);
            // completed runs size should be 1
            url = String.format("apps/%s/%s/%s/runs?status=killed", program.getApplicationId(),
                    program.getType().getCategoryName(), program.getId());
            historyStatusWithRetry(getVersionedAPIPath(url, Constants.Gateway.API_VERSION_3_TOKEN, namespace), 1);

            stopProgram(program);
            waitState(program, STOPPED);

            historyStatusWithRetry(getVersionedAPIPath(url, Constants.Gateway.API_VERSION_3_TOKEN, namespace), 2);

        } catch (Exception e) {
            // Log exception before finally block is called
            LOG.error("Got exception: ", e);
        } finally {
            HttpResponse httpResponse = doDelete(getVersionedAPIPath("apps/" + program.getApplicationId(),
                    Constants.Gateway.API_VERSION_3_TOKEN, program.getNamespaceId()));
            Assert.assertEquals(EntityUtils.toString(httpResponse.getEntity()), 200,
                    httpResponse.getStatusLine().getStatusCode());
        }
    }

    private void historyStatusWithRetry(String url, int size) throws Exception {
        int trials = 0;
        while (trials++ < 5) {
            HttpResponse response = doGet(url);
            List<RunRecord> result = GSON.fromJson(EntityUtils.toString(response.getEntity()), LIST_OF_RUN_RECORD);
            if (result != null && result.size() >= size) {
                for (RunRecord m : result) {
                    assertRunRecord(String.format("%s/%s", url.substring(0, url.indexOf("?")), m.getPid()),
                            GSON.fromJson(GSON.toJson(m), RunRecord.class));
                }
                break;
            }
            TimeUnit.SECONDS.sleep(1);
        }
        Assert.assertTrue(trials < 5);
    }

    private void assertRunRecord(String url, RunRecord expectedRunRecord) throws Exception {
        HttpResponse response = doGet(url);
        RunRecord actualRunRecord = GSON.fromJson(EntityUtils.toString(response.getEntity()), RunRecord.class);
        Assert.assertEquals(expectedRunRecord, actualRunRecord);
    }

    private void testRuntimeArgs(Class<?> app, String namespace, String appId, String programType, String programId)
            throws Exception {
        deploy(app, Constants.Gateway.API_VERSION_3_TOKEN, namespace);

        Map<String, String> args = Maps.newHashMap();
        args.put("Key1", "Val1");
        args.put("Key2", "Val1");
        args.put("Key2", "Val1");

        HttpResponse response;
        String argString = GSON.toJson(args, new TypeToken<Map<String, String>>() {
        }.getType());
        String versionedRuntimeArgsUrl = getVersionedAPIPath(
                "apps/" + appId + "/" + programType + "/" + programId + "/runtimeargs",
                Constants.Gateway.API_VERSION_3_TOKEN, namespace);
        response = doPut(versionedRuntimeArgsUrl, argString);

        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        response = doGet(versionedRuntimeArgsUrl);

        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        Map<String, String> argsRead = GSON.fromJson(EntityUtils.toString(response.getEntity()),
                new TypeToken<Map<String, String>>() {
                }.getType());

        Assert.assertEquals(args.size(), argsRead.size());

        for (Map.Entry<String, String> entry : args.entrySet()) {
            Assert.assertEquals(entry.getValue(), argsRead.get(entry.getKey()));
        }

        //test empty runtime args
        response = doPut(versionedRuntimeArgsUrl, "");
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        response = doGet(versionedRuntimeArgsUrl);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        argsRead = GSON.fromJson(EntityUtils.toString(response.getEntity()), new TypeToken<Map<String, String>>() {
        }.getType());
        Assert.assertEquals(0, argsRead.size());

        //test null runtime args
        response = doPut(versionedRuntimeArgsUrl, null);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());

        response = doGet(versionedRuntimeArgsUrl);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        argsRead = GSON.fromJson(EntityUtils.toString(response.getEntity()), new TypeToken<Map<String, String>>() {
        }.getType());
        Assert.assertEquals(0, argsRead.size());
    }
}