Java tutorial
/* * Copyright 2014-2015 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.cdap.internal.app.services.http.handlers; import co.cask.cdap.AppWithMultipleScheduledWorkflows; import co.cask.cdap.AppWithServices; import co.cask.cdap.AppWithWorker; import co.cask.cdap.AppWithWorkflow; import co.cask.cdap.DummyAppWithTrackingTable; import co.cask.cdap.SleepingWorkflowApp; import co.cask.cdap.WordCountApp; import co.cask.cdap.api.schedule.ScheduleSpecification; import co.cask.cdap.api.service.ServiceSpecification; import co.cask.cdap.api.service.http.HttpServiceHandlerSpecification; import co.cask.cdap.api.service.http.ServiceHttpEndpoint; import co.cask.cdap.api.workflow.WorkflowActionSpecification; import co.cask.cdap.common.NotFoundException; import co.cask.cdap.common.conf.Constants; import co.cask.cdap.common.queue.QueueName; import co.cask.cdap.common.utils.Tasks; import co.cask.cdap.data2.queue.ConsumerConfig; import co.cask.cdap.data2.queue.DequeueStrategy; import co.cask.cdap.data2.queue.QueueClientFactory; import co.cask.cdap.data2.queue.QueueConsumer; import co.cask.cdap.data2.queue.QueueEntry; import co.cask.cdap.data2.queue.QueueProducer; import co.cask.cdap.gateway.handlers.ProgramLifecycleHttpHandler; import co.cask.cdap.internal.app.ServiceSpecificationCodec; import co.cask.cdap.internal.app.services.http.AppFabricTestBase; import co.cask.cdap.proto.ApplicationDetail; import co.cask.cdap.proto.Id; import co.cask.cdap.proto.Instances; import co.cask.cdap.proto.ProgramRecord; import co.cask.cdap.proto.ProgramRunStatus; import co.cask.cdap.proto.ProgramType; import co.cask.cdap.proto.RunRecord; import co.cask.cdap.proto.ServiceInstances; import co.cask.cdap.proto.codec.ScheduleSpecificationCodec; import co.cask.cdap.proto.codec.WorkflowActionSpecificationCodec; import co.cask.cdap.test.SlowTests; import co.cask.cdap.test.XSlowTests; import co.cask.common.http.HttpMethod; import co.cask.tephra.TransactionAware; import co.cask.tephra.TransactionExecutor; import co.cask.tephra.TransactionExecutorFactory; import com.google.common.base.Charsets; import com.google.common.base.Predicate; import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; import org.apache.http.HttpResponse; import org.apache.http.util.EntityUtils; import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.lang.reflect.Type; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; /** * Tests for {@link ProgramLifecycleHttpHandler} */ public class ProgramLifecycleHttpHandlerTest extends AppFabricTestBase { private static final Logger LOG = LoggerFactory.getLogger(ProgramLifecycleHttpHandlerTest.class); private static final Gson GSON = new GsonBuilder() .registerTypeAdapter(ScheduleSpecification.class, new ScheduleSpecificationCodec()) .registerTypeAdapter(WorkflowActionSpecification.class, new WorkflowActionSpecificationCodec()) .create(); private static final Type LIST_OF_JSONOBJECT_TYPE = new TypeToken<List<JsonObject>>() { }.getType(); private static final Type LIST_OF_RUN_RECORD = new TypeToken<List<RunRecord>>() { }.getType(); private static final String WORDCOUNT_APP_NAME = "WordCountApp"; private static final String WORDCOUNT_FLOW_NAME = "WordCountFlow"; private static final String WORDCOUNT_MAPREDUCE_NAME = "VoidMapReduceJob"; private static final String WORDCOUNT_FLOWLET_NAME = "StreamSource"; private static final String DUMMY_APP_ID = "dummy"; private static final String DUMMY_MR_NAME = "dummy-batch"; private static final String SLEEP_WORKFLOW_APP_ID = "SleepWorkflowApp"; private static final String SLEEP_WORKFLOW_NAME = "SleepWorkflow"; private static final String APP_WITH_SERVICES_APP_ID = "AppWithServices"; private static final String APP_WITH_SERVICES_SERVICE_NAME = "NoOpService"; private static final String APP_WITH_WORKFLOW_APP_ID = "AppWithWorkflow"; private static final String APP_WITH_WORKFLOW_WORKFLOW_NAME = "SampleWorkflow"; private static final String APP_WITH_MULTIPLE_WORKFLOWS_APP_NAME = "AppWithMultipleScheduledWorkflows"; private static final String APP_WITH_MULTIPLE_WORKFLOWS_SOMEWORKFLOW = "SomeWorkflow"; private static final String APP_WITH_MULTIPLE_WORKFLOWS_ANOTHERWORKFLOW = "AnotherWorkflow"; private static final String EMPTY_ARRAY_JSON = "[]"; private static final String STOPPED = "STOPPED"; @Category(XSlowTests.class) @Test public void testProgramStartStopStatus() throws Exception { // deploy, check the status HttpResponse response = deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); Id.Flow wordcountFlow1 = Id.Flow.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME); Id.Flow wordcountFlow2 = Id.Flow.from(TEST_NAMESPACE2, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME); // flow is stopped initially Assert.assertEquals(STOPPED, getProgramStatus(wordcountFlow1)); // start flow in the wrong namespace and verify that it does not start startProgram(wordcountFlow2, 404); Assert.assertEquals(STOPPED, getProgramStatus(wordcountFlow1)); // start a flow and check the status startProgram(wordcountFlow1); waitState(wordcountFlow1, ProgramRunStatus.RUNNING.toString()); // stop the flow and check the status stopProgram(wordcountFlow1); waitState(wordcountFlow1, STOPPED); // deploy another app in a different namespace and verify response = deploy(DummyAppWithTrackingTable.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); Id.Program dummyMR1 = Id.Program.from(TEST_NAMESPACE1, DUMMY_APP_ID, ProgramType.MAPREDUCE, DUMMY_MR_NAME); Id.Program dummyMR2 = Id.Program.from(TEST_NAMESPACE2, DUMMY_APP_ID, ProgramType.MAPREDUCE, DUMMY_MR_NAME); // mapreduce is stopped initially Assert.assertEquals(STOPPED, getProgramStatus(dummyMR2)); // start mapreduce in the wrong namespace and verify it does not start startProgram(dummyMR1, 404); Assert.assertEquals(STOPPED, getProgramStatus(dummyMR2)); // start map-reduce and verify status startProgram(dummyMR2); waitState(dummyMR2, ProgramRunStatus.RUNNING.toString()); // stop the mapreduce program and check the status stopProgram(dummyMR2); waitState(dummyMR2, STOPPED); // start multiple runs of the map-reduce program startProgram(dummyMR2); startProgram(dummyMR2); // verify that more than one map-reduce program runs are running verifyProgramRuns(dummyMR2, "running", 1); // get run records corresponding to the program runs List<RunRecord> historyRuns = getProgramRuns(dummyMR2, "running"); Assert.assertTrue(2 == historyRuns.size()); // stop individual runs of the map-reduce program String runId = historyRuns.get(0).getPid(); stopProgram(dummyMR2, runId, 200); runId = historyRuns.get(1).getPid(); stopProgram(dummyMR2, runId, 200); waitState(dummyMR2, STOPPED); // deploy an app containing a workflow response = deploy(SleepingWorkflowApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); Id.Program sleepWorkflow1 = Id.Program.from(TEST_NAMESPACE1, SLEEP_WORKFLOW_APP_ID, ProgramType.WORKFLOW, SLEEP_WORKFLOW_NAME); Id.Program sleepWorkflow2 = Id.Program.from(TEST_NAMESPACE2, SLEEP_WORKFLOW_APP_ID, ProgramType.WORKFLOW, SLEEP_WORKFLOW_NAME); // workflow is stopped initially Assert.assertEquals(STOPPED, getProgramStatus(sleepWorkflow2)); // start workflow in the wrong namespace and verify that it does not start startProgram(sleepWorkflow1, 404); Assert.assertEquals(STOPPED, getProgramStatus(sleepWorkflow2)); // start workflow and check status startProgram(sleepWorkflow2); waitState(sleepWorkflow2, ProgramRunStatus.RUNNING.toString()); // workflow will stop itself waitState(sleepWorkflow2, STOPPED); // cleanup response = doDelete(getVersionedAPIPath("apps/", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1)); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); response = doDelete(getVersionedAPIPath("apps/", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2)); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); } @Category(XSlowTests.class) @Test public void testProgramStartStopStatusErrors() throws Exception { // deploy, check the status HttpResponse response = deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); // start unknown program startProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, "noexist"), 404); // start program in unknonw app startProgram(Id.Program.from(TEST_NAMESPACE1, "noexist", ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 404); // start program in unknown namespace startProgram(Id.Program.from("noexist", WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 404); // debug unknown program debugProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, "noexist"), 404); // debug a program that does not support it debugProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.MAPREDUCE, WORDCOUNT_MAPREDUCE_NAME), 501); // not implemented // status for unknown program programStatus(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, "noexist"), 404); // status for program in unknonw app programStatus(Id.Program.from(TEST_NAMESPACE1, "noexist", ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 404); // status for program in unknown namespace programStatus(Id.Program.from("noexist", WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 404); // stop unknown program stopProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, "noexist"), 404); // stop program in unknonw app stopProgram(Id.Program.from(TEST_NAMESPACE1, "noexist", ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 404); // stop program in unknown namespace stopProgram(Id.Program.from("noexist", WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 404); // stop program that is not running stopProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 400); // stop run of a program with ill-formed run id stopProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME), "norunid", 400); // start program twice startProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME)); waitState(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME), "RUNNING"); startProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 409); // conflict // get run records for later use List<RunRecord> runs = getProgramRuns( Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME), "running"); Assert.assertEquals(1, runs.size()); String runId = runs.get(0).getPid(); // stop program stopProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME), 200); waitState(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME), "STOPPED"); // get run records again, should be empty now Tasks.waitFor(true, new Callable<Boolean>() { @Override public Boolean call() throws Exception { Id.Program id = Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME); return getProgramRuns(id, "running").isEmpty(); } }, 10, TimeUnit.SECONDS); // stop run of the program that is not running stopProgram(Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME), runId, 404); // active run not found // cleanup response = doDelete(getVersionedAPIPath("apps/", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1)); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); } /** * Tests history of a flow. */ @Test public void testFlowHistory() throws Exception { testHistory(WordCountApp.class, Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME)); } /** * Tests history of a mapreduce. */ @Category(XSlowTests.class) @Test public void testMapreduceHistory() throws Exception { testHistory(DummyAppWithTrackingTable.class, Id.Program.from(TEST_NAMESPACE2, DUMMY_APP_ID, ProgramType.MAPREDUCE, DUMMY_MR_NAME)); } /** * Tests history of a workflow. */ @Category(SlowTests.class) @Test public void testWorkflowHistory() throws Exception { try { deploy(SleepingWorkflowApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1); Id.Program sleepWorkflow1 = Id.Program.from(TEST_NAMESPACE1, SLEEP_WORKFLOW_APP_ID, ProgramType.WORKFLOW, SLEEP_WORKFLOW_NAME); // first run startProgram(sleepWorkflow1); waitState(sleepWorkflow1, ProgramRunStatus.RUNNING.toString()); // workflow stops by itself after actions are done waitState(sleepWorkflow1, STOPPED); // second run startProgram(sleepWorkflow1); waitState(sleepWorkflow1, ProgramRunStatus.RUNNING.toString()); // workflow stops by itself after actions are done waitState(sleepWorkflow1, STOPPED); String url = String.format("apps/%s/%s/%s/runs?status=completed", SLEEP_WORKFLOW_APP_ID, ProgramType.WORKFLOW.getCategoryName(), SLEEP_WORKFLOW_NAME); historyStatusWithRetry(getVersionedAPIPath(url, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1), 2); } finally { Assert.assertEquals(200, doDelete(getVersionedAPIPath("apps/" + SLEEP_WORKFLOW_APP_ID, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1)).getStatusLine() .getStatusCode()); } } @Test public void testFlowRuntimeArgs() throws Exception { testRuntimeArgs(WordCountApp.class, TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW.getCategoryName(), WORDCOUNT_FLOW_NAME); } @Test public void testWorkflowRuntimeArgs() throws Exception { testRuntimeArgs(SleepingWorkflowApp.class, TEST_NAMESPACE2, SLEEP_WORKFLOW_APP_ID, ProgramType.WORKFLOW.getCategoryName(), SLEEP_WORKFLOW_NAME); } @Test public void testMapreduceRuntimeArgs() throws Exception { testRuntimeArgs(DummyAppWithTrackingTable.class, TEST_NAMESPACE1, DUMMY_APP_ID, ProgramType.MAPREDUCE.getCategoryName(), DUMMY_MR_NAME); } @Test public void testBatchStatus() throws Exception { final String statusUrl1 = getVersionedAPIPath("status", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1); final String statusUrl2 = getVersionedAPIPath("status", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2); // invalid json must return 400 Assert.assertEquals(400, doPost(statusUrl1, "").getStatusLine().getStatusCode()); Assert.assertEquals(400, doPost(statusUrl2, "").getStatusLine().getStatusCode()); // empty array is valid args Assert.assertEquals(200, doPost(statusUrl1, EMPTY_ARRAY_JSON).getStatusLine().getStatusCode()); Assert.assertEquals(200, doPost(statusUrl2, EMPTY_ARRAY_JSON).getStatusLine().getStatusCode()); // deploy an app in namespace1 deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1); // deploy another app in namespace2 deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2); // data requires appId, programId, and programType. Test missing fields/invalid programType Assert.assertEquals(400, doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'Flow'}]") .getStatusLine().getStatusCode()); Assert.assertEquals(400, doPost(statusUrl1, "[{'appId':'WordCountApp', 'programId':'WordCountFlow'}]") .getStatusLine().getStatusCode()); Assert.assertEquals(400, doPost(statusUrl1, "[{'programType':'Flow', 'programId':'WordCountFlow'}, {'appId':" + "'AppWithServices', 'programType': 'service', 'programId': 'NoOpService'}]") .getStatusLine().getStatusCode()); Assert.assertEquals(400, doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'Flow' " + "'programId':'WordCountFlow'}]") .getStatusLine().getStatusCode()); // Test missing app, programType, etc List<JsonObject> returnedBody = readResponse( doPost(statusUrl1, "[{'appId':'NotExist', 'programType':'Flow', " + "'programId':'WordCountFlow'}]"), LIST_OF_JSONOBJECT_TYPE); Assert.assertEquals(new NotFoundException(Id.Application.from("testnamespace1", "NotExist")).getMessage(), returnedBody.get(0).get("error").getAsString()); returnedBody = readResponse( doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'flow', 'programId':'NotExist'}," + "{'appId':'WordCountApp', 'programType':'flow', 'programId':'WordCountFlow'}]"), LIST_OF_JSONOBJECT_TYPE); Assert.assertEquals(new NotFoundException( Id.Program.from("testnamespace1", "WordCountApp", ProgramType.FLOW, "NotExist")).getMessage(), returnedBody.get(0).get("error").getAsString()); Assert.assertEquals(new NotFoundException( Id.Program.from("testnamespace1", "WordCountApp", ProgramType.FLOW, "NotExist")).getMessage(), returnedBody.get(0).get("error").getAsString()); // The programType should be consistent. Second object should have proper status Assert.assertEquals("Flow", returnedBody.get(1).get("programType").getAsString()); Assert.assertEquals(STOPPED, returnedBody.get(1).get("status").getAsString()); // test valid cases for namespace1 HttpResponse response = doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow'}," + "{'appId': 'WordCountApp', 'programType': 'Service', 'programId': " + "'WordFrequencyService'}]"); verifyInitialBatchStatusOutput(response); // test valid cases for namespace2 response = doPost(statusUrl2, "[{'appId': 'AppWithServices', 'programType': 'Service', 'programId': " + "'NoOpService'}]"); verifyInitialBatchStatusOutput(response); // start the flow Id.Program wordcountFlow1 = Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME); Id.Program service2 = Id.Program.from(TEST_NAMESPACE2, APP_WITH_SERVICES_APP_ID, ProgramType.SERVICE, APP_WITH_SERVICES_SERVICE_NAME); startProgram(wordcountFlow1); // test status API after starting the flow response = doPost(statusUrl1, "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow'}," + "{'appId': 'WordCountApp', 'programType': 'Mapreduce', 'programId': 'VoidMapReduceJob'}]"); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE); Assert.assertEquals(ProgramRunStatus.RUNNING.toString(), returnedBody.get(0).get("status").getAsString()); Assert.assertEquals(STOPPED, returnedBody.get(1).get("status").getAsString()); // start the service startProgram(service2); // test status API after starting the service response = doPost(statusUrl2, "[{'appId': 'AppWithServices', 'programType': 'Service', 'programId': " + "'NoOpService'}]"); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE); Assert.assertEquals(ProgramRunStatus.RUNNING.toString(), returnedBody.get(0).get("status").getAsString()); // stop the flow stopProgram(wordcountFlow1); waitState(wordcountFlow1, STOPPED); // stop the service stopProgram(service2); waitState(service2, STOPPED); // try posting a status request with namespace2 for apps in namespace1 response = doPost(statusUrl2, "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow'}," + "{'appId': 'WordCountApp', 'programType': 'Service', 'programId': 'WordFrequencyService'}," + "{'appId': 'WordCountApp', 'programType': 'Mapreduce', 'programId': 'VoidMapReduceJob'}]"); returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE); Assert.assertEquals( new NotFoundException(Id.Application.from("testnamespace2", "WordCountApp")).getMessage(), returnedBody.get(0).get("error").getAsString()); Assert.assertEquals( new NotFoundException(Id.Application.from("testnamespace2", "WordCountApp")).getMessage(), returnedBody.get(1).get("error").getAsString()); Assert.assertEquals( new NotFoundException(Id.Application.from("testnamespace2", "WordCountApp")).getMessage(), returnedBody.get(2).get("error").getAsString()); } @Test public void testBatchInstances() throws Exception { final String instancesUrl1 = getVersionedAPIPath("instances", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1); final String instancesUrl2 = getVersionedAPIPath("instances", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2); Assert.assertEquals(400, doPost(instancesUrl1, "").getStatusLine().getStatusCode()); Assert.assertEquals(400, doPost(instancesUrl2, "").getStatusLine().getStatusCode()); // empty array is valid args Assert.assertEquals(200, doPost(instancesUrl1, "[]").getStatusLine().getStatusCode()); Assert.assertEquals(200, doPost(instancesUrl2, "[]").getStatusLine().getStatusCode()); deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1); deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2); // data requires appId, programId, and programType. Test missing fields/invalid programType // TODO: These json strings should be replaced with JsonObjects so it becomes easier to refactor in future Assert.assertEquals(400, doPost(instancesUrl1, "[{'appId':'WordCountApp', 'programType':'Flow'}]") .getStatusLine().getStatusCode()); Assert.assertEquals(400, doPost(instancesUrl1, "[{'appId':'WordCountApp', 'programId':'WordCountFlow'}]") .getStatusLine().getStatusCode()); Assert.assertEquals(400, doPost(instancesUrl1, "[{'programType':'Flow', 'programId':'WordCountFlow'}," + "{'appId': 'WordCountApp', 'programType': 'Mapreduce', 'programId': 'WordFrequency'}]") .getStatusLine().getStatusCode()); Assert.assertEquals(400, doPost(instancesUrl1, "[{'appId':'WordCountApp', 'programType':'NotExist', " + "'programId':'WordCountFlow'}]") .getStatusLine().getStatusCode()); // Test malformed json Assert.assertEquals(400, doPost(instancesUrl1, "[{'appId':'WordCountApp', 'programType':'Flow' 'programId':'WordCountFlow'}]") .getStatusLine().getStatusCode()); // Test missing app, programType, etc List<JsonObject> returnedBody = readResponse( doPost(instancesUrl1, "[{'appId':'NotExist', 'programType':'Flow', 'programId':'WordCountFlow'}]"), LIST_OF_JSONOBJECT_TYPE); Assert.assertEquals(404, returnedBody.get(0).get("statusCode").getAsInt()); returnedBody = readResponse(doPost(instancesUrl1, "[{'appId':'WordCountApp', 'programType':'flow', 'programId':'WordCountFlow', " + "'runnableId': " + "NotExist'}]"), LIST_OF_JSONOBJECT_TYPE); Assert.assertEquals(404, returnedBody.get(0).get("statusCode").getAsInt()); // valid test in namespace1 HttpResponse response = doPost(instancesUrl1, "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow', " + "'runnableId': 'StreamSource'}," + "{'appId': 'WordCountApp', 'programType': 'Service', 'programId': " + "'WordFrequencyService', 'runnableId': 'WordFrequencyService'}]"); verifyInitialBatchInstanceOutput(response); // valid test in namespace2 response = doPost(instancesUrl2, "[{'appId': 'AppWithServices', 'programType':'Service', 'programId':'NoOpService', " + "'runnableId':'NoOpService'}]"); verifyInitialBatchInstanceOutput(response); // start the flow Id.Program wordcountFlow1 = Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME); startProgram(wordcountFlow1); response = doPost(instancesUrl1, "[{'appId':'WordCountApp', 'programType':'Flow', 'programId':'WordCountFlow'," + "'runnableId': 'StreamSource'}]"); returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE); Assert.assertEquals(1, returnedBody.get(0).get("provisioned").getAsInt()); // start the service Id.Program service2 = Id.Program.from(TEST_NAMESPACE2, APP_WITH_SERVICES_APP_ID, ProgramType.SERVICE, APP_WITH_SERVICES_SERVICE_NAME); startProgram(service2); response = doPost(instancesUrl2, "[{'appId':'AppWithServices', 'programType':'Service','programId':'NoOpService'," + " 'runnableId':'NoOpService'}]"); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE); Assert.assertEquals(1, returnedBody.get(0).get("provisioned").getAsInt()); // request for 2 more instances of the flowlet Assert.assertEquals(200, requestFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME, WORDCOUNT_FLOWLET_NAME, 2)); returnedBody = readResponse( doPost(instancesUrl1, "[{'appId':'WordCountApp', 'programType':'Flow'," + "'programId':'WordCountFlow', 'runnableId': 'StreamSource'}]"), LIST_OF_JSONOBJECT_TYPE); // verify that 2 more instances were requested Assert.assertEquals(2, returnedBody.get(0).get("requested").getAsInt()); stopProgram(wordcountFlow1); stopProgram(service2); waitState(wordcountFlow1, STOPPED); waitState(service2, STOPPED); } /** * Tests for program list calls */ @Test public void testProgramList() throws Exception { // test initial state testListInitialState(TEST_NAMESPACE1, ProgramType.FLOW); testListInitialState(TEST_NAMESPACE2, ProgramType.MAPREDUCE); testListInitialState(TEST_NAMESPACE1, ProgramType.WORKFLOW); testListInitialState(TEST_NAMESPACE2, ProgramType.SPARK); testListInitialState(TEST_NAMESPACE1, ProgramType.SERVICE); // deploy WordCountApp in namespace1 and verify HttpResponse response = deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); // deploy AppWithServices in namespace2 and verify response = deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); // verify list by namespace verifyProgramList(TEST_NAMESPACE1, ProgramType.FLOW, 1); verifyProgramList(TEST_NAMESPACE1, ProgramType.MAPREDUCE, 1); verifyProgramList(TEST_NAMESPACE2, ProgramType.SERVICE, 1); // verify list by app verifyProgramList(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, 1); verifyProgramList(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.MAPREDUCE, 1); verifyProgramList(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.WORKFLOW, 0); verifyProgramList(TEST_NAMESPACE2, APP_WITH_SERVICES_APP_ID, ProgramType.SERVICE, 1); // verify invalid namespace Assert.assertEquals(404, getAppFDetailResponseCode(TEST_NAMESPACE1, APP_WITH_SERVICES_APP_ID, ProgramType.SERVICE.getCategoryName())); // verify invalid app Assert.assertEquals(404, getAppFDetailResponseCode(TEST_NAMESPACE1, "random", ProgramType.FLOW.getCategoryName())); } /** * Worker Specification tests */ @Test public void testWorkerSpecification() throws Exception { // deploy AppWithWorker in namespace1 and verify HttpResponse response = deploy(AppWithWorker.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); verifyProgramSpecification(TEST_NAMESPACE1, AppWithWorker.NAME, ProgramType.WORKER.getCategoryName(), AppWithWorker.WORKER); Assert.assertEquals(404, getProgramSpecificationResponseCode(TEST_NAMESPACE2, AppWithWorker.NAME, ProgramType.WORKER.getCategoryName(), AppWithWorker.WORKER)); } @Test public void testServiceSpecification() throws Exception { deploy(AppWithServices.class); HttpResponse response = doGet("/v3/namespaces/default/apps/AppWithServices/services/NoOpService"); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); Set<ServiceHttpEndpoint> expectedEndpoints = ImmutableSet.of(new ServiceHttpEndpoint("GET", "/ping"), new ServiceHttpEndpoint("POST", "/multi"), new ServiceHttpEndpoint("GET", "/multi"), new ServiceHttpEndpoint("GET", "/multi/ping")); GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.registerTypeAdapter(ServiceSpecification.class, new ServiceSpecificationCodec()); Gson gson = gsonBuilder.create(); ServiceSpecification specification = readResponse(response, ServiceSpecification.class, gson); Set<ServiceHttpEndpoint> returnedEndpoints = new HashSet<>(); for (HttpServiceHandlerSpecification httpServiceHandlerSpecification : specification.getHandlers() .values()) { returnedEndpoints.addAll(httpServiceHandlerSpecification.getEndpoints()); } Assert.assertEquals("NoOpService", specification.getName()); Assert.assertTrue(returnedEndpoints.equals(expectedEndpoints)); } /** * Program specification tests through appfabric apis. */ @Test public void testProgramSpecification() throws Exception { // deploy WordCountApp in namespace1 and verify HttpResponse response = deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); // deploy AppWithServices in namespace2 and verify response = deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); // deploy AppWithWorkflow in namespace2 and verify response = deploy(AppWithWorkflow.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); // deploy AppWithWorker in namespace1 and verify response = deploy(AppWithWorker.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); // verify program specification verifyProgramSpecification(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW.getCategoryName(), WORDCOUNT_FLOW_NAME); verifyProgramSpecification(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.MAPREDUCE.getCategoryName(), WORDCOUNT_MAPREDUCE_NAME); verifyProgramSpecification(TEST_NAMESPACE2, APP_WITH_SERVICES_APP_ID, ProgramType.SERVICE.getCategoryName(), APP_WITH_SERVICES_SERVICE_NAME); verifyProgramSpecification(TEST_NAMESPACE2, APP_WITH_WORKFLOW_APP_ID, ProgramType.WORKFLOW.getCategoryName(), APP_WITH_WORKFLOW_WORKFLOW_NAME); verifyProgramSpecification(TEST_NAMESPACE1, AppWithWorker.NAME, ProgramType.WORKER.getCategoryName(), AppWithWorker.WORKER); // verify invalid namespace Assert.assertEquals(404, getProgramSpecificationResponseCode(TEST_NAMESPACE1, APP_WITH_SERVICES_APP_ID, ProgramType.SERVICE.getCategoryName(), APP_WITH_SERVICES_SERVICE_NAME)); // verify invalid app Assert.assertEquals(404, getProgramSpecificationResponseCode(TEST_NAMESPACE2, APP_WITH_SERVICES_APP_ID, ProgramType.WORKFLOW.getCategoryName(), APP_WITH_WORKFLOW_WORKFLOW_NAME)); // verify invalid program type Assert.assertEquals(405, getProgramSpecificationResponseCode(TEST_NAMESPACE2, APP_WITH_SERVICES_APP_ID, "random", APP_WITH_WORKFLOW_WORKFLOW_NAME)); // verify invalid program type Assert.assertEquals(404, getProgramSpecificationResponseCode(TEST_NAMESPACE2, AppWithWorker.NAME, ProgramType.WORKER.getCategoryName(), AppWithWorker.WORKER)); } @Test public void testFlows() throws Exception { // deploy WordCountApp in namespace1 and verify HttpResponse response = deploy(WordCountApp.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); // check initial flowlet instances int initial = getFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME, WORDCOUNT_FLOWLET_NAME); Assert.assertEquals(1, initial); // request two more instances Assert.assertEquals(200, requestFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME, WORDCOUNT_FLOWLET_NAME, 3)); // verify int after = getFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME, WORDCOUNT_FLOWLET_NAME); Assert.assertEquals(3, after); // invalid namespace Assert.assertEquals(404, requestFlowletInstances(TEST_NAMESPACE2, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME, WORDCOUNT_FLOWLET_NAME, 3)); // invalid app Assert.assertEquals(404, requestFlowletInstances(TEST_NAMESPACE1, APP_WITH_SERVICES_APP_ID, WORDCOUNT_FLOW_NAME, WORDCOUNT_FLOWLET_NAME, 3)); // invalid flow Assert.assertEquals(404, requestFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, "random", WORDCOUNT_FLOWLET_NAME, 3)); // invalid flowlet Assert.assertEquals(404, requestFlowletInstances(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME, "random", 3)); // test live info // send invalid program type to live info response = sendLiveInfoRequest(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, "random", WORDCOUNT_FLOW_NAME); Assert.assertEquals(405, response.getStatusLine().getStatusCode()); // test valid live info JsonObject liveInfo = getLiveInfo(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW.getCategoryName(), WORDCOUNT_FLOW_NAME); Assert.assertEquals(WORDCOUNT_APP_NAME, liveInfo.get("app").getAsString()); Assert.assertEquals(ProgramType.FLOW.getPrettyName(), liveInfo.get("type").getAsString()); Assert.assertEquals(WORDCOUNT_FLOW_NAME, liveInfo.get("id").getAsString()); // start flow Id.Program wordcountFlow1 = Id.Program.from(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW, WORDCOUNT_FLOW_NAME); startProgram(wordcountFlow1); liveInfo = getLiveInfo(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, ProgramType.FLOW.getCategoryName(), WORDCOUNT_FLOW_NAME); Assert.assertEquals(WORDCOUNT_APP_NAME, liveInfo.get("app").getAsString()); Assert.assertEquals(ProgramType.FLOW.getPrettyName(), liveInfo.get("type").getAsString()); Assert.assertEquals(WORDCOUNT_FLOW_NAME, liveInfo.get("id").getAsString()); Assert.assertEquals("in-memory", liveInfo.get("runtime").getAsString()); // should not delete queues while running Assert.assertEquals(403, deleteQueues(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME)); Assert.assertEquals(403, deleteQueues(TEST_NAMESPACE1)); // stop stopProgram(wordcountFlow1); // delete queues Assert.assertEquals(200, deleteQueues(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME)); } @Test public void testMultipleWorkflowSchedules() throws Exception { // Deploy the app HttpResponse response = deploy(AppWithMultipleScheduledWorkflows.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); List<ScheduleSpecification> someSchedules = getSchedules(TEST_NAMESPACE2, APP_WITH_MULTIPLE_WORKFLOWS_APP_NAME, APP_WITH_MULTIPLE_WORKFLOWS_SOMEWORKFLOW); Assert.assertEquals(2, someSchedules.size()); Assert.assertEquals(APP_WITH_MULTIPLE_WORKFLOWS_SOMEWORKFLOW, someSchedules.get(0).getProgram().getProgramName()); Assert.assertEquals(APP_WITH_MULTIPLE_WORKFLOWS_SOMEWORKFLOW, someSchedules.get(1).getProgram().getProgramName()); List<ScheduleSpecification> anotherSchedules = getSchedules(TEST_NAMESPACE2, APP_WITH_MULTIPLE_WORKFLOWS_APP_NAME, APP_WITH_MULTIPLE_WORKFLOWS_ANOTHERWORKFLOW); Assert.assertEquals(3, anotherSchedules.size()); Assert.assertEquals(APP_WITH_MULTIPLE_WORKFLOWS_ANOTHERWORKFLOW, anotherSchedules.get(0).getProgram().getProgramName()); Assert.assertEquals(APP_WITH_MULTIPLE_WORKFLOWS_ANOTHERWORKFLOW, anotherSchedules.get(1).getProgram().getProgramName()); Assert.assertEquals(APP_WITH_MULTIPLE_WORKFLOWS_ANOTHERWORKFLOW, anotherSchedules.get(2).getProgram().getProgramName()); } @Test public void testServices() throws Exception { HttpResponse response = deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); Id.Service service1 = Id.Service.from(Id.Namespace.from(TEST_NAMESPACE1), APP_WITH_SERVICES_APP_ID, APP_WITH_SERVICES_SERVICE_NAME); Id.Service service2 = Id.Service.from(Id.Namespace.from(TEST_NAMESPACE2), APP_WITH_SERVICES_APP_ID, APP_WITH_SERVICES_SERVICE_NAME); // start service in wrong namespace startProgram(service1, 404); startProgram(service2); // verify instances try { getServiceInstances(service1); Assert.fail("Should not find service in " + TEST_NAMESPACE1); } catch (AssertionError expected) { // expected } ServiceInstances instances = getServiceInstances(service2); Assert.assertEquals(1, instances.getRequested()); Assert.assertEquals(1, instances.getProvisioned()); // request 2 additional instances int code = setServiceInstances(service1, 3); Assert.assertEquals(404, code); code = setServiceInstances(service2, 3); Assert.assertEquals(200, code); // verify that additional instances were provisioned instances = getServiceInstances(service2); Assert.assertEquals(3, instances.getRequested()); Assert.assertEquals(3, instances.getProvisioned()); // verify that endpoints are not available in the wrong namespace response = callService(service1, HttpMethod.POST, "multi"); code = response.getStatusLine().getStatusCode(); Assert.assertEquals(404, code); response = callService(service1, HttpMethod.GET, "multi/ping"); code = response.getStatusLine().getStatusCode(); Assert.assertEquals(404, code); // stop service stopProgram(service1, 404); stopProgram(service2); } @Test public void testDeleteQueues() throws Exception { QueueName queueName = QueueName.fromFlowlet(TEST_NAMESPACE1, WORDCOUNT_APP_NAME, WORDCOUNT_FLOW_NAME, WORDCOUNT_FLOWLET_NAME, "out"); // enqueue some data enqueue(queueName, new QueueEntry("x".getBytes(Charsets.UTF_8))); // verify it exists Assert.assertTrue(dequeueOne(queueName)); // clear queue in wrong namespace Assert.assertEquals(200, doDelete("/v3/namespaces/" + TEST_NAMESPACE2 + "/queues").getStatusLine().getStatusCode()); // verify queue is still here Assert.assertTrue(dequeueOne(queueName)); // clear queue in the right namespace Assert.assertEquals(200, doDelete("/v3/namespaces/" + TEST_NAMESPACE1 + "/queues").getStatusLine().getStatusCode()); // verify queue is gone Assert.assertFalse(dequeueOne(queueName)); } @After public void cleanup() throws Exception { doDelete(getVersionedAPIPath("apps/", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE1)); doDelete(getVersionedAPIPath("apps/", Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2)); } // TODO: Duplicate from AppFabricHttpHandlerTest, remove the AppFabricHttpHandlerTest method after deprecating v2 APIs private void enqueue(QueueName queueName, final QueueEntry queueEntry) throws Exception { QueueClientFactory queueClientFactory = AppFabricTestBase.getInjector() .getInstance(QueueClientFactory.class); final QueueProducer producer = queueClientFactory.createProducer(queueName); // doing inside tx TransactionExecutorFactory txExecutorFactory = AppFabricTestBase.getInjector() .getInstance(TransactionExecutorFactory.class); txExecutorFactory.createExecutor(ImmutableList.of((TransactionAware) producer)) .execute(new TransactionExecutor.Subroutine() { @Override public void apply() throws Exception { // write more than one so that we can dequeue multiple times for multiple checks // we only dequeue twice, but ensure that the drop queues call drops the rest of the entries as well int numEntries = 0; while (numEntries++ < 5) { producer.enqueue(queueEntry); } } }); } private boolean dequeueOne(QueueName queueName) throws Exception { QueueClientFactory queueClientFactory = AppFabricTestBase.getInjector() .getInstance(QueueClientFactory.class); final QueueConsumer consumer = queueClientFactory.createConsumer(queueName, new ConsumerConfig(1L, 0, 1, DequeueStrategy.ROUND_ROBIN, null), 1); // doing inside tx TransactionExecutorFactory txExecutorFactory = AppFabricTestBase.getInjector() .getInstance(TransactionExecutorFactory.class); return txExecutorFactory.createExecutor(ImmutableList.of((TransactionAware) consumer)) .execute(new Callable<Boolean>() { @Override public Boolean call() throws Exception { return !consumer.dequeue(1).isEmpty(); } }); } private ServiceInstances getServiceInstances(Id.Service serviceId) throws Exception { String instanceUrl = String.format("apps/%s/services/%s/instances", serviceId.getApplicationId(), serviceId.getId()); String versionedInstanceUrl = getVersionedAPIPath(instanceUrl, Constants.Gateway.API_VERSION_3_TOKEN, serviceId.getNamespaceId()); HttpResponse response = doGet(versionedInstanceUrl); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); return readResponse(response, ServiceInstances.class); } private int setServiceInstances(Id.Service serviceId, int instances) throws Exception { String instanceUrl = String.format("apps/%s/services/%s/instances", serviceId.getApplicationId(), serviceId.getId()); String versionedInstanceUrl = getVersionedAPIPath(instanceUrl, Constants.Gateway.API_VERSION_3_TOKEN, serviceId.getNamespaceId()); String instancesBody = GSON.toJson(new Instances(instances)); return doPut(versionedInstanceUrl, instancesBody).getStatusLine().getStatusCode(); } private HttpResponse callService(Id.Service serviceId, HttpMethod method, String endpoint) throws Exception { String serviceUrl = String.format("apps/%s/service/%s/methods/%s", serviceId.getApplicationId(), serviceId.getId(), endpoint); String versionedServiceUrl = getVersionedAPIPath(serviceUrl, Constants.Gateway.API_VERSION_3_TOKEN, serviceId.getNamespaceId()); if (HttpMethod.GET.equals(method)) { return doGet(versionedServiceUrl); } else if (HttpMethod.POST.equals(method)) { return doPost(versionedServiceUrl); } throw new IllegalArgumentException("Only GET and POST supported right now."); } private int deleteQueues(String namespace) throws Exception { String versionedDeleteUrl = getVersionedAPIPath("queues", Constants.Gateway.API_VERSION_3_TOKEN, namespace); HttpResponse response = doDelete(versionedDeleteUrl); return response.getStatusLine().getStatusCode(); } private int deleteQueues(String namespace, String appId, String flow) throws Exception { String deleteQueuesUrl = String.format("apps/%s/flows/%s/queues", appId, flow); String versionedDeleteUrl = getVersionedAPIPath(deleteQueuesUrl, Constants.Gateway.API_VERSION_3_TOKEN, namespace); HttpResponse response = doDelete(versionedDeleteUrl); return response.getStatusLine().getStatusCode(); } private JsonObject getLiveInfo(String namespace, String appId, String programType, String programId) throws Exception { HttpResponse response = sendLiveInfoRequest(namespace, appId, programType, programId); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); return readResponse(response, JsonObject.class); } private HttpResponse sendLiveInfoRequest(String namespace, String appId, String programType, String programId) throws Exception { String liveInfoUrl = String.format("apps/%s/%s/%s/live-info", appId, programType, programId); String versionedUrl = getVersionedAPIPath(liveInfoUrl, Constants.Gateway.API_VERSION_3_TOKEN, namespace); return doGet(versionedUrl); } private int requestFlowletInstances(String namespace, String appId, String flow, String flowlet, int noRequested) throws Exception { String flowletInstancesVersionedUrl = getFlowletInstancesVersionedUrl(namespace, appId, flow, flowlet); JsonObject instances = new JsonObject(); instances.addProperty("instances", noRequested); String body = GSON.toJson(instances); return doPut(flowletInstancesVersionedUrl, body).getStatusLine().getStatusCode(); } private int getFlowletInstances(String namespace, String appId, String flow, String flowlet) throws Exception { String flowletInstancesUrl = getFlowletInstancesVersionedUrl(namespace, appId, flow, flowlet); String response = readResponse(doGet(flowletInstancesUrl)); Assert.assertNotNull(response); JsonObject instances = GSON.fromJson(response, JsonObject.class); Assert.assertTrue(instances.has("instances")); return instances.get("instances").getAsInt(); } private String getFlowletInstancesVersionedUrl(String namespace, String appId, String flow, String flowlet) { String flowletInstancesUrl = String.format("apps/%s/%s/%s/flowlets/%s/instances", appId, ProgramType.FLOW.getCategoryName(), flow, flowlet); return getVersionedAPIPath(flowletInstancesUrl, Constants.Gateway.API_VERSION_3_TOKEN, namespace); } private void verifyProgramSpecification(String namespace, String appId, String programType, String programId) throws Exception { JsonObject programSpec = getProgramSpecification(namespace, appId, programType, programId); Assert.assertTrue( programSpec.has("className") && programSpec.has("name") && programSpec.has("description")); Assert.assertEquals(programId, programSpec.get("name").getAsString()); } private JsonObject getProgramSpecification(String namespace, String appId, String programType, String programId) throws Exception { HttpResponse response = requestProgramSpecification(namespace, appId, programType, programId); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); String result = EntityUtils.toString(response.getEntity()); Assert.assertNotNull(result); return GSON.fromJson(result, JsonObject.class); } private int getProgramSpecificationResponseCode(String namespace, String appId, String programType, String programId) throws Exception { HttpResponse response = requestProgramSpecification(namespace, appId, programType, programId); return response.getStatusLine().getStatusCode(); } private HttpResponse requestProgramSpecification(String namespace, String appId, String programType, String programId) throws Exception { String uri = getVersionedAPIPath(String.format("apps/%s/%s/%s", appId, programType, programId), Constants.Gateway.API_VERSION_3_TOKEN, namespace); return doGet(uri); } private void testListInitialState(String namespace, ProgramType programType) throws Exception { HttpResponse response = doGet(getVersionedAPIPath(programType.getCategoryName(), Constants.Gateway.API_VERSION_3_TOKEN, namespace)); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); Assert.assertEquals(EMPTY_ARRAY_JSON, readResponse(response)); } private void verifyProgramList(String namespace, ProgramType programType, int expected) throws Exception { HttpResponse response = requestProgramList(namespace, programType.getCategoryName()); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); String json = EntityUtils.toString(response.getEntity()); List<Map<String, String>> programs = GSON.fromJson(json, LIST_MAP_STRING_STRING_TYPE); Assert.assertEquals(expected, programs.size()); } private void verifyProgramList(String namespace, String appName, final ProgramType programType, int expected) throws Exception { HttpResponse response = requestAppDetail(namespace, appName); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); String json = EntityUtils.toString(response.getEntity()); ApplicationDetail appDetail = GSON.fromJson(json, ApplicationDetail.class); Collection<ProgramRecord> programs = Collections2.filter(appDetail.getPrograms(), new Predicate<ProgramRecord>() { @Override public boolean apply(@Nullable ProgramRecord record) { return programType.getCategoryName().equals(record.getType().getCategoryName()); } }); Assert.assertEquals(expected, programs.size()); } private int getAppFDetailResponseCode(String namespace, @Nullable String appName, String programType) throws Exception { HttpResponse response = requestAppDetail(namespace, appName); return response.getStatusLine().getStatusCode(); } private HttpResponse requestProgramList(String namespace, String programType) throws Exception { return doGet(getVersionedAPIPath(programType, Constants.Gateway.API_VERSION_3_TOKEN, namespace)); } private HttpResponse requestAppDetail(String namespace, String appName) throws Exception { String uri = getVersionedAPIPath(String.format("apps/%s", appName), Constants.Gateway.API_VERSION_3_TOKEN, namespace); return doGet(uri); } private void verifyInitialBatchStatusOutput(HttpResponse response) throws IOException { Assert.assertEquals(200, response.getStatusLine().getStatusCode()); List<JsonObject> returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE); for (JsonObject obj : returnedBody) { Assert.assertEquals(200, obj.get("statusCode").getAsInt()); Assert.assertEquals(STOPPED, obj.get("status").getAsString()); } } private void verifyInitialBatchInstanceOutput(HttpResponse response) throws IOException { Assert.assertEquals(200, response.getStatusLine().getStatusCode()); List<JsonObject> returnedBody = readResponse(response, LIST_OF_JSONOBJECT_TYPE); for (JsonObject obj : returnedBody) { Assert.assertEquals(200, obj.get("statusCode").getAsInt()); Assert.assertEquals(1, obj.get("requested").getAsInt()); Assert.assertEquals(0, obj.get("provisioned").getAsInt()); } } private void testHistory(Class<?> app, Id.Program program) throws Exception { try { String namespace = program.getNamespaceId(); deploy(app, Constants.Gateway.API_VERSION_3_TOKEN, namespace); // first run startProgram(program); waitState(program, ProgramRunStatus.RUNNING.toString()); stopProgram(program); waitState(program, STOPPED); // second run startProgram(program); waitState(program, ProgramRunStatus.RUNNING.toString()); String url = String.format("apps/%s/%s/%s/runs?status=running", program.getApplicationId(), program.getType().getCategoryName(), program.getId()); //active size should be 1 historyStatusWithRetry(getVersionedAPIPath(url, Constants.Gateway.API_VERSION_3_TOKEN, namespace), 1); // completed runs size should be 1 url = String.format("apps/%s/%s/%s/runs?status=killed", program.getApplicationId(), program.getType().getCategoryName(), program.getId()); historyStatusWithRetry(getVersionedAPIPath(url, Constants.Gateway.API_VERSION_3_TOKEN, namespace), 1); stopProgram(program); waitState(program, STOPPED); historyStatusWithRetry(getVersionedAPIPath(url, Constants.Gateway.API_VERSION_3_TOKEN, namespace), 2); } catch (Exception e) { // Log exception before finally block is called LOG.error("Got exception: ", e); } finally { HttpResponse httpResponse = doDelete(getVersionedAPIPath("apps/" + program.getApplicationId(), Constants.Gateway.API_VERSION_3_TOKEN, program.getNamespaceId())); Assert.assertEquals(EntityUtils.toString(httpResponse.getEntity()), 200, httpResponse.getStatusLine().getStatusCode()); } } private void historyStatusWithRetry(String url, int size) throws Exception { int trials = 0; while (trials++ < 5) { HttpResponse response = doGet(url); List<RunRecord> result = GSON.fromJson(EntityUtils.toString(response.getEntity()), LIST_OF_RUN_RECORD); if (result != null && result.size() >= size) { for (RunRecord m : result) { assertRunRecord(String.format("%s/%s", url.substring(0, url.indexOf("?")), m.getPid()), GSON.fromJson(GSON.toJson(m), RunRecord.class)); } break; } TimeUnit.SECONDS.sleep(1); } Assert.assertTrue(trials < 5); } private void assertRunRecord(String url, RunRecord expectedRunRecord) throws Exception { HttpResponse response = doGet(url); RunRecord actualRunRecord = GSON.fromJson(EntityUtils.toString(response.getEntity()), RunRecord.class); Assert.assertEquals(expectedRunRecord, actualRunRecord); } private void testRuntimeArgs(Class<?> app, String namespace, String appId, String programType, String programId) throws Exception { deploy(app, Constants.Gateway.API_VERSION_3_TOKEN, namespace); Map<String, String> args = Maps.newHashMap(); args.put("Key1", "Val1"); args.put("Key2", "Val1"); args.put("Key2", "Val1"); HttpResponse response; String argString = GSON.toJson(args, new TypeToken<Map<String, String>>() { }.getType()); String versionedRuntimeArgsUrl = getVersionedAPIPath( "apps/" + appId + "/" + programType + "/" + programId + "/runtimeargs", Constants.Gateway.API_VERSION_3_TOKEN, namespace); response = doPut(versionedRuntimeArgsUrl, argString); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); response = doGet(versionedRuntimeArgsUrl); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); Map<String, String> argsRead = GSON.fromJson(EntityUtils.toString(response.getEntity()), new TypeToken<Map<String, String>>() { }.getType()); Assert.assertEquals(args.size(), argsRead.size()); for (Map.Entry<String, String> entry : args.entrySet()) { Assert.assertEquals(entry.getValue(), argsRead.get(entry.getKey())); } //test empty runtime args response = doPut(versionedRuntimeArgsUrl, ""); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); response = doGet(versionedRuntimeArgsUrl); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); argsRead = GSON.fromJson(EntityUtils.toString(response.getEntity()), new TypeToken<Map<String, String>>() { }.getType()); Assert.assertEquals(0, argsRead.size()); //test null runtime args response = doPut(versionedRuntimeArgsUrl, null); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); response = doGet(versionedRuntimeArgsUrl); Assert.assertEquals(200, response.getStatusLine().getStatusCode()); argsRead = GSON.fromJson(EntityUtils.toString(response.getEntity()), new TypeToken<Map<String, String>>() { }.getType()); Assert.assertEquals(0, argsRead.size()); } }