org.finra.dm.service.EmrServiceTest.java Source code

Java tutorial

Introduction

Here is the source code for org.finra.dm.service.EmrServiceTest.java

Source

/*
* Copyright 2015 herd contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.finra.dm.service;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.Serializable;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.xml.bind.JAXBException;

import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.elasticmapreduce.model.ListStepsRequest;
import com.amazonaws.services.elasticmapreduce.model.ListStepsResult;
import com.amazonaws.services.elasticmapreduce.model.StepSummary;
import org.apache.commons.io.IOUtils;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.client.WorkflowJob;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;

import org.finra.dm.dao.EmrOperations;
import org.finra.dm.dao.impl.MockAwsOperationsHelper;
import org.finra.dm.dao.impl.MockEc2OperationsImpl;
import org.finra.dm.dao.impl.MockEmrOperationsImpl;
import org.finra.dm.dao.impl.MockOozieOperationsImpl;
import org.finra.dm.dao.impl.OozieDaoImpl;
import org.finra.dm.model.ObjectNotFoundException;
import org.finra.dm.model.dto.ConfigurationValue;
import org.finra.dm.model.dto.EmrClusterAlternateKeyDto;
import org.finra.dm.model.jpa.EmrClusterCreationLogEntity;
import org.finra.dm.model.jpa.NamespaceEntity;
import org.finra.dm.model.api.xml.ConfigurationFiles;
import org.finra.dm.model.api.xml.EmrCluster;
import org.finra.dm.model.api.xml.EmrClusterCreateRequest;
import org.finra.dm.model.api.xml.EmrClusterDefinition;
import org.finra.dm.model.api.xml.EmrClusterDefinitionApplication;
import org.finra.dm.model.api.xml.EmrClusterDefinitionConfiguration;
import org.finra.dm.model.api.xml.EmrHadoopJarStep;
import org.finra.dm.model.api.xml.EmrHadoopJarStepAddRequest;
import org.finra.dm.model.api.xml.EmrHiveStepAddRequest;
import org.finra.dm.model.api.xml.EmrMasterSecurityGroup;
import org.finra.dm.model.api.xml.EmrMasterSecurityGroupAddRequest;
import org.finra.dm.model.api.xml.EmrOozieStepAddRequest;
import org.finra.dm.model.api.xml.EmrPigStepAddRequest;
import org.finra.dm.model.api.xml.EmrShellStep;
import org.finra.dm.model.api.xml.EmrShellStepAddRequest;
import org.finra.dm.model.api.xml.HadoopJarStep;
import org.finra.dm.model.api.xml.InstanceDefinition;
import org.finra.dm.model.api.xml.KeyValuePairConfigurations;
import org.finra.dm.model.api.xml.NodeTag;
import org.finra.dm.model.api.xml.OozieWorkflowAction;
import org.finra.dm.model.api.xml.OozieWorkflowJob;
import org.finra.dm.model.api.xml.Parameter;
import org.finra.dm.model.api.xml.RunOozieWorkflowRequest;
import org.finra.dm.model.api.xml.ScriptDefinition;
import org.finra.dm.service.helper.EmrStepHelper;

/**
 * This class tests functionality within the EmrService.
 */
public class EmrServiceTest extends AbstractServiceTest {
    @Autowired
    @Qualifier(value = "emrServiceImpl")
    private EmrService emrServiceImpl;

    @Autowired
    EmrOperations emrOperations;

    /**
     * This method tests the happy path scenario for adding security groups
     */
    @Test
    public void testAddSecurityGroup() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);

        // Create the Add security group.
        EmrMasterSecurityGroupAddRequest emrMasterSecurityGroupAddRequest = getNewEmrAddSecurityGroupMasterRequest(
                request.getEmrClusterName());
        EmrMasterSecurityGroup emrMasterSecurityGroup = emrService
                .addSecurityGroupsToClusterMaster(emrMasterSecurityGroupAddRequest);

        // Validate the returned object against the input.
        assertNotNull(emrMasterSecurityGroup);
        assertTrue(emrMasterSecurityGroup.getNamespace().equals(request.getNamespace()));
        assertTrue(
                emrMasterSecurityGroup.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrMasterSecurityGroup.getEmrClusterName().equals(request.getEmrClusterName()));
    }

    /**
     * This method tests the scenario where cluster does not exist
     */
    @Test(expected = IllegalArgumentException.class)
    public void testAddSecurityGroupClusterDoesNotExist() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        // Create the Add security group.
        EmrMasterSecurityGroupAddRequest emrMasterSecurityGroupAddRequest = getNewEmrAddSecurityGroupMasterRequest(
                "DOES_NOT_EXIST");
        emrService.addSecurityGroupsToClusterMaster(emrMasterSecurityGroupAddRequest);
    }

    /**
     * This method tests the scenario where EC2 instances have not been provisioned yet
     */
    @Test(expected = IllegalArgumentException.class)
    public void testAddSecurityGroupEC2InstanceNotProvisioned() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        request.setEmrClusterName(MockEmrOperationsImpl.MOCK_CLUSTER_NOT_PROVISIONED_NAME);
        emrService.createCluster(request);

        // Create the Add security group.
        EmrMasterSecurityGroupAddRequest emrMasterSecurityGroupAddRequest = getNewEmrAddSecurityGroupMasterRequest(
                request.getEmrClusterName());
        emrService.addSecurityGroupsToClusterMaster(emrMasterSecurityGroupAddRequest);
    }

    /**
     * This method tests the scenario where AmazonServiceException is thrown
     */
    @Test(expected = AmazonServiceException.class)
    public void testAddSecurityGroupAmazonException() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);

        // Create the Add security group.
        EmrMasterSecurityGroupAddRequest emrMasterSecurityGroupAddRequest = getNewEmrAddSecurityGroupMasterRequest(
                request.getEmrClusterName());
        emrMasterSecurityGroupAddRequest.getSecurityGroupIds().clear();
        emrMasterSecurityGroupAddRequest.getSecurityGroupIds()
                .add(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION);
        emrService.addSecurityGroupsToClusterMaster(emrMasterSecurityGroupAddRequest);
    }

    /**
     * This method tests the scenario when at least one security group must be specified
     */
    @Test(expected = IllegalArgumentException.class)
    public void testAddSecurityGroupNoneSpecified() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);

        // Create the Add security group.
        EmrMasterSecurityGroupAddRequest emrMasterSecurityGroupAddRequest = getNewEmrAddSecurityGroupMasterRequest(
                request.getEmrClusterName());
        emrMasterSecurityGroupAddRequest.setSecurityGroupIds(null);
        emrService.addSecurityGroupsToClusterMaster(emrMasterSecurityGroupAddRequest);
    }

    /**
     * This method tests the happy path scenario by providing all the parameters
     */
    @Test
    public void testCreateEmrCluster() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String definitionXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
        EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper
                .unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        EmrCluster emrCluster = emrService.createCluster(request);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
        assertNotNull(emrCluster.getId());
        assertNull(emrCluster.isDryRun());
        assertNotNull(emrCluster.getEmrClusterDefinition());
        assertTrue(emrCluster.isEmrClusterCreated());
        assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());

        validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
    }

    /**
     * This method tests the cluster creation with the startup hadoop jar steps being added.
     */
    @Test
    public void testCreateEmrClusterStartupSteps() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String definitionXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
        EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper
                .unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        EmrCluster emrCluster = emrService.createCluster(request);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
        assertNotNull(emrCluster.getId());
        assertNull(emrCluster.isDryRun());
        assertNotNull(emrCluster.getEmrClusterDefinition());
        assertTrue(emrCluster.isEmrClusterCreated());
        assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());

        validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);

        ListStepsRequest listStepsRequest = (new ListStepsRequest()).withClusterId(emrCluster.getId());

        ListStepsResult listStepsResult = emrOperations.listStepsRequest(null, listStepsRequest);
        List<StepSummary> addedSteps = listStepsResult.getSteps();
        // Validate that the step was added.
        for (HadoopJarStep hadoopJarStep : expectedEmrClusterDefinition.getHadoopJarSteps()) {
            boolean stepFound = false;

            for (StepSummary stepSummary : addedSteps) {
                if (stepSummary.getName().equals(hadoopJarStep.getStepName())) {
                    // Step found
                    stepFound = true;
                    break;
                }
            }

            assertTrue(stepFound);
        }
    }

    /**
     * <p> Validates whether EMR cluster creation log entries for the given cluster's namespace, definition name and cluster name. </p> <p> Asserts that: </p>
     * <ul> <li>There is exactly 1 log for the given data.</li> <li>The contents of the log matches the given expected.</li> </ul>
     *
     * @param emrCluster the EMR cluster.
     * @param expectedEmrClusterDefinition the expected EMR cluster definition.
     *
     * @throws JAXBException
     */
    private void validateEmrClusterCreationLogUnique(EmrCluster emrCluster,
            EmrClusterDefinition expectedEmrClusterDefinition) throws JAXBException {
        String namespace = emrCluster.getNamespace();
        String emrClusterDefinitionName = emrCluster.getEmrClusterDefinitionName();
        String emrClusterName = emrCluster.getEmrClusterName();

        System.out.println(String.format("%s, %s, %s", namespace, emrClusterDefinitionName, emrClusterName));

        List<EmrClusterCreationLogEntity> list = getEmrClusterCreationLogEntities(namespace,
                emrClusterDefinitionName, emrClusterName);
        assertEquals("EMR cluster creation log size", 1, list.size());
        EmrClusterCreationLogEntity log = list.get(0);
        assertEquals("EMR cluster creation log cluster ID", emrCluster.getId(), log.getEmrClusterId());
        assertEquals("EMR cluster creation log namespace", namespace, log.getNamespace().getCode());

        String expectedDefinitionXml = xmlHelper.objectToXml(expectedEmrClusterDefinition);
        assertEquals("EMR cluster creation log definition", expectedDefinitionXml, log.getEmrClusterDefinition());
    }

    /**
     * Asserts that a log entry for the given cluster's namespace, definition name, and cluster name does not exist.
     *
     * @param emrCluster the EMR cluster.
     */
    private void assertEmrClusterCreationLogNotExist(EmrCluster emrCluster) {
        List<EmrClusterCreationLogEntity> list = getEmrClusterCreationLogEntities(emrCluster.getNamespace(),
                emrCluster.getEmrClusterDefinitionName(), emrCluster.getEmrClusterName());
        assertTrue("EMR cluster creation log empty", list.isEmpty());
    }

    /**
     * This method tests the multiple bootstrap scripts.
     */
    @Test
    public void testCreateEmrClusterMultipleBootstrap() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String configXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
        EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class,
                configXml);

        ScriptDefinition allScript = emrClusterDefinition.getCustomBootstrapActionAll().get(0);
        ScriptDefinition secondAllScript = new ScriptDefinition();
        secondAllScript.setScriptName(allScript.getScriptName() + "_second");
        secondAllScript.setScriptLocation(allScript.getScriptLocation());
        secondAllScript.setScriptArguments(allScript.getScriptArguments());
        emrClusterDefinition.getCustomBootstrapActionAll().add(secondAllScript);

        ScriptDefinition masterScript = emrClusterDefinition.getCustomBootstrapActionMaster().get(0);
        ScriptDefinition secondMasterScript = new ScriptDefinition();
        secondMasterScript.setScriptName(masterScript.getScriptName() + "_second");
        secondMasterScript.setScriptLocation(masterScript.getScriptLocation());
        secondMasterScript.setScriptArguments(masterScript.getScriptArguments());
        emrClusterDefinition.getCustomBootstrapActionMaster().add(secondMasterScript);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME,
                xmlHelper.objectToXml(emrClusterDefinition));

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        EmrCluster emrCluster = emrService.createCluster(request);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
    }

    /**
     * This method tests the blank values for various parameters
     */
    @Test
    public void testCreateEmrClusterBlankParams() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String configXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());

        EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class,
                configXml);
        List<ScriptDefinition> scriptDefinitions = emrClusterDefinition.getCustomBootstrapActionAll();
        scriptDefinitions.get(0).setScriptArguments(null);
        emrClusterDefinition.setCustomBootstrapActionAll(scriptDefinitions);
        scriptDefinitions = emrClusterDefinition.getCustomBootstrapActionMaster();
        scriptDefinitions.get(0).setScriptArguments(null);
        emrClusterDefinition.setCustomBootstrapActionMaster(scriptDefinitions);
        emrClusterDefinition.setVisibleToAll(null);
        emrClusterDefinition.setServiceIamRole(null);
        emrClusterDefinition.setAmiVersion(null);
        emrClusterDefinition.setServiceIamRole(emrClusterDefinition.getEc2NodeIamProfileName());

        configXml = xmlHelper.objectToXml(emrClusterDefinition);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);
    }

    /**
     * This method tests the service IAM role
     */
    @Test
    public void testCreateEmrClusterServiceRole() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String configXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());

        EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class,
                configXml);
        emrClusterDefinition.setServiceIamRole(emrClusterDefinition.getEc2NodeIamProfileName());

        configXml = xmlHelper.objectToXml(emrClusterDefinition);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);

        // Create a new EMR cluster create request.
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);
    }

    /**
     * This method tests supported product
     */
    @Test
    public void testCreateEmrClusterSupportedProduct() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String configXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream());

        EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class,
                configXml);

        // Set the supported product
        emrClusterDefinition.setSupportedProduct("mapr-m3");
        configXml = xmlHelper.objectToXml(emrClusterDefinition);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);

        // Create a new EMR cluster create request.
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);
    }

    /**
     * This method tests additionalInfo
     */
    @Test
    public void testCreateEmrClusterAdditionalInfo() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String configXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream());

        EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class,
                configXml);

        // Set the additional Info
        emrClusterDefinition.setAdditionalInfo(
                "{ami64: \"ami-e82af080\", amiHvm64: \"ami-e82af080\", hadoopVersion: \"2.4.0\", hadoopConfigurationVersion: \"3.1\"}");

        configXml = xmlHelper.objectToXml(emrClusterDefinition);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);

        // Create a new EMR cluster create request.
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);
    }

    /**
     * This method tests the error cases for AmazonExceptions for Illegal Argument
     */
    @Test(expected = IllegalArgumentException.class)
    public void testCreateEmrClusterAmazonBadRequest() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String configXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream());

        EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class,
                configXml);
        emrClusterDefinition.setAmiVersion(MockAwsOperationsHelper.AMAZON_BAD_REQUEST);

        configXml = xmlHelper.objectToXml(emrClusterDefinition);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);

        // Create a new EMR cluster create request.
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);
    }

    /**
     * This method tests the error cases for Illegal Arguments for Instances not defined properly
     */
    @Test
    public void testCreateEmrClusterInstanceNotDefined() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        // Create the test EMR cluster definition entity with missing instance definitions.
        String configXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream());
        EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class,
                configXml);
        emrClusterDefinition.setInstanceDefinitions(null);
        configXml = xmlHelper.objectToXml(emrClusterDefinition);
        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);

        // Try to create a new EMR cluster using EMR cluster definition with missing instance definitions.
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        try {
            emrService.createCluster(request);
            fail("Should throw an IllegalArgumentException when instance definitions are not specified.");
        } catch (IllegalArgumentException e) {
            assertEquals("Instance definitions must be specified.", e.getMessage());
        }
    }

    /**
     * This method tests the error cases for AmazonExceptions for Illegal Argument
     */
    @Test(expected = ObjectNotFoundException.class)
    public void testCreateEmrClusterAmazonObjectNotFound() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String configXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream());

        EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class,
                configXml);
        emrClusterDefinition.setAmiVersion(MockAwsOperationsHelper.AMAZON_NOT_FOUND);

        configXml = xmlHelper.objectToXml(emrClusterDefinition);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);

        // Create a new EMR cluster create request.
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);
    }

    /**
     * This method tests the error cases for AmazonExceptions for AmazonServiceException
     */
    @Test(expected = AmazonServiceException.class)
    public void testCreateEmrClusterAmazonOtherException() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String configXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream());

        EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class,
                configXml);
        emrClusterDefinition.setAmiVersion(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION);

        configXml = xmlHelper.objectToXml(emrClusterDefinition);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);

        // Create a new EMR cluster create request.
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);
    }

    /**
     * This method tests some of the negative test cases
     */
    @Test
    public void testCreateEmrClusterWrongInstanceConfigs() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream()));

        // Create a new EMR cluster create request.
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        EmrCluster emrCluster = emrService.createCluster(request);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));

        // Calling again must return the same cluster id
        emrService.createCluster(request);
    }

    /**
     * This method tests the scenario in which the namespaceCd is invalid ObjectNotFoundException is expected to be thrown
     */
    @Test(expected = ObjectNotFoundException.class)
    public void testCreateEmrClusterInvalidNamespace() throws Exception {
        // Create the emr cluster request without registering the namespace entity - ${TEST_ACTIVITI_NAMESPACE_CD}
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();

        // Following method must throw ObjectNotFoundException, as the namespace entity ${TEST_ACTIVITI_NAMESPACE_CD} does not exist.
        emrService.createCluster(request);
    }

    /**
     * This method tests the scenario in which the cluster name is blank for the method getEmrClusterIdByName
     */
    @Test
    public void testGetEmrClusterIdByNameForBlank() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream()));

        // Create a new EMR cluster create request.
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);
    }

    /**
     * This method tests the scenario in which the cluster definition is invalid ObjectNotFoundException is expected to be thrown
     */
    @Test(expected = ObjectNotFoundException.class)
    public void testCreateEmrClusterInvalidDefinition() throws Exception {
        // Create the namespace entity.
        createNamespaceEntity(NAMESPACE_CD);

        // Create the emr cluster request without registering the namespace entity - ${TEST_ACTIVITI_NAMESPACE_CD}
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();

        // Following method must throw ObjectNotFoundException, as the namespace entity ${TEST_ACTIVITI_NAMESPACE_CD} does not exist.
        emrService.createCluster(request);
    }

    /**
     * This method tests the scenario where task instances are there
     */
    @Test
    public void testCreateEmrClusterWithTaskInstances() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String configXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());

        EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class,
                configXml);
        InstanceDefinition taskDef = new InstanceDefinition();
        taskDef.setInstanceCount(1);

        // This could be any EC2 instance type supported in EMR.
        taskDef.setInstanceType(MockEc2OperationsImpl.INSTANCE_TYPE_1);
        emrClusterDefinition.getInstanceDefinitions().setTaskInstances(taskDef);

        configXml = xmlHelper.objectToXml(emrClusterDefinition);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);

        // Create a new EMR cluster create request.
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        EmrCluster emrCluster = emrService.createCluster(request);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
    }

    /**
     * This method tests the scenario where task instances are there
     */
    @Test
    public void testCreateEmrClusterMandatoryTagsNull() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        // Create a new EMR cluster create request.
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();

        // Remove property source so that we can test mandatory tags config being null
        Map<String, Object> overrideMap = new HashMap<>();
        overrideMap.put(ConfigurationValue.MANDATORY_AWS_TAGS.getKey(), null);
        modifyPropertySourceInEnvironment(overrideMap);

        EmrCluster emrCluster = emrService.createCluster(request);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));

        restorePropertySourceInEnvironment();
    }

    @Test
    public void testCreateEmrClusterDryRunFalseNoOverride() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String definitionXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
        EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper
                .unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();

        // with dry run set to false
        request.setDryRun(false);

        EmrCluster emrCluster = emrService.createCluster(request);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
        assertNotNull(emrCluster.getId());
        assertNotNull(emrCluster.isDryRun());
        assertFalse(emrCluster.isDryRun());
        assertTrue(emrCluster.isEmrClusterCreated());
        assertNotNull(emrCluster.getEmrClusterDefinition());
        assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());

        validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
    }

    @Test
    public void testCreateEmrClusterDryRunTrueNoOverride() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String definitionXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
        EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper
                .unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();

        // with dry run set to true
        request.setDryRun(true);

        EmrCluster emrCluster = emrService.createCluster(request);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
        assertNull(emrCluster.getId());
        assertNotNull(emrCluster.isDryRun());
        assertTrue(emrCluster.isDryRun());
        assertFalse(emrCluster.isEmrClusterCreated());
        assertNotNull(emrCluster.getEmrClusterDefinition());
        assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());

        assertEmrClusterCreationLogNotExist(emrCluster);
    }

    @Test
    public void testCreateEmrClusterOverrideAllNull() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String definitionXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
        EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper
                .unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();

        EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
        request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);

        EmrCluster emrCluster = emrService.createCluster(request);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
        assertNotNull(emrCluster.getId());
        assertNull(emrCluster.isDryRun());
        assertTrue(emrCluster.isEmrClusterCreated());
        assertNotNull(emrCluster.getEmrClusterDefinition());
        assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());

        validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
    }

    @Test
    public void testCreateEmrClusterOverrideScalar() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);
        String definitionXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
        EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper
                .unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();

        EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
        emrClusterDefinitionOverride.setReleaseLabel("test" + Math.random());
        expectedEmrClusterDefinition.setReleaseLabel(emrClusterDefinitionOverride.getReleaseLabel());
        emrClusterDefinitionOverride.setAdditionalInfo("test" + Math.random());
        expectedEmrClusterDefinition.setAdditionalInfo(emrClusterDefinitionOverride.getAdditionalInfo());
        emrClusterDefinitionOverride.setAmiVersion("test" + Math.random());
        expectedEmrClusterDefinition.setAmiVersion(emrClusterDefinitionOverride.getAmiVersion());
        emrClusterDefinitionOverride.setEc2NodeIamProfileName("test" + Math.random());
        expectedEmrClusterDefinition
                .setEc2NodeIamProfileName(emrClusterDefinitionOverride.getEc2NodeIamProfileName());
        emrClusterDefinitionOverride.setEncryptionEnabled(!expectedEmrClusterDefinition.isEncryptionEnabled());
        expectedEmrClusterDefinition.setEncryptionEnabled(emrClusterDefinitionOverride.isEncryptionEnabled());
        emrClusterDefinitionOverride.setHadoopVersion("test" + Math.random());
        expectedEmrClusterDefinition.setHadoopVersion(emrClusterDefinitionOverride.getHadoopVersion());
        emrClusterDefinitionOverride.setHiveVersion("test" + Math.random());
        expectedEmrClusterDefinition.setHiveVersion(emrClusterDefinitionOverride.getHiveVersion());
        emrClusterDefinitionOverride.setInstallOozie(!expectedEmrClusterDefinition.isInstallOozie());
        expectedEmrClusterDefinition.setInstallOozie(emrClusterDefinitionOverride.isInstallOozie());
        emrClusterDefinitionOverride.setKeepAlive(!expectedEmrClusterDefinition.isKeepAlive());
        expectedEmrClusterDefinition.setKeepAlive(emrClusterDefinitionOverride.isKeepAlive());
        emrClusterDefinitionOverride.setLogBucket("test" + Math.random());
        expectedEmrClusterDefinition.setLogBucket(emrClusterDefinitionOverride.getLogBucket());
        emrClusterDefinitionOverride.setPigVersion("test" + Math.random());
        expectedEmrClusterDefinition.setPigVersion(emrClusterDefinitionOverride.getPigVersion());
        emrClusterDefinitionOverride.setServiceIamRole("test" + Math.random());
        expectedEmrClusterDefinition.setServiceIamRole(emrClusterDefinitionOverride.getServiceIamRole());
        emrClusterDefinitionOverride.setSshKeyPairName("test" + Math.random());
        expectedEmrClusterDefinition.setSshKeyPairName(emrClusterDefinitionOverride.getSshKeyPairName());
        emrClusterDefinitionOverride.setSubnetId(MockEc2OperationsImpl.SUBNET_1);
        expectedEmrClusterDefinition.setSubnetId(emrClusterDefinitionOverride.getSubnetId());
        emrClusterDefinitionOverride.setSupportedProduct("test" + Math.random());
        expectedEmrClusterDefinition.setSupportedProduct(emrClusterDefinitionOverride.getSupportedProduct());
        emrClusterDefinitionOverride
                .setTerminationProtection(!expectedEmrClusterDefinition.isTerminationProtection());
        expectedEmrClusterDefinition
                .setTerminationProtection(emrClusterDefinitionOverride.isTerminationProtection());
        emrClusterDefinitionOverride.setVisibleToAll(!expectedEmrClusterDefinition.isVisibleToAll());
        expectedEmrClusterDefinition.setVisibleToAll(emrClusterDefinitionOverride.isVisibleToAll());
        request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);

        EmrCluster emrCluster = emrService.createCluster(request);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
        assertNotNull(emrCluster.getId());
        assertNull(emrCluster.isDryRun());
        assertTrue(emrCluster.isEmrClusterCreated());
        assertNotNull(emrCluster.getEmrClusterDefinition());
        assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());

        validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
    }

    @Test
    public void testCreateEmrClusterOverrideList() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String definitionXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
        EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper
                .unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();

        EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
        emrClusterDefinitionOverride.setApplications(Collections.<EmrClusterDefinitionApplication>emptyList());
        expectedEmrClusterDefinition.setApplications(emrClusterDefinitionOverride.getApplications());
        emrClusterDefinitionOverride.setConfigurations(Collections.<EmrClusterDefinitionConfiguration>emptyList());
        expectedEmrClusterDefinition.setConfigurations(emrClusterDefinitionOverride.getConfigurations());
        emrClusterDefinitionOverride.setCustomBootstrapActionAll(Collections.<ScriptDefinition>emptyList());
        expectedEmrClusterDefinition
                .setCustomBootstrapActionAll(emrClusterDefinitionOverride.getCustomBootstrapActionAll());
        emrClusterDefinitionOverride.setCustomBootstrapActionMaster(Collections.<ScriptDefinition>emptyList());
        expectedEmrClusterDefinition
                .setCustomBootstrapActionMaster(emrClusterDefinitionOverride.getCustomBootstrapActionMaster());
        emrClusterDefinitionOverride.setDaemonConfigurations(Collections.<Parameter>emptyList());
        expectedEmrClusterDefinition
                .setDaemonConfigurations(emrClusterDefinitionOverride.getDaemonConfigurations());
        emrClusterDefinitionOverride.setHadoopConfigurations(Collections.<Serializable>emptyList());
        expectedEmrClusterDefinition
                .setHadoopConfigurations(emrClusterDefinitionOverride.getHadoopConfigurations());
        emrClusterDefinitionOverride.setHadoopJarSteps(Collections.<HadoopJarStep>emptyList());
        expectedEmrClusterDefinition.setHadoopJarSteps(emrClusterDefinitionOverride.getHadoopJarSteps());
        List<NodeTag> nodeTags = expectedEmrClusterDefinition.getNodeTags();
        nodeTags.add(new NodeTag("testTag", "test"));
        emrClusterDefinitionOverride.setNodeTags(nodeTags);
        expectedEmrClusterDefinition.setNodeTags(nodeTags);
        request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);

        EmrCluster emrCluster = emrService.createCluster(request);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
        assertNotNull(emrCluster.getId());
        assertNull(emrCluster.isDryRun());
        assertTrue(emrCluster.isEmrClusterCreated());
        assertNotNull(emrCluster.getEmrClusterDefinition());
        assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());

        validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
    }

    @Test
    public void testCreateEmrClusterOverrideHadoopConfigurations() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String definitionXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
        EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper
                .unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();

        EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
        ArrayList<Serializable> hadoopConfigurations = new ArrayList<Serializable>();
        hadoopConfigurations.add(new ConfigurationFiles());
        hadoopConfigurations.add(new KeyValuePairConfigurations());
        emrClusterDefinitionOverride.setHadoopConfigurations(hadoopConfigurations);
        expectedEmrClusterDefinition
                .setHadoopConfigurations(emrClusterDefinitionOverride.getHadoopConfigurations());
        request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);

        EmrCluster emrCluster = emrService.createCluster(request);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
        assertNotNull(emrCluster.getId());
        assertNull(emrCluster.isDryRun());
        assertTrue(emrCluster.isEmrClusterCreated());
        assertNotNull(emrCluster.getEmrClusterDefinition());
        assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());

        validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
    }

    @Test
    public void testCreateEmrClusterOverrideObject() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String definitionXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
        EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper
                .unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();

        int instanceCount = expectedEmrClusterDefinition.getInstanceDefinitions().getMasterInstances()
                .getInstanceCount();
        expectedEmrClusterDefinition.getInstanceDefinitions().getMasterInstances()
                .setInstanceCount(instanceCount + 1);

        EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
        emrClusterDefinitionOverride.setInstanceDefinitions(expectedEmrClusterDefinition.getInstanceDefinitions());
        request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);

        EmrCluster emrCluster = emrService.createCluster(request);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
        assertNotNull(emrCluster.getId());
        assertNull(emrCluster.isDryRun());
        assertTrue(emrCluster.isEmrClusterCreated());
        assertNotNull(emrCluster.getEmrClusterDefinition());
        assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());

        validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
    }

    @Test
    public void testCreateEmrClusterDuplicate() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String definitionXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
        EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper
                .unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        EmrCluster emrCluster = emrService.createCluster(request);
        emrCluster = emrService.createCluster(request);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
        assertNotNull(emrCluster.getId());
        assertNull(emrCluster.isDryRun());
        assertFalse(emrCluster.isEmrClusterCreated());
        assertNull(emrCluster.getEmrClusterDefinition());

        // should throw if not unique
        validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
    }

    /**
     * This method fills-up the parameters required for the EMR cluster create request. This is called from all the other test methods.
     */
    private EmrClusterCreateRequest getNewEmrClusterCreateRequest() throws Exception {
        // Create the definition.
        EmrClusterCreateRequest request = new EmrClusterCreateRequest();

        // Fill in the parameters.
        request.setNamespace(NAMESPACE_CD);
        request.setEmrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME);
        request.setEmrClusterName("UT_EMR_CLUSTER-" + Math.random());

        return request;
    }

    /**
     * This method tests the happy path scenario by providing all the parameters
     */
    @Test
    public void testGetEmrClusterById() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String configXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream());

        EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class,
                configXml);
        emrClusterDefinition.setAmiVersion(MockAwsOperationsHelper.AMAZON_CLUSTER_STATUS_WAITING);

        configXml = xmlHelper.objectToXml(emrClusterDefinition);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        EmrCluster emrCluster = emrService.createCluster(request);

        EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = EmrClusterAlternateKeyDto.builder()
                .namespace(NAMESPACE_CD).emrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME)
                .emrClusterName(request.getEmrClusterName()).build();

        EmrCluster emrClusterGet = emrService.getCluster(emrClusterAlternateKeyDto, emrCluster.getId(), null, true,
                true);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertNotNull(emrClusterGet);
        assertTrue(emrCluster.getId().equals(emrClusterGet.getId()));
        assertTrue(emrCluster.getNamespace().equals(emrClusterGet.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(emrClusterGet.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(emrClusterGet.getEmrClusterName()));

        // Validate the oozie jobs
        assertNotNull(emrClusterGet.getOozieWorkflowJobs());
        assertTrue(emrClusterGet.getOozieWorkflowJobs().size() == dmStringHelper
                .getConfigurationValueAsInteger(ConfigurationValue.EMR_OOZIE_JOBS_TO_INCLUDE_IN_CLUSTER_STATUS));

        int clientRunningCount = 0;
        int clientNotRunningCount = 0;

        for (OozieWorkflowJob oozieWorkflowJob : emrClusterGet.getOozieWorkflowJobs()) {
            if (oozieWorkflowJob.getStatus().equals(OozieDaoImpl.OOZIE_WORKFLOW_JOB_STATUS_DM_PREP)) {
                clientNotRunningCount++;
            } else if (oozieWorkflowJob.getStatus().equals(WorkflowJob.Status.RUNNING.toString())) {
                clientRunningCount++;
            }
        }

        assertTrue(50 == clientRunningCount);
        assertTrue(50 == clientNotRunningCount);

        // Terminate the cluster and validate.
        emrService.terminateCluster(emrClusterAlternateKeyDto, true);

        emrClusterGet = emrService.getCluster(emrClusterAlternateKeyDto, emrCluster.getId(), null, true, true);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertNotNull(emrClusterGet);
        assertTrue(emrCluster.getId().equals(emrClusterGet.getId()));
        assertTrue(emrCluster.getNamespace().equals(emrClusterGet.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(emrClusterGet.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(emrClusterGet.getEmrClusterName()));
    }

    /**
     * This method tests the scenario when cluster specified does not exists.
     */
    @Test(expected = IllegalArgumentException.class)
    public void testGetEmrClusterByIdDoesNotExist() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = EmrClusterAlternateKeyDto.builder()
                .namespace(NAMESPACE_CD).emrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME)
                .emrClusterName("cluster_does_not_exist").build();

        emrService.getCluster(emrClusterAlternateKeyDto, "cluster_does_not_exist", null, true, true);

        fail("Should throw an IllegalArgumentException.");
    }

    /**
     * This method tests the scenario when cluster specified does not exists.
     */
    @Test(expected = IllegalArgumentException.class)
    public void testGetEmrClusterByIdDoesNotExistForNamespace() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        EmrCluster emrCluster = emrService.createCluster(request);

        //        EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = EmrClusterAlternateKeyDto.builder().namespace(NAMESPACE_CD)
        //                .emrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME).emrClusterName(request.getEmrClusterName()).build();

        // Create the second namespace entity.
        NamespaceEntity namespaceEntity_2 = createNamespaceEntity(NAMESPACE_CD_2);

        createEmrClusterDefinitionEntity(namespaceEntity_2, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterAlternateKeyDto emrClusterAlternateKeyDto_2 = EmrClusterAlternateKeyDto.builder()
                .namespace(NAMESPACE_CD_2).emrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME)
                .emrClusterName(request.getEmrClusterName()).build();

        emrService.getCluster(emrClusterAlternateKeyDto_2, emrCluster.getId(), null, true, true);

        fail("Should throw an IllegalArgumentException.");
    }

    /**
     * This method tests the scenario with AmazonServiceException.
     */
    @Test(expected = AmazonServiceException.class)
    public void testGetEmrClusterByIdAmazonException() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = EmrClusterAlternateKeyDto.builder()
                .namespace(NAMESPACE_CD).emrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME)
                .emrClusterName("test").build();

        emrService.getCluster(emrClusterAlternateKeyDto, MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION, null,
                true, true);

        fail("Should throw an AmazonServiceException.");
    }

    /**
     * This method tests the happy path scenario by providing all the parameters
     */
    @Test
    public void testGetEmrClusterByName() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        EmrCluster emrCluster = emrService.createCluster(request);

        EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = EmrClusterAlternateKeyDto.builder()
                .namespace(NAMESPACE_CD).emrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME)
                .emrClusterName(request.getEmrClusterName()).build();

        EmrCluster emrClusterGet = emrService.getCluster(emrClusterAlternateKeyDto, null, null, true, true);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertNotNull(emrClusterGet);
        assertTrue(emrCluster.getNamespace().equals(emrClusterGet.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(emrClusterGet.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(emrClusterGet.getEmrClusterName()));
    }

    /**
     * This method tests the scenario with providing step Id.
     */
    @Test
    public void testGetEmrClusterByIdWithStepId() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        EmrCluster emrCluster = emrService.createCluster(request);

        // Add a running step
        // Create the Add step request.
        EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName());
        stepRequest.setStepName(MockEmrOperationsImpl.MOCK_STEP_RUNNING_NAME);
        EmrShellStep emrShellStep = (EmrShellStep) emrService.addStepToCluster(stepRequest);

        String stepId = emrShellStep.getId();

        EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = EmrClusterAlternateKeyDto.builder()
                .namespace(NAMESPACE_CD).emrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME)
                .emrClusterName(request.getEmrClusterName()).build();

        EmrCluster emrClusterGet = emrService.getCluster(emrClusterAlternateKeyDto, emrCluster.getId(), stepId,
                true, true);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertNotNull(emrClusterGet);
        assertTrue(emrCluster.getId().equals(emrClusterGet.getId()));
        assertTrue(emrCluster.getNamespace().equals(emrClusterGet.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(emrClusterGet.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(emrClusterGet.getEmrClusterName()));
        assertTrue(stepId.equals(emrClusterGet.getActiveStep().getId()));
        assertTrue(stepId.equals(emrClusterGet.getStep().getId()));

        // Test the non verbose flow
        emrClusterGet = emrService.getCluster(emrClusterAlternateKeyDto, emrCluster.getId(), stepId, false, false);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertNotNull(emrClusterGet);
        assertTrue(emrCluster.getId().equals(emrClusterGet.getId()));
        assertTrue(emrCluster.getNamespace().equals(emrClusterGet.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName().equals(emrClusterGet.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(emrClusterGet.getEmrClusterName()));
        assertTrue(stepId.equals(emrClusterGet.getActiveStep().getId()));
        assertTrue(stepId.equals(emrClusterGet.getStep().getId()));
    }

    /**
     * This method tests the happy path scenario by providing all the parameters
     */
    @Test
    public void testTerminateEmrCluster() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        // Create a new EMR cluster create request
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        EmrCluster emrCluster = emrService.createCluster(request);

        EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = EmrClusterAlternateKeyDto.builder()
                .namespace(NAMESPACE_CD).emrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME)
                .emrClusterName(request.getEmrClusterName()).build();

        EmrCluster emrClusterTerminated = emrService.terminateCluster(emrClusterAlternateKeyDto, true);

        // Validate the returned object against the input.
        assertNotNull(emrCluster);
        assertNotNull(emrClusterTerminated);
        assertTrue(emrCluster.getNamespace().equals(emrClusterTerminated.getNamespace()));
        assertTrue(emrCluster.getEmrClusterDefinitionName()
                .equals(emrClusterTerminated.getEmrClusterDefinitionName()));
        assertTrue(emrCluster.getEmrClusterName().equals(emrClusterTerminated.getEmrClusterName()));
    }

    /**
     * This method tests the scenario when no active cluster exists.
     */
    @Test(expected = IllegalArgumentException.class)
    public void testTerminateEmrClusterNoCluster() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = EmrClusterAlternateKeyDto.builder()
                .namespace(NAMESPACE_CD).emrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME)
                .emrClusterName("cluster_not_found").build();

        emrService.terminateCluster(emrClusterAlternateKeyDto, true);
    }

    /**
     * This method tests the scenario where AmazonServiceException is thrown
     */
    @Test(expected = AmazonServiceException.class)
    public void testTerminateEmrClusterAmazonException() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        request.setEmrClusterName(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION);
        emrService.createCluster(request);

        EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = EmrClusterAlternateKeyDto.builder()
                .namespace(NAMESPACE_CD).emrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME)
                .emrClusterName(request.getEmrClusterName()).build();

        emrService.terminateCluster(emrClusterAlternateKeyDto, true);
    }

    /**
     * This method tests the happy path scenario by testing all the step types
     */
    @Test
    public void testEmrAddStepsAllTypes() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);

        List<Object> emrSteps = new ArrayList<>();

        List<String> shellScriptArgs = new ArrayList<>();
        shellScriptArgs.add("Hello");
        shellScriptArgs.add("DM");
        shellScriptArgs.add("How Are You");

        EmrShellStepAddRequest shellStepRequest = new EmrShellStepAddRequest();
        shellStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/test/test_script.sh");
        shellStepRequest.setStepName("Test Shell Script");
        shellStepRequest.setScriptArguments(shellScriptArgs);
        emrSteps.add(shellStepRequest);

        EmrHiveStepAddRequest hiveStepRequest = new EmrHiveStepAddRequest();
        List<String> scriptArgs1 = new ArrayList<>();
        scriptArgs1.add("arg2=sampleArg");
        scriptArgs1.add("arg1=tables");
        hiveStepRequest.setStepName("Test Hive");
        hiveStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/test/test_hive.hql");
        hiveStepRequest.setScriptArguments(scriptArgs1);
        emrSteps.add(hiveStepRequest);

        EmrPigStepAddRequest pigStepRequest = new EmrPigStepAddRequest();
        pigStepRequest.setStepName("Test Pig");
        pigStepRequest.setScriptArguments(shellScriptArgs);
        pigStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/test/test_pig.pig");
        emrSteps.add(pigStepRequest);

        shellStepRequest = new EmrShellStepAddRequest();
        shellStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/bootstrap/install_oozie.sh");
        shellStepRequest.setStepName("Install Oozie");
        List<String> shellScriptArgsOozie = new ArrayList<>();
        shellScriptArgsOozie.add("s3://test-bucket-managed/app-a/bootstrap/oozie-4.0.1-distro.tar");
        shellStepRequest.setScriptArguments(shellScriptArgsOozie);

        emrSteps.add(shellStepRequest);

        EmrOozieStepAddRequest oozieStep = new EmrOozieStepAddRequest();
        oozieStep.setStepName("Test Oozie");
        oozieStep.setWorkflowXmlLocation("s3://test-bucket-managed/app-a/test/workflow.xml");
        oozieStep.setOoziePropertiesFileLocation("s3://test-bucket-managed/app-a/test/job.properties");
        emrSteps.add(oozieStep);

        EmrHadoopJarStepAddRequest hadoopJarStepRequest = new EmrHadoopJarStepAddRequest();
        List<String> scriptArgs2 = new ArrayList<>();
        scriptArgs2.add("oozie_run");
        scriptArgs2.add("wordcountOutput");
        hadoopJarStepRequest.setStepName("Hadoop Jar");
        hadoopJarStepRequest
                .setJarLocation("s3://test-bucket-managed/app-a/test/hadoop-mapreduce-examples-2.4.0.jar");
        hadoopJarStepRequest.setMainClass("wordcount");
        hadoopJarStepRequest.setScriptArguments(scriptArgs2);
        emrSteps.add(hadoopJarStepRequest);

        EmrStepHelper stepHelper = null;

        for (Object emrStepAddRequest : emrSteps) {
            stepHelper = emrStepHelperFactory.getStepHelper(emrStepAddRequest.getClass().getName());
            stepHelper.setRequestNamespace(emrStepAddRequest, NAMESPACE_CD);
            stepHelper.setRequestEmrClusterDefinitionName(emrStepAddRequest, EMR_CLUSTER_DEFINITION_NAME);
            stepHelper.setRequestEmrClusterName(emrStepAddRequest, request.getEmrClusterName());

            Object emrStep = emrService.addStepToCluster(emrStepAddRequest);

            assertNotNull(emrStep);
            assertNotNull(stepHelper.getStepId(emrStep));

            Method getNameMethod = emrStep.getClass().getMethod("getStepName");
            String emrStepName = (String) getNameMethod.invoke(emrStep);

            assertEquals(stepHelper.getRequestStepName(emrStepAddRequest), emrStepName);

            Method isContinueOnErrorMethod = emrStep.getClass().getMethod("isContinueOnError");
            Object emrStepIsContinueOnError = isContinueOnErrorMethod.invoke(emrStep);

            assertEquals(stepHelper.isRequestContinueOnError(emrStepAddRequest), emrStepIsContinueOnError);
        }
    }

    /**
     * This method tests the happy path scenario by testing all the step types
     */
    @Test
    public void testEmrAddStepsHadoopNoMainClass() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);

        // TODO: Why are we adding EMR steps to a list, but not doing anything with them once they're added?
        List<Serializable> emrSteps = new ArrayList<>();

        EmrHadoopJarStepAddRequest hadoopJarStepRequest = new EmrHadoopJarStepAddRequest();
        hadoopJarStepRequest.setNamespace(request.getNamespace());
        hadoopJarStepRequest.setEmrClusterDefinitionName(request.getEmrClusterDefinitionName());
        hadoopJarStepRequest.setEmrClusterName(request.getEmrClusterName());
        hadoopJarStepRequest.setStepName("Hadoop Jar");
        hadoopJarStepRequest
                .setJarLocation("s3://test-bucket-managed/app-a/test/hadoop-mapreduce-examples-2.4.0.jar");
        emrSteps.add(hadoopJarStepRequest);

        EmrHadoopJarStep emrHadoopJarStep = (EmrHadoopJarStep) emrService.addStepToCluster(hadoopJarStepRequest);

        assertNotNull(emrHadoopJarStep);
        assertNotNull(emrHadoopJarStep.getId());
    }

    /**
     * This method tests the happy path scenario by providing all the parameters
     */
    @Test(expected = IllegalArgumentException.class)
    public void testEmrAddStepsInvalidCluster() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);

        // Create a new EMR cluster create request.
        EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName());
        stepRequest.setEmrClusterName("InvalidName");
        emrService.addStepToCluster(stepRequest);
    }

    /**
     * This method tests the happy path scenario by providing all the parameters
     */
    @Test
    public void testEmrAddSteps() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);

        // Create the Add step request.
        EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName());
        EmrShellStep emrShellStep = (EmrShellStep) emrService.addStepToCluster(stepRequest);

        // Validate the returned object against the input.
        assertNotNull(emrShellStep);
        assertTrue(emrShellStep.getNamespace().equals(request.getNamespace()));
        assertTrue(emrShellStep.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(emrShellStep.getEmrClusterName().equals(request.getEmrClusterName()));
    }

    /**
     * This method tests the Amazon BAD_REQUEST exception
     */
    @Test(expected = IllegalArgumentException.class)
    public void testEmrAddStepsAmazonBadRequest() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);

        // Create a new EMR cluster create request.
        EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName());
        stepRequest.setStepName(MockAwsOperationsHelper.AMAZON_BAD_REQUEST);
        emrService.addStepToCluster(stepRequest);
    }

    /**
     * This method tests the error cases for AmazonExceptions for Illegal Argument
     */
    @Test(expected = ObjectNotFoundException.class)
    public void testEmrAddStepsAmazonObjectNotFound() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);

        // Create a new EMR cluster create request.
        EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName());
        stepRequest.setStepName(MockAwsOperationsHelper.AMAZON_NOT_FOUND);
        emrService.addStepToCluster(stepRequest);
    }

    /**
     * This method tests the Amazon exceptions
     */
    @Test(expected = AmazonServiceException.class)
    public void testEmrAddStepsAmazonOtherException() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        emrService.createCluster(request);

        // Create a new EMR cluster create request.
        EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName());
        stepRequest.setStepName(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION);
        emrService.addStepToCluster(stepRequest);
    }

    /**
     * This method tests the Oozie job submission.
     */
    @Test
    public void testRunOozieJob() throws Exception {
        // Create the namespace entity.
        NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

        String configXml = IOUtils.toString(
                resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream());

        EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class,
                configXml);
        emrClusterDefinition.setAmiVersion(MockAwsOperationsHelper.AMAZON_CLUSTER_STATUS_WAITING);

        configXml = xmlHelper.objectToXml(emrClusterDefinition);

        createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);

        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        EmrCluster emrCluster = emrService.createCluster(request);

        // Create the Run Oozie Job.
        RunOozieWorkflowRequest runOozieRequest = new RunOozieWorkflowRequest(namespaceEntity.getCode(),
                EMR_CLUSTER_DEFINITION_NAME, emrCluster.getEmrClusterName(), OOZIE_WORKFLOW_LOCATION, null);
        OozieWorkflowJob oozieWorkflowJob = emrService.runOozieWorkflow(runOozieRequest);

        // Validate the returned object against the input.
        assertNotNull(oozieWorkflowJob);
        assertNotNull(oozieWorkflowJob.getId());
        assertTrue(oozieWorkflowJob.getNamespace().equals(request.getNamespace()));
        assertTrue(oozieWorkflowJob.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
        assertTrue(oozieWorkflowJob.getEmrClusterName().equals(request.getEmrClusterName()));
    }

    @Test
    public void testRunOozieJobMissingRequiredParameters() throws Exception {
        // Try to run oozie job when namespace is not specified.
        try {
            RunOozieWorkflowRequest runOozieRequest = new RunOozieWorkflowRequest(BLANK_TEXT,
                    EMR_CLUSTER_DEFINITION_NAME, "test_cluster", OOZIE_WORKFLOW_LOCATION, null);
            emrService.runOozieWorkflow(runOozieRequest);
            fail("Should throw an IllegalArgumentException when namespace is not specified.");
        } catch (IllegalArgumentException e) {
            assertEquals("A namespace must be specified.", e.getMessage());
        }

        // Try to run oozie job when cluster definition name is not specified.
        try {
            RunOozieWorkflowRequest runOozieRequest = new RunOozieWorkflowRequest(NAMESPACE_CD, BLANK_TEXT,
                    "test_cluster", OOZIE_WORKFLOW_LOCATION, null);
            emrService.runOozieWorkflow(runOozieRequest);
            fail("Should throw an IllegalArgumentException when cluster definition name is not specified.");
        } catch (IllegalArgumentException e) {
            assertEquals("An EMR cluster definition name must be specified.", e.getMessage());
        }

        // Try to run oozie job when cluster name is not specified.
        try {
            RunOozieWorkflowRequest runOozieRequest = new RunOozieWorkflowRequest(NAMESPACE_CD,
                    EMR_CLUSTER_DEFINITION_NAME, BLANK_TEXT, OOZIE_WORKFLOW_LOCATION, null);
            emrService.runOozieWorkflow(runOozieRequest);
            fail("Should throw an IllegalArgumentException when cluster name is not specified.");
        } catch (IllegalArgumentException e) {
            assertEquals("An EMR cluster name must be specified.", e.getMessage());
        }

        // Try to run oozie job when workflow location is not specified.
        try {
            RunOozieWorkflowRequest runOozieRequest = new RunOozieWorkflowRequest(NAMESPACE_CD,
                    EMR_CLUSTER_DEFINITION_NAME, "test_cluster", BLANK_TEXT, null);
            emrService.runOozieWorkflow(runOozieRequest);
            fail("Should throw an IllegalArgumentException when workflow location is not specified.");
        } catch (IllegalArgumentException e) {
            assertEquals("An oozie workflow location must be specified.", e.getMessage());
        }
    }

    @Test
    public void testRunOozieJobWrongParameters() throws Exception {
        // Try to run oozie job when namespace does not exist.
        try {
            RunOozieWorkflowRequest runOozieRequest = new RunOozieWorkflowRequest(NAMESPACE_CD,
                    EMR_CLUSTER_DEFINITION_NAME, "test_cluster", OOZIE_WORKFLOW_LOCATION, null);
            emrService.runOozieWorkflow(runOozieRequest);
            fail("Should throw an ObjectNotFoundException when specified namespace does not exist.");
        } catch (ObjectNotFoundException ex) {
            assertEquals(String.format("Namespace \"%s\" doesn't exist.", NAMESPACE_CD), ex.getMessage());
        }

        // Try to run oozie job when cluster definition does not exist.
        try {
            createNamespaceEntity(NAMESPACE_CD);

            RunOozieWorkflowRequest runOozieRequest = new RunOozieWorkflowRequest(NAMESPACE_CD,
                    EMR_CLUSTER_DEFINITION_NAME, "test_cluster", OOZIE_WORKFLOW_LOCATION, null);
            emrService.runOozieWorkflow(runOozieRequest);
            fail("Should throw an ObjectNotFoundException when specified cluster definition does not exist.");
        } catch (ObjectNotFoundException ex) {
            assertEquals(
                    String.format("EMR cluster definition with name \"%s\" doesn't exist for namespace \"%s\".",
                            EMR_CLUSTER_DEFINITION_NAME, NAMESPACE_CD),
                    ex.getMessage());
        }

        // Try to run oozie job when duplicate parameters are specified.
        try {
            List<Parameter> parameters = new ArrayList<>();
            parameters.add(new Parameter("PARAM_NAME", ""));
            parameters.add(new Parameter("PARAM_NAME", ""));

            RunOozieWorkflowRequest runOozieRequest = new RunOozieWorkflowRequest(NAMESPACE_CD,
                    EMR_CLUSTER_DEFINITION_NAME, "test_cluster", OOZIE_WORKFLOW_LOCATION, parameters);
            emrService.runOozieWorkflow(runOozieRequest);
            fail("Should throw an IllegalArgumentException when duplicate parameters are specified.");
        } catch (IllegalArgumentException e) {
            assertEquals("Duplicate parameter name found: PARAM_NAME", e.getMessage());
        }
    }

    @Test
    public void testRunOozieJobClusterNotValid() throws Exception {
        // Try to run oozie job when cluster does not exist.
        try {
            NamespaceEntity namespaceEntity = createNamespaceEntity(NAMESPACE_CD);

            createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(
                    resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));

            RunOozieWorkflowRequest runOozieRequest = new RunOozieWorkflowRequest(NAMESPACE_CD,
                    EMR_CLUSTER_DEFINITION_NAME, "test_cluster", OOZIE_WORKFLOW_LOCATION, null);
            emrService.runOozieWorkflow(runOozieRequest);
            fail("Should throw an ObjectNotFoundException when specified cluster does not exist.");
        } catch (ObjectNotFoundException ex) {
            assertEquals(String.format(
                    "Either the cluster \"%s\" does not exist or not in RUNNING or WAITING state.",
                    emrHelper.buildEmrClusterName(NAMESPACE_CD, EMR_CLUSTER_DEFINITION_NAME, "test_cluster")),
                    ex.getMessage());
        }

        // Try to run oozie job when cluster is not in running or waiting.
        EmrCluster emrCluster = null;
        try {
            EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
            emrCluster = emrService.createCluster(request);

            RunOozieWorkflowRequest runOozieRequest = new RunOozieWorkflowRequest(NAMESPACE_CD,
                    EMR_CLUSTER_DEFINITION_NAME, emrCluster.getEmrClusterName(), OOZIE_WORKFLOW_LOCATION, null);
            emrService.runOozieWorkflow(runOozieRequest);
            fail("Should throw an ObjectNotFoundException when specified cluster is not in running or waiting state.");
        } catch (ObjectNotFoundException ex) {
            assertEquals(
                    String.format("Either the cluster \"%s\" does not exist or not in RUNNING or WAITING state.",
                            emrHelper.buildEmrClusterName(NAMESPACE_CD, EMR_CLUSTER_DEFINITION_NAME,
                                    emrCluster.getEmrClusterName())),
                    ex.getMessage());
        }
    }

    /**
     * This method is to get the coverage for the emr service method that starts the new transaction.
     */
    @Test
    public void testEmrServiceMethodsNewTx() throws Exception {
        EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
        try {
            emrServiceImpl.createCluster(request);
            fail("Should throw a ObjectNotFoundException.");
        } catch (ObjectNotFoundException e) {
            assertEquals("Namespace \"" + NAMESPACE_CD + "\" doesn't exist.", e.getMessage());
        }

        try {
            EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName());
            emrServiceImpl.addStepToCluster(stepRequest);
            fail("Should throw a ObjectNotFoundException.");
        } catch (ObjectNotFoundException e) {
            assertEquals("Namespace \"" + NAMESPACE_CD + "\" doesn't exist.", e.getMessage());
        }

        try {
            EmrMasterSecurityGroupAddRequest emrMasterSecurityGroupAddRequest = getNewEmrAddSecurityGroupMasterRequest(
                    request.getEmrClusterName());
            emrServiceImpl.addSecurityGroupsToClusterMaster(emrMasterSecurityGroupAddRequest);
            fail("Should throw a ObjectNotFoundException.");
        } catch (ObjectNotFoundException e) {
            assertEquals("Namespace \"" + NAMESPACE_CD + "\" doesn't exist.", e.getMessage());
        }

        try {
            EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = EmrClusterAlternateKeyDto.builder()
                    .namespace(NAMESPACE_CD).emrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME)
                    .emrClusterName("test_cluster").build();
            emrServiceImpl.getCluster(emrClusterAlternateKeyDto, null, null, false, false);
            fail("Should throw a ObjectNotFoundException.");
        } catch (ObjectNotFoundException e) {
            assertEquals("Namespace \"" + NAMESPACE_CD + "\" doesn't exist.", e.getMessage());
        }

        try {
            EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = EmrClusterAlternateKeyDto.builder()
                    .namespace(NAMESPACE_CD).emrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME)
                    .emrClusterName("test_cluster").build();
            emrServiceImpl.terminateCluster(emrClusterAlternateKeyDto, false);
            fail("Should throw a ObjectNotFoundException.");
        } catch (ObjectNotFoundException e) {
            assertEquals("Namespace \"" + NAMESPACE_CD + "\" doesn't exist.", e.getMessage());
        }

        try {
            RunOozieWorkflowRequest runOozieWorkflowRequest = new RunOozieWorkflowRequest(NAMESPACE_CD,
                    EMR_CLUSTER_DEFINITION_NAME, "test_cluster", OOZIE_WORKFLOW_LOCATION, null);
            emrServiceImpl.runOozieWorkflow(runOozieWorkflowRequest);
            fail("Should throw a ObjectNotFoundException.");
        } catch (ObjectNotFoundException e) {
            assertEquals("Namespace \"" + NAMESPACE_CD + "\" doesn't exist.", e.getMessage());
        }

        try {
            emrServiceImpl.getEmrOozieWorkflowJob(NAMESPACE_CD, EMR_CLUSTER_DEFINITION_NAME, "test_cluster",
                    "test_oozie_id", false);
            fail("Should throw a ObjectNotFoundException.");
        } catch (ObjectNotFoundException e) {
            assertEquals("Namespace \"" + NAMESPACE_CD + "\" doesn't exist.", e.getMessage());
        }
    }

    /**
     * This method creates a EMR Shell step add request. This is called from all the other test methods.
     */
    private EmrShellStepAddRequest getNewEmrShellStepAddRequest(String clusterName) throws Exception {
        // Create the request.
        EmrShellStepAddRequest request = new EmrShellStepAddRequest();

        // Fill in the parameters.
        request.setNamespace(NAMESPACE_CD);
        request.setEmrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME);
        request.setEmrClusterName(clusterName);

        request.setStepName("A_SHELL_STEP");
        request.setScriptLocation("SCRIPT_LOCATION");
        request.setContinueOnError(false);
        List<String> arguments = new ArrayList<>();
        arguments.add("one");
        arguments.add("two");
        request.setScriptArguments(arguments);

        return request;
    }

    /**
     * This method fills-up the parameters required for the EMR add steps request. This is called from all the other test methods.
     */
    private EmrMasterSecurityGroupAddRequest getNewEmrAddSecurityGroupMasterRequest(String clusterName)
            throws Exception {
        // Create the EmrMasterSecurityGroupAddRequest object
        EmrMasterSecurityGroupAddRequest request = new EmrMasterSecurityGroupAddRequest();

        // Fill in the parameters.
        request.setNamespace(NAMESPACE_CD);
        request.setEmrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME);
        request.setEmrClusterName(clusterName);

        List<String> groupIds = new ArrayList<>();
        groupIds.add("A_TEST_SECURITY_GROUP");

        request.setSecurityGroupIds(groupIds);

        return request;
    }

    /**
     * A test case to validate that the XSD restrictions are loose enough to allow empty elements for nested elements.
     */
    @Test
    public void testUnmarshallXmlWithNestedElements() {
        String xml = "<emrClusterCreateRequest>" + "<namespace>" + NAMESPACE_CD + "</namespace>"
                + "<emrClusterDefinitionName>" + EMR_CLUSTER_DEFINITION_NAME + "</emrClusterDefinitionName>"
                + "<emrClusterName>cluster1</emrClusterName>" + "<dryRun>true</dryRun>"
                + "<emrClusterDefinitionOverride>" + "<customBootstrapActionMaster/>"
                + "<customBootstrapActionAll/>" + "<instanceDefinitions/>" + "<nodeTags/>"
                + "<daemonConfigurations/>" + "<hadoopConfigurations/>" + "</emrClusterDefinitionOverride>"
                + "</emrClusterCreateRequest>";
        try {
            xmlHelper.unmarshallXmlToObject(EmrClusterCreateRequest.class, xml);
        } catch (JAXBException e) {
            e.printStackTrace();
            fail("Failed to unmarshall XML");
        }
    }

    /**
     * A call to getEmrOozieWorkflowJob will return the oozie job status for a specified EMR cluster. If the verbose flag is false or null, only the job's basic
     * details are returned.
     * <p/>
     * This test uses CASE_1_JOB_ID, which is a job in a running state.
     */
    @Test
    public void testGetEmrOozieWorkflowJobNotVerbose() throws Exception {
        String oozieWorkflowJobId = MockOozieOperationsImpl.CASE_1_JOB_ID;
        Boolean verbose = null;
        EmrCluster emrCluster = createEmrClusterInWaitingState(NAMESPACE_CD, EMR_CLUSTER_DEFINITION_NAME);

        try {
            OozieWorkflowJob oozieWorkflowJob = emrService.getEmrOozieWorkflowJob(emrCluster.getNamespace(),
                    emrCluster.getEmrClusterDefinitionName(), emrCluster.getEmrClusterName(), oozieWorkflowJobId,
                    verbose);

            assertEquals("job ID", MockOozieOperationsImpl.CASE_1_JOB_ID, oozieWorkflowJob.getId());
            assertEquals("namespace", emrCluster.getNamespace(), oozieWorkflowJob.getNamespace());
            assertEquals("EMR cluster definition name", emrCluster.getEmrClusterDefinitionName(),
                    oozieWorkflowJob.getEmrClusterDefinitionName());
            assertEquals("EMR cluster name", emrCluster.getEmrClusterName(), oozieWorkflowJob.getEmrClusterName());
            assertNotNull("job start time is null", oozieWorkflowJob.getStartTime());
            assertNull("job end time is not null", oozieWorkflowJob.getEndTime());
            assertNull("actions is not null", oozieWorkflowJob.getWorkflowActions());
        } catch (Exception e) {
            e.printStackTrace();
            fail("unexpected exception: " + e);
        }
    }

    /**
     * getEmrOozieWorkflow with verbose flag set to true will return the job details, along with any action statuses in this job.
     * <p/>
     * This test uses CASE_1_JOB_ID, which is a job in a running state, with 3 actions, each in different statuses.
     */
    @Test
    public void testGetEmrOozieWorkflowJobVerbose() throws Exception {
        String oozieWorkflowJobId = MockOozieOperationsImpl.CASE_1_JOB_ID;
        Boolean verbose = true;
        EmrCluster emrCluster = createEmrClusterInWaitingState(NAMESPACE_CD, EMR_CLUSTER_DEFINITION_NAME);

        try {
            OozieWorkflowJob oozieWorkflowJob = emrService.getEmrOozieWorkflowJob(emrCluster.getNamespace(),
                    emrCluster.getEmrClusterDefinitionName(), emrCluster.getEmrClusterName(), oozieWorkflowJobId,
                    verbose);

            assertEquals("job ID", MockOozieOperationsImpl.CASE_1_JOB_ID, oozieWorkflowJob.getId());
            assertEquals("namespace", emrCluster.getNamespace(), oozieWorkflowJob.getNamespace());
            assertEquals("EMR cluster definition name", emrCluster.getEmrClusterDefinitionName(),
                    oozieWorkflowJob.getEmrClusterDefinitionName());
            assertEquals("EMR cluster name", emrCluster.getEmrClusterName(), oozieWorkflowJob.getEmrClusterName());
            assertNotNull("job start time is null", oozieWorkflowJob.getStartTime());
            assertNull("job end time is not null", oozieWorkflowJob.getEndTime());
            assertNotNull("actions is null", oozieWorkflowJob.getWorkflowActions());
            assertEquals("actions size", 3, oozieWorkflowJob.getWorkflowActions().size());
            {
                OozieWorkflowAction oozieWorkflowAction = oozieWorkflowJob.getWorkflowActions().get(0);
                assertEquals("action[0] id", "action1", oozieWorkflowAction.getId());
                assertEquals("action[0] name", "action1", oozieWorkflowAction.getName());
                assertEquals("action[0] status", WorkflowAction.Status.DONE.name(),
                        oozieWorkflowAction.getStatus());
                assertNotNull("action[0] start time is null", oozieWorkflowAction.getStartTime());
                assertNotNull("action[0] end time is null", oozieWorkflowAction.getEndTime());
                assertNull("action[0] error code is not null", oozieWorkflowAction.getErrorCode());
                assertNull("action[0] error message is not null", oozieWorkflowAction.getErrorMessage());
            }
            {
                OozieWorkflowAction oozieWorkflowAction = oozieWorkflowJob.getWorkflowActions().get(1);
                assertEquals("action[1] id", "action2", oozieWorkflowAction.getId());
                assertEquals("action[1] name", "action2", oozieWorkflowAction.getName());
                assertEquals("action[1] status", WorkflowAction.Status.RUNNING.name(),
                        oozieWorkflowAction.getStatus());
                assertNotNull("action[1] start time is null", oozieWorkflowAction.getStartTime());
                assertNull("action[1] end time is not null", oozieWorkflowAction.getEndTime());
                assertNull("action[1] error code is not null", oozieWorkflowAction.getErrorCode());
                assertNull("action[1] error message is not null", oozieWorkflowAction.getErrorMessage());
            }
            {
                OozieWorkflowAction oozieWorkflowAction = oozieWorkflowJob.getWorkflowActions().get(2);
                assertEquals("action[2] id", "action3", oozieWorkflowAction.getId());
                assertEquals("action[2] name", "action3", oozieWorkflowAction.getName());
                assertEquals("action[2] status", WorkflowAction.Status.ERROR.name(),
                        oozieWorkflowAction.getStatus());
                assertNotNull("action[2] start time is null", oozieWorkflowAction.getStartTime());
                assertNotNull("action[2] end time is null", oozieWorkflowAction.getEndTime());
                assertEquals("action[2] error code", "testErrorCode", oozieWorkflowAction.getErrorCode());
                assertEquals("action[2] error message", "testErrorMessage", oozieWorkflowAction.getErrorMessage());
            }
        } catch (Exception e) {
            e.printStackTrace();
            fail("unexpected exception: " + e);
        }
    }

    /**
     * GetEmrOozieWorkflowJob only works if the specified EMR cluster is in either WAITING or RUNNING state.
     */
    @Test
    public void testGetEmrOozieWorkflowJobClusterWaitingState() {
        String oozieWorkflowJobId = MockOozieOperationsImpl.CASE_1_JOB_ID;
        Boolean verbose = null;
        EmrCluster emrCluster = createEmrClusterInWaitingState(NAMESPACE_CD, EMR_CLUSTER_DEFINITION_NAME);

        try {
            emrService.getEmrOozieWorkflowJob(emrCluster.getNamespace(), emrCluster.getEmrClusterDefinitionName(),
                    emrCluster.getEmrClusterName(), oozieWorkflowJobId, verbose);
        } catch (Exception e) {
            e.printStackTrace();
            fail("unexpected exception: " + e);
        }
    }

    /**
     * GetEmrOozieWorkflowJob only works if the specified EMR cluster is in either WAITING or RUNNING state.
     */
    @Test
    public void testGetEmrOozieWorkflowJobClusterRunningState() {
        String oozieWorkflowJobId = MockOozieOperationsImpl.CASE_1_JOB_ID;
        Boolean verbose = null;
        EmrCluster emrCluster = createEmrClusterInRunningState(NAMESPACE_CD, EMR_CLUSTER_DEFINITION_NAME);

        try {
            emrService.getEmrOozieWorkflowJob(emrCluster.getNamespace(), emrCluster.getEmrClusterDefinitionName(),
                    emrCluster.getEmrClusterName(), oozieWorkflowJobId, verbose);
        } catch (Exception e) {
            e.printStackTrace();
            fail("unexpected exception: " + e);
        }
    }

    /**
     * getEmrOozieWorkflow will fail with an error if the EMR cluster specified has not yet started or has been stopped. The only valid statuses are RUNNING and
     * WAITING.
     */
    @Test
    public void testGetEmrOozieWorkflowJobClusterIsNotRunningOrWaiting() throws Exception {
        String oozieWorkflowJobId = MockOozieOperationsImpl.CASE_1_JOB_ID;
        Boolean verbose = false;
        EmrCluster emrCluster = createEmrClusterInBootstrappingState(NAMESPACE_CD, EMR_CLUSTER_DEFINITION_NAME);

        try {
            emrService.getEmrOozieWorkflowJob(emrCluster.getNamespace(), emrCluster.getEmrClusterDefinitionName(),
                    emrCluster.getEmrClusterName(), oozieWorkflowJobId, verbose);
            fail("expected a ObjectNotFoundException, but not exception was thrown");
        } catch (Exception e) {
            assertEquals("thrown exception type", ObjectNotFoundException.class, e.getClass());
            assertEquals("exception message",
                    "Either the cluster \"" + emrCluster.getNamespace() + "."
                            + emrCluster.getEmrClusterDefinitionName() + "." + emrCluster.getEmrClusterName()
                            + "\" does not exist or not in RUNNING or WAITING state.",
                    e.getMessage());
        }
    }

    /**
     * getEmrOozieWorkflow will fail if the specified EMR cluster does not exist.
     */
    @Test
    public void testGetEmrOozieWorkflowJobClusterDoesNotExist() throws Exception {
        String oozieWorkflowJobId = MockOozieOperationsImpl.CASE_1_JOB_ID;
        Boolean verbose = false;
        EmrCluster emrCluster = createEmrClusterInBootstrappingState(NAMESPACE_CD, EMR_CLUSTER_DEFINITION_NAME);

        try {
            emrService.getEmrOozieWorkflowJob(emrCluster.getNamespace(), "DOES_NOT_EXIST", "DOES_NOT_EXIST",
                    oozieWorkflowJobId, verbose);
            fail("expected a ObjectNotFoundException, but not exception was thrown");
        } catch (Exception e) {
            assertEquals("thrown exception type", ObjectNotFoundException.class, e.getClass());
            assertEquals("exception message",
                    "EMR cluster definition with name \"DOES_NOT_EXIST\" doesn't exist for namespace \""
                            + emrCluster.getNamespace() + "\".",
                    e.getMessage());
        }
    }

    /**
     * Test for when the workflow job is in DM_prep state.
     */
    @Test
    public void testGetEmrOozieWorkflowJobInDmPrep() throws Exception {
        String oozieWorkflowJobId = MockOozieOperationsImpl.CASE_4_JOB_ID;
        Boolean verbose = null;
        EmrCluster emrCluster = createEmrClusterInWaitingState(NAMESPACE_CD, EMR_CLUSTER_DEFINITION_NAME);

        try {
            OozieWorkflowJob oozieWorkflowJob = emrService.getEmrOozieWorkflowJob(emrCluster.getNamespace(),
                    emrCluster.getEmrClusterDefinitionName(), emrCluster.getEmrClusterName(), oozieWorkflowJobId,
                    verbose);

            assertEquals("job ID", MockOozieOperationsImpl.CASE_4_JOB_ID, oozieWorkflowJob.getId());
            assertEquals("namespace", emrCluster.getNamespace(), oozieWorkflowJob.getNamespace());
            assertEquals("EMR cluster definition name", emrCluster.getEmrClusterDefinitionName(),
                    oozieWorkflowJob.getEmrClusterDefinitionName());
            assertEquals("EMR cluster name", emrCluster.getEmrClusterName(), oozieWorkflowJob.getEmrClusterName());
            assertEquals("EMR status", OozieDaoImpl.OOZIE_WORKFLOW_JOB_STATUS_DM_PREP,
                    oozieWorkflowJob.getStatus());
            assertNull("job start time is not null", oozieWorkflowJob.getStartTime());
            assertNull("job end time is not null", oozieWorkflowJob.getEndTime());
            assertNull("actions is not null", oozieWorkflowJob.getWorkflowActions());
        } catch (Exception e) {
            e.printStackTrace();
            fail("unexpected exception: " + e);
        }
    }

    /**
     * Test for when the workflow job is in DM_FAILED state.
     */
    @Test
    public void testGetEmrOozieWorkflowJobInDmFailed() throws Exception {
        String oozieWorkflowJobId = MockOozieOperationsImpl.CASE_6_JOB_ID;
        Boolean verbose = null;
        EmrCluster emrCluster = createEmrClusterInWaitingState(NAMESPACE_CD, EMR_CLUSTER_DEFINITION_NAME);

        try {
            OozieWorkflowJob oozieWorkflowJob = emrService.getEmrOozieWorkflowJob(emrCluster.getNamespace(),
                    emrCluster.getEmrClusterDefinitionName(), emrCluster.getEmrClusterName(), oozieWorkflowJobId,
                    verbose);

            assertEquals("job ID", MockOozieOperationsImpl.CASE_6_JOB_ID, oozieWorkflowJob.getId());
            assertEquals("namespace", emrCluster.getNamespace(), oozieWorkflowJob.getNamespace());
            assertEquals("EMR cluster definition name", emrCluster.getEmrClusterDefinitionName(),
                    oozieWorkflowJob.getEmrClusterDefinitionName());
            assertEquals("EMR cluster name", emrCluster.getEmrClusterName(), oozieWorkflowJob.getEmrClusterName());
            assertEquals("EMR status", OozieDaoImpl.OOZIE_WORKFLOW_JOB_STATUS_DM_FAILED,
                    oozieWorkflowJob.getStatus());
            assertNull("job start time is not null", oozieWorkflowJob.getStartTime());
            assertNull("job end time is not null", oozieWorkflowJob.getEndTime());
            assertNull("actions is not null", oozieWorkflowJob.getWorkflowActions());
        } catch (Exception e) {
            e.printStackTrace();
            fail("unexpected exception: " + e);
        }
    }

    /**
     * Creates a new EMR cluster in WAITING state. Any namespace and cluster definition are also created in current persistence. AMI version is used to hint
     * mock implementation to create cluster in WAITING state.
     *
     * @param namespace - namespace code to create
     * @param emrClusterDefinitionName - name of the EMR cluster definition to create
     *
     * @return newly created EMR cluster, with generated cluster name
     */
    private EmrCluster createEmrClusterInWaitingState(String namespace, String emrClusterDefinitionName) {
        String amiVersion = MockAwsOperationsHelper.AMAZON_CLUSTER_STATUS_WAITING;
        return createEmrCluster(namespace, emrClusterDefinitionName, amiVersion);
    }

    /**
     * Creates a new EMR cluster in WAITING state. Any namespace and cluster definition are also created in current persistence. AMI version is used to hint
     * mock implementation to create cluster in RUNNING state.
     *
     * @param namespace - namespace code to create
     * @param emrClusterDefinitionName - name of the EMR cluster definition to create
     *
     * @return newly created EMR cluster, with generated cluster name
     */
    private EmrCluster createEmrClusterInRunningState(String namespace, String emrClusterDefinitionName) {
        String amiVersion = MockAwsOperationsHelper.AMAZON_CLUSTER_STATUS_RUNNING;
        return createEmrCluster(namespace, emrClusterDefinitionName, amiVersion);
    }

    /**
     * Creates a new EMR cluster in BOOTSTRAPPING state. Any namespace and cluster definition are also created in current persistence. Bootstrapping is the
     * default state that the mock implementation uses.
     *
     * @param namespace - namespace code to create
     * @param emrClusterDefinitionName - name of the EMR cluster definition to create
     *
     * @return newly created EMR cluster, with generated cluster name
     */
    private EmrCluster createEmrClusterInBootstrappingState(String namespace, String emrClusterDefinitionName) {
        return createEmrCluster(namespace, emrClusterDefinitionName, null);
    }

    /**
     * Creates a new EMR cluster along with the specified namespace and definition name. A AMI version may be specified to hint the mock for certain behaviors.
     * This method uses the cluster definition specified by EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH.
     *
     * @param namespace - namespace to create
     * @param emrClusterDefinitionName - EMR cluster definition name
     * @param amiVersion - AMI version used to hint. Default behavior will be used if set to null.
     *
     * @return newly created EMR cluster, with generated cluster name
     */
    private EmrCluster createEmrCluster(String namespace, String emrClusterDefinitionName, String amiVersion) {
        EmrCluster emrCluster;

        try {
            // Create the namespace entity.
            NamespaceEntity namespaceEntity = createNamespaceEntity(namespace);

            String configXml = IOUtils.toString(
                    resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream());

            EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class,
                    configXml);

            if (amiVersion != null) {
                emrClusterDefinition.setAmiVersion(amiVersion);
            }

            configXml = xmlHelper.objectToXml(emrClusterDefinition);

            createEmrClusterDefinitionEntity(namespaceEntity, emrClusterDefinitionName, configXml);

            EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
            emrCluster = emrService.createCluster(request);
        } catch (Exception e) {
            throw new IllegalArgumentException("Error staging data", e);
        }
        return emrCluster;
    }
}