com.vmware.photon.controller.deployer.xenon.workflow.DeploymentWorkflowService.java Source code

Java tutorial

Introduction

Here is the source code for com.vmware.photon.controller.deployer.xenon.workflow.DeploymentWorkflowService.java

Source

/*
 * Copyright 2015 VMware, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License.  You may obtain a copy of
 * the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed
 * under the License is distributed on an "AS IS" BASIS, without warranties or
 * conditions of any kind, EITHER EXPRESS OR IMPLIED.  See the License for the
 * specific language governing permissions and limitations under the License.
 */

package com.vmware.photon.controller.deployer.xenon.workflow;

import com.vmware.photon.controller.api.model.DeploymentState;
import com.vmware.photon.controller.api.model.IpRange;
import com.vmware.photon.controller.api.model.UsageTag;
import com.vmware.photon.controller.cloudstore.xenon.entity.DeploymentService;
import com.vmware.photon.controller.cloudstore.xenon.entity.DhcpSubnetService;
import com.vmware.photon.controller.cloudstore.xenon.entity.HostService;
import com.vmware.photon.controller.cloudstore.xenon.entity.SubnetAllocatorService;
import com.vmware.photon.controller.common.Constants;
import com.vmware.photon.controller.common.IpHelper;
import com.vmware.photon.controller.common.xenon.ControlFlags;
import com.vmware.photon.controller.common.xenon.InitializationUtils;
import com.vmware.photon.controller.common.xenon.PatchUtils;
import com.vmware.photon.controller.common.xenon.QueryTaskUtils;
import com.vmware.photon.controller.common.xenon.ServiceUtils;
import com.vmware.photon.controller.common.xenon.TaskUtils;
import com.vmware.photon.controller.common.xenon.ValidationUtils;
import com.vmware.photon.controller.common.xenon.deployment.NoMigrationDuringDeployment;
import com.vmware.photon.controller.common.xenon.migration.DeploymentMigrationInformation;
import com.vmware.photon.controller.common.xenon.migration.NoMigrationDuringUpgrade;
import com.vmware.photon.controller.common.xenon.validation.DefaultInteger;
import com.vmware.photon.controller.common.xenon.validation.DefaultTaskState;
import com.vmware.photon.controller.common.xenon.validation.Immutable;
import com.vmware.photon.controller.common.xenon.validation.Positive;
import com.vmware.photon.controller.common.xenon.validation.WriteOnce;
import com.vmware.photon.controller.deployer.DeployerConfig;
import com.vmware.photon.controller.deployer.xenon.ContainersConfig;
import com.vmware.photon.controller.deployer.xenon.entity.ContainerService;
import com.vmware.photon.controller.deployer.xenon.entity.ContainerTemplateService;
import com.vmware.photon.controller.deployer.xenon.entity.VmService;
import com.vmware.photon.controller.deployer.xenon.task.AllocateClusterManagerResourcesTaskFactoryService;
import com.vmware.photon.controller.deployer.xenon.task.AllocateClusterManagerResourcesTaskService;
import com.vmware.photon.controller.deployer.xenon.task.CopyStateTaskFactoryService;
import com.vmware.photon.controller.deployer.xenon.task.CopyStateTaskService;
import com.vmware.photon.controller.deployer.xenon.util.HostUtils;
import com.vmware.photon.controller.nsxclient.NsxClient;
import com.vmware.photon.controller.nsxclient.datatypes.LogicalServiceResourceType;
import com.vmware.photon.controller.nsxclient.datatypes.ServiceProfileResourceType;
import com.vmware.photon.controller.nsxclient.models.DhcpRelayProfile;
import com.vmware.photon.controller.nsxclient.models.DhcpRelayProfileCreateSpec;
import com.vmware.photon.controller.nsxclient.models.DhcpRelayService;
import com.vmware.photon.controller.nsxclient.models.DhcpRelayServiceCreateSpec;
import com.vmware.photon.controller.nsxclient.models.LogicalRouter;
import com.vmware.photon.controller.nsxclient.utils.NameUtils;
import com.vmware.xenon.common.Operation;
import com.vmware.xenon.common.OperationJoin;
import com.vmware.xenon.common.Service;
import com.vmware.xenon.common.ServiceDocument;
import com.vmware.xenon.common.StatefulService;
import com.vmware.xenon.common.UriUtils;
import com.vmware.xenon.common.Utils;
import com.vmware.xenon.services.common.QueryTask;
import com.vmware.xenon.services.common.ServiceUriPaths;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.FutureCallback;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.jetty.util.BlockingArrayQueue;

import static com.google.common.base.Preconditions.checkState;

import javax.annotation.Nullable;

import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

/**
 * This class implements a Xenon service representing the end-to-end deployment workflow.
 */
public class DeploymentWorkflowService extends StatefulService {

    private static boolean inUnitTests = false;

    /**
     * This class defines the state of a {@link DeploymentWorkflowService} task.
     */
    public static class TaskState extends com.vmware.xenon.common.TaskState {

        /**
         * This value represents the current sub-stage for the task.
         */
        public SubStage subStage;

        /**
         * This enum represents the possible sub-states for this task.
         */
        public enum SubStage {
            PROVISION_MANAGEMENT_HOSTS, CREATE_MANAGEMENT_PLANE, PROVISION_ALL_HOSTS, ALLOCATE_CM_RESOURCES, GET_EDGE_CLUSTER_ID, CREATE_SUBNET_ALLOCATOR, CREATE_DHCP_SUBNET, CREATE_DHCP_RELAY_PROFILE, CREATE_DHCP_RELAY_SERVICE, MIGRATE_DEPLOYMENT_DATA, SET_DEPLOYMENT_STATE
        }
    }

    /**
     * This class defines the document state associated with a single
     * {@link DeploymentWorkflowService} instance.
     */
    @NoMigrationDuringUpgrade
    @NoMigrationDuringDeployment
    public static class State extends ServiceDocument {

        /**
         * This value represents the file name of the management VM image.
         */
        @Immutable
        public String managementVmImageFile;

        /**
         * This value represents the state of the current task.
         */
        @DefaultTaskState(value = TaskState.TaskStage.CREATED)
        public TaskState taskState;

        /**
         * This value represents the states of individual task sub-stages.
         * <p>
         * N.B. This value is not actually immutable, but it should never be set in a patch; instead, it is updated
         * synchronously in the start and patch handlers.
         */
        @Immutable
        public List<TaskState.TaskStage> taskSubStates;

        /**
         * This value represents the control flags for the operation.
         */
        @DefaultInteger(value = 0)
        public Integer controlFlags;

        /**
         * This value represents the interval, in milliseconds, to use when polling
         * the state of a task object returned by an API call.
         */
        @Positive
        public Integer taskPollDelay;

        /**
         * This value represents the override value to use for the pollInterval field
         * in child tasks.
         */
        @DefaultInteger(value = 3000)
        @Immutable
        public Integer childPollInterval;

        /**
         * This value represents the link to the {@link DeploymentService.State} entity.
         */
        @WriteOnce
        public String deploymentServiceLink;

        /**
         * This value represents the desired state of the deployed management plane.
         */
        public DeploymentState desiredState;

        /**
         * This value represents dhcp relay profile.
         */
        public String dhcpRelayProfileId;
    }

    public DeploymentWorkflowService() {
        super(State.class);
        super.toggleOption(ServiceOption.PERSISTENCE, true);
        // It is intentional to leave out the OWNER_SELECTED and REPLICATION options, because
        // this task is only intended to run on the initial deployer, and should not be
        // replicated among the new deployers in the management plane we bring up.
    }

    public static void setInUnitTests(boolean inUnitTests) {
        DeploymentWorkflowService.inUnitTests = inUnitTests;
    }

    /**
     * This method is called when a start operation is performed for the current
     * service instance.
     *
     * @param start Supplies the start operation object.
     */
    @Override
    public void handleStart(Operation start) {
        ServiceUtils.logInfo(this, "Handling start for service %s", getSelfLink());
        State startState = start.getBody(State.class);
        InitializationUtils.initialize(startState);

        if (null == startState.taskPollDelay) {
            startState.taskPollDelay = HostUtils.getDeployerContext(this).getTaskPollDelay();
        }

        if (null == startState.desiredState) {
            startState.desiredState = DeploymentState.PAUSED;
        }

        if (null == startState.managementVmImageFile) {
            startState.managementVmImageFile = DeployerConfig.getManagementImageFile();
        }

        if (TaskState.TaskStage.CREATED == startState.taskState.stage) {
            checkState(null == startState.taskSubStates);
            startState.taskSubStates = new ArrayList<>(TaskState.SubStage.values().length);
            for (TaskState.SubStage subStage : TaskState.SubStage.values()) {
                startState.taskSubStates.add(subStage.ordinal(), TaskState.TaskStage.CREATED);
            }
        }

        validateState(startState);

        if (TaskState.TaskStage.CREATED == startState.taskState.stage) {
            startState.taskState.stage = TaskState.TaskStage.STARTED;
            startState.taskState.subStage = TaskState.SubStage.PROVISION_MANAGEMENT_HOSTS;
            startState.taskSubStates.set(0, TaskState.TaskStage.STARTED);
        }

        if (startState.documentExpirationTimeMicros <= 0) {
            startState.documentExpirationTimeMicros = ServiceUtils
                    .computeExpirationTime(ServiceUtils.DEFAULT_DOC_EXPIRATION_TIME_MICROS);
        }

        start.setBody(startState).complete();

        try {
            if (ControlFlags.isOperationProcessingDisabled(startState.controlFlags)) {
                ServiceUtils.logInfo(this, "Skipping start operation processing (disabled)");
            } else if (TaskState.TaskStage.STARTED == startState.taskState.stage) {
                sendStageProgressPatch(startState.taskState);
            }
        } catch (Throwable t) {
            failTask(t);
        }
    }

    /**
     * This method is called when a patch operation is performed for the current
     * service instance.
     *
     * @param patch Supplies the start operation object.
     */
    @Override
    public void handlePatch(Operation patch) {
        ServiceUtils.logInfo(this, "Handling patch for service %s", getSelfLink());
        State startState = getState(patch);
        State patchState = patch.getBody(State.class);
        validatePatchState(startState, patchState);
        State currentState = applyPatch(startState, patchState);
        validateState(currentState);
        patch.complete();

        try {
            if (ControlFlags.isOperationProcessingDisabled(currentState.controlFlags)) {
                ServiceUtils.logInfo(this, "Skipping patch operation processing (disabled)");
            } else if (TaskState.TaskStage.STARTED == currentState.taskState.stage) {
                processStartedState(currentState);
            }
        } catch (Throwable t) {
            failTask(t);
        }
    }

    /**
     * This method validates a state object for internal consistency.
     *
     * @param currentState Supplies current state object.
     */
    private void validateState(State currentState) {
        ValidationUtils.validateState(currentState);
        ValidationUtils.validateTaskStage(currentState.taskState);
        validateTaskSubStage(currentState.taskState);

        if (TaskState.TaskStage.STARTED == currentState.taskState.stage) {
            switch (currentState.taskState.subStage) {
            case PROVISION_MANAGEMENT_HOSTS:
            case CREATE_MANAGEMENT_PLANE:
            case PROVISION_ALL_HOSTS:
            case ALLOCATE_CM_RESOURCES:
            case GET_EDGE_CLUSTER_ID:
            case CREATE_SUBNET_ALLOCATOR:
            case CREATE_DHCP_SUBNET:
            case CREATE_DHCP_RELAY_PROFILE:
            case CREATE_DHCP_RELAY_SERVICE:
            case MIGRATE_DEPLOYMENT_DATA:
            case SET_DEPLOYMENT_STATE:
                break;
            default:
                throw new IllegalStateException("Unknown task sub-stage: " + currentState.taskState.subStage);
            }
        }

        checkState(null != currentState.taskSubStates);
        checkState(TaskState.SubStage.values().length == currentState.taskSubStates.size());
        for (TaskState.SubStage subStage : TaskState.SubStage.values()) {
            try {
                TaskState.TaskStage value = currentState.taskSubStates.get(subStage.ordinal());
                checkState(null != value);
                if (null != currentState.taskState.subStage) {
                    if (currentState.taskState.subStage.ordinal() > subStage.ordinal()) {
                        checkState(TaskState.TaskStage.FINISHED == value);
                    } else if (currentState.taskState.subStage.ordinal() == subStage.ordinal()) {
                        checkState(TaskState.TaskStage.STARTED == value);
                    } else {
                        checkState(TaskState.TaskStage.CREATED == value);
                    }
                }
                if (null != currentState.taskState.subStage
                        && currentState.taskState.subStage.ordinal() >= subStage.ordinal()) {
                    checkState(value != TaskState.TaskStage.CREATED);
                }
            } catch (IndexOutOfBoundsException e) {
                throw new IllegalStateException(e);
            }
        }
    }

    private void validateTaskSubStage(TaskState taskState) {
        switch (taskState.stage) {
        case CREATED:
            checkState(null == taskState.subStage);
            break;
        case STARTED:
            checkState(null != taskState.subStage);
            break;
        case FINISHED:
        case FAILED:
        case CANCELLED:
            checkState(null == taskState.subStage);
            break;
        }
    }

    /**
     * This method checks a patch object for validity against a document state object.
     *
     * @param startState Supplies the start state object.
     * @param patchState Supplies the patch state object.
     */
    private void validatePatchState(State startState, State patchState) {
        ValidationUtils.validatePatch(startState, patchState);
        ValidationUtils.validateTaskStage(patchState.taskState);
        validateTaskSubStage(patchState.taskState);
        ValidationUtils.validateTaskStageProgression(startState.taskState, patchState.taskState);

        if (null != startState.taskState.subStage && null != patchState.taskState.subStage) {
            checkState(patchState.taskState.subStage.ordinal() >= startState.taskState.subStage.ordinal());
        }
    }

    /**
     * This method applies a patch to a state object.
     *
     * @param startState Supplies the start state object.
     * @param patchState Supplies the patch state object.
     */
    private State applyPatch(State startState, State patchState) {
        if (patchState.taskState.stage != startState.taskState.stage
                || patchState.taskState.subStage != startState.taskState.subStage) {
            ServiceUtils.logInfo(this, "Moving from %s:%s to stage %s:%s", startState.taskState.stage,
                    startState.taskState.subStage, patchState.taskState.stage, patchState.taskState.subStage);

            switch (patchState.taskState.stage) {
            case STARTED:
                startState.taskSubStates.set(patchState.taskState.subStage.ordinal(), TaskState.TaskStage.STARTED);
                // fall through
            case FINISHED:
                startState.taskSubStates.set(startState.taskState.subStage.ordinal(), TaskState.TaskStage.FINISHED);
                break;
            case FAILED:
                startState.taskSubStates.set(startState.taskState.subStage.ordinal(), TaskState.TaskStage.FAILED);
                break;
            case CANCELLED:
                startState.taskSubStates.set(startState.taskState.subStage.ordinal(),
                        TaskState.TaskStage.CANCELLED);
                break;
            }
        }

        PatchUtils.patchState(startState, patchState);
        return startState;
    }

    /**
     * This method performs document state updates in response to an operation which
     * sets the state to STARTED.
     *
     * @param currentState Supplies the current state object.
     */
    private void processStartedState(final State currentState) throws Throwable {
        switch (currentState.taskState.subStage) {
        case PROVISION_MANAGEMENT_HOSTS:
            addHosts(currentState);
            break;
        case CREATE_MANAGEMENT_PLANE:
            batchCreateManagementPlane(currentState);
            break;
        case PROVISION_ALL_HOSTS:
            processProvisionAllHosts(currentState);
            break;
        case ALLOCATE_CM_RESOURCES:
            allocateClusterManagerResources(currentState);
            break;
        case GET_EDGE_CLUSTER_ID:
            getEdgeClusterId(currentState);
            break;
        case CREATE_SUBNET_ALLOCATOR:
            createSubnetAllocator(currentState);
            break;
        case CREATE_DHCP_SUBNET:
            createDhcpSubnet(currentState);
            break;
        case CREATE_DHCP_RELAY_PROFILE:
            createDhcpRelayProfile(currentState);
            break;
        case CREATE_DHCP_RELAY_SERVICE:
            createDhcpRelayService(currentState);
            break;
        case MIGRATE_DEPLOYMENT_DATA:
            migrateData(currentState);
            break;
        case SET_DEPLOYMENT_STATE:
            setDesiredDeploymentState(currentState);
            break;
        }
    }

    private void addHosts(final State currentState) {
        ServiceUtils.logInfo(this, "Adding management hosts..");
        final Service service = this;

        FutureCallback<AddManagementHostWorkflowService.State> callback = new FutureCallback<AddManagementHostWorkflowService.State>() {
            @Override
            public void onSuccess(@Nullable AddManagementHostWorkflowService.State result) {
                switch (result.taskState.stage) {
                case FINISHED:
                    TaskUtils.sendSelfPatch(service, buildPatch(TaskState.TaskStage.STARTED,
                            TaskState.SubStage.CREATE_MANAGEMENT_PLANE, null));
                    break;
                case FAILED:
                    State patchState = buildPatch(TaskState.TaskStage.FAILED, null, null);
                    patchState.taskState.failure = result.taskState.failure;
                    TaskUtils.sendSelfPatch(service, patchState);
                    break;
                case CANCELLED:
                    TaskUtils.sendSelfPatch(service, buildPatch(TaskState.TaskStage.CANCELLED, null, null));
                    break;
                }
            }

            @Override
            public void onFailure(Throwable t) {
                failTask(t);
            }
        };

        AddManagementHostWorkflowService.State startState = new AddManagementHostWorkflowService.State();
        startState.deploymentServiceLink = currentState.deploymentServiceLink;
        startState.childPollInterval = currentState.childPollInterval;
        startState.isNewDeployment = true;

        TaskUtils.startTaskAsync(this, AddManagementHostWorkflowFactoryService.SELF_LINK, startState,
                (state) -> TaskUtils.finalTaskStages.contains(state.taskState.stage),
                AddManagementHostWorkflowService.State.class, currentState.taskPollDelay, callback);
    }

    private void batchCreateManagementPlane(final State currentState) {
        ServiceUtils.logInfo(this, "Bulk provisioning management plane");
        sendRequest(HostUtils.getCloudStoreHelper(this).createGet(currentState.deploymentServiceLink)
                .setCompletion((operation, throwable) -> {
                    if (null != throwable) {
                        failTask(throwable);
                        return;
                    }

                    DeploymentService.State deploymentState = operation.getBody(DeploymentService.State.class);
                    try {
                        batchCreateManagementPlane(currentState, deploymentState);
                    } catch (Throwable t) {
                        failTask(t);
                    }
                }));
    }

    private void batchCreateManagementPlane(final State currentState, DeploymentService.State deploymentService) {

        final Service service = this;

        FutureCallback<BatchCreateManagementWorkflowService.State> callback = new FutureCallback<BatchCreateManagementWorkflowService.State>() {
            @Override
            public void onSuccess(@Nullable BatchCreateManagementWorkflowService.State result) {
                switch (result.taskState.stage) {
                case FINISHED:
                    TaskUtils.sendSelfPatch(service,
                            buildPatch(TaskState.TaskStage.STARTED, TaskState.SubStage.PROVISION_ALL_HOSTS, null));
                    break;
                case FAILED:
                    State patchState = buildPatch(TaskState.TaskStage.FAILED, null, null);
                    patchState.taskState.failure = result.taskState.failure;
                    TaskUtils.sendSelfPatch(service, patchState);
                    break;
                case CANCELLED:
                    TaskUtils.sendSelfPatch(service, buildPatch(TaskState.TaskStage.CANCELLED, null, null));
                    break;
                }
            }

            @Override
            public void onFailure(Throwable t) {
                failTask(t);
            }
        };

        BatchCreateManagementWorkflowService.State startState = new BatchCreateManagementWorkflowService.State();

        startState.imageFile = currentState.managementVmImageFile;
        startState.deploymentServiceLink = currentState.deploymentServiceLink;
        startState.isAuthEnabled = deploymentService.oAuthEnabled;
        startState.taskPollDelay = currentState.taskPollDelay;
        startState.ntpEndpoint = deploymentService.ntpEndpoint;
        startState.childPollInterval = currentState.childPollInterval;
        startState.oAuthServerAddress = deploymentService.oAuthServerAddress;
        startState.oAuthTenantName = deploymentService.oAuthTenantName;

        TaskUtils.startTaskAsync(this, BatchCreateManagementWorkflowFactoryService.SELF_LINK, startState,
                (state) -> TaskUtils.finalTaskStages.contains(state.taskState.stage),
                BatchCreateManagementWorkflowService.State.class, currentState.taskPollDelay, callback);
    }

    private void allocateClusterManagerResources(final State currentState) {
        ServiceUtils.logInfo(this, "Allocating ClusterManager resources");
        sendRequest(HostUtils.getCloudStoreHelper(this).createGet(currentState.deploymentServiceLink)
                .setCompletion((operation, throwable) -> {
                    if (null != throwable) {
                        failTask(throwable);
                        return;
                    }

                    DeploymentService.State deploymentState = operation.getBody(DeploymentService.State.class);
                    try {
                        allocateClusterManagerResources(currentState, deploymentState);
                    } catch (Throwable t) {
                        failTask(t);
                    }
                }));
    }

    private void allocateClusterManagerResources(final State currentState, DeploymentService.State deploymentState)
            throws Throwable {

        final Service service = this;

        FutureCallback<AllocateClusterManagerResourcesTaskService.State> callback = new FutureCallback<AllocateClusterManagerResourcesTaskService.State>() {
            @Override
            public void onSuccess(@Nullable AllocateClusterManagerResourcesTaskService.State result) {
                switch (result.taskState.stage) {
                case FINISHED:
                    TaskUtils.sendSelfPatch(service,
                            buildPatch(TaskState.TaskStage.STARTED, TaskState.SubStage.GET_EDGE_CLUSTER_ID, null));
                    break;
                case FAILED:
                    State patchState = buildPatch(TaskState.TaskStage.FAILED, null, null);
                    patchState.taskState.failure = result.taskState.failure;
                    TaskUtils.sendSelfPatch(service, patchState);
                    break;
                case CANCELLED:
                    TaskUtils.sendSelfPatch(service, buildPatch(TaskState.TaskStage.CANCELLED, null, null));
                    break;
                }
            }

            @Override
            public void onFailure(Throwable t) {
                failTask(t);
            }
        };

        AllocateClusterManagerResourcesTaskService.State startState = new AllocateClusterManagerResourcesTaskService.State();
        if (deploymentState.oAuthEnabled) {
            startState.apifeProtocol = "https";
        }

        TaskUtils.startTaskAsync(this, AllocateClusterManagerResourcesTaskFactoryService.SELF_LINK, startState,
                (state) -> TaskUtils.finalTaskStages.contains(state.taskState.stage),
                AllocateClusterManagerResourcesTaskService.State.class, currentState.taskPollDelay, callback);
    }

    private void getEdgeClusterId(State currentState) {
        ServiceUtils.logInfo(this, "Getting the edge cluster ID if sdn is enabled");

        sendRequest(HostUtils.getCloudStoreHelper(this).createGet(currentState.deploymentServiceLink)
                .setCompletion((operation, throwable) -> {
                    if (null != throwable) {
                        failTask(throwable);
                        return;
                    }

                    DeploymentService.State deploymentState = operation.getBody(DeploymentService.State.class);
                    try {
                        getEdgeClusterId(currentState, deploymentState);
                    } catch (Throwable t) {
                        failTask(t);
                    }
                }));
    }

    private void getEdgeClusterId(State currentState, DeploymentService.State deploymentState) throws IOException {
        if (!deploymentState.sdnEnabled || StringUtils.isBlank(deploymentState.networkTopRouterId)) {
            TaskUtils.sendSelfPatch(this,
                    buildPatch(TaskState.TaskStage.STARTED, TaskState.SubStage.CREATE_SUBNET_ALLOCATOR, null));
            return;
        }

        NsxClient nsxClient = HostUtils.getNsxClientFactory(this).create(deploymentState.networkManagerAddress,
                deploymentState.networkManagerUsername, deploymentState.networkManagerPassword);

        nsxClient.getLogicalRouterApi().getLogicalRouter(deploymentState.networkTopRouterId,
                new FutureCallback<LogicalRouter>() {
                    @Override
                    public void onSuccess(LogicalRouter logicalRouter) {
                        saveEdgeClusterId(currentState, logicalRouter.getEdgeClusterId());
                    }

                    @Override
                    public void onFailure(Throwable t) {
                        failTask(t);
                    }
                });
    }

    private void saveEdgeClusterId(State currentState, String edgeClusterId) {
        DeploymentService.State deploymentState = new DeploymentService.State();
        deploymentState.edgeClusterId = edgeClusterId;

        sendRequest(HostUtils.getCloudStoreHelper(this).createPatch(currentState.deploymentServiceLink)
                .addRequestHeader(Operation.REPLICATION_QUORUM_HEADER,
                        Operation.REPLICATION_QUORUM_HEADER_VALUE_ALL)
                .setBody(deploymentState).setCompletion((completedOp, failure) -> {
                    if (null != failure) {
                        failTask(failure);
                    } else {
                        TaskUtils.sendSelfPatch(this, buildPatch(TaskState.TaskStage.STARTED,
                                TaskState.SubStage.CREATE_SUBNET_ALLOCATOR, null));
                    }
                }));
    }

    private void createSubnetAllocator(State currentState) {
        ServiceUtils.logInfo(this, "Creating Subnet Allocator if sdn is enabled");
        sendRequest(HostUtils.getCloudStoreHelper(this).createGet(currentState.deploymentServiceLink)
                .setCompletion((operation, throwable) -> {
                    if (null != throwable) {
                        failTask(throwable);
                        return;
                    }

                    DeploymentService.State deploymentState = operation.getBody(DeploymentService.State.class);
                    try {
                        createSubnetAllocator(deploymentState);
                    } catch (Throwable t) {
                        failTask(t);
                    }
                }));
    }

    private void createSubnetAllocator(DeploymentService.State deploymentState) {
        if (!deploymentState.sdnEnabled || deploymentState.ipRange == null || deploymentState.ipRange.isEmpty()) {
            TaskUtils.sendSelfPatch(this,
                    buildPatch(TaskState.TaskStage.STARTED, TaskState.SubStage.CREATE_DHCP_SUBNET, null));
            return;
        }

        SubnetAllocatorService.State subnetAllocatorServiceState = new SubnetAllocatorService.State();
        subnetAllocatorServiceState.rootCidr = deploymentState.ipRange;
        subnetAllocatorServiceState.documentSelfLink = SubnetAllocatorService.SINGLETON_LINK;
        subnetAllocatorServiceState.dhcpAgentEndpoint = String.format("http://%s:%d",
                // Selecting first index in Dhcp server list since expecting only one entry in this iteration
                deploymentState.dhcpServers.get(0), Constants.DHCP_AGENT_PORT);

        sendRequest(HostUtils.getCloudStoreHelper(this).createPost(SubnetAllocatorService.FACTORY_LINK)
                .setBody(subnetAllocatorServiceState).setCompletion((completedOp, failure) -> {
                    if (null != failure) {
                        failTask(failure);
                    } else {
                        TaskUtils.sendSelfPatch(this, buildPatch(TaskState.TaskStage.STARTED,
                                TaskState.SubStage.CREATE_DHCP_SUBNET, null));
                    }
                }));
    }

    private void createDhcpSubnet(State currentState) {
        ServiceUtils.logInfo(this, "Creating Dhcp Subnet if sdn is enabled");
        sendRequest(HostUtils.getCloudStoreHelper(this).createGet(currentState.deploymentServiceLink)
                .setCompletion((operation, throwable) -> {
                    if (null != throwable) {
                        failTask(throwable);
                        return;
                    }

                    DeploymentService.State deploymentState = operation.getBody(DeploymentService.State.class);
                    try {
                        createDhcpSubnet(deploymentState);
                    } catch (Throwable t) {
                        failTask(t);
                    }
                }));
    }

    private void createDhcpSubnet(DeploymentService.State deploymentState) {
        if (!deploymentState.sdnEnabled || deploymentState.floatingIpRange == null) {
            TaskUtils.sendSelfPatch(this,
                    buildPatch(TaskState.TaskStage.STARTED, TaskState.SubStage.CREATE_DHCP_RELAY_PROFILE, null));
            return;
        }

        DhcpSubnetService.State dhcpSubnetServiceState = createDhcpSubnetServiceState(
                deploymentState.floatingIpRange,
                // Selecting first index in Dhcp server list since expecting only one entry in this iteration
                deploymentState.dhcpServers.get(0));
        dhcpSubnetServiceState.documentSelfLink = DhcpSubnetService.FLOATING_IP_SUBNET_SINGLETON_LINK;
        sendRequest(HostUtils.getCloudStoreHelper(this).createPost(DhcpSubnetService.FACTORY_LINK)
                .setBody(dhcpSubnetServiceState).setCompletion((completedOp, failure) -> {
                    if (null != failure) {
                        failTask(failure);
                    } else {
                        TaskUtils.sendSelfPatch(this, buildPatch(TaskState.TaskStage.STARTED,
                                TaskState.SubStage.CREATE_DHCP_RELAY_PROFILE, null));
                    }
                }));
    }

    private void createDhcpRelayProfile(State currentState) {
        ServiceUtils.logInfo(this, "Creating Dhcp Relay Profile if sdn is enabled");
        sendRequest(HostUtils.getCloudStoreHelper(this).createGet(currentState.deploymentServiceLink)
                .setCompletion((operation, throwable) -> {
                    if (null != throwable) {
                        failTask(throwable);
                        return;
                    }

                    DeploymentService.State deploymentState = operation.getBody(DeploymentService.State.class);
                    try {
                        createDhcpRelayProfile(deploymentState);
                    } catch (Throwable t) {
                        failTask(t);
                    }
                }));
    }

    private void createDhcpRelayProfile(DeploymentService.State deploymentState) {
        if (!deploymentState.sdnEnabled || deploymentState.dhcpServers == null
                || deploymentState.dhcpServers.isEmpty()) {
            TaskUtils.sendSelfPatch(this,
                    buildPatch(TaskState.TaskStage.STARTED, TaskState.SubStage.CREATE_DHCP_RELAY_SERVICE, null));
            return;
        }

        final Service service = this;

        DhcpRelayProfileCreateSpec request = new DhcpRelayProfileCreateSpec();
        request.setResourceType(ServiceProfileResourceType.DHCP_RELAY_PROFILE);
        request.setServerAddresses(deploymentState.dhcpServers);
        request.setDisplayName(NameUtils
                .getDhcpRelayProfileName(ServiceUtils.getIDFromDocumentSelfLink(deploymentState.documentSelfLink)));
        request.setDescription(NameUtils.getDhcpRelayProfileDescription(
                ServiceUtils.getIDFromDocumentSelfLink(deploymentState.documentSelfLink)));

        try {
            NsxClient nsxClient = HostUtils.getNsxClientFactory(this).create(deploymentState.networkManagerAddress,
                    deploymentState.networkManagerUsername, deploymentState.networkManagerPassword);

            nsxClient.getDhcpServiceApi().createDhcpRelayProfile(request, new FutureCallback<DhcpRelayProfile>() {
                @Override
                public void onSuccess(DhcpRelayProfile result) {
                    try {
                        DeploymentWorkflowService.State patchState = buildPatch(TaskState.TaskStage.STARTED,
                                TaskState.SubStage.CREATE_DHCP_RELAY_SERVICE, null);
                        patchState.dhcpRelayProfileId = result.getId();
                        TaskUtils.sendSelfPatch(service, patchState);
                    } catch (Throwable t) {
                        failTask(t);
                    }
                }

                @Override
                public void onFailure(Throwable t) {
                    failTask(t);
                }
            });
        } catch (Throwable t) {
            failTask(t);
        }
    }

    private void createDhcpRelayService(State currentState) {
        ServiceUtils.logInfo(this, "Creating Dhcp Relay Service if sdn is enabled");
        sendRequest(HostUtils.getCloudStoreHelper(this).createGet(currentState.deploymentServiceLink)
                .setCompletion((operation, throwable) -> {
                    if (null != throwable) {
                        failTask(throwable);
                        return;
                    }

                    DeploymentService.State deploymentState = operation.getBody(DeploymentService.State.class);
                    try {
                        createDhcpRelayService(currentState, deploymentState);
                    } catch (Throwable t) {
                        failTask(t);
                    }
                }));
    }

    private void createDhcpRelayService(State currentState, DeploymentService.State deploymentState) {
        if (!deploymentState.sdnEnabled || currentState.dhcpRelayProfileId == null
                || currentState.dhcpRelayProfileId.isEmpty()) {
            TaskUtils.sendSelfPatch(this,
                    buildPatch(TaskState.TaskStage.STARTED, TaskState.SubStage.MIGRATE_DEPLOYMENT_DATA, null));
            return;
        }

        DhcpRelayServiceCreateSpec request = new DhcpRelayServiceCreateSpec();
        request.setResourceType(LogicalServiceResourceType.DHCP_RELAY_SERVICE);
        request.setProfileId(currentState.dhcpRelayProfileId);
        request.setDisplayName(NameUtils.getDhcpRelayServiceName(
                ServiceUtils.getIDFromDocumentSelfLink(currentState.deploymentServiceLink)));
        request.setDescription(NameUtils.getDhcpRelayServiceDescription(
                ServiceUtils.getIDFromDocumentSelfLink(currentState.deploymentServiceLink)));

        try {
            NsxClient nsxClient = HostUtils.getNsxClientFactory(this).create(deploymentState.networkManagerAddress,
                    deploymentState.networkManagerUsername, deploymentState.networkManagerPassword);

            nsxClient.getDhcpServiceApi().createDhcpRelayService(request, new FutureCallback<DhcpRelayService>() {
                @Override
                public void onSuccess(DhcpRelayService result) {
                    try {
                        updateDeploymentServiceStateDhcpRelay(currentState, result.getId());
                    } catch (Throwable t) {
                        failTask(t);
                    }
                }

                @Override
                public void onFailure(Throwable t) {
                    failTask(t);
                }
            });
        } catch (Throwable t) {
            failTask(t);
        }
    }

    private void updateDeploymentServiceStateDhcpRelay(State currentState, String dhcpRelayServiceId) {
        ServiceUtils.logInfo(this, "Setting DeploymentServiceState's Dhcp Relay Profile/Service if sdn is enabled");

        DeploymentService.State deploymentServiceState = new DeploymentService.State();
        deploymentServiceState.dhcpRelayProfileId = currentState.dhcpRelayProfileId;
        deploymentServiceState.dhcpRelayServiceId = dhcpRelayServiceId;

        try {
            sendRequest(Operation.createPatch(UriUtils.buildUri(getHost(), currentState.deploymentServiceLink))
                    .setBody(deploymentServiceState).setCompletion((completedOp, failure) -> {
                        if (null != failure) {
                            failTask(failure);
                        } else {
                            TaskUtils.sendSelfPatch(this, buildPatch(TaskState.TaskStage.STARTED,
                                    TaskState.SubStage.MIGRATE_DEPLOYMENT_DATA, null));
                        }
                    }));
        } catch (Throwable t) {
            failTask(t);
        }
    }

    private static DhcpSubnetService.State createDhcpSubnetServiceState(IpRange ipRange, String dhcpAgentIP) {
        DhcpSubnetService.State state = new DhcpSubnetService.State();
        state.lowIp = IpHelper.ipStringToLong(ipRange.getStart());
        state.highIp = IpHelper.ipStringToLong(ipRange.getEnd());
        state.isFloatingIpSubnet = true;
        state.documentSelfLink = DhcpSubnetService.FLOATING_IP_SUBNET_SINGLETON_LINK;
        state.subnetId = ServiceUtils.getIDFromDocumentSelfLink(state.documentSelfLink);
        state.dhcpAgentEndpoint = String.format("http://%s:%d", dhcpAgentIP, Constants.DHCP_AGENT_PORT);

        return state;
    }

    private void migrateData(State currentState) {
        ServiceUtils.logInfo(this, "Migrating deployment data");

        sendRequest(HostUtils.getCloudStoreHelper(this).createGet(currentState.deploymentServiceLink)
                .setCompletion((operation, throwable) -> {
                    if (null != throwable) {
                        failTask(throwable);
                        return;
                    }

                    DeploymentService.State deploymentState = operation.getBody(DeploymentService.State.class);
                    String destinationProtocol = "http";
                    if (deploymentState.oAuthEnabled && !DeploymentWorkflowService.inUnitTests) {
                        destinationProtocol = "https";
                    }
                    migrateData(currentState, destinationProtocol);
                }));
    }

    private void migrateData(State currentState, String destinationProtocol) {
        // get all container
        Operation queryContainersOp = buildBroadcastKindQuery(ContainerService.State.class);
        // get all container templates
        Operation queryTemplatesOp = buildBroadcastKindQuery(ContainerTemplateService.State.class);
        // get all vms
        Operation queryVmsOp = buildBroadcastKindQuery(VmService.State.class);

        OperationJoin.create(queryContainersOp, queryTemplatesOp, queryVmsOp).setCompletion((os, ts) -> {
            if (ts != null && !ts.isEmpty()) {
                failTask(ts.values());
                return;
            }
            List<ContainerService.State> containers = QueryTaskUtils
                    .getBroadcastQueryDocuments(ContainerService.State.class, os.get(queryContainersOp.getId()));
            List<ContainerTemplateService.State> templates = QueryTaskUtils.getBroadcastQueryDocuments(
                    ContainerTemplateService.State.class, os.get(queryTemplatesOp.getId()));
            List<VmService.State> vms = QueryTaskUtils.getBroadcastQueryDocuments(VmService.State.class,
                    os.get(queryVmsOp.getId()));

            String templateLink = templates.stream().filter(
                    template -> template.name.equals(ContainersConfig.ContainerType.PhotonControllerCore.name()))
                    .findFirst().get().documentSelfLink;
            List<String> vmServiceLinks = containers.stream()
                    .filter(container -> container.containerTemplateServiceLink.equals(templateLink))
                    .map(container -> container.vmServiceLink).collect(Collectors.toList());
            List<VmService.State> photonControllerCoreVms = vms.stream()
                    .filter(vm -> vmServiceLinks.contains(vm.documentSelfLink)).collect(Collectors.toList());

            migrateData(currentState, photonControllerCoreVms, destinationProtocol);
        }).sendWith(this);
    }

    private void migrateData(State currentState, List<VmService.State> managementVms,
            final String destinationProtocol) {
        Collection<DeploymentMigrationInformation> migrationInformation = HostUtils.getDeployerContext(this)
                .getDeploymentMigrationInformation();

        final AtomicInteger latch = new AtomicInteger(migrationInformation.size());
        final List<Throwable> errors = new BlockingArrayQueue<>();

        Set<InetSocketAddress> sourceServers = new HashSet<>();
        Set<InetSocketAddress> destinationServers = new HashSet<>();

        for (DeploymentMigrationInformation entry : migrationInformation) {
            if (sourceServers.size() == 0) {
                sourceServers.add(new InetSocketAddress(getHost().getPreferredAddress(), getHost().getPort()));
                for (VmService.State vm : managementVms) {
                    destinationServers.add(new InetSocketAddress(vm.ipAddress, vm.deployerXenonPort));
                }
            }

            String factory = entry.factoryServicePath;
            if (!factory.endsWith("/")) {
                factory += "/";
            }

            CopyStateTaskService.State startState = new CopyStateTaskService.State();
            startState.sourceURIs = Collections.singletonList(getHost().getUri());
            startState.sourceFactoryLink = factory;
            startState.destinationURI = UriUtils.buildUri(destinationProtocol, managementVms.get(0).ipAddress,
                    managementVms.get(0).deployerXenonPort, null, null);
            startState.destinationFactoryLink = factory;

            TaskUtils.startTaskAsync(this, CopyStateTaskFactoryService.SELF_LINK, startState,
                    state -> TaskUtils.finalTaskStages.contains(state.taskState.stage),
                    CopyStateTaskService.State.class, currentState.taskPollDelay,
                    new FutureCallback<CopyStateTaskService.State>() {

                        @Override
                        public void onSuccess(@Nullable CopyStateTaskService.State result) {
                            switch (result.taskState.stage) {
                            case FINISHED:
                                break;
                            case FAILED:
                            case CANCELLED:
                                errors.add(new Throwable("service: " + result.documentSelfLink + " did not finish. "
                                        + result.taskState.failure.message));
                                break;
                            default:
                                errors.add(new Throwable("service: " + result.documentSelfLink
                                        + " ended in unexpected stage " + result.taskState.stage.name()));
                                break;
                            }

                            if (latch.decrementAndGet() == 0) {
                                if (!errors.isEmpty()) {
                                    failTask(errors);
                                } else {
                                    updateDeploymentServiceState(destinationServers, currentState,
                                            destinationProtocol);
                                }
                            }
                        }

                        @Override
                        public void onFailure(Throwable t) {
                            errors.add(t);
                            if (latch.decrementAndGet() == 0) {
                                failTask(errors);
                            }
                        }
                    });
        }
    }

    private void updateDeploymentServiceState(Set<InetSocketAddress> remoteCloudStoreServers, State currentState,
            String destinationProtocol) {

        DeploymentService.State deploymentServiceState = new DeploymentService.State();
        deploymentServiceState.state = DeploymentState.READY;

        try {
            sendRequest(Operation
                    .createPatch(ServiceUtils.createUriFromServerSet(remoteCloudStoreServers,
                            currentState.deploymentServiceLink, destinationProtocol))
                    .addRequestHeader(Operation.REPLICATION_QUORUM_HEADER,
                            Operation.REPLICATION_QUORUM_HEADER_VALUE_ALL)
                    .setBody(deploymentServiceState).setCompletion((completedOp, failure) -> {
                        if (null != failure) {
                            failTask(failure);
                        } else {
                            TaskUtils.sendSelfPatch(this, buildPatch(TaskState.TaskStage.STARTED,
                                    TaskState.SubStage.SET_DEPLOYMENT_STATE, null));
                        }
                    }));
        } catch (URISyntaxException e) {
            failTask(e);
        }
    }

    private void processProvisionAllHosts(final State currentState) throws Throwable {

        final Service service = this;

        FutureCallback<BulkProvisionHostsWorkflowService.State> callback = new FutureCallback<BulkProvisionHostsWorkflowService.State>() {
            @Override
            public void onSuccess(@Nullable BulkProvisionHostsWorkflowService.State result) {
                switch (result.taskState.stage) {
                case FINISHED:
                    TaskUtils.sendSelfPatch(service, buildPatch(TaskState.TaskStage.STARTED,
                            TaskState.SubStage.ALLOCATE_CM_RESOURCES, null));
                    break;
                case FAILED:
                    State patchState = buildPatch(TaskState.TaskStage.FAILED, null, null);
                    patchState.taskState.failure = result.taskState.failure;
                    TaskUtils.sendSelfPatch(service, patchState);
                    break;
                case CANCELLED:
                    TaskUtils.sendSelfPatch(service, buildPatch(TaskState.TaskStage.CANCELLED, null, null));
                    break;
                }
            }

            @Override
            public void onFailure(Throwable t) {
                failTask(t);
            }
        };

        BulkProvisionHostsWorkflowService.State startState = new BulkProvisionHostsWorkflowService.State();
        startState.deploymentServiceLink = currentState.deploymentServiceLink;

        sendRequest(HostUtils.getCloudStoreHelper(this).createGet(currentState.deploymentServiceLink)
                .setCompletion((o, e) -> {
                    try {
                        if (e != null) {
                            failTask(e);
                        } else {
                            DeploymentService.State deploymentState = o.getBody(DeploymentService.State.class);
                            ServiceUtils.logInfo(this, "Deployment Auth Enabled: %s",
                                    deploymentState.oAuthEnabled.toString());
                            ServiceUtils.logInfo(this,
                                    "Create cert flag being set to %s in BulkProvisionHostService for all host",
                                    deploymentState.oAuthEnabled.toString());
                            startState.createCert = deploymentState.oAuthEnabled;
                            // Here even if we are going to provision all the host, the tag is being set to CLOUD.
                            startState.usageTag = UsageTag.CLOUD.name();

                            QueryTask.Query kindClause = new QueryTask.Query()
                                    .setTermPropertyName(ServiceDocument.FIELD_NAME_KIND)
                                    .setTermMatchValue(Utils.buildKind(HostService.State.class));
                            QueryTask.QuerySpecification querySpecification = new QueryTask.QuerySpecification();
                            querySpecification.query.addBooleanClause(kindClause);
                            startState.querySpecification = querySpecification;

                            TaskUtils.startTaskAsync(this, BulkProvisionHostsWorkflowFactoryService.SELF_LINK,
                                    startState,
                                    (state) -> TaskUtils.finalTaskStages.contains(state.taskState.stage),
                                    BulkProvisionHostsWorkflowService.State.class, currentState.taskPollDelay,
                                    callback);
                        }
                    } catch (Throwable t) {
                        failTask(t);
                    }
                }));

    }

    private void setDesiredDeploymentState(State currentState) {
        DeploymentService.State deployment = new DeploymentService.State();
        deployment.state = currentState.desiredState;

        sendRequest(HostUtils.getCloudStoreHelper(this).createPatch(currentState.deploymentServiceLink)
                .addRequestHeader(Operation.REPLICATION_QUORUM_HEADER,
                        Operation.REPLICATION_QUORUM_HEADER_VALUE_ALL)
                .setBody(deployment).setCompletion((completedOp, failure) -> {
                    if (null != failure) {
                        failTask(failure);
                    } else {
                        TaskUtils.sendSelfPatch(this, buildPatch(TaskState.TaskStage.FINISHED, null, null));
                    }
                }));
    }

    private Operation buildBroadcastKindQuery(Class<? extends ServiceDocument> type) {
        QueryTask.Query query = QueryTask.Query.Builder.create().addKindFieldClause(type).build();
        return Operation
                .createPost(UriUtils.buildBroadcastRequestUri(
                        UriUtils.buildUri(getHost(), ServiceUriPaths.CORE_LOCAL_QUERY_TASKS),
                        ServiceUriPaths.DEFAULT_NODE_SELECTOR))
                .setBody(QueryTask.Builder.createDirectTask()
                        .addOptions(EnumSet.of(QueryTask.QuerySpecification.QueryOption.EXPAND_CONTENT))
                        .setQuery(query).build());
    }

    /**
     * This method sends a patch operation to the current service instance to
     * move to a new state.
     *
     * @param state
     */
    private void sendStageProgressPatch(TaskState state) {
        ServiceUtils.logInfo(this, "Sending self-patch to stage %s", state.stage, state.subStage);
        TaskUtils.sendSelfPatch(this, buildPatch(state.stage, state.subStage, null));
    }

    /**
     * This method sends a patch operation to the current service instance to
     * move to the FAILED state in response to the specified exception.
     *
     * @param e
     */
    private void failTask(Throwable e) {
        ServiceUtils.logSevere(this, e);
        TaskUtils.sendSelfPatch(this, buildPatch(TaskState.TaskStage.FAILED, null, e));
    }

    private void failTask(Collection<Throwable> failures) {
        failures.forEach((throwable) -> ServiceUtils.logSevere(this, throwable));
        TaskUtils.sendSelfPatch(this, buildPatch(TaskState.TaskStage.FAILED, null, failures.iterator().next()));
    }

    /**
     * This method builds a patch state object which can be used to submit a
     * self-patch.
     *
     * @param patchStage
     * @param patchSubStage
     * @param t
     * @return
     */
    @VisibleForTesting
    protected static State buildPatch(TaskState.TaskStage patchStage, @Nullable TaskState.SubStage patchSubStage,
            @Nullable Throwable t) {

        State patchState = new State();
        patchState.taskState = new TaskState();
        patchState.taskState.stage = patchStage;
        patchState.taskState.subStage = patchSubStage;

        if (null != t) {
            patchState.taskState.failure = Utils.toServiceErrorResponse(t);
        }

        return patchState;
    }
}