com.vmware.bdd.manager.ClusterManager.java Source code

Introduction

Here is the source code for com.vmware.bdd.manager.ClusterManager.java

Source

/***************************************************************************
 * Copyright (c) 2012-2015 VMware, Inc. All Rights Reserved.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ***************************************************************************/

package com.vmware.bdd.manager;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileFilter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.*;
import java.util.regex.Pattern;

import com.vmware.aurora.vc.VcResourcePool;
import com.vmware.bdd.apitypes.*;
import com.vmware.bdd.exception.*;
import com.vmware.bdd.placement.entity.BaseNode;
import com.vmware.bdd.service.impl.ClusteringService;
import com.vmware.bdd.utils.*;
import com.vmware.vim.binding.vim.Folder;
import org.apache.commons.collections.CollectionUtils;
import com.vmware.bdd.service.resmgmt.IVcInventorySyncService;
import com.vmware.bdd.service.resmgmt.sync.filter.VcResourceFilters;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.BooleanUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.batch.core.JobParameter;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.vmware.aurora.global.Configuration;
import com.vmware.aurora.vc.VcVirtualMachine;
import com.vmware.aurora.vc.vcservice.VcContext;
import com.vmware.bdd.aop.annotation.ClusterManagerPointcut;
import com.vmware.bdd.apitypes.ClusterCreate;
import com.vmware.bdd.apitypes.ClusterRead;
import com.vmware.bdd.apitypes.ClusterStatus;
import com.vmware.bdd.apitypes.ClusterType;
import com.vmware.bdd.apitypes.LimitInstruction;
import com.vmware.bdd.apitypes.NodeGroupCreate;
import com.vmware.bdd.apitypes.NodeGroupRead;
import com.vmware.bdd.apitypes.NodeRead;
import com.vmware.bdd.apitypes.NodeStatus;
import com.vmware.bdd.apitypes.PlacementPolicy.GroupAssociation;
import com.vmware.bdd.apitypes.Priority;
import com.vmware.bdd.apitypes.TaskRead;
import com.vmware.bdd.apitypes.VcClusterMap;
import com.vmware.bdd.entity.ClusterEntity;
import com.vmware.bdd.entity.NetworkEntity;
import com.vmware.bdd.entity.NodeEntity;
import com.vmware.bdd.entity.NodeGroupEntity;
import com.vmware.bdd.entity.VcResourcePoolEntity;
import com.vmware.bdd.manager.intf.IClusterEntityManager;
import com.vmware.bdd.service.IClusterHealService;
import com.vmware.bdd.service.IClusteringService;
import com.vmware.bdd.service.IExecutionService;
import com.vmware.bdd.service.impl.ClusterUserMgmtValidService;
import com.vmware.bdd.service.job.JobConstants;
import com.vmware.bdd.service.resmgmt.INetworkService;
import com.vmware.bdd.service.resmgmt.INodeTemplateService;
import com.vmware.bdd.service.resmgmt.IResourceService;
import com.vmware.bdd.service.utils.VcResourceUtils;
import com.vmware.bdd.software.mgmt.plugin.intf.SoftwareManager;
import com.vmware.bdd.software.mgmt.plugin.model.HadoopStack;
import com.vmware.bdd.software.mgmt.plugin.model.NodeGroupInfo;
import com.vmware.bdd.specpolicy.ClusterSpecFactory;
import com.vmware.bdd.specpolicy.CommonClusterExpandPolicy;
import com.vmware.bdd.spectypes.IronfanStack;
import com.vmware.bdd.spectypes.VcCluster;
import com.vmware.bdd.usermgmt.UserMgmtConstants;

public class ClusterManager {
    static final Logger logger = Logger.getLogger(ClusterManager.class);

    private ClusterConfigManager clusterConfigMgr;

    private INetworkService networkManager;

    private JobManager jobManager;

    private IResourceService resMgr;

    private IClusterEntityManager clusterEntityMgr;

    private RackInfoManager rackInfoMgr;

    private IClusteringService clusteringService;

    private IClusterHealService clusterHealService;

    private IExecutionService executionService;
    private SoftwareManagerCollector softwareManagerCollector;

    private ShrinkManager shrinkManager;

    @Autowired
    private INodeTemplateService nodeTemplateService;

    @Autowired
    private ClusterUserMgmtValidService clusterUserMgmtValidService;

    @Autowired
    private VcResourceFilterBuilder vcResourceFilterBuilder;

    @Autowired
    private UnsupportedOpsBlocker opsBlocker;

    @Autowired
    private IVcInventorySyncService syncService;

    private static boolean extraPackagesExisted = false;
    private static HashSet<String> extraRequiredPackages = getExtraRequiredPackages();
    private static final String commRegex = "-[0-9]+\\.[0-9]+.*\\.rpm";

    private static HashSet<String> getExtraRequiredPackages() {
        HashSet<String> hs = new HashSet<String>();
        String extraPackStr = Configuration
                .getString(Constants.SERENGETI_YUM_EXTRA_PACKAGES_CONFIG, Constants.SERENGETI_YUM_EXTRA_PACKAGES)
                .trim();
        if (!extraPackStr.isEmpty()) {
            String[] packs = extraPackStr.split(",");
            hs.addAll(Arrays.asList(packs));
        }
        return hs;
    }

    public JobManager getJobManager() {
        return jobManager;
    }

    public void setJobManager(JobManager jobManager) {
        this.jobManager = jobManager;
    }

    public ClusterConfigManager getClusterConfigMgr() {
        return clusterConfigMgr;
    }

    public void setClusterConfigMgr(ClusterConfigManager clusterConfigMgr) {
        this.clusterConfigMgr = clusterConfigMgr;
    }

    public INetworkService getNetworkManager() {
        return networkManager;
    }

    public void setNetworkManager(INetworkService networkManager) {
        this.networkManager = networkManager;
    }

    public IResourceService getResMgr() {
        return resMgr;
    }

    @Autowired
    public void setResMgr(IResourceService resMgr) {
        this.resMgr = resMgr;
    }

    public IClusterEntityManager getClusterEntityMgr() {
        return clusterEntityMgr;
    }

    @Autowired
    public void setClusterEntityMgr(IClusterEntityManager clusterEntityMgr) {
        this.clusterEntityMgr = clusterEntityMgr;
    }

    public RackInfoManager getRackInfoMgr() {
        return rackInfoMgr;
    }

    @Autowired
    public void setRackInfoMgr(RackInfoManager rackInfoMgr) {
        this.rackInfoMgr = rackInfoMgr;
    }

    public IClusteringService getClusteringService() {
        return clusteringService;
    }

    @Autowired
    public void setClusteringService(IClusteringService clusteringService) {
        this.clusteringService = clusteringService;
    }

    public IClusterHealService getClusterHealService() {
        return clusterHealService;
    }

    @Autowired
    public void setClusterHealService(IClusterHealService clusterHealService) {
        this.clusterHealService = clusterHealService;
    }

    public IExecutionService getExecutionService() {
        return executionService;
    }

    @Autowired
    public void setExecutionService(IExecutionService executionService) {
        this.executionService = executionService;
    }

    @Autowired
    public void setSoftwareManagerCollector(SoftwareManagerCollector softwareManagerCollector) {
        this.softwareManagerCollector = softwareManagerCollector;
    }

    @Autowired
    public void setShrinkManager(ShrinkManager shrinkManager) {
        this.shrinkManager = shrinkManager;
    }

    public Map<String, Object> getClusterConfigManifest(String clusterName, List<String> targets,
            boolean needAllocIp) {
        ClusterCreate clusterConfig = clusterConfigMgr.getClusterConfig(clusterName, needAllocIp);
        Map<String, String> cloudProvider = resMgr.getCloudProviderAttributes();

        ClusterRead read = clusterEntityMgr.findClusterWithNodes(clusterName, true);

        Map<String, Object> attrs = new HashMap<String, Object>();

        if (Constants.IRONFAN.equalsIgnoreCase(clusterConfig.getAppManager())) {
            SoftwareManager softwareManager = clusterConfigMgr.getSoftwareManager(clusterConfig.getAppManager());
            IronfanStack stack = (IronfanStack) filterDistroFromAppManager(softwareManager,
                    clusterConfig.getDistro());
            CommonClusterExpandPolicy.expandDistro(clusterConfig, stack);

            attrs.put("cloud_provider", cloudProvider);
            attrs.put("cluster_definition", clusterConfig);
        }

        if (read != null) {
            attrs.put("cluster_data", read);
        }
        if (targets != null && !targets.isEmpty()) {
            attrs.put("targets", targets);
        }
        return attrs;
    }

    private void writeJsonFile(Map<String, Object> clusterConfig, File file) {
        Gson gson = new GsonBuilder().excludeFieldsWithoutExposeAnnotation().setPrettyPrinting().create();
        String jsonStr = gson.toJson(clusterConfig);

        AuAssert.check(jsonStr != null);
        logger.debug("writing cluster manifest in json " + jsonStr + " to file " + file);
        BufferedWriter out = null;
        try {
            out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "UTF-8"));
            out.write(jsonStr);
        } catch (IOException ex) {
            logger.error(ex.getMessage() + "\n failed to write cluster manifest to file " + file);
            throw BddException.INTERNAL(ex, "Failed to write cluster manifest.");
        } finally {
            if (out != null) {
                try {
                    out.close();
                } catch (IOException e) {
                    logger.error("falied to close writer" + out, e);
                }
            }
        }
    }

    public File writeClusterSpecFile(String targetName, File workDir, boolean needAllocIp) {
        String clusterName = targetName.split("-")[0];
        String fileName = clusterName + ".json";
        List<String> targets = new ArrayList<String>(1);
        targets.add(targetName);
        Map<String, Object> clusterConfig = getClusterConfigManifest(clusterName, targets, needAllocIp);

        File file = new File(workDir, fileName);
        writeJsonFile(clusterConfig, file);

        return file;
    }

    private void refreshClusterStatus(String clusterName) {
        List<String> clusterNames = new ArrayList<String>();
        clusterNames.add(clusterName);
        refreshClusterStatus(clusterNames);
    }

    private void refreshClusterStatus(List<String> clusterNames) {
        for (String clusterName : clusterNames) {
            Map<String, JobParameter> param = new TreeMap<String, JobParameter>();
            param.put(JobConstants.TIMESTAMP_JOB_PARAM, new JobParameter(new Date()));
            param.put(JobConstants.CLUSTER_NAME_JOB_PARAM, new JobParameter(clusterName));
            JobParameters jobParameters = new JobParameters(param);
            try {
                long jobExecutionId = jobManager.runJob(JobConstants.QUERY_CLUSTER_JOB_NAME, jobParameters);
                TaskRead status = jobManager.getJobExecutionStatus(jobExecutionId);
                while (status.getStatus() != TaskRead.Status.COMPLETED
                        && status.getStatus() != TaskRead.Status.FAILED
                        && status.getStatus() != TaskRead.Status.ABANDONED
                        && status.getStatus() != TaskRead.Status.STOPPED) {
                    Thread.sleep(1000);
                    status = jobManager.getJobExecutionStatus(jobExecutionId);
                }
            } catch (Exception ex) {
                logger.error("failed to run query cluster job: " + clusterName, ex);
            }
        }
    }

    public ClusterRead getClusterByName(String clusterName, boolean realTime) {
        ClusterEntity cluster = clusterEntityMgr.findByName(clusterName);
        if (cluster == null) {
            throw BddException.NOT_FOUND("Cluster", clusterName);
        }

        // return the latest data from db
        java.util.Date startTime = new java.util.Date();
        if (realTime && cluster.getStatus().isSyncServiceStatus()) {
            // for not running cluster, we don't sync up status from chef
            refreshClusterStatus(clusterName);
        }
        logger.debug("refreshClusterStatus time: " + (new java.util.Date().getTime() - startTime.getTime()));
        return realTime ? clusterEntityMgr.findClusterWithNodes(clusterName, false)
                : clusterEntityMgr.findClusterWithNodeGroups(clusterName);
    }

    public ClusterCreate getClusterSpec(String clusterName) {
        ClusterCreate spec = clusterConfigMgr.getClusterConfig(clusterName);
        spec.setVcClusters(null);
        spec.setDistroMap(null);
        spec.setSharedDatastorePattern(null);
        spec.setLocalDatastorePattern(null);
        spec.setNetworkings(null);
        spec.setNetworkConfig(null);
        spec.setName(null);
        spec.setValidateConfig(null);
        spec.setSpecFile(null);
        spec.setTopologyPolicy(null);
        spec.setHostToRackMap(null);
        spec.setHttpProxy(null);
        spec.setNoProxy(null);
        spec.setDistroVendor(null);
        spec.setDistroVersion(null);
        spec.setPassword(null);
        spec.setHostnamePrefix(null);
        NodeGroupCreate[] groups = spec.getNodeGroups();
        if (groups != null) {
            for (NodeGroupCreate group : groups) {
                group.setVcClusters(null);
                group.getStorage().setImagestoreNamePattern(null);
                group.getStorage().setDiskstoreNamePattern(null);
                group.setVmFolderPath(null);
                group.getStorage().setSplitPolicy(null);
                group.getStorage().setControllerType(null);
                group.getStorage().setAllocType(null);
                if (group.getPlacementPolicies() != null) {
                    List<GroupAssociation> associations = group.getPlacementPolicies().getGroupAssociations();
                    if (associations != null && associations.isEmpty()) {
                        group.getPlacementPolicies().setGroupAssociations(null);
                    }
                }
            }
        }
        return spec;
    }

    public List<ClusterRead> getClusters(Boolean realTime) {
        java.util.Date startTime = new java.util.Date();

        List<ClusterRead> clusters = new ArrayList<ClusterRead>();
        List<String> clusterNames = clusterEntityMgr.findAllClusterNames();
        logger.debug("findAllClusters time:" + (new java.util.Date().getTime() - startTime.getTime()));
        for (String clusterName : clusterNames) {
            clusters.add(getClusterByName(clusterName, realTime));
        }
        logger.debug("getClusters total time:" + (new java.util.Date().getTime() - startTime.getTime()));
        return clusters;
    }

    @ClusterManagerPointcut
    public Long createCluster(ClusterCreate createSpec, org.apache.commons.configuration.Configuration newParams)
            throws Exception {
        logger.debug("entering createCluster: " + createSpec.getName());

        SoftwareManager softMgr = softwareManagerCollector.getSoftwareManager(createSpec.getAppManager());
        // @ Todo if specify hadoop stack, we can get hadoop stack by stack name. Otherwise, we will get a default hadoop stack.
        HadoopStack stack = clusterConfigMgr.filterDistroFromAppManager(softMgr, createSpec.getDistro());

        // if the distro is not specified by the REST Client, add it.
        if (CommonUtil.isBlank(createSpec.getDistro())) {
            createSpec.setDistro(stack.getDistro());
        }
        createSpec.setDistroVendor(stack.getVendor());
        createSpec.setDistroVersion(stack.getFullVersion());
        if (createSpec.getType() == null) {
            createSpec.setType(ClusterType.DEFAULT);
        }

        // create auto rps if vc cluster/rp is specified
        createAutoRps(createSpec);
        ClusterCreate clusterSpec = ClusterSpecFactory.getCustomizedSpec(createSpec, softMgr.getType());
        verifyRequiredPackages(clusterSpec);
        clusterSpec.verifyClusterNameLength();
        clusterSpec.validateNodeGroupNames();
        //Check the cpu, memory max configuration according vm hardware version
        if (clusterSpec != null && clusterSpec.getNodeGroups() != null) {
            for (NodeGroupCreate ng : clusterSpec.getNodeGroups()) {
                String templateVmId = this.nodeTemplateService
                        .getNodeTemplateIdByName(clusterSpec.getTemplateName());
                if (templateVmId != null) {
                    VcResourceUtils.checkVmMaxConfiguration(templateVmId,
                            ng.getCpuNum() == null ? 0 : ng.getCpuNum(),
                            ng.getMemCapacityMB() == null ? 0 : ng.getMemCapacityMB());
                }
            }
        }
        String name = clusterSpec.getName();

        validateInfraConfig(clusterSpec);
        if (softMgr.getType().equalsIgnoreCase(Constants.CLOUDERA_MANAGER_PLUGIN_TYPE)) {
            validateServiceUserAndGroupsInLdap(clusterSpec);
        }
        logger.info("start to create a cluster: " + name);

        List<String> dsNames = getDsNamesToBeUsed(clusterSpec.getDsNames());
        if (dsNames.isEmpty()) {
            throw ClusterConfigException.NO_DATASTORE_ADDED();
        }
        List<VcCluster> vcClusters = getVcClustersToBeUsed(clusterSpec.getRpNames());
        if (vcClusters == null || vcClusters.isEmpty()) {
            throw ClusterConfigException.NO_RESOURCE_POOL_ADDED();
        }

        if (newParams.getBoolean(Constants.SKIP_REFRESH_VC, false)) {
            logger.info("skip refresh vc resources.");
        } else {
            VcResourceFilters filters = vcResourceFilterBuilder.build(dsNames, getRpNames(clusterSpec.getRpNames()),
                    createSpec.getNetworkNames());
            syncService.refreshInventory(filters);
        }

        // validate accessibility
        validateDatastore(dsNames, vcClusters);
        validateNetworkAccessibility(createSpec.getName(), createSpec.getNetworkNames(), vcClusters);

        // get the current cluster clone type from the configuration file
        // if it is not set in configuration file, then use INSTANT clone for VC6/ESXi6
        String type = getClusterCloneType();
        clusterSpec.setClusterCloneType(type);

        //save configuration into meta-db, and extend configuration using default spec
        clusterConfigMgr.createClusterConfig(clusterSpec);

        clusterEntityMgr.updateClusterStatus(name, ClusterStatus.PROVISIONING);
        Map<String, JobParameter> param = new TreeMap<String, JobParameter>();
        param.put(JobConstants.TIMESTAMP_JOB_PARAM, new JobParameter(new Date()));
        param.put(JobConstants.CLUSTER_SUCCESS_STATUS_JOB_PARAM, new JobParameter(ClusterStatus.RUNNING.name()));
        param.put(JobConstants.CLUSTER_FAILURE_STATUS_JOB_PARAM,
                new JobParameter(ClusterStatus.PROVISION_ERROR.name()));
        param.put(JobConstants.CLUSTER_NAME_JOB_PARAM, new JobParameter(createSpec.getName()));
        param.put(JobConstants.VERIFY_NODE_STATUS_SCOPE_PARAM,
                new JobParameter(JobConstants.CLUSTER_NODE_SCOPE_VALUE));
        JobParameters jobParameters = new JobParameters(param);
        return jobManager.runJob(JobConstants.CREATE_CLUSTER_JOB_NAME, jobParameters);
    }

    private Map<String, Set<String>> getServiceGroupUsers(ClusterCreate clusterSpec) {
        Map<String, Set<String>> groupUsers = new HashMap<>();
        Map<String, Object> configuration = clusterSpec.getConfiguration();
        if (MapUtils.isNotEmpty(configuration)) {
            Map<String, Map<String, String>> serviceUsersConfigs = (Map<String, Map<String, String>>) configuration
                    .get(UserMgmtConstants.SERVICE_USER_CONFIG_IN_SPEC_FILE);
            if (MapUtils.isNotEmpty(serviceUsersConfigs)) {
                for (Map<String, String> serviceUserConfig : serviceUsersConfigs.values()) {
                    String groupName = serviceUserConfig.get(UserMgmtConstants.SERVICE_USER_GROUP);
                    String userName = serviceUserConfig.get(UserMgmtConstants.SERVICE_USER_NAME);
                    if (groupName != null && !groupName.isEmpty() && userName != null && !userName.isEmpty()) {
                        if (groupUsers.get(groupName) == null) {
                            Set<String> users = new HashSet<>();
                            groupUsers.put(groupName, users);
                        }
                        groupUsers.get(groupName).add(userName);
                    }
                }
            }
        }
        return groupUsers;
    }

    private void validateServiceUserAndGroupsInLdap(ClusterCreate clusterSpec) {
        Map<String, Set<String>> groupUsers = getServiceGroupUsers(clusterSpec);
        if (!groupUsers.isEmpty()) {
            logger.info("going to validate Ldap user and groups for: " + new Gson().toJson(groupUsers));
            clusterUserMgmtValidService.validateGroupUsers(UserMgmtConstants.DEFAULT_USERMGMT_SERVER_NAME,
                    groupUsers);
            logger.info("validate service user in Ldap succeed");
        }
    }

    private void validateInfraConfig(ClusterCreate clusterSpec) {
        Map<String, Map<String, String>> infraConfig = clusterSpec.getInfrastructure_config();

        if (MapUtils.isEmpty(infraConfig)) {
            logger.info("no infra configuration in cluster create spec!");
            return;
        }

        Map<String, String> userMgmtCfg = infraConfig.get(UserMgmtConstants.LDAP_USER_MANAGEMENT);

        if (MapUtils.isEmpty(userMgmtCfg)) {
            logger.debug("no user management configuration section.");
        } else {
            clusterUserMgmtValidService.validateUserMgmtConfig(userMgmtCfg);

            logger.info("user management configuration validated successfully!");
        }
    }

    private void createAutoRps(ClusterCreate createSpec) {
        if (createSpec.getVcClusters() == null) {
            return;
        }
        // user specify vc resource pools directly, create auto rp in meta db dynamically
        List<VcCluster> vcClusters = createSpec.getVcClusters();
        List<String> rpNames = clusterConfigMgr.getRpMgr().addAutoResourcePools(vcClusters, true);
        logger.info("added automation resource pools: " + rpNames);
        createSpec.setRpNames(rpNames);
    }

    private void validateNetworkAccessibility(final String clusterName, List<String> networkList,
            List<VcCluster> clusters) {
        if (networkList == null || networkList.isEmpty()) {
            throw ClusterConfigException.NETWORK_IS_NOT_SPECIFIED(clusterName);
        }
        Set<String> networkNames = new HashSet<String>();
        networkNames.addAll(networkList);
        logger.info("start to validate network if exsit in db.");
        verifyNetworkNamesExsitInDB(networkNames, clusterName);
        logger.info("start to validate network accessibility.");
        if (!resMgr.isNetworkAccessibleByCluster(networkList, clusters)) {
            List<String> clusterNames = new ArrayList<String>();
            for (VcCluster cluster : clusters) {
                clusterNames.add(cluster.getName());
            }
            throw ClusterConfigException.NETWORK_UNACCESSIBLE(networkList, clusterNames);
        }
    }

    private void validateDatastore(List<String> dsNames, List<VcCluster> clusters) {
        // validate if there is any datastore is accessible by one cluster
        logger.info("start to validate accessibility for datastores: " + dsNames + ", and clusters: " + clusters);
        boolean found = false;
        for (String dsName : dsNames) {
            for (VcCluster vcCluster : clusters) {
                if (resMgr.isDatastoreAccessibleByCluster(dsName, vcCluster.getName())) {
                    found = true;
                    break;
                }
            }
        }
        if (!found) {
            // no any datastore is accessible by specified cluster
            List<String> vcClusterNames = new ArrayList<String>();
            for (VcCluster vcCluster : clusters) {
                vcClusterNames.add(vcCluster.getName());
            }
            throw ClusterConfigException.DATASTORE_UNACCESSIBLE(vcClusterNames, dsNames);
        }
    }

    /**
     * Get the dsNames to be used by the cluster
     */
    private List<String> getDsNamesToBeUsed(final List<String> specifiedDsNames) {
        if (specifiedDsNames == null || specifiedDsNames.isEmpty()) {
            List<String> allDsNames = new ArrayList<>();
            allDsNames.addAll(clusterConfigMgr.getDatastoreMgr().getAllDatastoreNames());
            return allDsNames;
        } else {
            return validateGivenDS(specifiedDsNames);
        }
    }

    private List<String> getRpNames(List<String> rpNames) {
        if (CollectionUtils.isEmpty(rpNames)) {
            List<String> newRpNameList = new ArrayList<>();
            newRpNameList.addAll(clusterConfigMgr.getRpMgr().getAllRPNames());
            return newRpNameList;
        }

        return rpNames;
    }

    /**
     * Get the vCenter clusters to be used by the cluster
     */
    private List<VcCluster> getVcClustersToBeUsed(List<String> rpNames) {
        List<VcCluster> clusters = null;
        if (rpNames == null || rpNames.isEmpty()) {
            clusters = clusterConfigMgr.getRpMgr().getAllVcResourcePool();
        } else {
            clusters = new ArrayList<VcCluster>();
            StringBuffer nonexistentRpNames = new StringBuffer();
            for (String rpName : rpNames) {
                List<VcCluster> vcClusters = clusterConfigMgr.getRpMgr().getVcResourcePoolByName(rpName);

                if (vcClusters == null) {
                    nonexistentRpNames.append(rpName).append(",");
                } else {
                    clusters.addAll(vcClusters);
                }
            }
            if (nonexistentRpNames.length() > 0) {
                nonexistentRpNames.delete(nonexistentRpNames.length() - 1, nonexistentRpNames.length());
                throw VcProviderException.RESOURCE_POOL_NOT_FOUND(nonexistentRpNames.toString());
            }
        }
        return clusters;
    }

    @ClusterManagerPointcut
    public Long configCluster(String clusterName, ClusterCreate createSpec) throws Exception {
        opsBlocker.blockUnsupportedOpsByCluster("configCluster", clusterName);

        logger.info("ClusterManager, config cluster " + clusterName);
        ClusterEntity cluster;

        if ((cluster = clusterEntityMgr.findByName(clusterName)) == null) {
            logger.error("cluster " + clusterName + " does not exist");
            throw BddException.NOT_FOUND("Cluster", clusterName);
        }

        ValidationUtils.validateVersion(clusterEntityMgr, clusterName);

        if (!cluster.getStatus().isActiveServiceStatus()
                && !ClusterStatus.CONFIGURE_ERROR.equals(cluster.getStatus())
                && !ClusterStatus.SERVICE_STOPPED.equals(cluster.getStatus())) {
            logger.error("can not config cluster: " + clusterName + ", " + cluster.getStatus());
            throw ClusterManagerException.UPDATE_NOT_ALLOWED_ERROR(clusterName,
                    "To update a cluster, its status must be RUNNING, CONFIGURE_ERROR or SERVICE_ERROR");
        }
        clusterConfigMgr.updateAppConfig(clusterName, createSpec);

        Map<String, JobParameter> param = new TreeMap<String, JobParameter>();
        param.put(JobConstants.CLUSTER_NAME_JOB_PARAM, new JobParameter(clusterName));
        param.put(JobConstants.TIMESTAMP_JOB_PARAM, new JobParameter(new Date()));
        param.put(JobConstants.CLUSTER_SUCCESS_STATUS_JOB_PARAM, new JobParameter(ClusterStatus.RUNNING.name()));
        param.put(JobConstants.CLUSTER_FAILURE_STATUS_JOB_PARAM,
                new JobParameter(ClusterStatus.CONFIGURE_ERROR.name()));
        JobParameters jobParameters = new JobParameters(param);
        clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.CONFIGURING);
        clusterEntityMgr.cleanupActionError(clusterName);
        try {
            return jobManager.runJob(JobConstants.CONFIG_CLUSTER_JOB_NAME, jobParameters);
        } catch (Exception e) {
            logger.error("Failed to configure cluster " + clusterName, e);
            clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.CONFIGURE_ERROR);
            throw e;
        }
    }

    public long enableLdap(String clusterName) throws Exception {
        Map<String, JobParameter> param = new TreeMap<String, JobParameter>();
        param.put(JobConstants.CLUSTER_NAME_JOB_PARAM, new JobParameter(clusterName));
        param.put(JobConstants.TIMESTAMP_JOB_PARAM, new JobParameter(new Date()));
        param.put(JobConstants.CLUSTER_SUCCESS_STATUS_JOB_PARAM, new JobParameter(ClusterStatus.RUNNING.name()));
        param.put(JobConstants.CLUSTER_FAILURE_STATUS_JOB_PARAM,
                new JobParameter(ClusterStatus.CONFIGURE_ERROR.name()));
        JobParameters jobParameters = new JobParameters(param);
        clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.CONFIGURING);
        clusterEntityMgr.cleanupActionError(clusterName);
        try {
            return jobManager.runJob("configLdapUserMgmtJob", jobParameters);
        } catch (Exception e) {
            logger.error("Failed to configure cluster " + clusterName, e);
            clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.CONFIGURE_ERROR);
            throw e;
        }
    }

    public Long resumeClusterCreation(String clusterName, org.apache.commons.configuration.Configuration newParams)
            throws Exception {
        logger.info("ClusterManager, resume cluster creation " + clusterName);

        ClusterEntity cluster = clusterEntityMgr.findByName(clusterName);

        if (cluster == null) {
            logger.error("cluster " + clusterName + " does not exist");
            throw BddException.NOT_FOUND("Cluster", clusterName);
        }

        ValidationUtils.validateVersion(clusterEntityMgr, clusterName);

        if (cluster.getStatus() != ClusterStatus.EXPANDING) {
            if (cluster.getStatus() != ClusterStatus.PROVISION_ERROR) {
                logger.error("can not resume creation of cluster: " + clusterName + ", " + cluster.getStatus());
                throw ClusterManagerException.UPDATE_NOT_ALLOWED_ERROR(clusterName,
                        "To update a cluster, its status must be PROVISION_ERROR or SERVICE_ERROR");
            }
        }

        List<String> dsNames = getDsNamesToBeUsed(cluster.getVcDatastoreNameList());
        if (dsNames.isEmpty()) {
            throw ClusterConfigException.NO_DATASTORE_ADDED();
        }
        List<VcCluster> vcClusters = getVcClustersToBeUsed(cluster.getVcRpNameList());
        if (vcClusters.isEmpty()) {
            throw ClusterConfigException.NO_RESOURCE_POOL_ADDED();
        }

        if (newParams.getBoolean(Constants.SKIP_REFRESH_VC, false)) {
            logger.info("skip refresh vc resources.");
        } else {
            VcResourceFilters filters = vcResourceFilterBuilder.build(dsNames,
                    getRpNames(cluster.getVcRpNameList()), cluster.fetchNetworkNameList());
            syncService.refreshInventory(filters);
        }

        // validate accessibility
        validateDatastore(dsNames, vcClusters);
        validateNetworkAccessibility(cluster.getName(), cluster.fetchNetworkNameList(), vcClusters);
        Map<String, JobParameter> param = new TreeMap<String, JobParameter>();
        param.put(JobConstants.CLUSTER_NAME_JOB_PARAM, new JobParameter(clusterName));
        param.put(JobConstants.TIMESTAMP_JOB_PARAM, new JobParameter(new Date()));
        param.put(JobConstants.CLUSTER_SUCCESS_STATUS_JOB_PARAM, new JobParameter(ClusterStatus.RUNNING.name()));
        param.put(JobConstants.CLUSTER_FAILURE_STATUS_JOB_PARAM,
                new JobParameter(ClusterStatus.PROVISION_ERROR.name()));
        JobParameters jobParameters = new JobParameters(param);
        clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.PROVISIONING);
        clusterEntityMgr.cleanupActionError(clusterName);
        try {
            return jobManager.runJob(JobConstants.RESUME_CLUSTER_JOB_NAME, jobParameters);
        } catch (Exception e) {
            logger.error("Failed to resume cluster creation for cluster " + clusterName, e);
            clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.PROVISION_ERROR);
            throw e;
        }
    }

    @ClusterManagerPointcut
    public Long deleteClusterByName(String clusterName) throws Exception {
        logger.info("ClusterManager, deleting cluster " + clusterName);

        ClusterEntity cluster = clusterEntityMgr.findByName(clusterName);

        if (cluster == null) {
            logger.error("cluster " + clusterName + " does not exist");
            throw BddException.NOT_FOUND("Cluster", clusterName);
        }

        if (!cluster.getStatus().isStableStatus()) {
            logger.error(
                    "cluster: " + clusterName + " cannot be deleted, it is in " + cluster.getStatus() + " status");
            throw ClusterManagerException.DELETION_NOT_ALLOWED_ERROR(clusterName,
                    "To delete a cluster, its status must be RUNNING, STOPPED, ERROR, PROVISION_ERROR, CONFIGURE_ERROR or UPGRADE_ERROR");
        }
        Map<String, JobParameter> param = new TreeMap<String, JobParameter>();
        param.put(JobConstants.CLUSTER_NAME_JOB_PARAM, new JobParameter(clusterName));
        param.put(JobConstants.TIMESTAMP_JOB_PARAM, new JobParameter(new Date()));
        param.put(JobConstants.CLUSTER_FAILURE_STATUS_JOB_PARAM, new JobParameter(ClusterStatus.ERROR.name()));
        JobParameters jobParameters = new JobParameters(param);
        clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.DELETING);
        clusterEntityMgr.cleanupActionError(clusterName);
        try {
            return jobManager.runJob(JobConstants.DELETE_CLUSTER_JOB_NAME, jobParameters);
        } catch (Exception e) {
            logger.error("Failed to delete cluster " + clusterName, e);
            cluster = clusterEntityMgr.findByName(clusterName);
            if (cluster != null) {
                clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.ERROR);
            }
            throw e;
        }
    }

    @ClusterManagerPointcut
    public Long upgradeClusterByName(String clusterName) throws Exception {
        logger.info("ClusterManager, upgrading cluster " + clusterName);

        ClusterEntity cluster = clusterEntityMgr.findByName(clusterName);

        if (cluster == null) {
            logger.error("cluster " + clusterName + " does not exist");
            throw BddException.NOT_FOUND("Cluster", clusterName);
        }

        if (!clusterEntityMgr.needUpgrade(clusterName)) {
            logger.error("cluster " + clusterName + " is the latest version already");
            throw ClusterManagerException.ALREADY_LATEST_VERSION_ERROR(clusterName);
        }

        if (!cluster.getStatus().isStableStatus()) {
            logger.error(
                    "cluster: " + clusterName + " cannot be upgraded, it is in " + cluster.getStatus() + " status");
            throw ClusterManagerException.UPGRADE_NOT_ALLOWED_ERROR(clusterName,
                    "To upgrade a cluster, its status must be RUNNING, STOPPED, ERROR, CONFIGURE_ERROR or UPGRADE_ERROR");
        }
        Map<String, JobParameter> param = new TreeMap<String, JobParameter>();
        param.put(JobConstants.CLUSTER_NAME_JOB_PARAM, new JobParameter(clusterName));
        param.put(JobConstants.TIMESTAMP_JOB_PARAM, new JobParameter(new Date()));
        param.put(JobConstants.CLUSTER_FAILURE_STATUS_JOB_PARAM,
                new JobParameter(ClusterStatus.UPGRADE_ERROR.name()));
        JobParameters jobParameters = new JobParameters(param);
        clusterEntityMgr.storeClusterLastStatus(clusterName);
        clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.UPGRADING);
        clusterEntityMgr.updateNodesActionForUpgrade(clusterName, Constants.NODE_ACTION_UPGRADING);
        clusterEntityMgr.cleanupErrorForClusterUpgrade(clusterName);
        try {
            return jobManager.runJob(JobConstants.UPGRADE_CLUSTER_JOB_NAME, jobParameters);
        } catch (Exception e) {
            logger.error("Failed to upgrade cluster " + clusterName, e);
            cluster = clusterEntityMgr.findByName(clusterName);
            if (cluster != null) {
                clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.UPGRADE_ERROR);
            }
            throw e;
        }
    }

    public Long startCluster(String clusterName, boolean force) throws Exception {
        logger.info("ClusterManager, starting cluster " + clusterName);

        ClusterEntity cluster = clusterEntityMgr.findByName(clusterName);
        if (cluster == null) {
            logger.error("cluster " + clusterName + " does not exist");
            throw BddException.NOT_FOUND("Cluster", clusterName);
        }

        ValidationUtils.validateVersion(clusterEntityMgr, clusterName);

        if (cluster.getStatus().isActiveServiceStatus()) {
            logger.error("cluster " + clusterName + " is running already");
            throw ClusterManagerException.ALREADY_STARTED_ERROR(clusterName);
        }

        if (!ClusterStatus.STOPPED.equals(cluster.getStatus())
                && !ClusterStatus.ERROR.equals(cluster.getStatus())) {
            logger.error(
                    "cluster " + clusterName + " cannot be started, it is in " + cluster.getStatus() + " status");
            throw ClusterManagerException.START_NOT_ALLOWED_ERROR(clusterName,
                    "To start a cluster, its status must be STOPPED or ERROR");
        }

        cluster.setVhmTargetNum(-1);
        clusterEntityMgr.update(cluster);
        clusterEntityMgr.cleanupActionError(clusterName);
        Map<String, JobParameter> param = new TreeMap<String, JobParameter>();
        param.put(JobConstants.CLUSTER_NAME_JOB_PARAM, new JobParameter(clusterName));
        param.put(Constants.FORCE_CLUSTER_OPERATION_JOB_PARAM, new JobParameter(String.valueOf(force)));
        param.put(JobConstants.TIMESTAMP_JOB_PARAM, new JobParameter(new Date()));
        param.put(JobConstants.CLUSTER_SUCCESS_STATUS_JOB_PARAM, new JobParameter(ClusterStatus.RUNNING.name()));
        param.put(JobConstants.CLUSTER_FAILURE_STATUS_JOB_PARAM, new JobParameter(ClusterStatus.ERROR.name()));
        JobParameters jobParameters = new JobParameters(param);
        clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.STARTING);
        try {
            return jobManager.runJob(JobConstants.START_CLUSTER_JOB_NAME, jobParameters);
        } catch (Exception e) {
            logger.error("Failed to start cluster " + clusterName, e);
            clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.ERROR);
            throw e;
        }
    }

    @ClusterManagerPointcut
    public Long stopCluster(String clusterName) throws Exception {
        logger.info("ClusterManager, stopping cluster " + clusterName);

        ClusterEntity cluster = clusterEntityMgr.findByName(clusterName);
        if (cluster == null) {
            logger.error("cluster " + clusterName + " does not exist");
            throw BddException.NOT_FOUND("Cluster", clusterName);
        }

        ValidationUtils.validateVersion(clusterEntityMgr, clusterName);

        if (ClusterStatus.STOPPED.equals(cluster.getStatus())) {
            logger.error("cluster " + clusterName + " is stopped already");
            throw ClusterManagerException.ALREADY_STOPPED_ERROR(clusterName);
        }

        if (!cluster.getStatus().isActiveServiceStatus()
                && !ClusterStatus.SERVICE_STOPPED.equals(cluster.getStatus())
                && !ClusterStatus.ERROR.equals(cluster.getStatus())
                && !ClusterStatus.SERVICE_WARNING.equals(cluster.getStatus())) {
            logger.error(
                    "cluster " + clusterName + " cannot be stopped, it is in " + cluster.getStatus() + " status");
            throw ClusterManagerException.STOP_NOT_ALLOWED_ERROR(clusterName,
                    "To stop a cluster, its status must be RUNNING, ERROR, SERVICE_WARNING or SERVICE_STOPPED");
        }
        Map<String, JobParameter> param = new TreeMap<String, JobParameter>();
        param.put(JobConstants.CLUSTER_NAME_JOB_PARAM, new JobParameter(clusterName));
        param.put(JobConstants.TIMESTAMP_JOB_PARAM, new JobParameter(new Date()));
        param.put(JobConstants.CLUSTER_SUCCESS_STATUS_JOB_PARAM, new JobParameter(ClusterStatus.STOPPED.name()));
        param.put(JobConstants.CLUSTER_FAILURE_STATUS_JOB_PARAM, new JobParameter(ClusterStatus.ERROR.name()));
        JobParameters jobParameters = new JobParameters(param);
        clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.STOPPING);
        clusterEntityMgr.cleanupActionError(clusterName);
        try {
            return jobManager.runJob(JobConstants.STOP_CLUSTER_JOB_NAME, jobParameters);
        } catch (Exception e) {
            logger.error("Failed to stop cluster " + clusterName, e);
            clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.ERROR);
            throw e;
        }
    }

    @ClusterManagerPointcut
    public Long resizeCluster(String clusterName, String nodeGroupName, int instanceNum, boolean force,
            org.apache.commons.configuration.Configuration newParams) throws Exception {
        logger.info("ClusterManager, updating node group " + nodeGroupName + " in cluster " + clusterName
                + " reset instance number to " + instanceNum);

        ClusterEntity cluster = clusterEntityMgr.findByName(clusterName);
        if (cluster == null) {
            logger.error("cluster " + clusterName + " does not exist");
            throw BddException.NOT_FOUND("Cluster", clusterName);
        }

        ValidationUtils.validateVersion(clusterEntityMgr, clusterName);

        List<String> dsNames = getDsNamesToBeUsed(cluster.getVcDatastoreNameList());
        if (dsNames.isEmpty()) {
            throw ClusterConfigException.NO_DATASTORE_ADDED();
        }

        List<VcCluster> vcClusters = getVcClustersToBeUsed(cluster.getVcRpNameList());
        if (vcClusters.isEmpty()) {
            throw ClusterConfigException.NO_RESOURCE_POOL_ADDED();
        }

        if (newParams.getBoolean(Constants.SKIP_REFRESH_VC, false)) {
            logger.info("skip refresh vc resources.");
        } else {
            VcResourceFilters filters = vcResourceFilterBuilder.build(dsNames,
                    getRpNames(cluster.getVcRpNameList()), cluster.fetchNetworkNameList());
            syncService.refreshInventory(filters);
        }

        // validate accessibility
        validateDatastore(dsNames, vcClusters);
        validateNetworkAccessibility(cluster.getName(), cluster.fetchNetworkNameList(), vcClusters);

        NodeGroupEntity group = clusterEntityMgr.findByName(cluster, nodeGroupName);
        if (group == null) {
            logger.error("nodegroup " + nodeGroupName + " of cluster " + clusterName + " does not exist");
            throw ClusterManagerException.NODEGROUP_NOT_FOUND_ERROR(nodeGroupName);
        }

        AuAssert.check(!group.getRoleNameList().isEmpty(), "roles should not be empty");
        SoftwareManager softMgr = softwareManagerCollector.getSoftwareManager(cluster.getAppManager());
        NodeGroupInfo groupInfo = clusterEntityMgr.toNodeGroupInfo(clusterName, nodeGroupName);
        List<String> unsupportedRoles = softMgr.validateRolesForScaleOut(groupInfo);
        if (!unsupportedRoles.isEmpty()) {
            logger.info("can not resize node group with role: " + unsupportedRoles);
            throw ClusterManagerException.ROLES_NOT_SUPPORTED(unsupportedRoles);
        }

        logger.info("check cluster status before resize action:" + cluster.getStatus());
        if (!cluster.getStatus().isActiveServiceStatus()) {
            logger.error("cluster " + clusterName + " can be resized only in RUNNING status, it is now in "
                    + cluster.getStatus() + " status");
            throw ClusterManagerException.UPDATE_NOT_ALLOWED_ERROR(clusterName,
                    "To update a cluster, its status must be RUNNING");
        }

        if (instanceNum < group.getDefineInstanceNum()) {
            try {
                softMgr.validateRolesForShrink(groupInfo);
                return shrinkManager.shrinkNodeGroup(clusterName, nodeGroupName, instanceNum);
            } catch (Exception e) {
                if (!(e instanceof ShrinkException)) {
                    logger.error("Failed to shrink cluster " + clusterName, e);
                    throw ShrinkException.SHRINK_NODE_GROUP_FAILED(e, clusterName, e.getMessage());
                }
            }
        }

        //when use force, we allow user to rerun resize to fix bad nodes
        if ((instanceNum == group.getDefineInstanceNum()) && !force) {
            logger.error("the new instanceNum " + instanceNum + " shouldn't be the same as the old one ");
            throw ClusterManagerException.NO_NEED_TO_RESIZE(clusterName, nodeGroupName, instanceNum);
        }

        Integer instancePerHost = group.getInstancePerHost();
        if (instancePerHost != null && instanceNum % instancePerHost != 0) {
            throw BddException.INVALID_PARAMETER("instance number", new StringBuilder(100).append(instanceNum)
                    .append(".instanceNum must be evenly divisible by instancePerHost").toString());
        }

        ValidationUtils.validHostNumber(clusterEntityMgr, group, instanceNum);
        ValidationUtils.hasEnoughHost(rackInfoMgr, clusterEntityMgr, group, instanceNum);

        int oldInstanceNum = group.getDefineInstanceNum();
        group.setDefineInstanceNum(instanceNum);

        clusterEntityMgr.update(group);
        clusterEntityMgr.cleanupActionError(clusterName);
        // create job
        Map<String, JobParameter> param = new TreeMap<String, JobParameter>();
        param.put(JobConstants.CLUSTER_NAME_JOB_PARAM, new JobParameter(clusterName));
        param.put(JobConstants.GROUP_NAME_JOB_PARAM, new JobParameter(nodeGroupName));
        param.put(JobConstants.GROUP_INSTANCE_NEW_NUMBER_JOB_PARAM, new JobParameter(Long.valueOf(instanceNum)));
        param.put(JobConstants.GROUP_INSTANCE_OLD_NUMBER_JOB_PARAM, new JobParameter(Long.valueOf(oldInstanceNum)));
        param.put(JobConstants.TIMESTAMP_JOB_PARAM, new JobParameter(new Date()));
        param.put(JobConstants.CLUSTER_SUCCESS_STATUS_JOB_PARAM, new JobParameter(ClusterStatus.RUNNING.name()));
        param.put(JobConstants.CLUSTER_FAILURE_STATUS_JOB_PARAM, new JobParameter(ClusterStatus.RUNNING.name()));
        param.put(JobConstants.VERIFY_NODE_STATUS_SCOPE_PARAM,
                new JobParameter(JobConstants.GROUP_NODE_SCOPE_VALUE));
        param.put(Constants.FORCE_CLUSTER_OPERATION_JOB_PARAM, new JobParameter(String.valueOf(force)));
        JobParameters jobParameters = new JobParameters(param);
        clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.UPDATING);
        try {
            return jobManager.runJob(JobConstants.RESIZE_CLUSTER_JOB_NAME, jobParameters);
        } catch (Exception e) {
            logger.error("Failed to resize cluster " + clusterName, e);
            clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.RUNNING);
            group.setDefineInstanceNum(oldInstanceNum);
            clusterEntityMgr.update(group);
            throw e;
        }
    }

    /**
     * set cluster parameters synchronously
     *
     * @param clusterName
     * @param activeComputeNodeNum
     * @param minComputeNodeNum
     * @param maxComputeNodeNum
     * @param enableAuto
     * @param ioPriority
     * @return
     * @throws Exception
     */
    @SuppressWarnings("unchecked")
    public List<String> syncSetParam(String clusterName, Integer activeComputeNodeNum, Integer minComputeNodeNum,
            Integer maxComputeNodeNum, Boolean enableAuto, Priority ioPriority) throws Exception {

        //allow set ioshare only.
        if (enableAuto != null || activeComputeNodeNum != null || maxComputeNodeNum != null
                || minComputeNodeNum != null) {
            opsBlocker.blockUnsupportedOpsByCluster("syncSetElasticity", clusterName);
        }

        ClusterEntity cluster = clusterEntityMgr.findByName(clusterName);
        ClusterRead clusterRead = getClusterByName(clusterName, false);
        if (cluster == null) {
            logger.error("cluster " + clusterName + " does not exist");
            throw BddException.NOT_FOUND("Cluster", clusterName);
        }

        ValidationUtils.validateVersion(clusterEntityMgr, clusterName);

        clusterEntityMgr.cleanupActionError(clusterName);
        //update vm ioshares
        if (ioPriority != null) {
            prioritizeCluster(clusterName, ioPriority);
        }

        // as prioritizeCluster will update clusterEntity, here we need to refresh to avoid overriding
        cluster = clusterEntityMgr.findByName(clusterName);

        if (enableAuto != null && enableAuto != cluster.getAutomationEnable()) {
            if (enableAuto && cluster.getDistroVendor().equalsIgnoreCase(Constants.MAPR_VENDOR)) {
                logger.error("cluster " + clusterName + " is a MAPR distro, which cannot be auto scaled");
                throw BddException.NOT_ALLOWED_SCALING("cluster", clusterName);
            }
            cluster.setAutomationEnable(enableAuto);
        }

        if (minComputeNodeNum != null && minComputeNodeNum != cluster.getVhmMinNum()) {
            cluster.setVhmMinNum(minComputeNodeNum);
        }

        if (maxComputeNodeNum != null && maxComputeNodeNum != cluster.getVhmMaxNum()) {
            cluster.setVhmMaxNum(maxComputeNodeNum);
        }

        List<String> nodeGroupNames = new ArrayList<String>();
        if ((enableAuto != null || minComputeNodeNum != null || maxComputeNodeNum != null
                || activeComputeNodeNum != null) && !clusterRead.validateSetManualElasticity(nodeGroupNames)) {
            throw BddException.INVALID_PARAMETER("cluster", clusterName);
        }

        if (activeComputeNodeNum != null) {
            if (!activeComputeNodeNum.equals(cluster.getVhmTargetNum())) {
                cluster.setVhmTargetNum(activeComputeNodeNum);
            }
        }

        //enableAuto is only set during cluster running status and
        //other elasticity attributes are only set during cluster running/stop status
        if ((enableAuto != null) && !cluster.getStatus().isActiveServiceStatus()) {
            logger.error("Cannot change elasticity mode, when cluster " + clusterName + " is in "
                    + cluster.getStatus() + " status");
            throw ClusterManagerException.SET_AUTO_ELASTICITY_NOT_ALLOWED_ERROR(clusterName,
                    "The cluster's status must be RUNNING");
        }
        if (!cluster.getStatus().isActiveServiceStatus()
                && !ClusterStatus.SERVICE_STOPPED.equals(cluster.getStatus())
                && !ClusterStatus.STOPPED.equals(cluster.getStatus())) {
            logger.error("Cannot change elasticity parameters, when cluster " + clusterName + " is in "
                    + cluster.getStatus() + " status");
            throw ClusterManagerException.SET_AUTO_ELASTICITY_NOT_ALLOWED_ERROR(clusterName,
                    "The cluster's status must be RUNNING, SERVICE_WARNING, SERVICE_ERROR or STOPPED");
        }

        clusterEntityMgr.update(cluster);

        //update vhm extra config file
        if (enableAuto != null || minComputeNodeNum != null || maxComputeNodeNum != null) {
            boolean success = clusteringService.setAutoElasticity(clusterName, false);
            if (!success) {
                throw ClusterManagerException.FAILED_TO_SET_AUTO_ELASTICITY_ERROR(clusterName,
                        "Could not update elasticity configuration file");
            }
        }

        //waitForManual if switch to Manual and targetNodeNum is null
        if (enableAuto != null && !enableAuto && cluster.getVhmTargetNum() == null) {
            JobUtils.waitForManual(clusterName, executionService);
        }

        return nodeGroupNames;
    }

    /**
     * set cluster parameters asynchronously
     *
     * @param clusterName
     * @param activeComputeNodeNum
     * @param minComputeNodeNum
     * @param maxComputeNodeNum
     * @param enableAuto
     * @param ioPriority
     * @return
     * @throws Exception
     */
    @ClusterManagerPointcut
    public Long asyncSetParam(String clusterName, Integer activeComputeNodeNum, Integer minComputeNodeNum,
            Integer maxComputeNodeNum, Boolean enableAuto, Priority ioPriority) throws Exception {
        //allow set ioshare only.
        if (enableAuto != null || activeComputeNodeNum != null || maxComputeNodeNum != null
                || minComputeNodeNum != null) {
            opsBlocker.blockUnsupportedOpsByCluster("asyncSetElasticity", clusterName);
        }

        ValidationUtils.validateVersion(clusterEntityMgr, clusterName);

        syncSetParam(clusterName, activeComputeNodeNum, minComputeNodeNum, maxComputeNodeNum, enableAuto,
                ioPriority);

        ClusterRead cluster = getClusterByName(clusterName, false);
        // cluster must be running status
        if (!cluster.getStatus().isActiveServiceStatus()) {
            String msg = "Cluster " + clusterName + " is not running.";
            logger.error(msg);
            throw ClusterManagerException.SET_MANUAL_ELASTICITY_NOT_ALLOWED_ERROR(msg);
        }

        Map<String, JobParameter> param = new TreeMap<String, JobParameter>();
        param.put(JobConstants.TIMESTAMP_JOB_PARAM, new JobParameter(new Date()));
        param.put(JobConstants.CLUSTER_NAME_JOB_PARAM, new JobParameter(clusterName));
        // TODO: transfer SET_TARGET/UNLIMIT from CLI directly
        if (activeComputeNodeNum == null) {
            param.put(JobConstants.VHM_ACTION_JOB_PARAM, new JobParameter(LimitInstruction.actionWaitForManual));
        } else if (activeComputeNodeNum == -1) {
            param.put(JobConstants.VHM_ACTION_JOB_PARAM, new JobParameter(LimitInstruction.actionUnlimit));
        } else {
            param.put(JobConstants.VHM_ACTION_JOB_PARAM, new JobParameter(LimitInstruction.actionSetTarget));
        }
        if (activeComputeNodeNum != null) {
            param.put(JobConstants.ACTIVE_COMPUTE_NODE_NUMBER_JOB_PARAM,
                    new JobParameter(Long.valueOf(activeComputeNodeNum)));
        }
        param.put(JobConstants.CLUSTER_SUCCESS_STATUS_JOB_PARAM, new JobParameter(ClusterStatus.RUNNING.name()));
        param.put(JobConstants.CLUSTER_FAILURE_STATUS_JOB_PARAM, new JobParameter(ClusterStatus.RUNNING.name()));
        JobParameters jobParameters = new JobParameters(param);
        try {
            clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.VHM_RUNNING);
            return jobManager.runJob(JobConstants.SET_MANUAL_ELASTICITY_JOB_NAME, jobParameters);
        } catch (Exception e) {
            logger.error("Failed to set manual elasticity for cluster " + clusterName, e);
            clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.RUNNING);
            throw e;
        }
    }

    @ClusterManagerPointcut
    @Transactional
    public void updateCluster(ClusterCreate clusterUpdate, boolean ignoreWarning, boolean appendResource)
            throws Exception {
        String clusterName = clusterUpdate.getName();
        ClusterEntity cluster = clusterEntityMgr.findByName(clusterName);
        if (cluster == null) {
            logger.error("cluster " + clusterName + " does not exist");
            throw BddException.NOT_FOUND("Cluster", clusterName);
        }
        ValidationUtils.validateVersion(clusterEntityMgr, clusterName);

        List<String> newRpList = clusterUpdate.getRpNames();

        if (!CollectionUtils.isEmpty(newRpList)) {
            //Check whether the new resourcepools are valid in vc_resource_pool
            List<String> existRPs = validateGivenRp(newRpList);
            if (CollectionUtils.isEmpty(existRPs)) {
                throw ClusterConfigException.NO_RESOURCE_POOL_ADDED();
            }
            //check whether the updating resourcepool only the new ones
            if (appendResource == true) {
                List<String> usingRpList = cluster.getVcRpNameList();
                if (CollectionUtils.isEmpty(usingRpList)) {
                    return;
                }
                newRpList = appendResource(usingRpList, newRpList);
            } else {
                //Check whether the new resourcepools include the current resourcepools which are used by this cluster
                Set<VcResourcePoolEntity> usedVCRps = cluster.getUsedRps();
                List<String> usedRpList = new ArrayList<String>(usedVCRps.size());
                for (VcResourcePoolEntity rp : usedVCRps) {
                    usedRpList.add(rp.getName());
                }
                logger.info("Updating resourcepools for cluster " + clusterName + " from " + usedRpList.toString()
                        + " to " + newRpList.toString());
                if (!newRpList.containsAll(usedRpList)) {
                    throw BddException.NEW_RP_EXCLUDE_OLD_RP(null, usedRpList.toString(), newRpList.toString());
                }
            }

            cluster.setVcRpNameList(newRpList);
        }

        List<String> newDsList = clusterUpdate.getDsNames();

        if (!CollectionUtils.isEmpty(newDsList)) {
            //Check whether the new datastores are valid vc_data_store
            if (CollectionUtils.isEmpty(validateGivenDS(newDsList))) {
                throw ClusterConfigException.NO_DATASTORE_ADDED();
            }
            //check whether the updating resourcepool only the new ones
            if (appendResource == true) {
                List<String> usingDsList = cluster.getVcDatastoreNameList();
                if (CollectionUtils.isEmpty(usingDsList))
                    return;
                newDsList = appendResource(usingDsList, newDsList);
            }
            //Check whether the new dsNames contain all datastores used by this cluster already
            List<String> usedDsList = cluster.getVcDatastoreNameList();
            logger.info("Updating dsNames for cluster " + clusterName + " from " + usedDsList + " to "
                    + newDsList.toString());
            if (!ignoreWarning) {
                if (CollectionUtils.isEmpty(usedDsList)) {
                    throw WarningMessageException.SET_EMPTY_DATASTORES_TO_NON_EMTPY(null, newDsList.toString());
                } else if (!newDsList.containsAll(usedDsList)) {
                    throw WarningMessageException.NEW_DATASTORES_EXCLUDE_OLD_DATASTORES(null, newDsList.toString(),
                            usedDsList.toString());
                }
            }

            cluster.setVcDatastoreNameList(newDsList);
        }

        clusterEntityMgr.update(cluster);
    }

    private List<String> validateGivenDS(List<String> specifiedDsNames) {
        List<String> exitsDs = new ArrayList<String>();
        Set<String> allDs = clusterConfigMgr.getDatastoreMgr().getAllDatastoreNames();
        StringBuffer nonexistentDsNames = new StringBuffer();

        for (String dsName : specifiedDsNames) {
            if (!allDs.contains(dsName))
                nonexistentDsNames.append(dsName).append(",");
            else
                exitsDs.add(dsName);
        }

        if (nonexistentDsNames.length() > 0) {
            nonexistentDsNames.delete(nonexistentDsNames.length() - 1, nonexistentDsNames.length());
            throw VcProviderException.DATASTORE_NOT_FOUND(nonexistentDsNames.toString());
        }

        return exitsDs;
    }

    private List<String> validateGivenRp(List<String> specifiedRpNames) {
        List<String> exitsRps = new ArrayList<String>();
        Set<String> allRps = clusterConfigMgr.getRpMgr().getAllRPNames();
        StringBuffer nonexistentRpNames = new StringBuffer();

        for (String rpName : specifiedRpNames) {
            if (!allRps.contains(rpName))
                nonexistentRpNames.append(rpName).append(",");
            else
                exitsRps.add(rpName);
        }

        if (nonexistentRpNames.length() > 0) {
            nonexistentRpNames.delete(nonexistentRpNames.length() - 1, nonexistentRpNames.length());
            throw VcProviderException.RESOURCE_POOL_NOT_FOUND(nonexistentRpNames.toString());
        }

        return exitsRps;
    }

    private List<String> appendResource(List<String> fullRs, List<String> addRs) {
        for (String rs : addRs) {
            if (!fullRs.contains(rs)) {
                fullRs.add(rs);
            }
        }
        return fullRs;
    }

    /*
     * Change the disk I/O priority of the cluster or a node group
     */
    public void prioritizeCluster(String clusterName, Priority ioShares) throws Exception {
        ClusterEntity cluster = clusterEntityMgr.findByName(clusterName);
        if (cluster == null) {
            logger.error("cluster " + clusterName + " does not exist");
            throw BddException.NOT_FOUND("Cluster", clusterName);
        }

        if (ioShares.equals(cluster.getIoShares())) {
            return;
        }

        logger.info("Change all nodes' disk I/O shares to " + ioShares + " in the cluster " + clusterName);

        // cluster must be in RUNNING or STOPPEED status
        if (!cluster.getStatus().isActiveServiceStatus() && !ClusterStatus.STOPPED.equals(cluster.getStatus())) {
            String msg = "The cluster's status must be RUNNING or STOPPED";
            logger.error(msg);
            throw ClusterManagerException.PRIORITIZE_CLUSTER_NOT_ALLOWED_ERROR(clusterName, msg);
        }

        // get target nodeuster
        List<NodeEntity> targetNodes;
        targetNodes = clusterEntityMgr.findAllNodes(clusterName);

        if (targetNodes.isEmpty()) {
            throw ClusterManagerException.PRIORITIZE_CLUSTER_NOT_ALLOWED_ERROR(clusterName,
                    "Target node set is empty");
        }

        clusterEntityMgr.cleanupActionError(clusterName);
        // call clustering service to set the io shares
        Map<String, String> failedNodes = clusteringService.configIOShares(clusterName, targetNodes, ioShares);
        if (failedNodes.isEmpty()) {
            logger.info("configured " + targetNodes.size() + " nodes' IO share level to " + ioShares.toString());
        } else {
            // update node table
            for (String name : failedNodes.keySet()) {
                NodeEntity node = clusterEntityMgr.findNodeByName(name);
                node.setActionFailed(true);
                node.setErrMessage(failedNodes.get(name));
                clusterEntityMgr.update(node);
            }
            throw ClusterManagerException.PRIORITIZE_CLUSTER_FAILED(clusterName, failedNodes.size(),
                    targetNodes.size());
        }

        // update io shares in db
        cluster.setIoShares(ioShares);
        clusterEntityMgr.update(cluster);
    }

    @ClusterManagerPointcut
    public Long fixDiskFailures(String clusterName, String groupName) throws Exception {
        opsBlocker.blockUnsupportedOpsByCluster("fixDisk", clusterName);

        ClusterEntity cluster = clusterEntityMgr.findByName(clusterName);
        if (cluster == null) {
            logger.error("cluster " + clusterName + " does not exist");
            throw BddException.NOT_FOUND("Cluster", clusterName);
        }
        SoftwareManager softMgr = softwareManagerCollector.getSoftwareManager(cluster.getAppManager());

        ValidationUtils.validateVersion(clusterEntityMgr, clusterName);

        ClusterStatus oldStatus = cluster.getStatus();

        if (!oldStatus.isActiveServiceStatus()) {
            throw ClusterHealServiceException.NOT_SUPPORTED(clusterName, "The cluster status must be RUNNING");
        }

        List<NodeGroupEntity> nodeGroups;

        if (groupName != null) {
            NodeGroupEntity nodeGroup = clusterEntityMgr.findByName(clusterName, groupName);
            if (nodeGroup == null) {
                logger.error("node group " + groupName + " does not exist");
                throw BddException.NOT_FOUND("group", groupName);
            }

            nodeGroups = new ArrayList<NodeGroupEntity>(1);
            nodeGroups.add(nodeGroup);
        } else {
            nodeGroups = clusterEntityMgr.findAllGroups(clusterName);
        }

        // only fix worker nodes that have datanode or tasktracker roles
        boolean workerNodesFound = false;
        JobParametersBuilder parametersBuilder = new JobParametersBuilder();
        List<JobParameters> jobParameterList = new ArrayList<JobParameters>();

        for (NodeGroupEntity nodeGroup : nodeGroups) {
            List<String> roles = nodeGroup.getRoleNameList();

            workerNodesFound = true;
            for (NodeEntity node : clusterEntityMgr.findAllNodes(clusterName, nodeGroup.getName())) {
                if (node.isObsoleteNode()) {
                    logger.info("Ingore node " + node.getVmName() + ", for it violate VM name convention."
                            + "or exceed defined group instance number. ");
                    continue;
                }
                if (clusterHealService.hasBadDisks(node.getVmName())) {
                    logger.warn("node " + node.getVmName() + " has bad disks. Fixing it..");

                    boolean vmPowerOn = (node.getStatus().ordinal() != NodeStatus.POWERED_OFF.ordinal());

                    JobParameters nodeParameters = parametersBuilder
                            .addString(JobConstants.CLUSTER_NAME_JOB_PARAM, clusterName)
                            .addString(JobConstants.TARGET_NAME_JOB_PARAM, node.getVmName())
                            .addString(JobConstants.GROUP_NAME_JOB_PARAM, nodeGroup.getName())
                            .addString(JobConstants.SUB_JOB_NODE_NAME, node.getVmName())
                            .addString(JobConstants.IS_VM_POWER_ON, String.valueOf(vmPowerOn)).toJobParameters();
                    jobParameterList.add(nodeParameters);
                }
            }
        }

        if (!workerNodesFound) {
            throw ClusterHealServiceException.NOT_SUPPORTED(clusterName,
                    "only support fixing disk failures for worker/non-management nodes");
        }

        // all target nodes are healthy, simply return
        if (jobParameterList.isEmpty()) {
            logger.info("all target nodes are healthy, simply return");
            throw ClusterHealServiceException.NOT_NEEDED(clusterName);
        }

        try {
            clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.MAINTENANCE);
            clusterEntityMgr.cleanupActionError(clusterName);
            return jobManager.runSubJobForNodes(JobConstants.FIX_NODE_DISK_FAILURE_JOB_NAME, jobParameterList,
                    clusterName, oldStatus, oldStatus);
        } catch (Exception e) {
            logger.error("failed to fix disk failures, " + e.getMessage());
            throw e;
        }
    }

    public Map<String, String> getRackTopology(String clusterName, String topology) {
        ClusterRead cluster = clusterEntityMgr.findClusterWithNodes(clusterName, false);
        Set<String> hosts = new HashSet<String>();
        List<NodeRead> nodes = new ArrayList<NodeRead>();
        for (NodeGroupRead nodeGroup : cluster.getNodeGroups()) {
            for (NodeRead node : nodeGroup.getInstances()) {
                if (node.getMoId() != null) {
                    hosts.add(node.getHostName());
                    nodes.add(node);
                }
            }
        }

        if (CommonUtil.isBlank(topology)) {
            topology = cluster.getTopologyPolicy().toString();
        }

        AuAssert.check(hosts.size() > 0);
        clusterConfigMgr.validateRackTopologyUploaded(hosts, topology);

        return clusterConfigMgr.buildTopology(nodes, topology);
    }

    public HadoopStack filterDistroFromAppManager(SoftwareManager softwareManager, String distroName) {
        return clusterConfigMgr.filterDistroFromAppManager(softwareManager, distroName);
    }

    private void verifyNetworkNamesExsitInDB(Set<String> networkNames, String clusterName) {
        NetworkEntity networkEntity = null;
        for (String networkName : networkNames) {
            networkEntity = networkManager.getNetworkEntityByName(networkName);
            if (networkEntity == null) {
                throw ClusterConfigException.NETWORK_IS_NOT_FOUND(networkName, clusterName);
            }
        }
    }

    private void verifyRequiredPackages(ClusterCreate createSpec) {
        // check if the cluster is hadoop cluster, to differentiate from other cluster with customized roles
        boolean isHadoopCluster = false;
        NodeGroupCreate[] ngcs = createSpec.getNodeGroups();
        for (NodeGroupCreate nodeGroup : ngcs) {
            List<String> roles = nodeGroup.getRoles();
            for (String role : roles) {
                if (role.indexOf("hadoop") == 0 || role.indexOf("hbase") == 0 || role.indexOf("mapr") == 0) {
                    isHadoopCluster = true;
                    break;
                }
            }
            if (isHadoopCluster) {
                break;
            }
        }

        // check if the 2 packages(mailx and wsdl4j) have been installed on the serengeti management server.
        // they are needed by cluster creation for Ironfan.
        if (isHadoopCluster && createSpec.getAppManager().equals(Constants.IRONFAN)) {
            checkExtraRequiredPackages();
        }
    }

    private void checkExtraRequiredPackages() {
        logger.info("check if extra required packages(mailx and wsdl4j) have been installed for Ironfan.");
        if (!extraPackagesExisted) {
            File yumRepoPath = new File(Constants.SERENGETI_YUM_REPO_PATH);

            // use hs to record the packages that have not been added
            final HashSet<String> hs = new HashSet<String>();
            hs.addAll(extraRequiredPackages);

            // scan the files under the serengeti yum repo directory
            File[] rpmList = yumRepoPath.listFiles(new FileFilter() {
                @Override
                public boolean accept(File f) {
                    String fname = f.getName();
                    int idx = fname.indexOf("-");

                    if (idx > 0) {
                        String packName = fname.substring(0, idx);
                        if (extraRequiredPackages.contains(packName)) {
                            String regx = packName + commRegex;
                            Pattern pat = Pattern.compile(regx);
                            if (pat.matcher(fname).matches()) {
                                hs.remove(packName);
                                return true;
                            }
                        }
                    }
                    return false;
                }
            });

            if (!hs.isEmpty()) {
                logger.info("cannot find all the needed packages, stop and return error now. ");
                throw BddException.EXTRA_PACKAGES_NOT_FOUND(hs.toString());
            }

            logger.info("the check is successful: all needed packages are there.");
            extraPackagesExisted = true;
        }
    }

    private String getClusterCloneType() {
        String type = Configuration.getString("cluster.clone.service");
        if (StringUtils.isBlank(type)) {
            String version = "";
            VcContext.initVcContext();
            version = VcContext.getVcVersion();
            if (!CommonUtil.isBlank(version)) {
                if (Version.compare(version, Constants.VCENTER_VERSION_6) < 0) {
                    type = Constants.CLUSTER_CLONE_TYPE_FAST_CLONE;
                } else {
                    logger.info(
                            "The VCenter version is equal or higher than 6.0. Set cluster clone type to instant.");
                    type = Constants.CLUSTER_CLONE_TYPE_INSTANT_CLONE;
                }
            }
        }
        return type;
    }

    @ClusterManagerPointcut
    public Long expandCluster(String clusterName, NodeGroupCreate[] nodeGroupsAdd) throws Exception {
        Long taskId = null;
        List<BaseNode> vNodes = new ArrayList<BaseNode>();
        String rpNames = null;
        boolean readyExpand = false;

        ClusterCreate clusterSpec = clusterConfigMgr.getClusterConfig(clusterName);
        List<NodeEntity> nodes = clusterEntityMgr.findAllNodes(clusterName);
        ClusterEntity clusterEntity = clusterEntityMgr.findByName(clusterName);

        if (clusterEntity == null) {
            throw ClusterConfigException.CLUSTER_CONFIG_NOT_FOUND(clusterName);
        }

        for (NodeEntity node : nodes) {
            rpNames = node.getVcRp().getVcResourcePool();
            if (null != rpNames)
                break;
        }

        if (!ClusterStatus.RUNNING.equals(clusterEntity.getStatus())
                && !ClusterStatus.PROVISION_ERROR.equals(clusterEntity.getStatus())) {
            readyExpand = false;
            logger.error("To expand cluster " + clusterSpec.getName()
                    + ", status should be RUNNING or PROVISION_ERROR.");
            throw ClusterManagerException.CLUSTER_STATUS_NOT_READY_FOR_EXPAND(clusterName);
        }

        if (null == nodeGroupsAdd) {
            readyExpand = false;
            logger.error("Node group should be defined in spec file, when expand " + clusterName);
            throw ClusterManagerException.NO_NODE_GROUP_DEFINDED_IN_SPECFILE(clusterName);
        } else {
            clusterSpec.setNodeGroups(nodeGroupsAdd);

            for (NodeGroupCreate ng : nodeGroupsAdd) {
                BaseNode node = new BaseNode();
                if (0 == ng.getInstanceNum()) {
                    throw ClusterManagerException.NODE_GROUP_CANNOT_BE_ZERO(ng.getName());
                }
                node.setCluster(clusterSpec);
                node.setTargetVcCluster(clusterSpec.getVcClusters().get(0).getName());
                node.setNodeGroup(ng);
                node.setTargetRp(rpNames);
                vNodes.add(node);
            }
            for (NodeGroupCreate ng : nodeGroupsAdd) {
                NodeGroupEntity group = clusterEntityMgr.findByName(clusterName, ng.getName());
                if (group == null) {
                    NodeGroupEntity addNodeGroupEntity = clusterConfigMgr.convertGroup(new Gson(), clusterEntity,
                            ng, null, true);
                    addNodeGroupEntity.setIsProvisioning(true);
                    clusterEntityMgr.insert(addNodeGroupEntity);
                    readyExpand = true;
                } else if (group.getIsProvisioning() == Boolean.TRUE) {
                    readyExpand = true;
                } else {
                    readyExpand = false;
                    logger.error("Can not add new node group " + ng.getName() + " into cluster " + clusterName
                            + " , because the node group already exists");
                    throw ClusterManagerException.NODE_GROUP_ALREADY_EXISTED(clusterName, ng.getName());

                }
            }
        }
        if (readyExpand) {
            clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.EXPANDING);
            if (clusteringService.addNodeGroups(clusterSpec, nodeGroupsAdd, vNodes)) {
                taskId = clusterExpandExecute(clusterName, nodeGroupsAdd, clusterSpec);
            } else {
                logger.error("Cluster " + clusterName + " failed to add node groups.");
                clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.PROVISION_ERROR);
                throw ClusterManagerException.ADD_NODE_GROUP_FAILED(clusterName);
            }
        } else {
            clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.PROVISION_ERROR);
        }
        return taskId;
    }

    public Long clusterExpandExecute(String clusterName, NodeGroupCreate[] nodeGroupsAdd, ClusterCreate clusterSpec)
            throws Exception {

        ClusterEntity cluster = clusterEntityMgr.findByName(clusterName);
        StringBuffer nodeGroupNameList = new StringBuffer();
        for (NodeGroupCreate nodeGroup : nodeGroupsAdd) {
            nodeGroupNameList.append(nodeGroup.getName());
            nodeGroupNameList.append(",");
        }

        if (cluster.getStatus() != ClusterStatus.EXPANDING) {
            if (cluster.getStatus() != ClusterStatus.PROVISION_ERROR) {
                logger.error("Failed to add node group for cluster: " + clusterName + ", " + cluster.getStatus());
                throw ClusterManagerException.UPDATE_NOT_ALLOWED_ERROR(clusterName,
                        "To update a cluster, its status must be PROVISION_ERROR");
            }
        }
        List<String> dsNames = getDsNamesToBeUsed(cluster.getVcDatastoreNameList());
        if (dsNames.isEmpty()) {
            throw ClusterConfigException.NO_DATASTORE_ADDED();
        }
        List<VcCluster> vcClusters = getVcClustersToBeUsed(cluster.getVcRpNameList());
        if (vcClusters.isEmpty()) {
            throw ClusterConfigException.NO_RESOURCE_POOL_ADDED();
        }

        VcResourceFilters filters = vcResourceFilterBuilder.build(dsNames, getRpNames(clusterSpec.getRpNames()),
                clusterSpec.getNetworkNames());
        syncService.refreshInventory(filters);

        // validate accessibility
        validateDatastore(dsNames, vcClusters);
        validateNetworkAccessibility(cluster.getName(), cluster.fetchNetworkNameList(), vcClusters);

        Map<String, JobParameter> param = new TreeMap<String, JobParameter>();
        param.put(JobConstants.NEW_NODE_GROUP_LIST_JOB_PARAM, new JobParameter(nodeGroupNameList.toString()));
        param.put(JobConstants.CLUSTER_NAME_JOB_PARAM, new JobParameter(clusterName));
        param.put(JobConstants.TIMESTAMP_JOB_PARAM, new JobParameter(new Date()));
        param.put(JobConstants.CLUSTER_SUCCESS_STATUS_JOB_PARAM, new JobParameter(ClusterStatus.RUNNING.name()));
        param.put(JobConstants.CLUSTER_FAILURE_STATUS_JOB_PARAM,
                new JobParameter(ClusterStatus.PROVISION_ERROR.name()));
        JobParameters jobParameters = new JobParameters(param);
        clusterEntityMgr.cleanupActionError(clusterName);
        try {
            return jobManager.runJob(JobConstants.EXPAND_CLUSTER_JOB_NAME, jobParameters);
        } catch (Exception e) {
            logger.error("Failed to expand node group for cluster " + clusterName, e);
            clusterEntityMgr.updateClusterStatus(clusterName, ClusterStatus.PROVISION_ERROR);
            throw e;
        }
    }

    public void recoverClusters(List<VcClusterMap> clstMaps) throws Exception {
        List<String> nodesNotFound = new ArrayList<String>();
        List<ClusterEntity> clusters = clusterEntityMgr.findAllClusters();
        for (ClusterEntity cluster : clusters) {
            List<NodeEntity> nodes = clusterEntityMgr.findAllNodes(cluster.getName());
            for (NodeEntity node : nodes) {
                VcVirtualMachine vcVm = ClusterUtil.getVcVm(clusterEntityMgr, node);
                if (null == vcVm) {
                    logger.info("The cluster node vm " + node.getVmName() + " is not found.");
                    nodesNotFound.add(node.getVmName());
                } else {
                    if (null != clstMaps) {
                        // for different data centers, we need to update the host name as well
                        // the moid has been updated during the getVcVm() method, for different data centers,
                        // we need to update the esxi host name as well, sine the esxi host name are generally
                        // different between 2 data centers
                        String srcHostName = node.getHostName();
                        for (VcClusterMap clstMap : clstMaps) {
                            Map<String, String> hostMap = clstMap.getHosts();
                            String tgtHostName = hostMap.get(srcHostName);
                            if (tgtHostName != null) {
                                node.setHostName(tgtHostName);
                                clusterEntityMgr.update(node);
                                break;
                            }
                        }
                    }
                    if (node.isNotExist()) {
                        clusterEntityMgr.refreshNodeByMobId(vcVm.getId(), false);
                    }
                }
            }
        }
        if (nodesNotFound.size() > 0) {
            String errMsg = "The following cluster node vms " + nodesNotFound.toString()
                    + " cannot be found in vCenter Server.";
            logger.info(errMsg);
            throw BddException.CLUSTER_RECOVER_FAILED(nodesNotFound.toString());
        }
    }
}