org.cloudifysource.esc.shell.installer.CloudGridAgentBootstrapper.java Source code

Java tutorial

Introduction

Here is the source code for org.cloudifysource.esc.shell.installer.CloudGridAgentBootstrapper.java

Source

/*******************************************************************************
 * Copyright (c) 2011 GigaSpaces Technologies Ltd. All rights reserved
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 *******************************************************************************/
package org.cloudifysource.esc.shell.installer;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.apache.commons.lang.StringUtils;
import org.cloudifysource.domain.cloud.Cloud;
import org.cloudifysource.domain.cloud.compute.ComputeTemplate;
import org.cloudifysource.dsl.internal.CloudifyConstants;
import org.cloudifysource.dsl.internal.CloudifyErrorMessages;
import org.cloudifysource.dsl.rest.response.ControllerDetails;
import org.cloudifysource.dsl.utils.IPUtils;
import org.cloudifysource.esc.driver.provisioning.BaseComputeDriver;
import org.cloudifysource.esc.driver.provisioning.CloudProvisioningException;
import org.cloudifysource.esc.driver.provisioning.ComputeDriverConfiguration;
import org.cloudifysource.esc.driver.provisioning.ComputeDriverProvisioningAdapter;
import org.cloudifysource.esc.driver.provisioning.MachineDetails;
import org.cloudifysource.esc.driver.provisioning.ProvisioningContextAccess;
import org.cloudifysource.esc.driver.provisioning.ProvisioningContextImpl;
import org.cloudifysource.esc.driver.provisioning.context.DefaultProvisioningDriverClassContext;
import org.cloudifysource.esc.driver.provisioning.context.ValidationContext;
import org.cloudifysource.esc.driver.provisioning.jclouds.ManagementWebServiceInstaller;
import org.cloudifysource.esc.driver.provisioning.storage.BaseStorageDriver;
import org.cloudifysource.esc.driver.provisioning.storage.StorageProvisioningException;
import org.cloudifysource.esc.driver.provisioning.validation.ValidationMessageType;
import org.cloudifysource.esc.installer.AgentlessInstaller;
import org.cloudifysource.esc.installer.InstallationDetails;
import org.cloudifysource.esc.installer.InstallerException;
import org.cloudifysource.esc.shell.ValidationContextImpl;
import org.cloudifysource.esc.shell.listener.CliAgentlessInstallerListener;
import org.cloudifysource.esc.shell.listener.CliProvisioningDriverListener;
import org.cloudifysource.esc.util.CalcUtils;
import org.cloudifysource.esc.util.InstallationDetailsBuilder;
import org.cloudifysource.esc.util.ProvisioningDriverClassBuilder;
import org.cloudifysource.esc.util.Utils;
import org.cloudifysource.restclient.utils.NewRestClientUtils;
import org.cloudifysource.shell.AdminFacade;
import org.cloudifysource.shell.ConditionLatch;
import org.cloudifysource.shell.ShellUtils;
import org.cloudifysource.shell.exceptions.CLIException;
import org.cloudifysource.shell.exceptions.CLIStatusException;
import org.cloudifysource.shell.rest.RestAdminFacade;
import org.cloudifysource.shell.rest.inspect.CLIApplicationUninstaller;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.type.TypeFactory;
import org.openspaces.admin.gsa.GSAReservationId;
import org.openspaces.admin.zone.config.ExactZonesConfig;
import org.openspaces.admin.zone.config.ExactZonesConfigurer;

/**
 * This class handles the bootstrapping of machines, activation of management processes and cloud tear-down.
 * 
 * @author barakm, adaml
 * @since 2.0.0
 * 
 */
public class CloudGridAgentBootstrapper {

    private static final String MANAGEMENT_APPLICATION = ManagementWebServiceInstaller.MANAGEMENT_APPLICATION_NAME;
    private static final String MANAGEMENT_GSA_ZONE = "management";

    private static final String OPERATION_TIMED_OUT = "The operation timed out. "
            + "Try to increase the timeout using the -timeout flag";

    private static final Logger logger = Logger.getLogger(CloudGridAgentBootstrapper.class.getName());

    private File providerDirectory;

    private AdminFacade adminFacade;

    private boolean verbose;

    private boolean force;
    private boolean terminateNow;

    private int progressInSeconds;

    private BaseComputeDriver provisioning;
    private BaseStorageDriver storageDriver;

    private Cloud cloud;

    private File cloudFile;

    private boolean noWebServices;
    private boolean useExistingManagers;
    private File existingManagersFile;
    private boolean noManagementSpace;

    public void setProviderDirectory(final File providerDirectory) {
        this.providerDirectory = providerDirectory;
    }

    public void setAdminFacade(final AdminFacade adminFacade) {
        this.adminFacade = adminFacade;
    }

    public void setVerbose(final boolean verbose) {
        this.verbose = verbose;
    }

    public void setProgressInSeconds(final int progressInSeconds) {
        this.progressInSeconds = progressInSeconds;
    }

    public void setForce(final boolean force) {
        this.force = force;
    }

    public void setTerminateNow(final boolean terminateNow) {
        this.terminateNow = terminateNow;
    }

    private static String nodePrefix(final MachineDetails node) {
        return "[" + node.getMachineId() + "] ";
    }

    private static void logServerDetails(final MachineDetails server) {
        if (logger.isLoggable(Level.FINE)) {
            logger.fine(nodePrefix(server) + "Cloud Server was created.");

            logger.fine(nodePrefix(server) + "Public IP: "
                    + (server.getPublicAddress() == null ? "" : server.getPublicAddress()));
            logger.fine(nodePrefix(server) + "Private IP: "
                    + (server.getPrivateAddress() == null ? "" : server.getPrivateAddress()));

        }
    }

    /**
     * Closes the provisioning and storage drivers.
     */
    public void close() {
        if (this.provisioning != null) {
            this.provisioning.close();
        }

        if (this.storageDriver != null) {
            this.storageDriver.close();
        }
    }

    /**
     * Bootstraps and waits until the management machines are running, or until the timeout is reached.
     * 
     * @param securityProfile
     *            set security profile (nonsecure/secure/ssl)
     * @param username
     *            The username for a secure connection to the server
     * @param password
     *            The password for a secure connection to the server
     * @param keystorePassword
     *            The password to the keystore to set on the rest server
     * @param performValidation
     *            true to perform configuration validations before bootstrap, false otherwise
     * @param timeout
     *            The number of {@link TimeUnit}s to wait before timing out
     * @param timeoutUnit
     *            The time unit to use (seconds, minutes etc.)
     * @throws InstallerException
     *             Indicates the provisioning driver failed to start management machines or that the management
     *             processes failed to start
     * @throws CLIException
     *             Indicates a basic failure or a time out. a detailed message is included
     * @throws InterruptedException
     *             Indicates a thread was interrupted while waiting
     */
    public void bootstrapCloudAndWait(final String securityProfile, final String username, final String password,
            final String keystorePassword, final boolean performValidation, final long timeout,
            final TimeUnit timeoutUnit) throws InstallerException, CLIException, InterruptedException {

        final long end = System.currentTimeMillis() + timeoutUnit.toMillis(timeout);

        createProvisioningDriver(performValidation);

        // Start the cloud machines!!!
        final MachineDetails[] servers = getOrCreateManagementServers(timeout, timeoutUnit, keystorePassword,
                securityProfile);

        // from this point on - close machines if an exception is thrown (to
        // avoid leaks).
        try {

            // log details in FINE
            if (logger.isLoggable(Level.FINE)) {
                for (final MachineDetails server : servers) {
                    logServerDetails(server);
                }
            }

            validateServers(servers);

            // Start the management agents and other processes
            // if (servers[0].isAgentRunning()) {
            // // must be using existing machines.
            // throw new IllegalStateException(
            // "Cloud bootstrapper found existing management machines with the same name. "
            // + "Please shut them down before continuing");
            // }

            if (servers[0].isAgentRunning()) {
                logger.fine(
                        "Management servers are already running Cloudify agent - skipping default bootstrapping");
            } else {
                startManagememntProcesses(servers, securityProfile, keystorePassword, end);
            }

            if (!isNoWebServices()) {
                final Integer restPort = getRestPort(cloud.getConfiguration().getComponents().getRest().getPort(),
                        ShellUtils.isSecureConnection(securityProfile));
                final Integer webuiPort = getWebuiPort(
                        cloud.getConfiguration().getComponents().getWebui().getPort(),
                        ShellUtils.isSecureConnection(securityProfile));
                waitForManagementWebServices(ShellUtils.isSecureConnection(securityProfile), username, password,
                        restPort, webuiPort, end, servers);
            }

        } catch (final IOException e) {
            stopManagementMachines();
            throw new CLIException("Cloudify bootstrap on provider " + this.cloud.getProvider().getProvider()
                    + " failed. Reason: " + e.getMessage(), e);
        } catch (final URISyntaxException e) {
            stopManagementMachines();
            throw new CLIException("Bootstrap-cloud failed. Reason: " + e.getMessage(), e);
        } catch (final TimeoutException e) {
            stopManagementMachines();
            throw new CLIException("Cloudify bootstrap on provider " + this.cloud.getProvider().getProvider()
                    + " timed-out. " + "Please try to run again using the timeout option.", e);
        } catch (final CLIException e) {
            stopManagementMachines();
            throw e;
        } catch (final InstallerException e) {
            stopManagementMachines();
            throw e;
        } catch (final InterruptedException e) {
            stopManagementMachines();
            throw e;
        }
    }

    private MachineDetails[] getOrCreateManagementServers(final long timeout, final TimeUnit timeoutUnit,
            final String keystorePassword, final String securityProfile) throws CLIException {

        MachineDetails[] servers;
        if (this.existingManagersFile != null) {
            servers = locateManagementMachinesFromFile();
        } else if (this.useExistingManagers) {
            servers = locateManagementMachines();
        } else {
            servers = createManagementServers(timeout, timeoutUnit, keystorePassword, securityProfile);
        }

        final ComputeTemplate template = this.cloud.getCloudCompute().getTemplates()
                .get(cloud.getConfiguration().getManagementMachineTemplate());
        for (final MachineDetails machineDetails : servers) {
            if (machineDetails.getInstallerConfiguration() == null) {
                machineDetails.setInstallerConfigutation(template.getInstaller());
            }
        }
        return servers;

    }

    private ProvisioningContextImpl setUpProvisioningContext(final String keystorePassword,
            final String securityProfile) {

        final ProvisioningContextImpl ctx = new ProvisioningContextImpl();
        ctx.setLocationId(null);
        ctx.setCloudFile(this.cloudFile);
        final InstallationDetailsBuilder builder = ctx.getInstallationDetailsBuilder();

        builder.setReservationId(null);
        builder.setAdmin(null);

        builder.setAuthGroups(null);
        builder.setCloud(this.cloud);
        builder.setCloudFile(this.cloudFile);
        builder.setKeystorePassword(keystorePassword);
        builder.setLookupLocators(null);
        builder.setManagement(true);
        builder.setRebootstrapping(isRebootstrapping());
        builder.setReservationId(null);
        builder.setSecurityProfile(securityProfile);
        builder.setTemplate(this.cloud.getCloudCompute().getTemplates()
                .get(this.cloud.getConfiguration().getManagementMachineTemplate()));
        builder.setTemplateName(this.cloud.getConfiguration().getManagementMachineTemplate());
        builder.setZones(new HashSet<String>(Arrays.asList(MANAGEMENT_GSA_ZONE)));

        return ctx;

    }

    private MachineDetails[] createManagementServers(final long timeout, final TimeUnit timeoutUnit,
            final String keystorePassword, final String securityProfile) throws CLIException {
        MachineDetails[] servers;

        logger.fine("Creating provisioning Context");
        final ProvisioningContextImpl context = setUpProvisioningContext(keystorePassword, securityProfile);

        try {
            servers = provisioning.startManagementMachines(context, timeout, timeoutUnit);
        } catch (final CloudProvisioningException e) {
            final CLIStatusException cliStatusException = new CLIStatusException(e,
                    CloudifyErrorMessages.CLOUD_API_ERROR.getName(), e.getMessage());
            throw cliStatusException;
        } catch (final TimeoutException e) {
            throw new CLIException("Cloudify bootstrap on provider " + this.cloud.getProvider().getProvider()
                    + " timed-out. " + "Please try to run again using the timeout option.", e);
        } finally {
            ProvisioningContextAccess.setCurrentProvisioingContext(null);
        }

        if (servers.length == 0) {
            throw new IllegalArgumentException("Received zero management servers from provisioning implementation");
        }
        return servers;
    }

    private MachineDetails[] locateManagementMachines() throws CLIStatusException {

        MachineDetails[] mds;
        try {
            mds = provisioning.getExistingManagementServers();
        } catch (final CloudProvisioningException e) {
            throw new CLIStatusException(e, CloudifyErrorMessages.MANAGEMENT_SERVERS_FAILED_TO_READ.getName(),
                    e.getMessage());
        } catch (final UnsupportedOperationException e) {
            throw new CLIStatusException(CloudifyErrorMessages.MANAGEMENT_LOCATOR_NOT_SUPPORTED.getName(),
                    this.cloud.getName());
        }
        if (mds.length == 0) {
            throw new CLIStatusException(CloudifyErrorMessages.MANAGEMENT_SERVERS_NOT_LOCATED.getName());
        }
        if (mds.length != this.cloud.getProvider().getNumberOfManagementMachines()) {
            throw new CLIStatusException(CloudifyErrorMessages.MANAGEMENT_SERVERS_NUMBER_NOT_MATCH.getName(),
                    cloud.getProvider().getNumberOfManagementMachines(), mds.length);
        }

        return mds;

    }

    @SuppressWarnings("deprecation")
    private MachineDetails[] locateManagementMachinesFromFile() throws CLIStatusException {

        final ObjectMapper mapper = new ObjectMapper();
        ControllerDetails[] controllers = null;
        try {
            controllers = mapper.readValue(this.existingManagersFile,
                    TypeFactory.arrayType(ControllerDetails.class));
        } catch (final IOException e) {
            throw new IllegalArgumentException("Failed to read managers file: "
                    + this.existingManagersFile.getAbsolutePath() + ". Error was: " + e.getMessage(), e);
        }

        MachineDetails[] mds;
        try {
            mds = provisioning.getExistingManagementServers(controllers);
        } catch (final CloudProvisioningException e) {
            throw new CLIStatusException(e, CloudifyErrorMessages.MANAGEMENT_SERVERS_FAILED_TO_READ.getName(),
                    e.getMessage());
        } catch (final UnsupportedOperationException e) {
            throw new CLIStatusException(CloudifyErrorMessages.MANAGEMENT_LOCATOR_NOT_SUPPORTED.getName(),
                    this.cloud.getName());
        }
        if (mds.length == 0) {
            throw new CLIStatusException(CloudifyErrorMessages.MANAGEMENT_SERVERS_NOT_LOCATED.getName());
        }
        if (mds.length != this.cloud.getProvider().getNumberOfManagementMachines()) {
            throw new CLIStatusException(CloudifyErrorMessages.MANAGEMENT_SERVERS_NUMBER_NOT_MATCH.getName(),
                    cloud.getProvider().getNumberOfManagementMachines(), mds.length);
        }

        return mds;

    }

    private void validateServers(final MachineDetails[] servers) throws CLIException {
        if (servers.length != this.cloud.getProvider().getNumberOfManagementMachines()) {
            throw new CLIException("Bootstrap required " + this.cloud.getProvider().getNumberOfManagementMachines()
                    + " machines, but recieved " + servers.length);
        }

        for (final MachineDetails machineDetails : servers) {
            if (this.cloud.getConfiguration().isBootstrapManagementOnPublicIp()) {
                if (machineDetails.getPublicAddress() == null) {
                    throw new CLIException(
                            "Missing a public address which is required for bootstrap in node with ID: "
                                    + machineDetails.getMachineId());
                }
            } else {
                if (machineDetails.getPrivateAddress() == null) {
                    throw new CLIException(
                            "Missing a private address which is required for bootstrap in node with ID: "
                                    + machineDetails.getMachineId());
                }
            }

            if (this.cloud.getConfiguration().isConnectToPrivateIp()) {
                if (machineDetails.getPrivateAddress() == null) {
                    throw new CLIException(
                            "Missing a private address which is required for server setup in node with ID: "
                                    + machineDetails.getMachineId());
                }
            } else {
                if (machineDetails.getPublicAddress() == null) {
                    throw new CLIException(
                            "Missing a public address which is required for server setup in node with ID: "
                                    + machineDetails.getMachineId());
                }
            }
        }

    }

    private void waitForManagementWebServices(final boolean isSecureConnection, final String username,
            final String password, final Integer restPort, final Integer webuiPort, final long end,
            final MachineDetails[] servers)
            throws MalformedURLException, URISyntaxException, InterruptedException, TimeoutException, CLIException {
        // Wait for rest to become available
        // When the rest gateway is up and running, the cloud is ready to go
        for (final MachineDetails server : servers) {
            String ipAddress = null;
            if (cloud.getConfiguration().isBootstrapManagementOnPublicIp()) {
                ipAddress = server.getPublicAddress();
            } else {
                ipAddress = server.getPrivateAddress();
            }

            final URL restAdminUrl = new URI(ShellUtils.getRestProtocol(isSecureConnection), null, ipAddress,
                    restPort, null, null, null).toURL();
            final URL webUIUrl = new URI(ShellUtils.getRestProtocol(isSecureConnection), null, ipAddress, webuiPort,
                    null, null, null).toURL();

            // We are relying on start-management command to be run on the
            // new machine, so everything should be up if the rest admin is up
            waitForConnection(username, password, restAdminUrl, isSecureConnection, CalcUtils.millisUntil(end),
                    TimeUnit.MILLISECONDS);

            logger.info("Rest service is available at: " + restAdminUrl + '.');
            logger.info("Webui service is available at: " + webUIUrl + '.');
        }
    }

    // if rest port was configured we return the config value
    private Integer getRestPort(final Integer configuredRestPort, final boolean isSecureConnection) {
        if (configuredRestPort != null) {
            return configuredRestPort;
        }
        if (isSecureConnection) {
            return CloudifyConstants.SECURE_REST_PORT;
        } else {
            return CloudifyConstants.DEFAULT_REST_PORT;
        }
    }

    // if webui port was configured we return the config value
    private Integer getWebuiPort(final Integer configuredWebuiPort, final boolean isSecureConnection) {
        if (configuredWebuiPort != null) {
            return configuredWebuiPort;
        }
        if (isSecureConnection) {
            return CloudifyConstants.SECURE_WEBUI_PORT;
        } else {
            return CloudifyConstants.DEFAULT_WEBUI_PORT;
        }
    }

    private void stopManagementMachines() {
        if (this.useExistingManagers || this.existingManagersFile != null) {
            // if we did not start the machines, we will not close them.
            return;
        }
        try {
            provisioning.stopManagementMachines();
        } catch (final CloudProvisioningException e) {
            // log a warning, don't throw an exception on this failure
            logger.warning("Failed to clean management machines after provisioning failure, reported error: "
                    + e.getMessage());
        } catch (final TimeoutException e) {
            // log a warning, don't throw an exception on this failure
            logger.warning(
                    "Failed to clean management machines after provisioning failure, the operation timed out ("
                            + e.getMessage() + ")");
        }
    }

    /**
     * loads the provisioning driver class and sets it up.
     * 
     * @param performValidation
     *            true to perform cloud configuration validations before bootstrap, false otherwise
     * @throws CLIException
     *             Indicates the configured could not be found and instantiated
     */
    private void createProvisioningDriver(final boolean performValidation) throws CLIException {
        try {
            final ProvisioningDriverClassBuilder builder = new ProvisioningDriverClassBuilder();
            final Object computeProvisioningInstance = builder.build(providerDirectory.getAbsolutePath(),
                    cloud.getConfiguration().getClassName());
            provisioning = ComputeDriverProvisioningAdapter.create(computeProvisioningInstance);
        } catch (final ClassNotFoundException e) {
            throw new CLIException("Failed to load provisioning class for cloud: " + cloud.getName()
                    + ". Class not found: " + cloud.getConfiguration().getClassName(), e);
        } catch (final Exception e) {
            throw new CLIException("Failed to load provisioning class for cloud: " + cloud.getName(), e);
        }

        provisioning.setProvisioningDriverClassContext(new DefaultProvisioningDriverClassContext());

        provisioning.addListener(new CliProvisioningDriverListener());

        final String serviceName = null;
        final ValidationContext validationContext = new ValidationContextImpl();

        try {

            if (performValidation) {
                new BootstrapUrlValidator(cloud).validateCloudifyUrls(validationContext);
            }

            final ComputeDriverConfiguration configuration = new ComputeDriverConfiguration();
            configuration.setAdmin(null);
            configuration.setCloud(cloud);
            configuration.setCloudTemplate(cloud.getConfiguration().getManagementMachineTemplate());
            configuration.setManagement(true);
            configuration.setServiceName(serviceName);

            provisioning.setConfig(configuration);

            if (performValidation) {
                validationContext.validationEvent(ValidationMessageType.TOP_LEVEL_VALIDATION_MESSAGE, ShellUtils
                        .getFormattedMessage(CloudifyErrorMessages.EVENT_VALIDATE_CLOUD_CONFIG.getName()));
                try {
                    provisioning.validateCloudConfiguration(validationContext);
                } catch (final UnsupportedOperationException e) {
                    // ignore
                }
                validationContext.validationEvent(ValidationMessageType.TOP_LEVEL_VALIDATION_MESSAGE, ShellUtils
                        .getFormattedMessage(CloudifyErrorMessages.EVENT_CLOUD_CONFIG_VALIDATED.getName()));
            }
        } catch (final CloudProvisioningException e) {
            throw new CLIException(e.getMessage(), e);
        }

    }

    private void createStorageDriver() throws CLIException {

        String storageClassName = cloud.getConfiguration().getStorageClassName();
        logger.fine("creating storage provisioning driver.");
        try {
            final ProvisioningDriverClassBuilder builder = new ProvisioningDriverClassBuilder();
            Object storageProvisioningInstance = builder.build(providerDirectory.getAbsolutePath(),
                    storageClassName);
            if (storageProvisioningInstance instanceof BaseStorageDriver) {
                storageDriver = (BaseStorageDriver) storageProvisioningInstance;
                logger.fine("storage provisioning driver created successfully.");
                storageDriver.setComputeContext(provisioning.getComputeContext());
                storageDriver.setConfig(cloud, cloud.getConfiguration().getManagementMachineTemplate());
            }

        } catch (final ClassNotFoundException e) {
            throw new CLIException("Failed to load storage provisioning class for cloud: " + cloud.getName()
                    + ". Class not found: " + storageClassName, e);
        } catch (final Exception e) {
            throw new CLIException("Failed to load storage provisioning class for cloud: " + cloud.getName()
                    + " class name: " + storageClassName, e);
        }
    }

    /**
     * 
     * @param timeout
     *            The number of {@link TimeUnit}s to wait before timing out
     * @param timeoutUnit
     *            The time unit to use (seconds, minutes etc.)
     * @throws TimeoutException
     *             Indicates the time out was reached before the tear-down completed
     * @throws CLIException
     *             Indicates a basic failure tear-down the cloud. a detailed message is included
     * @throws InterruptedException
     *             Indicates a thread was interrupted while waiting
     */
    public void teardownCloudAndWait(final long timeout, final TimeUnit timeoutUnit)
            throws TimeoutException, CLIException, InterruptedException {

        final long end = System.currentTimeMillis() + timeoutUnit.toMillis(timeout);

        createProvisioningDriver(false /* performValidation */);
        ShellUtils.checkNotNull("providerDirectory", providerDirectory);

        if (terminateNow) {
            try {
                provisioning.terminateAllResources(timeout, timeoutUnit);
            } catch (final CloudProvisioningException e) {
                throw new CLIException("Failed to terminate resources: " + e.getMessage(), e);
            }
        } else {
            destroyManagementServers(CalcUtils.millisUntil(end), TimeUnit.MILLISECONDS);
        }

        if (StringUtils.isNotBlank(cloud.getConfiguration().getStorageClassName())) {
            createStorageDriver();
            destroyVolumes(CalcUtils.millisUntil(end), TimeUnit.MILLISECONDS);
        }
    }

    private void destroyManagementServers(final long timeout, final TimeUnit timeoutUnit)
            throws CLIException, InterruptedException, TimeoutException {

        final long end = System.currentTimeMillis() + timeoutUnit.toMillis(timeout);

        if (!force) {

            if (!adminFacade.isConnected()) {
                throw new CLIException("Please connect to the cloud before tearing down");
            }
            uninstallApplications(end);

        } else {
            // try to shutdown gracefully - uninstall applications
            if (adminFacade.isConnected()) {
                try {
                    uninstallApplications(end);
                } catch (final InterruptedException e) {
                    throw e;
                } catch (final TimeoutException e) {
                    logger.fine("Failed to uninstall applications. Shut down of management machines will continue");
                } catch (final CLIException e) {
                    logger.fine("Failed to uninstall applications. Shut down of management machines will continue");
                }
            } else {
                logger.info("Teardown performed without connection to the cloud, only management machines will be "
                        + "terminated.");
            }
        }

        logger.info("Terminating cloud machines");

        try {
            provisioning.stopManagementMachines();
        } catch (final CloudProvisioningException e) {
            throw new CLIException(
                    "Failed to shut down management machine during tear down of cloud: " + e.getMessage(), e);
        }
        adminFacade.disconnect();

    }

    private void destroyVolumes(final long timeout, final TimeUnit timeoutUnit)
            throws CLIException, TimeoutException {
        if (storageDriver != null) {
            try {
                storageDriver.terminateAllVolumes(timeout, timeoutUnit);
            } catch (final StorageProvisioningException e) {
                throw new CLIException("Failed to terminate volume during tear down of cloud: " + e.getMessage(),
                        e);
            }
        } else {
            logger.fine("Storage driver is not initialized, volumes are not distroyed");
        }
    }

    private void uninstallApplications(final long end) throws CLIException, InterruptedException, TimeoutException {
        final Collection<String> applicationsList = adminFacade.getApplicationNamesList();

        final long startTime = System.currentTimeMillis();
        final long millisToEnd = end - startTime;

        if (NewRestClientUtils.isNewRestClientEnabled()) {
            uninstallNewRestClient(applicationsList, millisToEnd);
        } else {
            uninstall(applicationsList, millisToEnd);
        }

    }

    private void uninstall(final Collection<String> applicationsList, final long millisToEnd)
            throws CLIException, InterruptedException, TimeoutException {
        final int minutesToEnd = (int) TimeUnit.MILLISECONDS.toMinutes(millisToEnd);
        final Map<String, String> lifeCycleEventContainersIdsByApplicationName = new HashMap<String, String>();

        if (applicationsList.size() > 0) {
            logger.info("Uninstalling the currently deployed applications");
            for (final String application : applicationsList) {
                if (!application.equals(MANAGEMENT_APPLICATION)) {
                    final Map<String, String> uninstallApplicationResponse = adminFacade
                            .uninstallApplication(application, minutesToEnd);
                    lifeCycleEventContainersIdsByApplicationName.put(
                            uninstallApplicationResponse.get(CloudifyConstants.LIFECYCLE_EVENT_CONTAINER_ID),
                            application);
                }
            }
        }

        // now we need to wait for all the application to be uninstalled
        for (final Map.Entry<String, String> entry : lifeCycleEventContainersIdsByApplicationName.entrySet()) {
            logger.info("Waiting for application " + entry.getValue() + " to uninstall.");
            adminFacade.waitForLifecycleEvents(entry.getKey(), minutesToEnd,
                    CloudifyConstants.TIMEOUT_ERROR_MESSAGE);
        }

    }

    private void uninstallNewRestClient(final Collection<String> applicationsList, final long millisToEnd)
            throws CLIException {
        for (final String application : applicationsList) {
            if (!application.equals(MANAGEMENT_APPLICATION)) {
                final CLIApplicationUninstaller uninstaller = new CLIApplicationUninstaller();
                uninstaller.setRestClient(((RestAdminFacade) adminFacade).getNewRestClient());
                uninstaller.setApplicationName(application);
                uninstaller.setAskOnTimeout(false);
                uninstaller.setInitialTimeout((int) millisToEnd);
                try {
                    uninstaller.uninstall();
                } catch (final Exception e) {
                    if (force) {
                        logger.warning(
                                "Failed uninstalling application " + application + ". Teardown will continue");
                    } else {
                        throw new CLIException(e.getMessage(), e);
                    }
                }
            }
        }
    }

    private MachineDetails[] startManagememntProcesses(final MachineDetails[] machines,
            final String securityProfile, final String keystorePassword, final long endTime)
            throws InterruptedException, TimeoutException, InstallerException, IOException {

        final AgentlessInstaller installer = new AgentlessInstaller();
        installer.addListener(new CliAgentlessInstallerListener(this.verbose));

        // Update the logging level of jsch used by the AgentlessInstaller
        Logger.getLogger(AgentlessInstaller.SSH_LOGGER_NAME)
                .setLevel(Level.parse(cloud.getProvider().getSshLoggingLevel()));

        final ComputeTemplate template = cloud.getCloudCompute().getTemplates()
                .get(cloud.getConfiguration().getManagementMachineTemplate());

        // fixConfigRelativePaths(cloud, template);

        final int numOfManagementMachines = machines.length;

        final InstallationDetails[] installations = createInstallationDetails(numOfManagementMachines, machines,
                template, securityProfile, keystorePassword);
        // only one machine should try and deploy the WebUI and Rest Admin unless
        // noWebServices is true

        //Used for ESM testing purposes.
        if (installations.length > 0) {
            if (isNoWebServices()) {
                installations[0].setNoWebServices(true);
            }
            if (isNoManagementSpace()) {
                installations[0].setNoManagementSpace(true);
            }
        }

        //They were installed by the first management machine
        //and will automatically deployed on the other machines. 
        for (int i = 1; i < installations.length; i++) {
            installations[i].setNoWebServices(true);
            installations[i].setNoManagementSpace(true);
        }

        final String lookup = createLocatorsString(installations);
        for (final InstallationDetails detail : installations) {
            detail.setLocator(lookup);
        }

        // executes the agentless installer on all of the machines,
        // asynchronously
        installOnMachines(endTime, installer, numOfManagementMachines, installations);

        return machines;

    }

    private void installOnMachines(final long endTime, final AgentlessInstaller installer,
            final int numOfManagementMachines, final InstallationDetails[] installations)
            throws InterruptedException, TimeoutException, InstallerException {
        final ExecutorService executors = Executors.newFixedThreadPool(numOfManagementMachines);

        final BootstrapLogsFilters bootstrapLogs = new BootstrapLogsFilters(verbose);

        try {

            bootstrapLogs.applyLogFilters();

            final List<Future<Exception>> futures = new ArrayList<Future<Exception>>();

            for (final InstallationDetails detail : installations) {
                final Future<Exception> future = executors.submit(new Callable<Exception>() {

                    @Override
                    public Exception call() {
                        try {
                            installer.installOnMachineWithIP(detail, CalcUtils.millisUntil(endTime),
                                    TimeUnit.MILLISECONDS);
                        } catch (final TimeoutException e) {
                            logger.log(Level.INFO, "Failed accessing management VM " + detail.getPublicIp()
                                    + " Reason: " + e.getMessage(), e);
                            return e;
                        } catch (final InterruptedException e) {
                            logger.log(Level.INFO, "Failed accessing management VM " + detail.getPublicIp()
                                    + " Reason: " + e.getMessage(), e);
                            return e;
                        } catch (final InstallerException e) {
                            logger.log(Level.INFO, "Failed accessing management VM " + detail.getPublicIp()
                                    + " Reason: " + e.getMessage(), e);
                            return e;
                        }
                        return null;
                    }
                });
                futures.add(future);

            }

            for (final Future<Exception> future : futures) {
                try {
                    final Exception e = future.get(CalcUtils.millisUntil(endTime), TimeUnit.MILLISECONDS);
                    if (e != null) {
                        if (e instanceof TimeoutException) {
                            throw (TimeoutException) e;
                        }
                        if (e instanceof InterruptedException) {
                            throw (InterruptedException) e;
                        }
                        if (e instanceof InstallerException) {
                            throw (InstallerException) e;
                        }
                        throw new InstallerException("Failed creating machines.", e);
                    }
                } catch (final ExecutionException e) {
                    throw new InstallerException("Failed creating machines.", e);
                }
            }

        } finally {
            executors.shutdown();
            bootstrapLogs.restoreLogFilters();
        }
    }

    private String createLocatorsString(final InstallationDetails[] installations) {

        final String[] locators = new String[installations.length];
        final Integer port = cloud.getConfiguration().getComponents().getDiscovery().getDiscoveryPort();

        for (int i = 0; i < installations.length; i++) {
            final InstallationDetails detail = installations[i];
            locators[i] = cloud.getConfiguration().isConnectToPrivateIp() ? detail.getPrivateIp()
                    : detail.getPublicIp();
        }

        final String locatorsString = IPUtils.createLocatorsString(locators, port);

        return locatorsString;
    }

    private InstallationDetails[] createInstallationDetails(final int numOfManagementMachines,
            final MachineDetails[] machineDetails, final ComputeTemplate template, final String securityProfile,
            final String keystorePassword) throws FileNotFoundException {
        final InstallationDetails[] details = new InstallationDetails[numOfManagementMachines];

        final GSAReservationId reservationId = null;
        final String managementAuthGroups = null;

        for (int i = 0; i < details.length; i++) {
            final ExactZonesConfig zones = new ExactZonesConfigurer().addZone(MANAGEMENT_GSA_ZONE).create();
            details[i] = Utils.createInstallationDetails(machineDetails[i], cloud, template, zones, null, null,
                    true, this.cloudFile, reservationId, cloud.getConfiguration().getManagementMachineTemplate(),
                    securityProfile, keystorePassword, managementAuthGroups, isRebootstrapping(),
                    isNoManagementSpace());
        }

        return details;
    }

    private boolean isRebootstrapping() {
        return this.useExistingManagers || this.existingManagersFile != null;
    }

    /**
     * Waits for a connection to be established with the service. If the timeout is reached before a connection could be
     * established, a {@link TimeoutException} is thrown.
     * 
     * @param username
     *            The username for a secure connection to the rest server
     * @param password
     *            The password for a secure connection to the rest server
     * @param restAdminUrl
     *            The URL of the service
     * @param isSecureConnection
     *            Is this a secure connection (SSL)
     * @param timeout
     *            number of {@link TimeUnit}s to wait
     * @param timeunit
     *            The {@link TimeUnit} to use
     * @throws InterruptedException
     *             Reporting the thread is interrupted while waiting
     * @throws TimeoutException
     *             Reporting the time out was reached
     * @throws CLIException
     *             Reporting different errors while creating the connection to the service
     */
    private void waitForConnection(final String username, final String password, final URL restAdminUrl,
            final boolean isSecureConnection, final long timeout, final TimeUnit timeunit)
            throws InterruptedException, TimeoutException, CLIException {

        adminFacade.disconnect();

        createConditionLatch(timeout, timeunit).waitFor(new ConditionLatch.Predicate() {

            @Override
            public boolean isDone() throws CLIException, InterruptedException {

                try {
                    adminFacade.connect(username, password, restAdminUrl.toString(), isSecureConnection);
                    return true;
                } catch (final Exception e) {
                    if (verbose) {
                        logger.log(Level.INFO, "Error connecting to rest service.", e);
                    }
                }
                logger.log(Level.INFO, "Connecting to rest service.");
                return false;
            }
        });
    }

    private ConditionLatch createConditionLatch(final long timeout, final TimeUnit timeunit) {
        return new ConditionLatch().timeout(timeout, timeunit).pollingInterval(progressInSeconds, TimeUnit.SECONDS)
                .timeoutErrorMessage(OPERATION_TIMED_OUT).verbose(verbose);
    }

    public void setCloud(final Cloud cloud) {
        this.cloud = cloud;

    }

    public void setCloudFile(final File cloudFile) {
        this.cloudFile = cloudFile;
    }

    public boolean isNoWebServices() {
        return noWebServices;
    }

    public void setNoWebServices(final boolean noWebServices) {
        this.noWebServices = noWebServices;
    }

    public void setNoManagementSpace(final boolean noManagementSpace) {
        this.noManagementSpace = noManagementSpace;
    }

    public boolean isNoManagementSpace() {
        return noManagementSpace;
    }

    public void setUseExisting(final boolean useExistingManagers) {
        this.useExistingManagers = useExistingManagers;

    }

    /******
     * Returns existing cloud managers.
     * 
     * @return details of existing cloud managers.
     * @throws CLIException
     *             if failed to read cloudify managers from Cloud API.
     */
    public MachineDetails[] getCloudManagers() throws CLIException {
        createProvisioningDriver(false /* performValidation */);
        return locateManagementMachines();
    }

    public void setExistingManagersFile(final File existingManagersFile) {
        this.existingManagersFile = existingManagersFile;
    }

}