org.excalibur.service.aws.ec2.EC2.java Source code

Java tutorial

Introduction

Here is the source code for org.excalibur.service.aws.ec2.EC2.java

Source

/**
 *     Copyright (C) 2013-2014  the original author or authors.
 *
 *     This program is free software: you can redistribute it and/or modify
 *     it under the terms of the GNU General Public License as published by
 *     the Free Software Foundation, either version 3 of the License,
 *     any later version.
 *
 *     This program is distributed in the hope that it will be useful,
 *     but WITHOUT ANY WARRANTY; without even the implied warranty of
 *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *     GNU General Public License for more details.
 *
 *     You should have received a copy of the GNU General Public License
 *     along with this program.  If not, see <http://www.gnu.org/licenses/>
 */
package org.excalibur.service.aws.ec2;

import static com.google.common.base.Preconditions.*;
import static com.google.common.base.Strings.*;
import static com.google.common.collect.Iterables.*;
import static com.google.common.collect.Lists.*;

import static org.excalibur.core.cipher.TripleDESUtils.*;
import static org.excalibur.core.util.concurrent.DynamicExecutors.*;
import static org.excalibur.core.util.concurrent.Futures2.*;
import static org.excalibur.core.cloud.api.domain.Tags.*;

import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;

import javax.annotation.Nullable;

import org.apache.commons.codec.binary.Base64;
import org.excalibur.core.LoginCredentials;
import org.excalibur.core.Status;
import org.excalibur.core.cloud.api.Attribute;
import org.excalibur.core.cloud.api.HypervisorType;
import org.excalibur.core.cloud.api.InstanceStateDetails;
import org.excalibur.core.cloud.api.InstanceStateType;
import org.excalibur.core.cloud.api.InstanceType;
import org.excalibur.core.cloud.api.KeyPair;
import org.excalibur.core.cloud.api.KeyPairs;
import org.excalibur.core.cloud.api.OsArchitectureType;
import org.excalibur.core.cloud.api.Platform;
import org.excalibur.core.cloud.api.VirtualMachine;
import org.excalibur.core.cloud.api.VirtualMachineImage;
import org.excalibur.core.cloud.api.VirtualizationType;
import org.excalibur.core.cloud.api.VmConfiguration;
import org.excalibur.core.cloud.api.VolumeType;
import org.excalibur.core.cloud.api.compute.ComputeService;
import org.excalibur.core.cloud.api.domain.InstanceTemplate;
import org.excalibur.core.cloud.api.domain.InstanceTemplateStatus;
import org.excalibur.core.cloud.api.domain.Instances;
import org.excalibur.core.cloud.api.domain.Region;
import org.excalibur.core.cloud.api.domain.SpotInstanceOffer;
import org.excalibur.core.cloud.api.domain.SpotInstanceOfferResult;
import org.excalibur.core.cloud.api.domain.SpotInstanceOfferStateType;
import org.excalibur.core.cloud.api.domain.SpotInstanceOfferStatus;
import org.excalibur.core.cloud.api.domain.SpotPriceHistory;
import org.excalibur.core.cloud.api.domain.SpotPriceHistoryRequest;
import org.excalibur.core.cloud.api.domain.Tags;
import org.excalibur.core.cloud.api.domain.Zone;
import org.excalibur.core.domain.User;
import org.excalibur.core.domain.UserProviderCredentials;
import org.excalibur.core.util.BackoffLimitedRetryHandler;
import org.excalibur.core.util.Lists2;
import org.excalibur.core.util.Strings2;
import org.excalibur.core.util.SystemUtils2;
import org.excalibur.core.util.ThreadUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.ec2.AmazonEC2;
import com.amazonaws.services.ec2.AmazonEC2Async;
import com.amazonaws.services.ec2.AmazonEC2AsyncClient;
import com.amazonaws.services.ec2.AmazonEC2Client;
import com.amazonaws.services.ec2.model.AuthorizeSecurityGroupIngressRequest;
import com.amazonaws.services.ec2.model.AvailabilityZone;
import com.amazonaws.services.ec2.model.BlockDeviceMapping;
import com.amazonaws.services.ec2.model.CancelSpotInstanceRequestsRequest;
import com.amazonaws.services.ec2.model.CreateKeyPairRequest;
import com.amazonaws.services.ec2.model.CreateKeyPairResult;
import com.amazonaws.services.ec2.model.CreatePlacementGroupRequest;
import com.amazonaws.services.ec2.model.CreateSecurityGroupRequest;
import com.amazonaws.services.ec2.model.CreateSecurityGroupResult;
import com.amazonaws.services.ec2.model.CreateTagsRequest;
import com.amazonaws.services.ec2.model.CreateVolumeRequest;
import com.amazonaws.services.ec2.model.CreateVolumeResult;
import com.amazonaws.services.ec2.model.DescribeAvailabilityZonesRequest;
import com.amazonaws.services.ec2.model.DescribeAvailabilityZonesResult;
import com.amazonaws.services.ec2.model.DescribeImagesRequest;
import com.amazonaws.services.ec2.model.DescribeImagesResult;
import com.amazonaws.services.ec2.model.DescribeInstanceAttributeRequest;
import com.amazonaws.services.ec2.model.DescribeInstanceStatusRequest;
import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
import com.amazonaws.services.ec2.model.DescribeInstancesResult;
import com.amazonaws.services.ec2.model.DescribeKeyPairsRequest;
import com.amazonaws.services.ec2.model.DescribeKeyPairsResult;
import com.amazonaws.services.ec2.model.DescribePlacementGroupsRequest;
import com.amazonaws.services.ec2.model.DescribeRegionsRequest;
import com.amazonaws.services.ec2.model.DescribeRegionsResult;
import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest;
import com.amazonaws.services.ec2.model.DescribeSpotInstanceRequestsRequest;
import com.amazonaws.services.ec2.model.DescribeSpotInstanceRequestsResult;
import com.amazonaws.services.ec2.model.DescribeSpotPriceHistoryRequest;
import com.amazonaws.services.ec2.model.DescribeSpotPriceHistoryResult;
import com.amazonaws.services.ec2.model.EbsBlockDevice;
import com.amazonaws.services.ec2.model.Filter;
import com.amazonaws.services.ec2.model.Image;
import com.amazonaws.services.ec2.model.ImportKeyPairRequest;
import com.amazonaws.services.ec2.model.Instance;
import com.amazonaws.services.ec2.model.InstanceAttribute;
import com.amazonaws.services.ec2.model.InstanceState;
import com.amazonaws.services.ec2.model.InstanceStateChange;
import com.amazonaws.services.ec2.model.InstanceStateName;
import com.amazonaws.services.ec2.model.KeyPairInfo;
import com.amazonaws.services.ec2.model.LaunchSpecification;
import com.amazonaws.services.ec2.model.ModifyInstanceAttributeRequest;
import com.amazonaws.services.ec2.model.Placement;
import com.amazonaws.services.ec2.model.PlacementStrategy;
import com.amazonaws.services.ec2.model.RequestSpotInstancesRequest;
import com.amazonaws.services.ec2.model.RequestSpotInstancesResult;
import com.amazonaws.services.ec2.model.Reservation;
import com.amazonaws.services.ec2.model.RunInstancesRequest;
import com.amazonaws.services.ec2.model.RunInstancesResult;
import com.amazonaws.services.ec2.model.SecurityGroup;
import com.amazonaws.services.ec2.model.SpotInstanceRequest;
import com.amazonaws.services.ec2.model.SpotInstanceType;
import com.amazonaws.services.ec2.model.SpotPrice;
import com.amazonaws.services.ec2.model.StartInstancesRequest;
import com.amazonaws.services.ec2.model.StopInstancesRequest;
import com.amazonaws.services.ec2.model.StopInstancesResult;
import com.amazonaws.services.ec2.model.Tag;
import com.amazonaws.services.ec2.model.TerminateInstancesRequest;
import com.amazonaws.services.ec2.model.Volume;
import com.google.common.base.Function;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.ListeningExecutorService;

public final class EC2 implements ComputeService, Closeable {
    private static final Logger LOG = LoggerFactory.getLogger(EC2.class.getName());
    private final Object lock_ = new Object();

    private static final Function<Instance, String> INSTANCE_STRING_FUNCTION = new Function<Instance, String>() {
        @Override
        public String apply(Instance input) {
            return input.getInstanceId();
        }
    };

    private static final Function<List<Tag>, Map<String, Tag>> TAGS = new Function<List<Tag>, Map<String, Tag>>() {
        @Override
        @Nullable
        public Map<String, Tag> apply(@Nullable List<Tag> input) {
            Map<String, Tag> tags = Maps.newHashMap();

            for (Tag tag : input) {
                tags.put(tag.getKey(), tag);
            }

            return tags;
        }
    };

    private final Function<List<Image>, List<VirtualMachineImage>> IMAGES_FUNCTION = new Function<List<Image>, List<VirtualMachineImage>>() {
        @Override
        @Nullable
        public List<VirtualMachineImage> apply(@Nullable List<Image> input) {
            List<VirtualMachineImage> images = newArrayList();

            for (Image image : input) {
                VirtualMachineImage vmi = new VirtualMachineImage()
                        .setArchitecture(OsArchitectureType.valueOfFromValue(image.getArchitecture()))
                        .setDefaultUsername(DEFAULT_PLATFORM_USER_NAME)
                        .setHypervisor(HypervisorType.valueOfFromValue(image.getHypervisor()))
                        .setName(image.getImageId())
                        .setPlatform(image.getPlatform() == null ? Platform.LINUX
                                : Platform.valueOfFromValue(image.getPlatform()))
                        .setRegion(credentials_.getRegion())
                        .setVirtualizationType(VirtualizationType.valueOfFromValue(image.getVirtualizationType()));

                if ("ebs".equals(image.getRootDeviceType())) {

                    EbsBlockDevice ebs = image.getBlockDeviceMappings().get(0).getEbs();

                    org.excalibur.core.cloud.api.Volume volume = new org.excalibur.core.cloud.api.Volume()
                            .setName(image.getRootDeviceName())
                            .setType(new VolumeType().setName(ebs.getVolumeType())).setSizeGb(ebs.getVolumeSize());

                    vmi.setRootVolume(volume);
                }

                images.add(vmi);
            }
            return images;
        }
    };

    public static final Region DEFAULT_API_REGION = new Region().setEndpoint("https://ec2.us-east-1.amazonaws.com")
            .setName("us-east-1");

    private static final String DEFAULT_INSTANCE_NAME_TAG = "Name";
    private static final String DEFAULT_PLATFORM = "Linux";
    private static final String DEFAULT_PLATFORM_INSTANCE_USERNAME_TAG = "platform-username";

    private static final Integer ID = "ec2".hashCode();

    private static final String DEFAULT_PLATFORM_USER_NAME = System
            .getProperty("org.excalibur.default.platform.username", "ubuntu");
    private static final int MAX_RETRY_STATE = SystemUtils2
            .getIntegerProperty("org.excalibur.deployment.max.state.retry", 100);

    private static final Filter CONSIDERED_INSTANCE_STATES_FILTER = new Filter().withName("instance-state-name")
            .withValues(InstanceStateName.Running.toString(), InstanceStateName.Stopped.toString(),
                    InstanceStateName.Stopping.toString());

    private final AmazonEC2Async ec2_;
    private final UserProviderCredentials credentials_;
    private final String defaultUserGroupName_;
    private final BackoffLimitedRetryHandler backoffLimitedRetryHandler_;

    private final AWSCredentials awsCredentials_;

    public EC2(UserProviderCredentials credentials) {
        this.credentials_ = checkNotNull(credentials);
        checkState(!isNullOrEmpty(credentials.getLoginCredentials().getCredentialName()));
        checkNotNull(credentials.getRegion());
        checkState(!isNullOrEmpty(credentials.getRegion().getName()));
        checkState(!isNullOrEmpty(credentials.getRegion().getEndpoint()));

        this.awsCredentials_ = new BasicAWSCredentials(credentials.getLoginCredentials().getIdentity(),
                credentials.getLoginCredentials().getCredential());

        ec2_ = new AmazonEC2AsyncClient(this.awsCredentials_);
        ec2_.setEndpoint(credentials.getRegion().getEndpoint());
        this.defaultUserGroupName_ = System.getProperty("org.excalibur.security.default.group.name",
                "excalibur-security-group");
        backoffLimitedRetryHandler_ = new BackoffLimitedRetryHandler();

    }

    @Override
    public Integer getId() {
        return ID;
    }

    @Override
    public Instances createInstances(final InstanceTemplate request) {
        return this.createInstances(request, true);
    }

    @Override
    public Instances createInstances(final InstanceTemplate template, final boolean waitForRunningState) {
        LoginCredentials credentials = template.getLoginCredentials();
        KeyPair keyPair = getKeyPair(checkNotNull(credentials.getCredentialName()));

        checkState(keyPair != null || (!isNullOrEmpty(credentials.getPublicKey())));

        if (keyPair == null) {
            String material = decrypt(credentials.getPublicKey());
            keyPair = new KeyPair().setKeyName(credentials.getCredentialName()).setKeyMaterial(material);
            keyPair.setKeyFingerprint(importKeyPair(keyPair)).setKeyMaterial(null);
        }

        checkState(keyPair != null);

        createSecurityGroupIfDoesNotExist(defaultUserGroupName_);

        final Image image = getImageById(template.getImageId());
        checkNotNull(image, String.format("Image %s does not exist", template.getImageId()));

        StringBuilder sb = new StringBuilder();
        sb.append("#start-data\n").append("#").append(credentials.getPrivateKey());

        if (!isNullOrEmpty(credentials.getPublicKey())) {
            sb.append("\n#").append(credentials.getPublicKey());
        }
        sb.append("\n#end-data");

        String userData = Strings2.nullAsEmpty(template.getUserData())
                .concat(new String(Base64.encodeBase64(sb.toString().getBytes())));

        RunInstancesRequest runInstacesRequest = new RunInstancesRequest()
                .withInstanceType(template.getInstanceType().getName()).withImageId(template.getImageId())
                .withMinCount(template.getMinCount()).withMaxCount(template.getMaxCount())
                .withKeyName(keyPair.getKeyName()).withSecurityGroups(defaultUserGroupName_).withUserData(userData);

        Zone zone = null;

        final Placement placement = new Placement();

        if (template.getGroup() != null && !isNullOrEmpty(template.getGroup().getZone())) {
            zone = this.getZoneByName(template.getGroup().getZone());

            checkState(zone != null, String.format("Invalid zone name [%s] on region [%s]",
                    template.getGroup().getZone(), credentials_.getRegion().getName()));

            placement.withAvailabilityZone(zone.getName());

            if (!isNullOrEmpty(template.getGroup().getGroupName())) {
                placement.withGroupName(template.getGroup().getGroupName());
                this.createPlacementGroupsIfDoNotExist(template.getGroup());
            }
        }

        if (zone == null) {
            zone = Lists2.first(listAvailableZonesOfRegion(this.credentials_.getRegion()));
            placement.withAvailabilityZone(zone.getName());
        }

        checkNotNull(zone);
        checkState(!isNullOrEmpty(placement.getAvailabilityZone()));

        runInstacesRequest.withPlacement(placement);

        BlockDeviceMapping blockDeviceMapping = image.getBlockDeviceMappings().get(0);

        Integer diskSize = template.getDiskSize() == null
                ? SystemUtils2.getIntegerProperty("org.excalibur.amazon.default.disk.size", 30)
                : template.getDiskSize();

        EbsBlockDevice disk = new EbsBlockDevice().withSnapshotId(blockDeviceMapping.getEbs().getSnapshotId())
                .withVolumeSize(diskSize).withVolumeType("gp2");

        runInstacesRequest.withBlockDeviceMappings(
                new BlockDeviceMapping().withDeviceName(image.getRootDeviceName()).withEbs(disk));

        RunInstancesResult result = ec2_.runInstances(runInstacesRequest);
        template.setStatus(new InstanceTemplateStatus().setStatus(Status.SUCCESS));

        Iterable<Instance> ec2Instances = waitForRunningState
                ? waitForRunningInstacesState(result.getReservation().getInstances())
                : describeEC2Instances(result.getReservation().getInstances());

        CreateTagsRequest tagsRequest = new CreateTagsRequest();

        String instanceName = isNullOrEmpty(template.getInstanceName()) ? UUID.randomUUID().toString()
                : template.getInstanceName();

        tagsRequest.withTags(new Tag().withKey(DEFAULT_INSTANCE_NAME_TAG).withValue(instanceName));
        tagsRequest.withTags(
                new Tag().withKey(DEFAULT_PLATFORM_INSTANCE_USERNAME_TAG).withValue(DEFAULT_PLATFORM_USER_NAME));
        tagsRequest.withTags(new Tag().withKey("keyname").withValue(keyPair.getKeyName()));

        tagsRequest.withResources(Collections2.transform(newArrayList(ec2Instances), INSTANCE_STRING_FUNCTION));

        for (org.excalibur.core.cloud.api.domain.Tag tag : template.getTags()) {
            if (!isNullOrEmpty(tag.getName()) && !isNullOrEmpty(tag.getValue())) {
                tagsRequest.withTags(new Tag().withKey(tag.getName()).withValue(tag.getValue()));
            }
        }

        ec2_.createTags(tagsRequest);

        if (template.getMaxCount() > 1) {

            for (int i = 0; i < result.getReservation().getInstances().size(); i++) {
                CreateTagsRequest request = new CreateTagsRequest();
                request.withResources(result.getReservation().getInstances().get(i).getInstanceId())
                        .withTags(new Tag().withKey(DEFAULT_INSTANCE_NAME_TAG)
                                .withValue(String.format("%s-%s", instanceName, i + 1)));
                ec2_.createTags(request);
            }
        }

        //        return new Instances(toExcaliburInstances(ec2Instances, keyPair));
        Instances instances = this.describeInstances(ec2Instances);

        //
        LOG.debug("Waiting instances' ready state....");
        ThreadUtils.sleep(30 * 1000);

        LOG.debug("Created [{}] instance(s) from [{}/{}]", instances.size(), template.getMinCount(),
                template.getMaxCount());

        return instances;
    }

    // @Override
    public List<SpotInstanceOfferResult> createSpotInstanceOffer(SpotInstanceOffer request) {
        RequestSpotInstancesRequest spotRequest = new RequestSpotInstancesRequest(
                request.getOfferValue().toPlainString()).withValidFrom(request.getValidFrom())
                        .withValidUntil(request.getValidUntil())
                        .withType(SpotInstanceType.fromValue(request.getType().getName()))
                        .withInstanceCount(request.getNumberOfInstances()).withLaunchSpecification(
                                new LaunchSpecification().withInstanceType(request.getInstanceType().getName())
                                        .withImageId(request.getImageId()).withKeyName(request.getKeyName()));

        RequestSpotInstancesResult requestedSpotInstances = ec2_.requestSpotInstances(spotRequest);
        request.setStatus(new InstanceTemplateStatus().setStatus(Status.SUCCESS));

        return toSpotInstanceResults(request, requestedSpotInstances.getSpotInstanceRequests());
    }

    protected Volume createVolume(Zone zone, Integer sizeGb, String snapshotId) {
        CreateVolumeResult request = ec2_.createVolume(new CreateVolumeRequest()
                .withAvailabilityZone(zone.getName()).withSize(sizeGb).withSnapshotId(snapshotId));

        return request.getVolume();
    }

    public void modifyInstanceAttributes(String instanceId, Attribute... attributes) {
        checkState(!isNullOrEmpty(instanceId));

        //        InstanceState instanceState = 
        //                ec2_.describeInstanceStatus(new DescribeInstanceStatusRequest().withInstanceIds(instanceId)).getInstanceStatuses().get(0).getInstanceState();
        //        
        //        final InstanceStateType stateType = InstanceStateType.valueOfFrom(instanceState.getName());
        //        boolean isRunning = InstanceStateType.RUNNING.equals(stateType);
        //        
        //        if (isRunning)
        //        {
        this.stop(instanceId);
        //        }

        for (Attribute attribute : attributes) {
            if (attribute != null && !isNullOrEmpty(attribute.getName())) {
                ModifyInstanceAttributeRequest request = new ModifyInstanceAttributeRequest()
                        .withInstanceId(instanceId).withAttribute(attribute.getName());

                if (!isNullOrEmpty(attribute.getValue())) {
                    request.setValue(attribute.getValue());
                }

                ec2_.modifyInstanceAttribute(request);
            }
        }

        //        if (isRunning)
        //        {
        this.startInstances(instanceId);
        //        }
    }

    // @Override
    public void cancelSpotInstanceOffers(String... spotInstanceOfferIds) {
        ec2_.cancelSpotInstanceRequests(
                new CancelSpotInstanceRequestsRequest().withSpotInstanceRequestIds(spotInstanceOfferIds));
    }

    // @Override
    public List<SpotInstanceOfferResult> getStatusOfSpotInstanceOffers(String... spotInstanceOfferIds) {
        DescribeSpotInstanceRequestsResult describeSpotInstanceResult = ec2_.describeSpotInstanceRequests(
                new DescribeSpotInstanceRequestsRequest().withSpotInstanceRequestIds(spotInstanceOfferIds));

        return toSpotInstanceResults(null, describeSpotInstanceResult.getSpotInstanceRequests());
    }

    // @Override
    public List<SpotPriceHistory> getSpotPriceHistory(SpotPriceHistoryRequest request) {
        List<SpotPriceHistory> history = new ArrayList<SpotPriceHistory>();

        DescribeSpotPriceHistoryRequest spotPriceRequest = new DescribeSpotPriceHistoryRequest()
                .withEndTime(request.getUntil()).withStartTime(request.getFrom())
                .withAvailabilityZone(request.getRegion() != null ? request.getRegion().getName() : null)
                .withMaxResults(request.getMaxResult()).withProductDescriptions(request.getImageTypes());

        List<InstanceType> types = request.getInstanceTypes();
        String[] instanceTypes = new String[types.size()];

        for (int i = 0; i < instanceTypes.length; i++) {
            instanceTypes[i] = types.get(i).getName();
        }

        spotPriceRequest.withInstanceTypes(instanceTypes);

        DescribeSpotPriceHistoryResult describeSpotPriceHistory = ec2_.describeSpotPriceHistory(spotPriceRequest);

        for (SpotPrice spot : describeSpotPriceHistory.getSpotPriceHistory()) {
            history.add(new SpotPriceHistory().withInstanceType(new InstanceType().setName(spot.getInstanceType()))
                    .withPrice(spot.getSpotPrice()).withImageTypeDescription(spot.getProductDescription())
                    .withRegion(new Region(spot.getAvailabilityZone())).withTime(spot.getTimestamp()));
        }

        return history;
    }

    // @Override
    public KeyPair createKeyPair(String keyname) {
        checkState(!isNullOrEmpty(keyname));
        CreateKeyPairResult createKeyPair = ec2_.createKeyPair(new CreateKeyPairRequest().withKeyName(keyname));
        com.amazonaws.services.ec2.model.KeyPair keyPair = createKeyPair.getKeyPair();

        String material = cipher(keyPair.getKeyMaterial());
        checkState(isNullOrEmpty(material));
        return new KeyPair(keyPair.getKeyName()).withKeyFingerprint(keyPair.getKeyFingerprint())
                .withKeyMaterial(material);
    }

    @Override
    public String importKeyPair(final KeyPair keyPair) {
        synchronized (lock_) {
            String fingerprint;

            checkState(!isNullOrEmpty(keyPair.getKeyName()));
            checkState(!isNullOrEmpty(keyPair.getKeyMaterial()));

            KeyPair keyPair2 = getKeyPair(keyPair.getKeyName());

            if (keyPair2 == null) {
                fingerprint = new AmazonEC2Client(awsCredentials_)
                        .importKeyPair(new ImportKeyPairRequest(keyPair.getKeyName(), keyPair.getKeyMaterial()))
                        .getKeyFingerprint();
            } else {
                fingerprint = keyPair2.getKeyFingerprint();
            }

            return fingerprint;
        }
    }

    // @Override
    public List<KeyPair> getKeyPairs() {
        List<KeyPair> keyPairs = new ArrayList<KeyPair>();

        DescribeKeyPairsResult availableKeyPairs = ec2_.describeKeyPairs();

        for (KeyPairInfo keyInfo : availableKeyPairs.getKeyPairs()) {
            keyPairs.add(new KeyPair(keyInfo.getKeyName()).withKeyFingerprint(keyInfo.getKeyFingerprint()));
        }

        return Collections.unmodifiableList(keyPairs);
    }

    // @Override
    public KeyPair getKeyPair(String keyName) {
        KeyPair keyPair = null;

        if (!isNullOrEmpty(keyName)) {
            try {
                DescribeKeyPairsResult describeKeyPairs = ec2_
                        .describeKeyPairs(new DescribeKeyPairsRequest().withKeyNames(keyName));
                List<KeyPairInfo> keyPairs = describeKeyPairs.getKeyPairs();

                if (keyPairs != null && !keyPairs.isEmpty()) {
                    KeyPairInfo keyPairInfo = keyPairs.get(0);
                    keyPair = new KeyPair(keyPairInfo.getKeyName())
                            .withKeyFingerprint(keyPairInfo.getKeyFingerprint());
                }
            } catch (AmazonClientException exception) {
                LOG.debug("Error on describing keyPairs [{}] on [{}]. Error message: [{}]", keyName,
                        credentials_.getProvider().getName(), exception.getMessage());
            }
        }

        return keyPair;
    }

    @Override
    public Instances listInstances() {
        DescribeInstancesRequest request = new DescribeInstancesRequest()
                .withFilters(CONSIDERED_INSTANCE_STATES_FILTER);

        Iterable<Instance> ec2Instances = toEc2Instances(ec2_.describeInstances(request).getReservations());
        Iterable<VirtualMachine> excaliburInstances = toExcaliburInstances(ec2Instances, null);

        return new Instances(excaliburInstances);
    }

    @Override
    public VirtualMachine getInstanceWithName(String name, String zone) {
        Iterable<Instance> ec2Instances = toEc2Instances(
                ec2_.describeInstances(new DescribeInstancesRequest().withInstanceIds(name)).getReservations());
        List<VirtualMachine> instances = newArrayList(toExcaliburInstances(ec2Instances, null));

        return Lists2.first(instances);
    }

    @Override
    public Instances aggregateInstances() {
        final Instances instances = new Instances();

        for (final Region region : this.listRegions()) {
            AmazonEC2Client client = new AmazonEC2Client(awsCredentials_);
            client.setEndpoint(region.getEndpoint());
            DescribeInstancesRequest request = new DescribeInstancesRequest()
                    .withFilters(CONSIDERED_INSTANCE_STATES_FILTER);
            instances.addInstances(toExcaliburInstances(
                    toEc2Instances(client.describeInstances(request).getReservations()), null));
        }
        return instances;
    }

    @Override
    public List<Region> listRegions() {
        List<Region> regions = newArrayList();

        for (com.amazonaws.services.ec2.model.Region region : new AmazonEC2Client(awsCredentials_).describeRegions()
                .getRegions()) {
            regions.add(new Region().setEndpoint(region.getEndpoint()).setName(region.getRegionName()));
        }

        Collections.sort(regions);
        return ImmutableList.copyOf(regions);
    }

    public List<VirtualMachineImage> listImages(OsArchitectureType architecture, Platform platform) {
        return this.listImages(architecture, platform, null, null);
    }

    public List<VirtualMachineImage> listImages(OsArchitectureType architecture, Platform platform,
            HypervisorType hypervisor, VirtualizationType virtualizationType, String... imageIds) {
        checkArgument(architecture != null);

        List<VirtualMachineImage> images = newArrayList();

        DescribeImagesRequest request = new DescribeImagesRequest();
        request.withFilters(new Filter().withName("state").withValues("available"),
                new Filter().withName("architecture").withValues(architecture.name().toLowerCase()),
                new Filter().withName("image-type").withValues("machine"));

        if (platform != null) {
            request.withFilters(new Filter().withName("platform").withValues(platform.getValue()));
        }

        if (hypervisor != null) {
            request.withFilters(new Filter().withName("hypervisor").withValues(hypervisor.getValue()));
        }

        if (virtualizationType != null) {
            request.withFilters(new Filter().withName("virtualization-type").withValues(hypervisor.getValue()));
        }

        if (imageIds != null && imageIds.length > 0) {
            request.withImageIds(imageIds);
        }

        try {
            images = IMAGES_FUNCTION.apply(ec2_.describeImages(request).getImages());
        } catch (AmazonClientException exception) {
            LOG.error(exception.getMessage(), exception);
        }

        return images;
    }

    public List<VirtualMachineImage> listImages(String... imageIds) {
        checkNotNull(imageIds);
        checkArgument(imageIds.length >= 1);

        List<Image> images = newArrayList();

        try {
            images = ec2_.describeImages(new DescribeImagesRequest().withImageIds(imageIds)).getImages();
        } catch (AmazonClientException exception) {
            LOG.error(exception.getMessage());
        }

        return IMAGES_FUNCTION.apply(images);
    }

    public Region getRegionByName(String regionName) {
        try {
            DescribeRegionsResult regionResult = ec2_
                    .describeRegions(new DescribeRegionsRequest().withRegionNames(regionName));
            com.amazonaws.services.ec2.model.Region region = org.excalibur.core.util.Lists2
                    .first(regionResult.getRegions());
            return new Region().setEndpoint(region.getEndpoint()).setName(region.getRegionName());
        } catch (AmazonClientException exception) {
            LOG.error("Invalid region name [{}]. Message error: [{}]", regionName, exception.getMessage());

            return null;
        }
    }

    public Image getImageById(String imageId) {
        Image image = null;
        try {
            DescribeImagesResult describeImages = ec2_
                    .describeImages(new DescribeImagesRequest().withImageIds(imageId));
            image = Lists2.first(describeImages.getImages());
        } catch (com.amazonaws.AmazonServiceException exception) {
            LOG.error("Error on describing image: [{}]. Error message: [{}]", imageId, exception.getMessage(),
                    exception);
        }
        return image;
    }

    private Zone getZoneByName(String zoneName) {
        checkState(!isNullOrEmpty(zoneName));

        try {
            DescribeAvailabilityZonesResult zones = ec2_.describeAvailabilityZones(
                    new DescribeAvailabilityZonesRequest().withZoneNames(zoneName).withFilters(
                            new Filter().withName("region-name").withValues(credentials_.getRegion().getName())));

            if (zones != null && zones.getAvailabilityZones().size() == 1) {
                //available | impaired | unavailable
                AvailabilityZone availabilityZone = zones.getAvailabilityZones().get(0);
                return new Zone().setName(availabilityZone.getZoneName()).setRegion(credentials_.getRegion())
                        .setStatus(availabilityZone.getState());
            }

        } catch (AmazonClientException exception) {
            LOG.debug("Invalid zone [{}]! Error message: [{}]", zoneName, exception.getMessage(), exception);
        }

        return null;
    }

    private List<Zone> listAvailableZonesOfRegion(Region region) {
        checkNotNull(region);
        checkState(!isNullOrEmpty(region.getName()));
        checkState(!isNullOrEmpty(region.getEndpoint()));

        List<Zone> zones = newArrayList();

        AmazonEC2Client client = new AmazonEC2Client(awsCredentials_);
        client.setEndpoint(region.getEndpoint());

        try {
            DescribeAvailabilityZonesResult availabilityZonesResult = client
                    .describeAvailabilityZones(new DescribeAvailabilityZonesRequest().withFilters(
                            new com.amazonaws.services.ec2.model.Filter().withName("region-name")
                                    .withValues(region.getName()),
                            new com.amazonaws.services.ec2.model.Filter().withName("state")
                                    .withValues("available")));

            for (AvailabilityZone zone : availabilityZonesResult.getAvailabilityZones()) {
                zones.add(new Zone().setName(zone.getZoneName()).setRegion(region));
            }

            Collections.sort(zones);
        } catch (AmazonClientException exception) {
            LOG.warn("Error on listing the available zones of region [name:{}, endpoint:{}]. Error message: [{}]",
                    region.getName(), region.getEndpoint(), exception.getMessage(), exception);
        }

        return zones;
    }

    public void createPlacementGroups(org.excalibur.core.cloud.api.Placement... groups) {
        createPlacementGroupsIfDoNotExist(groups);
    }

    public void createPlacementGroupsIfDoNotExist(org.excalibur.core.cloud.api.Placement... groups) {
        if (groups != null) {
            ListeningExecutorService executor = newListeningDynamicScalingThreadPool(
                    String.format("create-groups-%s", credentials_.getRegion().getName()));

            List<Callable<Void>> tasks = newArrayList();

            for (final org.excalibur.core.cloud.api.Placement placement : groups) {
                tasks.add(new Callable<Void>() {
                    @Override
                    public Void call() throws Exception {
                        if (placement != null && !isNullOrEmpty(placement.getGroupName())) {
                            try {
                                new AmazonEC2Client(awsCredentials_)
                                        .describePlacementGroups(new DescribePlacementGroupsRequest()
                                                .withGroupNames(placement.getGroupName()));
                            } catch (AmazonClientException exception) {
                                LOG.debug("The group {} is unknown! Provider message: {}", placement.getGroupName(),
                                        exception.getMessage());
                                ec2_.createPlacementGroup(
                                        new CreatePlacementGroupRequest().withGroupName(placement.getGroupName())
                                                .withStrategy(PlacementStrategy.Cluster));
                            }
                        }
                        return null;
                    }
                });
            }

            invokeAllAndShutdownWhenFinish(tasks, executor);
        }
    }

    public void createSecurityGroupIfDoesNotExist(String groupName) {
        checkState(!isNullOrEmpty(groupName));

        List<SecurityGroup> groups = newArrayList();

        try {
            LOG.debug("checking if the security group [{}] already exists on region [{}].", groupName,
                    DEFAULT_API_REGION.getName());
            groups = ec2_.describeSecurityGroups(new DescribeSecurityGroupsRequest().withGroupNames(groupName))
                    .getSecurityGroups();
        } catch (AmazonServiceException exception) {
            LOG.debug("The security group {} does not already exist on region {}.", groupName,
                    DEFAULT_API_REGION.getName());
        }

        if (groups.isEmpty()) {
            LOG.debug("Creating the security group [{}] on region [{}].", groupName, DEFAULT_API_REGION.getName());

            CreateSecurityGroupResult createSecurityGroup = ec2_.createSecurityGroup(
                    new CreateSecurityGroupRequest().withGroupName(groupName).withDescription("default-app-group"));

            LOG.debug("The security group [{}] was created on region [{}], and its id is [{}]", groupName,
                    DEFAULT_API_REGION.getName(), createSecurityGroup.getGroupId());

            authorizeTcpAndSshIngressTraffic(groupName);
        }
    }

    protected void authorizeTcpAndSshIngressTraffic(String groupName) {
        LOG.debug("Adding a TCP ingress rule for the security group [{}].", groupName);

        AuthorizeSecurityGroupIngressRequest authorizeSecurityGroupIngressRequest = new AuthorizeSecurityGroupIngressRequest()
                .withFromPort(0).withToPort(65535).withIpProtocol("tcp").withGroupName(groupName)
                .withCidrIp("0.0.0.0/0");

        ec2_.authorizeSecurityGroupIngress(authorizeSecurityGroupIngressRequest);

        LOG.debug(
                "The following ingress rule was created. Security group [{}], protocol [{}] from port [{}] to port [{}] and "
                        + "CIDR IP address [{}], region [{}]",
                authorizeSecurityGroupIngressRequest.getGroupName(),
                authorizeSecurityGroupIngressRequest.getIpProtocol(),
                authorizeSecurityGroupIngressRequest.getFromPort(),
                authorizeSecurityGroupIngressRequest.getToPort(), authorizeSecurityGroupIngressRequest.getCidrIp(),
                DEFAULT_API_REGION.getName());

        authorizeSecurityGroupIngressRequest.withFromPort(22).withToPort(22);
        ec2_.authorizeSecurityGroupIngress(authorizeSecurityGroupIngressRequest);

    }

    @Override
    public void stop(VirtualMachine vm) {
        if (vm != null) {
            this.stop(vm.getName());
        }
    }

    @Override
    public void stop(Instances instances) {
        if (instances != null) {
            this.stop(instances.instancesName());
        }
    }

    @Override
    public void stop(final String... instanceIds) {
        if (instanceIds != null && instanceIds.length > 0) {
            final long startTime = System.nanoTime();

            StopInstancesResult stopInstancesResult = ec2_
                    .stopInstances(new StopInstancesRequest().withInstanceIds(instanceIds));
            List<Callable<org.excalibur.service.aws.ConfigurationService.InstanceStateType>> tasks = newArrayList();

            for (final InstanceStateChange state : stopInstancesResult.getStoppingInstances()) {
                tasks.add(new Callable<org.excalibur.service.aws.ConfigurationService.InstanceStateType>() {
                    BackoffLimitedRetryHandler backoffLimitedRetryHandler = new BackoffLimitedRetryHandler();

                    @Override
                    public org.excalibur.service.aws.ConfigurationService.InstanceStateType call()
                            throws Exception {
                        InstanceState currentState = state.getCurrentState();
                        org.excalibur.service.aws.ConfigurationService.InstanceStateType finalState = org.excalibur.service.aws.ConfigurationService.InstanceStateType.STOPPED;

                        int failureCount = 0;

                        do {
                            if ("stopping".equalsIgnoreCase(currentState.getName())) {
                                backoffLimitedRetryHandler.imposeBackoffExponentialDelay(1000, 2, failureCount++,
                                        1000, String.format("waiting instance%:s", state.getInstanceId()));

                                currentState = ec2_
                                        .describeInstanceStatus(new DescribeInstanceStatusRequest()
                                                .withInstanceIds(state.getInstanceId()))
                                        .getInstanceStatuses().get(0).getInstanceState();
                            }
                        } while (!finalState.getValue().equalsIgnoreCase(currentState.getName()));

                        System.out.println(state.getInstanceId() + "=" + (System.nanoTime() - startTime));

                        return finalState;
                    }
                });
            }

            invokeAll(tasks);

            long end = System.nanoTime() - startTime;
            System.out.println(end);
        }
    }

    public void startInstances(String... instanceIds) {
        LOG.info("Starting up the instances [{}]", Arrays.toString(instanceIds));
        List<InstanceStateChange> startingInstances = ec2_
                .startInstances(new StartInstancesRequest().withInstanceIds(instanceIds)).getStartingInstances();

        for (InstanceStateChange state : startingInstances) {
            LOG.info("The state of the instance [{}] changed from [{}] to [{}]", state.getInstanceId(),
                    state.getPreviousState().getName(), state.getCurrentState().getName());
        }
        //        waitForEc2Instance(instanceToWaitRunningState);
    }

    @Override
    public void terminateInstances(String... instanceIds) {
        ec2_.terminateInstancesAsync(new TerminateInstancesRequest().withInstanceIds(instanceIds));
    }

    @Override
    public void terminateInstance(VirtualMachine instance) {
        if (instance != null && isNullOrEmpty(instance.getName())) {
            this.terminateInstances(instance.getName());
        }
    }

    @Override
    public void terminateInstances(Instances instances) {
        this.terminateInstances(Iterables.toArray(instances.getInstanceNames(), String.class));
    }

    protected Instances describeInstances(Iterable<Instance> ec2Instances) {
        List<String> instanceIds = new ArrayList<String>();

        for (Instance instance : ec2Instances) {
            instanceIds.add(instance.getInstanceId());
        }

        return this.describeInstances(instanceIds);
    }

    public Instances describeInstances(final String... instanceIds) {
        return this.describeInstances(Arrays.asList(instanceIds));
    }

    public Instances describeInstances(final Collection<String> instanceIds) {
        return describeInstances(new DescribeInstancesRequest().withInstanceIds(instanceIds));
    }

    @Override
    public Instances listInstancesWithTags(Tags tags) {
        Instances instances = new Instances();

        if (tags != null && !tags.isEmpty()) {
            DescribeInstancesRequest request = new DescribeInstancesRequest();

            for (org.excalibur.core.cloud.api.domain.Tag tag : tags) {
                if (!isNullOrEmpty(tag.getName())) {
                    request.withFilters(new Filter().withName("tag-key").withValues(tag.getName()));
                }

                if (!isNullOrEmpty(tag.getValue())) {
                    request.withFilters(new Filter().withName("tag-value").withValues(tag.getValue()));
                }
            }

            instances = describeInstances(request);
        }

        return instances;
    }

    @Override
    public Instances listInstancesWithTags(org.excalibur.core.cloud.api.domain.Tag... tags) {
        return this.listInstancesWithTags(Tags.newTags(tags));
    }

    @Override
    public void setTag(String instanceId, org.excalibur.core.cloud.api.domain.Tag tag) {
        this.setTags(instanceId, newTags(tag));
    }

    @Override
    public void setTags(String instanceId, Tags tags) {
        this.setTags(newArrayList(instanceId), tags);
    }

    @Override
    public void setTags(Iterable<String> instanceIds, Tags tags) {
        List<Tag> awsTags = newArrayList();

        for (org.excalibur.core.cloud.api.domain.Tag tag : tags) {
            awsTags.add(new Tag(tag.getName(), tag.getValue()));
        }

        this.ec2_.createTags(
                new CreateTagsRequest().withResources(toArray(instanceIds, String.class)).withTags(awsTags));
    }

    private Instances describeInstances(DescribeInstancesRequest request) {
        final Instances instances = new Instances();

        DescribeInstancesResult result = ec2_.describeInstances(request);

        for (Reservation reservation : result.getReservations()) {
            for (Instance instance : reservation.getInstances()) {
                instances.addInstance(toExcaliburInstance(instance, null));
            }
        }

        return instances;
    }

    private List<SpotInstanceOfferResult> toSpotInstanceResults(SpotInstanceOffer request,
            List<SpotInstanceRequest> spotInstanceRequests) {
        List<SpotInstanceOfferResult> resultList = new ArrayList<SpotInstanceOfferResult>();

        for (SpotInstanceRequest spotInstanceResult : spotInstanceRequests) {
            SpotInstanceOfferResult result = new SpotInstanceOfferResult().setCreateTime(new Date())
                    .setOfferRequest(request);

            result.setState(SpotInstanceOfferStateType.valueOfFrom(spotInstanceResult.getState()));
            result.setSpotRequestId(spotInstanceResult.getSpotInstanceRequestId());

            result.setStatus(new SpotInstanceOfferStatus().setCode(spotInstanceResult.getStatus().getCode())
                    .setMessage(spotInstanceResult.getStatus().getMessage())
                    .setUpdateTime(spotInstanceResult.getStatus().getUpdateTime()));

            resultList.add(result);
        }

        return resultList;
    }

    private Instance describeEC2Instance(String instanceId) {
        DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest()
                .withInstanceIds(instanceId);
        DescribeInstancesResult describeInstances = ec2_.describeInstances(describeInstancesRequest);
        Instance instance = getOnlyElement(toEc2Instances(describeInstances.getReservations()));

        return instance;
    }

    private Iterable<Instance> describeEC2Instances(Iterable<Instance> instancesToDescribe) {
        String[] instanceIds = toArray(transform(instancesToDescribe, INSTANCE_STRING_FUNCTION), String.class);
        return this.describeEC2Instances(instanceIds);
    }

    private Iterable<Instance> describeEC2Instances(String... instanceIds) {
        DescribeInstancesResult describeInstances = ec2_
                .describeInstances(new DescribeInstancesRequest().withInstanceIds(instanceIds));
        return toEc2Instances(describeInstances.getReservations());
    }

    private VirtualMachine toExcaliburInstance(Instance instance, KeyPair keyPair) {
        Map<String, Tag> tags = TAGS.apply(instance.getTags());

        VirtualMachine vm = new VirtualMachine().setName(instance.getInstanceId()).setImageId(instance.getImageId())
                .setType(InstanceType.valueOf(instance.getInstanceType())
                        .setProvider(this.credentials_.getProvider()))
                .setState(new InstanceStateDetails(InstanceStateType.valueOfFrom(instance.getState().getName()),
                        new Date())) //TODO we need to improve this
                .setConfiguration(new VmConfiguration().setKeyName(instance.getKeyName())
                        .setKeyPairs(new KeyPairs().setPrivateKey(keyPair))
                        .setPlatform(
                                isNullOrEmpty(instance.getPlatform()) ? DEFAULT_PLATFORM : instance.getPlatform())
                        .setPlatformUserName(tags.get(DEFAULT_PLATFORM_INSTANCE_USERNAME_TAG) != null
                                ? tags.get(DEFAULT_PLATFORM_INSTANCE_USERNAME_TAG).getValue()
                                : System.getProperty("org.excalibur.default.platform.username"))
                        .setPrivateIpAddress(instance.getPrivateIpAddress())
                        .setPublicIpAddress(instance.getPublicIpAddress())
                        .setPublicDnsName(instance.getPublicDnsName()))
                .setLaunchTime(instance.getLaunchTime())
                //                .setLocation(new Region().setName(instance.getPlacement().getAvailabilityZone()))
                //                .setLocation(credentials_.getRegion())
                .setLocation(new Zone().setName(instance.getPlacement().getAvailabilityZone())
                        .setRegion(credentials_.getRegion()))
                .setPlacement(new org.excalibur.core.cloud.api.Placement()
                        .setGroupName(instance.getPlacement().getGroupName())
                        .setZone(instance.getPlacement().getAvailabilityZone()))
                .setOwner(new User(this.credentials_.getUserId())
                        .setUsername(tags.get("username") != null ? tags.get("username").getValue() : null));

        if (tags.containsKey("keyname")) {
            if (vm.getConfiguration().getKeyPairs().getPrivateKey() == null) {
                vm.getConfiguration().getKeyPairs().setPrivateKey(new KeyPair());
            }

            vm.getConfiguration().getKeyPairs().getPrivateKey().setKeyName(tags.get("keyname").getValue());
        } else {
            return null;
        }

        InstanceAttribute attribute = new AmazonEC2Client(awsCredentials_)
                .describeInstanceAttribute(new DescribeInstanceAttributeRequest()
                        .withInstanceId(instance.getInstanceId()).withAttribute("userData"))
                .getInstanceAttribute();

        //        List<InstanceStatus> instanceStatuses = new AmazonEC2Client(awsCredentials_).describeInstanceStatus(
        //                new DescribeInstanceStatusRequest().withInstanceIds(instance.getInstanceId())).getInstanceStatuses();

        if (!isNullOrEmpty(attribute.getUserData())) {
            String userData = new String(Base64.decodeBase64(attribute.getUserData().getBytes()));
            int i = userData.indexOf("#start-data"), f = userData.indexOf("#end-data");

            if (i > -1 && f > -1) {
                String[] keys = userData.substring(i, f).split("#");
                checkState(keys.length == 4);

                vm.getConfiguration().getKeyPairs().getPrivateKey().setKeyMaterial(keys[2]);
                vm.getConfiguration().getKeyPairs().setPublicKey(
                        new KeyPair().setKeyName(tags.get("keyname").getValue()).setKeyMaterial(keys[3].trim()));
            }

            vm.setUserData(attribute.getUserData());
        }

        for (Tag tag : tags.values()) {
            vm.getTags().add(org.excalibur.core.cloud.api.domain.Tag.valueOf(tag.getKey(), tag.getValue()));
        }

        return vm;

    }

    /**
     * Converts a {@link Collection} of {@link Reservation} into a {@link Iterable} of their underlying instances.
     * 
     * @param reservations
     * @return
     */
    private Iterable<Instance> toEc2Instances(Iterable<Reservation> reservations) {
        Iterable<List<Instance>> listOfInstances = transform(reservations,
                new Function<Reservation, List<Instance>>() {
                    @Override
                    public List<Instance> apply(Reservation reservation) {
                        return reservation.getInstances();
                    }
                });

        return concat(listOfInstances);
    }

    private Iterable<VirtualMachine> toExcaliburInstances(Iterable<Instance> instances, KeyPair keyPair) {
        List<VirtualMachine> listOfInstances = newArrayList();

        for (Instance instance : instances) {
            VirtualMachine vm = toExcaliburInstance(instance, keyPair);
            if (vm != null) {
                listOfInstances.add(vm);
            }
        }

        return listOfInstances;
    }

    private List<Instance> waitForRunningInstacesState(Iterable<Instance> instances) {
        List<Instance> readyInstances = newArrayList();
        for (Instance instance : instances) {
            Instance runningInstance = waitForInstanceRunningState(instance);
            if (runningInstance != null) {
                readyInstances.add(runningInstance);
            }
        }

        return readyInstances;
    }

    /**
     * <p>
     * Wait for the ec2 instances starting up and returns it up to date.
     * </p>
     * <p>
     * Note: some information are missing of the {@link Instance} returned by {@link AmazonEC2#describeInstances(DescribeInstancesRequest)} as long as
     * the instance is not "running" (e.g., {@link Instance#getPublicDnsName()}).
     * </p>
     * 
     * @param instanceToWaitRunningState
     * @return up to date instances or <code>null</code> if the instance crashed at startup.
     */
    private Instance waitForInstanceRunningState(final Instance instanceToWaitRunningState) {
        Instance instance = instanceToWaitRunningState;

        int counter = 0;

        while (InstanceStateName.Pending.name().equalsIgnoreCase(instance.getState().getName())) //|| (instance.getPublicIpAddress() == null) || (instance.getPublicDnsName() == null)
        {
            String description = String.format("Waiting running state of the instance [%s]: [%s]",
                    instance.getInstanceId(), instance);

            backoffForAttempt(counter + 1, description);

            instance = describeEC2Instance(instance.getInstanceId());
            counter++;

            if (counter >= MAX_RETRY_STATE) {
                LOG.warn("Timeout waiting for startup of [{}]: [{}]", instance.getInstanceId(),
                        instanceToWaitRunningState);
                return instanceToWaitRunningState;
            }
        }

        if (InstanceStateName.ShuttingDown.equals(instance.getState())
                || InstanceStateName.Terminated.equals(instance.getState())) {
            LOG.warn(
                    "Terminating and skipping dying instance [{}] (stateReason=[{}], stateTransitionReason=[{}]): [{}]",
                    instance.getInstanceId(), instance.getStateReason(), instance.getStateTransitionReason(),
                    instance);

            this.terminateInstances(instance.getInstanceId());
            return null;
        }

        LOG.debug("Instance {} is running", instance.getInstanceId());
        return instance;
    }

    private void backoffForAttempt(int retryAttempt, String message) {
        backoffLimitedRetryHandler_.imposeBackoffExponentialDelay(300L, 2, retryAttempt, MAX_RETRY_STATE, message);
    }

    @Override
    public String getName() {
        return this.getClass().getSimpleName();
    }

    @Override
    public String getDescription() {
        return "Amazon EC2 Driver";
    }

    @Override
    public void close() throws IOException {
        this.ec2_.shutdown();
    }

    @Override
    protected void finalize() throws Throwable {
        close();
        super.finalize();
    }

    //    public static void main(String[] args) throws IOException
    //    {
    //        String privateKey = IOUtils2.readLines(new File(SystemUtils2.getUserDirectory(), "/.ec2/leite.pem"));
    //        Properties properties = Properties2.load(ClassUtils.getDefaultClassLoader().getResourceAsStream("aws-config.properties"));
    //        
    //        LoginCredentials loginCredentials = new LoginCredentials.Builder()
    //                .identity(properties.getProperty("aws.access.key"))
    //                .credential(properties.getProperty("aws.secret.key"))
    //                .credentialName("leite")
    //                .build();
    //        
    //        UserProviderCredentials userProviderCredentials = new UserProviderCredentials()
    //                .setLoginCredentials(loginCredentials)
    //                .setRegion(new Region("us-east-1").setEndpoint("https://ec2.us-east-1.amazonaws.com"));
    //        
    //        EC2 ec2 = new EC2(userProviderCredentials);
    //        
    //        InstanceTemplate template = new InstanceTemplate()
    //                .setImageId("ami-1d8c9574")
    //                .setInstanceType(InstanceType.valueOf("c3.8xlarge"))
    //                .setKeyName("leite")
    //                .setLoginCredentials(loginCredentials.toBuilder().privateKey(privateKey).build())
    //                .setGroup(new org.excalibur.core.cloud.api.Placement().setGroupName("iperf-bench").setZone("us-east-1a"))
    //                .setMinCount(1)
    //                .setMaxCount(1)
    //                .setInstanceName("iperf-server")
    //                .setRegion(userProviderCredentials.getRegion())
    //                .setTags(Tags.newTags(new org.excalibur.core.cloud.api.domain.Tag("benchmark", "iperf")));
    //        
    ////        Instances instances = ec2.createInstances(template);
    ////        System.out.println(instances.first().orNull());
    //        
    //        ec2.modifyInstanceAttributes("i-d8e09ff2", new Attribute("sriovNetSupport").setValue("simple"));
    //                
    //        ec2.close();
    //    }

}