Java tutorial
// Copyright (C) GridGain Systems, Inc. Licensed under GPLv3, http://www.gnu.org/licenses/gpl.html /* _________ _____ __________________ _____ * __ ____/___________(_)______ /__ ____/______ ____(_)_______ * _ / __ __ ___/__ / _ __ / _ / __ _ __ `/__ / __ __ \ * / /_/ / _ / _ / / /_/ / / /_/ / / /_/ / _ / _ / / / * \____/ /_/ /_/ \_,__/ \____/ \__,_/ /_/ /_/ /_/ */ package org.gridgain.grid.spi.cloud.ec2lite; import com.amazonaws.*; import com.amazonaws.auth.*; import com.amazonaws.services.ec2.*; import com.amazonaws.services.ec2.model.*; import org.apache.commons.codec.binary.*; import org.gridgain.grid.*; import org.gridgain.grid.events.*; import org.gridgain.grid.lang.*; import org.gridgain.grid.logger.*; import org.gridgain.grid.resources.*; import org.gridgain.grid.spi.*; import org.gridgain.grid.spi.cloud.*; import org.gridgain.grid.typedef.*; import org.gridgain.grid.typedef.internal.*; import org.gridgain.grid.util.ec2.*; import org.jetbrains.annotations.*; import java.util.*; import java.util.concurrent.atomic.*; import static org.gridgain.grid.GridCloudResourceType.*; import static org.gridgain.grid.GridEventType.*; import static org.gridgain.grid.spi.cloud.GridCloudSpiResourceAction.*; import static org.gridgain.grid.util.ec2.GridEc2Helper.*; /** * This class defines EC2-based implementation for {@link GridCloudSpi}. It supports * seven actions: * <ol> * <li>{@link #RUN_INST_ACT} run new Amazon EC2 instances;</li> * <li>{@link #TERMINATE_INST_ACT} terminate Amazon EC2 instances;</li> * <li>{@link #TERMINATE_ALL_INST_ACT} terminate all Amazon EC2 instances;</li> * <li>{@link #STOP_INST_ACT} stop Amazon EC2 instances;</li> * <li>{@link #STOP_ALL_INST_ACT} stop all Amazon EC2 instances;</li> * <li>{@link #START_INST_ACT} start Amazon EC2 instances;</li> * <li>{@link #START_ALL_INST_ACT} start all Amazon EC2 instances.</li> * </ol> * <p> * See below for details on each command. * <p> * For information about Amazon EC2 visit <a href="http://aws.amazon.com">aws.amazon.com</a>. * * <h1 class="header">Attention</h1> * <p> * You can configure only one cloud within one Amazon EC2 account. Be very careful when * using commands that affects all instances since execution will affect all instances * within account (not cloud as it may seem). * <p> * The most safe usage of EC2 clouds is when you run only cloud within one Amazon EC2 account. * If you want to add more clouds you should run them under separate accounts. If you use Amazon * EC2 for goals other than running clouds it is recommended you use separate accounts for that. * * <h1 class="header">Warning</h1> * <p> * Note that accessing Amazon EC2 service (i.e. running instances within) will result in charges * to your Amazon EC2 account. * <p> * Set test mode of the SPI to {@code true} to avoid calls (see {@link #setTestMode(boolean)}). * * <h1 class="header">Configuration</h1> * <h2 class="header">Mandatory</h2> * This SPI has no mandatory configuration parameters: * <h2 class="header">Optional</h2> * This SPI has following optional configuration parameters: * <ul> * <li>Cloud ID (see {@link #setCloudId(String)})</li> * <li>State check frequency (see {@link #setStateCheckFrequency(long)}, {@link #MIN_STATE_CHECK_FREQ})</li> * <li>Access key ID (see {@link #setAccessKeyId(String)}, {@link GridEc2Helper#getEc2Credentials(String)})</li> * <li>Secret access key (see {@link #setSecretAccessKey(String)}, {@link GridEc2Helper#getEc2Credentials(String)})</li> * <li>Proxy host (see {@link #setProxyHost(String)})</li> * <li>Proxy port (see {@link #setProxyPort(int)})</li> * <li>Proxy user name (see {@link #setProxyUsername(String)})</li> * <li>Proxy password (see {@link #setProxyPassword(String)})</li> * <li>Image IDs (see {@link #setImageIds(List)})</li> * <li>Enable monitoring by default (see {@link #setEnableMonitoring(boolean)})</li> * </ul> * <h1 class="header">Commands</h1> * Following commands are supported: * <table class="doctable"> * <tr> * <th>Command</th> * <th>Parameter</th> * <th>Description</th> * </tr> * <tr> * <td rowspan=4> * <b>Run Instance(s)</b> * <p> * Runs one or more EC2 instances within EC2 account. * </td> * <td>{@link GridCloudCommand#action()}</td> * <td>Should return {@link #RUN_INST_ACT}</td> * </tr> * <tr> * <td>{@link GridCloudCommand#number()}</td> * <td> * Value greater than zero indicating how many nodes to start. * If less or equal to zero exception is thrown. * </td> * </tr> * <tr> * <td>{@link GridCloudCommand#resources()}</td> * <td> * Collection containing one and only one {@link GridCloudResourceType#CLD_IMAGE} resource * and zero or more {@link GridCloudResourceType#CLD_SECURITY_GROUP} resources. * Other resources types as well as other combinations are not allowed - * exception will be thrown by {@link #process(GridCloudCommand, UUID)}. * </td> * </tr> * <tr> * <td>{@link GridCloudCommand#parameters()}</td> * <td> * Command parameters map. May be {@code null} or empty. If not SPI handles the following keys: * <ul> * <li>{@link #CMD_DRY_RUN_KEY} - If provided and {@link Boolean#parseBoolean(String)} * returns {@code true}, then SPI goes through all the motions of running a command, * but makes no actual changes (does not run any instances).</li> * <li>{@link #INST_TYPE} - Instance type (refer to Amazon EC2 API reference).</li> * <li>{@link #INST_PLACEMENT} - Instance placement (refer to Amazon EC2 API reference).</li> * <li>{@link #INST_KEY_PAIR_NAME} - Key pair name (refer to Amazon EC2 API reference).</li> * <li>{@link #INST_PASS_EC2_KEYS} - If provided and {@link Boolean#parseBoolean(String)} * returns {@code true}, then SPI will pass EC2 access credentials to instance via user data.</li> * <li>{@link #INST_MAIN_S3_BUCKET} - Main S3 bucket for instance (passed via user data).</li> * <li>{@link #INST_USER_S3_BUCKET} - User S3 bucket for instance (passed via user data).</li> * <li>{@link #INST_JVM_OPTS} - JVM options for instance (refer to Amazon EC2 API reference).</li> * <li>{@link #INST_MON} - By providing {@code true} or {@code false} string default monitor * setting may be overridden (see {@link #setEnableMonitoring(boolean)}).</li> * </ul> * </td> * </tr> * <tr> * <td rowspan=4> * <b>Terminate instance (s)</b> * <p> * Terminates instances. * </td> * <td>{@link GridCloudCommand#action()}</td> * <td>Should return {@link #TERMINATE_INST_ACT}</td> * </tr> * <tr> * <td>{@link GridCloudCommand#number()}</td> * <td> * Value greater or equal to zero. The value indicates how many instances to terminate. * See below for further details. * </td> * </tr> * <tr> * <td>{@link GridCloudCommand#resources()}</td> * <td> * Not {@code null} and not empty collection.<br> * Two variants are possible: * <ol> * <li>Collection containing the only element - resource of type {@link GridCloudResourceType#CLD_IMAGE}. * In this case {@link GridCloudCommand#number()} shows how many instances created out of this image * to terminate.</li> * <li>Collection containing resources of type {@link GridCloudResourceType#CLD_INSTANCE}. in this case * {@link GridCloudCommand#number()} should be zero or equal to {@link GridCloudCommand#resources()} * size; if not then exception is thrown)</li> * </ol> * <p> * SPI will search instances to terminate by a command among instances that are currently * in one of the following states {@link #INST_PENDING_STATE}, {@link #INST_RUNNING_STATE}, * {@link #INST_STOPPING_STATE} or {@link #INST_STOPPED_STATE}. * If concrete instances were provided but not found among instances with above states * then exception is thrown. * <p> * If cloud has less instances than it is required to terminate by a command then exception is thrown * at command execution time. * </td> * </tr> * <tr> * <td>{@link GridCloudCommand#parameters()}</td> * <td> * Command parameters map. May be {@code null} or empty. If not, SPI handles the following keys: * <ul> * <li>{@link #CMD_DRY_RUN_KEY} - If provided and {@link Boolean#parseBoolean(String)} * returns {@code true}, then SPI goes through all the motions of running a command, * but makes no actual changes (does not terminate any instances).</li> * <ul> * </td> * </tr> * <tr> * <td rowspan=4> * <b>Terminate all instances.</b> * <p> * Terminates all instances in the cloud that are currently * in one of the following states {@link #INST_PENDING_STATE}, {@link #INST_RUNNING_STATE}, * {@link #INST_STOPPING_STATE} or {@link #INST_STOPPED_STATE}. * <p> * If cloud has no instances to terminate then this command is no-op. * </td> * <td>{@link GridCloudCommand#action()}</td> * <td>Should return {@link #TERMINATE_ALL_INST_ACT}</td> * </tr> * <tr> * <td>{@link GridCloudCommand#number()}</td> * <td>Any value. Will be ignored silently.</td> * </tr> * <tr> * <td>{@link GridCloudCommand#resources()}</td> * <td>Any value. Will be ignored silently.</td> * </tr> * <tr> * <td>{@link GridCloudCommand#parameters()}</td> * <td> * Command parameters map. May be {@code null} or empty. If not, SPI handles the following keys: * <ul> * <li>{@link #CMD_DRY_RUN_KEY} - If provided and {@link Boolean#parseBoolean(String)} * returns true, than SPI goes through all the motions of running a command, but makes no actual * changes (does not terminate any instances).</li> * <ul> * </td> * </tr> * <tr> * <td rowspan=4> * <b>Stop instance (s)</b> * <p> * Stops instances. * </td> * <td>{@link GridCloudCommand#action()}</td> * <td>Should return {@link #STOP_INST_ACT}</td> * </tr> * <tr> * <td>{@link GridCloudCommand#number()}</td> * <td> * Value greater or equal to zero. If not zero then should be equal to {@link GridCloudCommand#resources()} * size (if not then exception is thrown). * </td> * </tr> * <tr> * <td>{@link GridCloudCommand#resources()}</td> * <td> * Not {@code null} and not empty collection containing resources of type * {@link GridCloudResourceType#CLD_INSTANCE} representing instances that are currently in * {@link #INST_RUNNING_STATE} state. * <p> * SPI will search instances to terminate by a command among instances that are currently in * {@link #INST_RUNNING_STATE} state. If provided instances were not found within * running instances then exception is thrown at command execution time. * </td> * </tr> * <tr> * <td>{@link GridCloudCommand#parameters()}</td> * <td> * Command parameters map. May be {@code null} or empty. If not, SPI handles the following keys: * <ul> * <li>{@link #CMD_DRY_RUN_KEY} - If provided and {@link Boolean#parseBoolean(String)} * returns {@code true}, then SPI goes through all the motions of running a command, * but makes no actual changes (does not stop any instances).</li> * <li>{@link #FORCE_STOP} - If provided and {@link Boolean#parseBoolean(String)} * returns {@code true}, then instances are forced to stop (refer to Amazon EC2 API reference).</li> * <ul> * </td> * </tr> * <tr> * <td rowspan=4> * <b>Stop all instances.</b> * <p> * Stops all instances in the cloud That are currently in {@link #INST_RUNNING_STATE} state. * <p> * If cloud has no running instances then this command is no-op. * </td> * <td>{@link GridCloudCommand#action()}</td> * <td>Should return {@link #STOP_ALL_INST_ACT}</td> * </tr> * <tr> * <td>{@link GridCloudCommand#number()}</td> * <td>Any value. Will be ignored silently.</td> * </tr> * <tr> * <td>{@link GridCloudCommand#resources()}</td> * <td>Any value. Will be ignored silently.</td> * </tr> * <tr> * <td>{@link GridCloudCommand#parameters()}</td> * <td> * Command parameters map. May be {@code null} or empty. If not, SPI handles the following keys: * <ul> * <li>{@link #CMD_DRY_RUN_KEY} - If provided and {@link Boolean#parseBoolean(String)} * returns {@code true}, then SPI goes through all the motions of running a command, * but makes no actual changes (does not stop any instances).</li> * <li>{@link #FORCE_STOP} - If provided and {@link Boolean#parseBoolean(String)} * returns {@code true}, then instances are forced to stop (refer to Amazon EC2 API reference).</li> * <ul> * </td> * </tr> * <tr> * <td rowspan=4> * <b>Start instance (s)</b> * <p> * Starts instances that are currently in {@link #INST_STOPPED_STATE}. * </td> * <td>{@link GridCloudCommand#action()}</td> * <td>Should return {@link #START_INST_ACT}</td> * </tr> * <tr> * <td>{@link GridCloudCommand#number()}</td> * <td> * Value greater or equal to zero. If not zero then should be equal to {@link GridCloudCommand#resources()} * size (if not then exception is thrown). * </td> * </tr> * <tr> * <td>{@link GridCloudCommand#resources()}</td> * <td> * Not {@code null} and not empty collection containing resources of type * {@link GridCloudResourceType#CLD_INSTANCE} representing instances that are currently in * {@link #INST_STOPPED_STATE} state. * <p> * SPI will search instances to terminate by a command among instances that are currently in * {@link #INST_STOPPED_STATE} state. If provided instances were not found within * stopped instances then exception is thrown at command execution time. * </td> * </tr> * <tr> * <td>{@link GridCloudCommand#parameters()}</td> * <td> * Command parameters map. May be {@code null} or empty. If not, SPI handles the following keys: * <ul> * <li>{@link #CMD_DRY_RUN_KEY} - If provided and {@link Boolean#parseBoolean(String)} * returns {@code true}, then SPI goes through all the motions of running a command, * but makes no actual changes (does not start any instances).</li> * <ul> * </td> * </tr> * <tr> * <td rowspan=4> * <b>Start all instances.</b> * <p> * Starts all instances in the cloud That are currently in {@link #INST_STOPPED_STATE} state. * <p> * If cloud has no stopped instances then this command is no-op. * </td> * <td>{@link GridCloudCommand#action()}</td> * <td>Should return {@link #STOP_ALL_INST_ACT}</td> * </tr> * <tr> * <td>{@link GridCloudCommand#number()}</td> * <td>Any value. Will be ignored silently.</td> * </tr> * <tr> * <td>{@link GridCloudCommand#resources()}</td> * <td>Any value. Will be ignored silently.</td> * </tr> * <tr> * <td>{@link GridCloudCommand#parameters()}</td> * <td> * Command parameters map. May be {@code null} or empty. If not, SPI handles the following keys: * <ul> * <li>{@link #CMD_DRY_RUN_KEY} - If provided and {@link Boolean#parseBoolean(String)} * returns {@code true}, then SPI goes through all the motions of running a command, * but makes no actual changes (does not start any instances).</li> * <ul> * </td> * </tr> * </table> * <p> * If command's action is not recognized exception is thrown by SPI. * * <h2 class="header">Java Example</h2> * GridEc2LiteCloudSpi can be configured as follows: * <pre name="code" class="java"> * GridEc2LiteCloudSpi cloudSpi = new GridEc2LiteCloudSpi(); * * // Override default cloud ID. * cloudSpi.setCloudId("myCloud"); * * GridConfigurationAdapter cfg = new GridConfigurationAdapter(); * * // Override default cloud SPI. * cfg.setCloudSpi(cloudSpi); * * // Starts grid. * G.start(cfg); * </pre> * <h2 class="header">Spring Example</h2> * GridEc2LiteCloudSpi can be configured from Spring XML configuration file: * <pre name="code" class="xml"> * <bean id="grid.custom.cfg" class="org.gridgain.grid.GridConfigurationAdapter" singleton="true"> * ... * <property name="cloudSpi"> * <bean class="org.gridgain.grid.spi.cloud.ec2lite.GridEc2LiteCloudSpi"> * <property name="cloudId" value="myCloud"/> * </bean> * </property> * ... * </bean> * </pre> * <p> * <img src="http://www.gridgain.com/images/spring-small.png"> * <br> * For information about Spring framework visit <a href="http://www.springframework.org/">www.springframework.org</a> * * @author 2005-2011 Copyright (C) GridGain Systems, Inc. * @version 3.1.1c.19062011 * @see GridCloudSpi */ @GridSpiInfo(author = "GridGain Systems, Inc.", url = "www.gridgain.com", email = "support@gridgain.com", version = "3.1.1c.19062011") @GridSpiMultipleInstancesSupport(true) public class GridEc2LiteCloudSpi extends GridSpiAdapter implements GridCloudSpi, GridEc2LiteCloudSpiMBean { /** Collection parameter value delimiter. */ public static final String VAL_DELIM = "|"; /** Image architecture. */ public static final String IMG_ARCH = "Architecture"; /** Image location. */ public static final String IMG_LOC = "Location"; /** Image visibility. */ public static final String IMG_PUBLIC = "Visibility"; /** Image state. */ public static final String IMG_STATE = "State"; /** Instance monitoring state. */ public static final String INST_MON_STATE = "MonitoringState"; /** Instance state name. */ public static final String INST_STATE = "StateName"; /** Instance state code. */ public static final String INST_STATE_CODE = "StateCode"; /** Instance state transition reason. */ public static final String INST_STATE_TRANS_REASON = "StateTransitionReason"; /** Instance key pair name. */ public static final String INST_KEY_PAIR_NAME = "KeyName"; /** Instance launch time. */ public static final String INST_LAUNCH_TIME = "LaunchTime"; /** Instance AMI launch index. */ public static final String INST_AMI_LAUNCH_IDX = "AmiLaunchIndex"; /** Instance private DNS name. */ public static final String INST_PRIV_DNS = "PrivateDnsName"; /** Instance public DNS name. */ public static final String INST_PUB_DNS = "PublicDnsName"; /** Instance placement. */ public static final String INST_PLACEMENT = "Placement"; /** Security group description. */ public static final String GRP_DESCR = "Description"; /** Image/security group owner ID. */ public static final String OWNER_ID = "OwnerId"; /** Image/instance product codes IDs. */ public static final String PRODUCT_CODE_IDS = "ProductCodeIds"; /** Image/instance type. Refer to Amazon EC2 documentation for possible values. */ public static final String INST_TYPE = "Type"; /** Image/instance platform. */ public static final String INST_PLATFORM = "Platform"; /** Image/instance kernel ID. */ public static final String IMG_KERNEL_ID = "KernelId"; /** Image/instance ramdisk ID. */ public static final String IMG_RAMDISK_ID = "RamdiskId"; /** Instance monitoring flag (command parameter). */ public static final String INST_MON = "Monitoring"; /** Instance main S3 bucket (command parameter). */ public static final String INST_MAIN_S3_BUCKET = "MainS3Bucket"; /** Instance user S3 bucket (command parameter). */ public static final String INST_USER_S3_BUCKET = "UserS3Bucket"; /** Flag of EC2 keys pass to instance (command parameter). */ public static final String INST_PASS_EC2_KEYS = "PassEc2Keys"; /** Force instance stop (command parameter). */ public static final String FORCE_STOP = "forceStop"; /** Force instance stop (command parameter). */ public static final String CMD_DRY_RUN_KEY = "gridgain.ec2.cloud.cmd.dry.run"; /** Instance JVM options (command parameter). */ public static final String INST_JVM_OPTS = "JvmOpts"; /** Security group IP permissions count. */ public static final String GRP_IP_PERMS_CNT = "IpPermissionSize"; /** Security group IP permission. */ public static final String GRP_IP_PERM = "IpPermission"; /** IP permission from port. */ public static final String IP_PERM_FROM_PORT = "FromPort"; /** IP permission to port. */ public static final String IP_PERM_TO_PORT = "ToPort"; /** IP permission IP range. */ public static final String IP_PERM_IP_RANGE = "IpRange"; /** IP permission IP protocol. */ public static final String IP_PERM_IP_PROTO = "IpProtocol"; /** IP permissions User ID - Group pair. Used when creating {@link GridCloudResource} for Amazon IpPermission. */ public static final String USER_ID_GRP_PAIR = "UserIdGroupPair"; /** Pair constant to represent pair when creating {@link GridCloudResource} for Amazon IpPermission. */ public static final String PAIR = "Pair"; /** User ID in pair. */ public static final String PAIR_USER_ID = "UserId"; /** Group name in pair. */ static final String PAIR_GRP = "GroupName"; /** Instance state 'pending'. */ public static final String INST_PENDING_STATE = "pending"; /** Instance state 'running'. */ public static final String INST_RUNNING_STATE = "running"; /** Instance state 'shutting-down'. */ @SuppressWarnings({ "UnusedDeclaration" }) public static final String INST_SHUTTING_DOWN_STATE = "shutting-down"; /** Instance state 'stopping'. */ public static final String INST_STOPPING_STATE = "stopping"; /** Instance state 'stopped'. */ public static final String INST_STOPPED_STATE = "stopped"; /** Instance state 'terminated'. */ public static final String INST_TERMINATED_STATE = "terminated"; /** EC2 instance ID node attribute. */ public static final String ATTR_EC2_INSTANCE_ID = "GG_EC2_INSTANCE_ID"; /** Run instance action. */ public static final String RUN_INST_ACT = "run"; /** Terminate instance action. */ public static final String TERMINATE_INST_ACT = "terminate"; /** Terminate all instances action. */ public static final String TERMINATE_ALL_INST_ACT = "terminate-all"; /** Start instance action. */ public static final String START_INST_ACT = "start"; /** Stop instance action. */ public static final String STOP_INST_ACT = "stop"; /** Start all instance action. */ public static final String START_ALL_INST_ACT = "start-all"; /** Stop all instance action. */ public static final String STOP_ALL_INST_ACT = "stop-all"; /** Cloud name prefix for unnamed clouds. */ private static final String CLOUD_ID_PREF = "ec2-cloud-"; /** Default check state frequency in milliseconds. */ public static final long DFLT_STATE_CHECK_FREQ = 10000; /** Minimum check state frequency in milliseconds. Values below are not recommended. */ public static final int MIN_STATE_CHECK_FREQ = 2000; /** Cloud EC2 API version parameter. */ private static final String CLOUD_API_VER_PARAM = "ec2-api-ver"; /** Cloud EC2 API version value. */ private static final String CLOUD_API_VER_VAL = "1.1.1"; /** Cloud parameter key for last cloud update time. */ private static final String CLOUD_LAST_UPD_TIME_PARAM = "last-cloud-upd-time"; /** Cloud index for unnamed clouds. */ private static AtomicInteger cloudIdGen = new AtomicInteger(); /** Cloud ID. */ private String cloudId = CLOUD_ID_PREF + cloudIdGen.incrementAndGet(); /** Frequency to check cloud state change. */ @SuppressWarnings({ "FieldAccessedSynchronizedAndUnsynchronized" }) private long stateCheckFreq = DFLT_STATE_CHECK_FREQ; /** Cloud SPI listener. */ private volatile GridCloudSpiListener lsnr; /** Grid name. */ private String gridName; /** Grid logger. */ @GridLoggerResource private GridLogger log; /** Cloud commands to execute. */ private Map<UUID, GridCloudCommand> cmds = new LinkedHashMap<UUID, GridCloudCommand>(); /** Mutex. */ private final Object mux = new Object(); /** Cloud control thread. */ private CloudControlThread ctrlThread; /** EC2 access key. */ private String accessKeyId; /** EC2 secret key. */ private String secretAccessKey; /** Proxy host. */ private String prxHost; /** Proxy port. */ private int prxPort = -1; /** Proxy user name. */ private String prxUser; /** Proxy password. */ private String prxPwd; /** EC2 client. */ private AmazonEC2 ec2; /** If testMode is <tt>true</tt> SPI will not call EC2 it will use mock to emulate calls. */ private boolean testMode; /** Enable monitoring by default flag. */ private boolean enableMon; /** IDs of EC2 images used in this cloud. */ private List<String> imgIds; private GridCloudSpiSnapshot lastSnp; /** */ private Collection<Image> imgs; /** */ private Collection<SecurityGroup> secGrps; /** Discovery listener. */ private final GridLocalEventListener discoLsnr = new GridLocalEventListener() { @SuppressWarnings({ "NakedNotify" }) @Override public void onEvent(GridEvent evt) { assert evt instanceof GridDiscoveryEvent; UUID nodeId = ((GridDiscoveryEvent) evt).eventNodeId(); if (evt.type() == EVT_NODE_JOINED) { GridNode node = getSpiContext().node(nodeId); if (node != null && node.attribute(ATTR_EC2_INSTANCE_ID) != null) synchronized (mux) { mux.notifyAll(); } } else synchronized (mux) { mux.notifyAll(); } } }; /** {@inheritDoc} */ @Override public String getCloudId() { return cloudId; } /** * Sets Cloud ID. * * @param cloudId Cloud ID. */ @GridSpiConfiguration(optional = false) public void setCloudId(String cloudId) { this.cloudId = cloudId; } /** {@inheritDoc} */ @Override public long getStateCheckFrequency() { return stateCheckFreq; } /** * Sets frequency to check cloud state change. * * @param stateCheckFreq Frequency in milliseconds. */ @GridSpiConfiguration(optional = true) public void setStateCheckFrequency(long stateCheckFreq) { this.stateCheckFreq = stateCheckFreq; } /** {@inheritDoc} */ @Override public void setListener(GridCloudSpiListener lsnr) { this.lsnr = lsnr; } /** {@inheritDoc} */ @Override public String getAccessKeyId() { return accessKeyId; } /** * Sets EC2 access key. * * @param accessKeyId EC2 access key. */ @GridSpiConfiguration(optional = true) public void setAccessKeyId(String accessKeyId) { this.accessKeyId = accessKeyId; } /** * Sets EC2 secret key. * * @param secretAccessKey EC2 access key. */ @GridSpiConfiguration(optional = true) public void setSecretAccessKey(String secretAccessKey) { this.secretAccessKey = secretAccessKey; } /** {@inheritDoc} */ @Override public String getProxyHost() { return prxHost; } /** * Sets HTTP proxy host to use to connect to EC2 service. This is not required parameter. * * @param prxHost HTTP proxy host. */ @GridSpiConfiguration(optional = true) public void setProxyHost(String prxHost) { this.prxHost = prxHost; } /** {@inheritDoc} */ @Override public int getProxyPort() { return prxPort; } /** * Sets HTTP proxy port to use to connect to EC2 service. This is not required parameter. * * @param prxPort HTTP proxy port. */ @GridSpiConfiguration(optional = true) public void setProxyPort(int prxPort) { this.prxPort = prxPort; } /** {@inheritDoc} */ @Override public String getProxyUsername() { return prxUser; } /** * Sets HTTP proxy user name to use to connect to EC2 service. This is not required parameter. * * @param prxUser HTTP proxy user name. */ @GridSpiConfiguration(optional = true) public void setProxyUsername(String prxUser) { this.prxUser = prxUser; } /** * Sets HTTP proxy password to use to connect to EC2 service. This is not required parameter. * * @param prxPwd HTTP proxy password. */ @GridSpiConfiguration(optional = true) public void setProxyPassword(String prxPwd) { this.prxPwd = prxPwd; } /** * Gets if test mode is set or not. SPI will not call EC2 in test mode * it will use mock to emulate calls. * * @return <tt>true</tt> if test mode is using, otherwise <tt>false</tt>. */ public boolean isTestMode() { return testMode; } /** * Sets test mode. SPI will not call EC2 in test mode it will use mock to emulate calls. * * @param testMode <tt>true</tt> if test mode is using, otherwise <tt>false</tt>. */ public void setTestMode(boolean testMode) { this.testMode = testMode; } /** * Sets EC2 image IDs for using in this cloud. * * @param imgIds EC2 images IDs. */ @GridSpiConfiguration(optional = true) public void setImageIds(List<String> imgIds) { this.imgIds = imgIds; } /** {@inheritDoc} */ @Override public List<String> getImageIds() { return imgIds; } /** * Sets monitoring flag. * * @param enableMon CloudWatch monitoring flag. */ @GridSpiConfiguration(optional = true) public void setEnableMonitoring(boolean enableMon) { this.enableMon = enableMon; } /** {@inheritDoc} */ @Override public boolean isEnableMonitoring() { return enableMon; } /** {@inheritDoc} */ @Override public void spiStart(String gridName) throws GridSpiException { this.gridName = gridName; if (accessKeyId == null && secretAccessKey == null && !testMode) { try { GridEc2Keys keys = GridEc2Helper.getEc2Credentials(null); accessKeyId = keys.getAccessKeyId(); secretAccessKey = keys.getSecretAccessKey(); } catch (GridException e) { throw new GridSpiException("Failed to get EC2 credentials.", e); } } assertParameter(!F.isEmpty(cloudId), "!F.isEmpty(cloudId)"); assertParameter(stateCheckFreq > 0, "stateCheckFreq > 0"); if (!testMode) { assertParameter(!F.isEmpty(accessKeyId), "!F.isEmpty(accessKeyId)"); assertParameter(!F.isEmpty(secretAccessKey), "!F.isEmpty(secretAccessKey)"); } registerMBean(gridName, this, GridEc2LiteCloudSpiMBean.class); // Ack parameters. if (log.isDebugEnabled()) { log.debug(configInfo("cloudId", cloudId)); log.debug(configInfo("stateCheckFreq", stateCheckFreq)); log.debug(configInfo("enableMon", enableMon)); log.debug(configInfo("testMode", testMode)); log.debug(configInfo("imgIds", imgIds)); if (prxHost != null) { log.debug(configInfo("prxHost", prxHost)); log.debug(configInfo("prxPort", prxPort)); log.debug(configInfo("prxUser", prxUser)); } } initEC2Client(); if (stateCheckFreq < MIN_STATE_CHECK_FREQ) U.warn(log, "State check frequency is too low (recommended " + MIN_STATE_CHECK_FREQ + " ms or higher): " + stateCheckFreq); if (log.isDebugEnabled()) log.debug(startInfo()); } /** {@inheritDoc} */ @Override public void spiStop() throws GridSpiException { deactivate(); unregisterMBean(); if (log.isDebugEnabled()) log.debug(stopInfo()); } /** {@inheritDoc} */ @Override public GridCloudSpiSnapshot activate() throws GridSpiException { getSpiContext().addLocalEventListener(discoLsnr, EVTS_DISCOVERY); lastSnp = makeSnapshot(); // Start control thread after snapshot is ready. ctrlThread = new CloudControlThread(); ctrlThread.start(); return lastSnp; } /** {@inheritDoc} */ @Override public void deactivate() { getSpiContext().removeLocalEventListener(discoLsnr); U.interrupt(ctrlThread); U.join(ctrlThread, log); } /** * Creates EC2 client based on current SPI parameters. */ private void initEC2Client() { ClientConfiguration ec2config = new ClientConfiguration(); if (prxHost != null) { ec2config.setProxyHost(prxHost); if (prxPort != -1) ec2config.setProxyPort(prxPort); if (prxUser != null) { ec2config.setProxyUsername(prxUser); if (prxPwd != null) ec2config.setProxyPassword(prxPwd); } } // In test mode, the test should inject client itself. ec2 = testMode ? null : new AmazonEC2Client(new BasicAWSCredentials(accessKeyId, secretAccessKey), ec2config); } /** {@inheritDoc} */ @Override public void process(GridCloudCommand cmd, UUID cmdExecId) throws GridSpiException { assert cmd != null; assert cmdExecId != null; String act = cmd.action(); int num = cmd.number(); Collection<GridCloudResource> rsrcs = cmd.resources(); if (RUN_INST_ACT.equalsIgnoreCase(act)) { if (num <= 0) throw new GridSpiException("Invalid command (number should be positive): " + cmd); int imgCnt = 0; if (rsrcs != null && !rsrcs.isEmpty()) for (GridCloudResource rsrc : rsrcs) { int type = rsrc.type(); if (type != CLD_IMAGE && type != CLD_SECURITY_GROUP) throw new GridSpiException( "Invalid command (resources contain unsupported resource): " + cmd); else if (type == CLD_IMAGE) { imgCnt++; if (imgCnt > 1) // Command is invalid, no need to continue. break; } } if (imgCnt != 1) throw new GridSpiException( "Invalid command (exactly one image resource should be provided): " + cmd); if (log.isDebugEnabled()) log.debug("Received command to run instances: " + cmd); } else if (TERMINATE_INST_ACT.equalsIgnoreCase(act)) { if (rsrcs == null || rsrcs.isEmpty()) throw new GridSpiException("Invalid command (resources cannot be empty): " + cmd); boolean terminateByImage = false; for (GridCloudResource rsrc : rsrcs) if (rsrc.type() == CLD_IMAGE) { if (rsrcs.size() != 1) throw new GridSpiException( "Invalid command (if image resource is provided it should be the " + "only resource): " + cmd); else terminateByImage = true; } else if (rsrc.type() != CLD_INSTANCE) throw new GridSpiException("Invalid command (resources contain unsupported resource): " + cmd); if (terminateByImage) { if (num <= 0) throw new GridSpiException("Invalid command (number should be positive): " + cmd); } else if (num != 0 && num != rsrcs.size()) throw new GridSpiException( "Invalid command (number should be zero or equal to resources size): " + cmd); if (log.isDebugEnabled()) log.debug("Received command to terminate instances: " + cmd); } else if (TERMINATE_ALL_INST_ACT.equalsIgnoreCase(act)) { if (log.isDebugEnabled()) log.debug("Received command to terminate all instances: " + cmd); } else if (START_INST_ACT.equalsIgnoreCase(act)) { if (rsrcs == null || rsrcs.isEmpty()) throw new GridSpiException("Invalid command (resources cannot be empty): " + cmd); if (num != 0 && num != rsrcs.size()) throw new GridSpiException( "Invalid command (number should be zero or equal to resources size): " + cmd); for (GridCloudResource rsrc : rsrcs) if (rsrc.type() != CLD_INSTANCE) throw new GridSpiException("Invalid command (resources contain unsupported resource): " + cmd); } else if (START_ALL_INST_ACT.equalsIgnoreCase(act)) { if (log.isDebugEnabled()) log.debug("Received command to start all instances: " + cmd); } else if (STOP_INST_ACT.equalsIgnoreCase(act)) { if (rsrcs == null || rsrcs.isEmpty()) throw new GridSpiException("Invalid command (resources cannot be empty): " + cmd); if (num != 0 && num != rsrcs.size()) throw new GridSpiException( "Invalid command (number should be zero or equal to resources size): " + cmd); for (GridCloudResource rsrc : rsrcs) if (rsrc.type() != CLD_INSTANCE) throw new GridSpiException("Invalid command (resources contain unsupported resource): " + cmd); if (log.isDebugEnabled()) log.debug("Received command to stop instances: " + cmd); } else if (STOP_ALL_INST_ACT.equalsIgnoreCase(act)) { if (log.isDebugEnabled()) log.debug("Received command to stop all instances: " + cmd); } else throw new GridSpiException("Invalid command (action unknown): " + cmd); synchronized (mux) { cmds.put(cmdExecId, cmd); mux.notifyAll(); } } /** {@inheritDoc} */ @Override public Collection<GridTuple2<GridCloudResource, GridCloudSpiResourceAction>> compare( GridCloudSpiSnapshot oldSnp, GridCloudSpiSnapshot newSnp) { assert oldSnp != null; assert newSnp != null; if (F.isEmpty(oldSnp.getResources()) && F.isEmpty(newSnp.getResources())) return Collections.emptyList(); Map<String, GridCloudResource> oldRsrcs = new LinkedHashMap<String, GridCloudResource>(); if (!F.isEmpty(oldSnp.getResources())) for (GridCloudResource rsrc : oldSnp.getResources()) { assert rsrc.id() != null; oldRsrcs.put(rsrc.id(), rsrc); } Map<String, GridCloudResource> newRsrcs = new LinkedHashMap<String, GridCloudResource>(); if (!F.isEmpty(newSnp.getResources())) for (GridCloudResource rsrc : newSnp.getResources()) { assert rsrc.id() != null; assert rsrc.type() == CLD_INSTANCE || rsrc.type() == CLD_IMAGE || rsrc.type() == CLD_SECURITY_GROUP || rsrc.type() == CLD_NODE; newRsrcs.put(rsrc.id(), rsrc); } Collection<GridTuple2<GridCloudResource, GridCloudSpiResourceAction>> res = new LinkedList<GridTuple2<GridCloudResource, GridCloudSpiResourceAction>>(); for (Iterator<Map.Entry<String, GridCloudResource>> iter = newRsrcs.entrySet().iterator(); iter .hasNext();) { Map.Entry<String, GridCloudResource> e = iter.next(); GridCloudResource newRsrc = e.getValue(); GridCloudResource oldRsrc = oldRsrcs.remove(e.getKey()); if (oldRsrc != null) { if (!newRsrc.equals(oldRsrc)) res.add(F.<GridCloudResource, GridCloudSpiResourceAction>t(newRsrc, CHANGED)); iter.remove(); } else res.add(F.<GridCloudResource, GridCloudSpiResourceAction>t(newRsrc, ADDED)); } if (!oldRsrcs.isEmpty()) for (GridCloudResource rsrc : oldRsrcs.values()) res.add(F.<GridCloudResource, GridCloudSpiResourceAction>t(rsrc, REMOVED)); return res; } /** * Checks cloud state and executes commands. */ private void checkCloud() { Map<UUID, GridCloudCommand> tmp = null; synchronized (mux) { if (!cmds.isEmpty()) { tmp = cmds; cmds = new LinkedHashMap<UUID, GridCloudCommand>(); } } if (!F.isEmpty(tmp)) { assert tmp != null; for (Map.Entry<UUID, GridCloudCommand> e : tmp.entrySet()) { UUID id = e.getKey(); GridCloudCommand cmd = e.getValue(); String act = cmd.action(); assert !F.isEmpty(act); if (RUN_INST_ACT.equalsIgnoreCase(act)) try { runInstances(cmd); notifySpiListenerOnCommand(true, id, cmd, null); } catch (GridSpiException ex) { log.error(ex.getMessage()); notifySpiListenerOnCommand(false, id, cmd, ex.getMessage()); } else if (TERMINATE_INST_ACT.equalsIgnoreCase(act)) try { terminateInstances(cmd); notifySpiListenerOnCommand(true, id, cmd, null); } catch (GridSpiException ex) { log.error(ex.getMessage()); notifySpiListenerOnCommand(false, id, cmd, ex.getMessage()); } else if (TERMINATE_ALL_INST_ACT.equalsIgnoreCase(act)) try { terminateAllInstances(cmd); notifySpiListenerOnCommand(true, id, cmd, null); } catch (GridSpiException ex) { log.error(ex.getMessage()); notifySpiListenerOnCommand(false, id, cmd, ex.getMessage()); } else if (STOP_INST_ACT.equalsIgnoreCase(act)) try { stopInstances(cmd); notifySpiListenerOnCommand(true, id, cmd, null); } catch (GridSpiException ex) { log.error(ex.getMessage()); notifySpiListenerOnCommand(false, id, cmd, ex.getMessage()); } else if (STOP_ALL_INST_ACT.equalsIgnoreCase(act)) try { stopAllInstances(cmd); notifySpiListenerOnCommand(true, id, cmd, null); } catch (GridSpiException ex) { log.error(ex.getMessage()); notifySpiListenerOnCommand(false, id, cmd, ex.getMessage()); } else if (START_INST_ACT.equalsIgnoreCase(act)) try { startInstances(cmd); notifySpiListenerOnCommand(true, id, cmd, null); } catch (GridSpiException ex) { log.error(ex.getMessage()); notifySpiListenerOnCommand(false, id, cmd, ex.getMessage()); } else if (START_ALL_INST_ACT.equalsIgnoreCase(act)) try { startAllInstances(cmd); notifySpiListenerOnCommand(true, id, cmd, null); } catch (GridSpiException ex) { log.error(ex.getMessage()); notifySpiListenerOnCommand(false, id, cmd, ex.getMessage()); } else assert false; } } notifySpiListenerOnChange(); } /** * Runs Amazon EC2 instances by command. * <p> * Command may be very complex and contain plenty of parameters. * Refer to class documentation for details. * * @param cmd Cloud command. * @throws GridSpiException Thrown if any exception occurs. */ private void runInstances(GridCloudCommand cmd) throws GridSpiException { assert cmd != null; RunInstancesRequest req = createRunInstancesRequest(cmd); Map<String, String> params = cmd.parameters(); if (params != null && Boolean.parseBoolean(params.get(CMD_DRY_RUN_KEY))) { if (log.isDebugEnabled()) log.debug("Dry run - instances run omitted."); return; } RunInstancesResult res; try { res = ec2.runInstances(req); } catch (AmazonClientException e) { throw new GridSpiException("Failed to perform run instances request.", e); } if (log.isDebugEnabled()) log.debug("Sent run instances request [imgId=" + req.getImageId() + ", minCount=" + req.getMinCount() + ", maxCount=" + req.getMaxCount() + ']'); Collection<String> instIds = new LinkedList<String>(); boolean throwEx = false; for (Instance item : res.getReservation().getInstances()) { String instId = item.getInstanceId(); String imgId = item.getImageId(); instIds.add(instId); if (log.isDebugEnabled()) log.debug("Added (ran) new instance [instId=" + instId + ", imgId=" + imgId + ']'); if (!req.getImageId().equals(imgId)) throwEx = true; } if (!instIds.isEmpty()) { if (req.isMonitoring()) { try { ec2.monitorInstances(new MonitorInstancesRequest().withInstanceIds(instIds)); } catch (AmazonClientException e) { U.error(log, "Failed to start instance monitoring.", e); } if (log.isDebugEnabled()) log.debug("Started instances monitoring: " + instIds); } } if (throwEx || instIds.size() != cmd.number()) throw new GridSpiException("Cloud command has not been successfully executed: " + cmd); } /** * Creates Amazon EC2 run instances request by provided command. * * @param cmd Cloud command. * @return EC2 run instances request. * @throws GridSpiException If any exception occurs. */ private RunInstancesRequest createRunInstancesRequest(GridCloudCommand cmd) throws GridSpiException { assert cmd != null; Collection<GridCloudResource> rsrcs = cmd.resources(); int num = cmd.number(); assert rsrcs != null && !rsrcs.isEmpty(); assert num > 0; GridCloudResource img = null; Collection<String> grps = new ArrayList<String>(); // Separate image and security groups for (GridCloudResource rsrc : rsrcs) if (rsrc.type() == CLD_IMAGE) img = rsrc; else if (rsrc.type() == CLD_SECURITY_GROUP) grps.add(rsrc.id()); assert img != null; Map<String, String> imgParams = img.parameters(); Map<String, String> cmdParams = cmd.parameters(); if (imgParams == null) throw new GridSpiException( "Unable to process command (image parameters are null) [cmd=" + cmd + ", image=" + img + ']'); RunInstancesRequest req = new RunInstancesRequest(); req.setImageId(img.id()); req.setMinCount(num); req.setMaxCount(num); if (!grps.isEmpty()) req.setSecurityGroups(grps); String val; if (!F.isEmpty(val = imgParams.get(IMG_KERNEL_ID))) req.setKernelId(val); if (!F.isEmpty(val = imgParams.get(IMG_RAMDISK_ID))) req.setRamdiskId(val); Collection<String> userDataList = new LinkedList<String>(); if (cmdParams != null) { if (!F.isEmpty(val = cmdParams.get(INST_TYPE))) req.setInstanceType(val); if (!F.isEmpty(val = cmdParams.get(INST_PLACEMENT))) req.setPlacement(new Placement(val)); if (!F.isEmpty(val = cmdParams.get(INST_KEY_PAIR_NAME))) req.setKeyName(val); if (!F.isEmpty(val = cmdParams.get(INST_MON))) req.setMonitoring(Boolean.parseBoolean(val)); if (!F.isEmpty(val = cmdParams.get(INST_PASS_EC2_KEYS)) && Boolean.parseBoolean(val)) { userDataList.add(GRIDGAIN_ACCESS_KEY_ID_KEY + '=' + accessKeyId); userDataList.add(GRIDGAIN_SECRET_KEY_KEY + '=' + secretAccessKey); } if (!F.isEmpty(val = cmdParams.get(INST_MAIN_S3_BUCKET))) userDataList.add(GRIDGAIN_MAIN_S3_BUCKET_KEY + '=' + val); if (!F.isEmpty(val = cmdParams.get(INST_USER_S3_BUCKET))) userDataList.add(GRIDGAIN_EXT_S3_BUCKET_KEY + '=' + val); if (!F.isEmpty(val = cmdParams.get(INST_JVM_OPTS))) userDataList.add(val); } if (req.isMonitoring() == null) // Monitoring was not set from params, set default value req.setMonitoring(enableMon); if (!userDataList.isEmpty()) req.setUserData(new String(Base64.encodeBase64(F.concat(userDataList, USER_DATA_DELIM).getBytes()))); return req; } /** * Terminates Amazon EC2 cloud instances by command. * <p> * Refer to class documentation for details on how to setup such kind of command. * * @param cmd Cloud command. * @throws GridSpiException Thrown if any exception occurs or no alive instances found * or not all provided instances found. */ private void terminateInstances(GridCloudCommand cmd) throws GridSpiException { assert cmd != null; assert TERMINATE_INST_ACT.equalsIgnoreCase(cmd.action()); Collection<GridCloudResource> rsrcs = cmd.resources(); int num = cmd.number(); assert rsrcs != null && !rsrcs.isEmpty(); assert num >= 0; Collection<Instance> runInsts = getEc2Instances(INST_PENDING_STATE, INST_RUNNING_STATE, INST_STOPPING_STATE, INST_STOPPED_STATE); if (F.isEmpty(runInsts)) throw new GridSpiException("There are no alive instances to terminate for command: " + cmd); Collection<String> instIds = new LinkedList<String>(); GridCloudResource rsrc = F.first(rsrcs); if (rsrc != null && rsrc.type() == CLD_IMAGE) { for (Instance inst : runInsts) if (inst.getImageId().equals(rsrc.id())) { instIds.add(inst.getInstanceId()); if (instIds.size() == num) break; } if (instIds.size() != num) throw new GridSpiException("Necessary instances number to stop was not found for command: " + cmd); } else { Collection<String> runInstIds = F.transform(runInsts, new C1<Instance, String>() { @Override public String apply(Instance inst) { return inst.getInstanceId(); } }); for (GridCloudResource rsrc0 : rsrcs) { assert rsrc0.type() == CLD_INSTANCE; String id = rsrc0.id(); if (runInstIds.contains(id)) instIds.add(id); else throw new GridSpiException( "Unable to find alive EC2 instance with given ID [id=" + id + ", cmd=" + cmd + ']'); } } Map<String, String> params = cmd.parameters(); if (params != null && Boolean.parseBoolean(params.get(CMD_DRY_RUN_KEY))) { if (log.isDebugEnabled()) log.debug("Dry run - instances termination omitted."); return; } terminateInstances(instIds); } /** * Terminates all EC2 cloud instances by command. * <p> * In order for command to execute properly instances provided in command * should be in one of the following states: * <ul> * <li>{@link #INST_RUNNING_STATE}; * <li>{@link #INST_STOPPED_STATE}; * <li>{@link #INST_STOPPING_STATE}; * <li>{@link #INST_PENDING_STATE}. * </ul> * * @param cmd Cloud command. * @throws GridSpiException Thrown if any exception occurs or no instances in required states found. */ private void terminateAllInstances(GridCloudCommand cmd) throws GridSpiException { assert cmd != null; assert TERMINATE_ALL_INST_ACT.equalsIgnoreCase(cmd.action()); Collection<String> instIds = F.transform( getEc2Instances(INST_PENDING_STATE, INST_RUNNING_STATE, INST_STOPPING_STATE, INST_STOPPED_STATE), new C1<Instance, String>() { @Override public String apply(Instance e) { return e.getInstanceId(); } }); if (instIds.isEmpty()) { if (log.isDebugEnabled()) log.debug("There are no instances to terminate by command (safely ignoring): " + cmd); return; } Map<String, String> params = cmd.parameters(); if (params != null && Boolean.parseBoolean(params.get(CMD_DRY_RUN_KEY))) { if (log.isDebugEnabled()) log.debug("Dry run - instances termination omitted."); return; } terminateInstances(instIds); } /** * Send request to Amazon EC2 to terminate given instances. * * @param instIds Instances IDs to terminate. Not {@code null} and not empty. * @throws GridSpiException Thrown if any exception occurs. */ private void terminateInstances(Collection<String> instIds) throws GridSpiException { assert !F.isEmpty(instIds); TerminateInstancesRequest req = new TerminateInstancesRequest().withInstanceIds(instIds); TerminateInstancesResult res; try { res = ec2.terminateInstances(req); } catch (AmazonClientException e) { throw new GridSpiException("Failed to perform terminate instances request.", e); } Collection<String> terminatedInstIds = F.transform(res.getTerminatingInstances(), new C1<InstanceStateChange, String>() { @Override public String apply(InstanceStateChange ist) { return ist.getInstanceId(); } }); if (instIds.size() != terminatedInstIds.size() || !instIds.containsAll(terminatedInstIds)) throw new GridSpiException("Instances were not successfully terminated."); } /** * Starts EC2 instances by command. * <p> * Instances provided in command should be in {@link #INST_STOPPED_STATE} state * in order for command to execute properly. * * @param cmd Cloud command. * @throws GridSpiException Thrown if any exception occurs or not all provided instances are in required state. */ private void startInstances(GridCloudCommand cmd) throws GridSpiException { assert cmd != null; assert START_INST_ACT.equalsIgnoreCase(cmd.action()); Collection<GridCloudResource> rsrcs = cmd.resources(); assert rsrcs != null && !rsrcs.isEmpty(); Collection<String> stoppedInstIds = F.transform(getEc2Instances(INST_STOPPED_STATE), new C1<Instance, String>() { @Override public String apply(Instance inst) { return inst.getInstanceId(); } }); Collection<String> startInstIds = new ArrayList<String>(rsrcs.size()); for (GridCloudResource rsrc : rsrcs) { assert rsrc.type() == CLD_INSTANCE; String id = rsrc.id(); if (stoppedInstIds.contains(id)) startInstIds.add(id); else throw new GridSpiException( "Unable to find stopped EC2 instance with given ID [id=" + id + ", cmd=" + cmd + ']'); } Map<String, String> params = cmd.parameters(); if (params != null && Boolean.parseBoolean(params.get(CMD_DRY_RUN_KEY))) { if (log.isDebugEnabled()) log.debug("Dry run - instances start omitted."); return; } startInstances(startInstIds); } /** * Starts all stopped instances. * <p> * This method gets all stopped instances from Amazon EC2 * and attempts to start them. * * @param cmd Cloud command. * @throws GridSpiException Thrown if any exception occurs or no instances in required states found. */ private void startAllInstances(GridCloudCommand cmd) throws GridSpiException { assert cmd != null; assert START_ALL_INST_ACT.equalsIgnoreCase(cmd.action()); Collection<String> instIds = F.transform(getEc2Instances(INST_STOPPED_STATE), new C1<Instance, String>() { @Override public String apply(Instance inst) { return inst.getInstanceId(); } }); if (instIds.isEmpty()) { if (log.isDebugEnabled()) log.debug("There are no instances to start by command (safely ignoring): " + cmd); return; } Map<String, String> params = cmd.parameters(); if (params != null && Boolean.parseBoolean(params.get(CMD_DRY_RUN_KEY))) { if (log.isDebugEnabled()) log.debug("Dry run - instances start omitted."); return; } startInstances(instIds); } /** * Sends request to Amazon EC2 to start given instances. * <p> * Provided instances should be in {@link #INST_STOPPED_STATE} state * in order for command to execute properly. * * @param instIds Instances IDs to start. Not {@code null} and not empty. * @throws GridSpiException Thrown if any exception occurs. */ private void startInstances(Collection<String> instIds) throws GridSpiException { assert !F.isEmpty(instIds); StartInstancesRequest req = new StartInstancesRequest().withInstanceIds(instIds); StartInstancesResult res; try { res = ec2.startInstances(req); } catch (AmazonClientException e) { throw new GridSpiException("Failed to perform start instances request.", e); } Collection<String> startInstIds = F.transform(res.getStartingInstances(), new C1<InstanceStateChange, String>() { @Override public String apply(InstanceStateChange ist) { return ist.getInstanceId(); } }); if (instIds.size() != startInstIds.size() || !instIds.containsAll(startInstIds)) throw new GridSpiException("Instances were not successfully started."); } /** * Stops EC2 instances by command. * <p> * Command may contain {@link #CMD_DRY_RUN_KEY} and {@link #FORCE_STOP} * parameters. Refer to class documentation for details. * <p> * Instances provided in command should be in {@link #INST_RUNNING_STATE} state * in order for command to execute properly. * * @param cmd Cloud command. * @throws GridSpiException Thrown if any exception occurs or not all * provided instances are in required states. */ private void stopInstances(GridCloudCommand cmd) throws GridSpiException { assert cmd != null; assert STOP_INST_ACT.equalsIgnoreCase(cmd.action()); Collection<GridCloudResource> rsrcs = cmd.resources(); assert rsrcs != null && !rsrcs.isEmpty(); Collection<String> runningInstIds = F.transform(getEc2Instances(INST_RUNNING_STATE), new C1<Instance, String>() { @Override public String apply(Instance inst) { return inst.getInstanceId(); } }); Collection<String> stopInstIds = new ArrayList<String>(rsrcs.size()); for (GridCloudResource rsrc : rsrcs) { assert rsrc.type() == CLD_INSTANCE; String id = rsrc.id(); if (runningInstIds.contains(id)) stopInstIds.add(id); else throw new GridSpiException( "Unable to find running EC2 instance with given ID [id=" + id + ", cmd=" + cmd + ']'); } Map<String, String> params = cmd.parameters(); boolean force = false; boolean dryRun = false; if (params != null) { force = Boolean.parseBoolean(params.get(FORCE_STOP)); dryRun = Boolean.parseBoolean(params.get(CMD_DRY_RUN_KEY)); } if (dryRun) { if (log.isDebugEnabled()) log.debug("Dry run - instances run omitted."); return; } stopInstances(stopInstIds, force); } /** * Stops all EC2 instances by command. * <p> * Command may contain {@link #CMD_DRY_RUN_KEY} and {@link #FORCE_STOP} * parameters. Refer to class documentation for details. * <p> * Instances provided in command should be in {@link #INST_RUNNING_STATE} state * in order for command to execute properly. * * @param cmd Cloud command. * @throws GridSpiException Thrown if any exception occurs. */ private void stopAllInstances(GridCloudCommand cmd) throws GridSpiException { assert cmd != null; assert STOP_ALL_INST_ACT.equalsIgnoreCase(cmd.action()); Collection<String> instIds = F.transform(getEc2Instances(INST_RUNNING_STATE), new C1<Instance, String>() { @Override public String apply(Instance inst) { return inst.getInstanceId(); } }); if (instIds.isEmpty()) { if (log.isDebugEnabled()) log.debug("There are no instances to stop by command (safely ignoring): " + cmd); return; } Map<String, String> params = cmd.parameters(); boolean force = false; if (params != null && !params.isEmpty()) force = Boolean.parseBoolean(params.get(FORCE_STOP)); boolean dryRun = false; if (params != null) { force = Boolean.parseBoolean(params.get(FORCE_STOP)); dryRun = Boolean.parseBoolean(params.get(CMD_DRY_RUN_KEY)); } if (dryRun) { if (log.isDebugEnabled()) log.debug("Dry run - instances run omitted."); return; } stopInstances(instIds, force); } /** * Sends request to Amazon EC2 to stop given instances. * <p> * Provided instances should be in {@link #INST_RUNNING_STATE} state * in order for command to execute properly. * * @param instIds Instances IDs to stop. Not {@code null} and not empty. * @param force Pass {@code true} to force instance stop. * @throws GridSpiException Thrown if any exception occurs. */ private void stopInstances(Collection<String> instIds, boolean force) throws GridSpiException { assert !F.isEmpty(instIds); StopInstancesRequest req = new StopInstancesRequest().withInstanceIds(instIds).withForce(force); StopInstancesResult res; try { res = ec2.stopInstances(req); } catch (AmazonClientException e) { throw new GridSpiException("Failed to perform start instances request.", e); } Collection<String> stopInstIds = F.transform(res.getStoppingInstances(), new C1<InstanceStateChange, String>() { @Override public String apply(InstanceStateChange ist) { return ist.getInstanceId(); } }); if (instIds.size() != stopInstIds.size() || !instIds.containsAll(stopInstIds)) throw new GridSpiException("Instances were not successfully stopped."); } /** * Notifies SPI listener on cloud change if there were any. */ private void notifySpiListenerOnChange() { try { GridCloudSpiSnapshot snp = makeSnapshot(); GridCloudSpiListener tmp = lsnr; if (tmp != null && (lastSnp == null || !compare(lastSnp, snp).isEmpty())) { lastSnp = snp; tmp.onChange(lastSnp); } } catch (GridSpiException e) { U.error(log, "Failed to make snapshot.", e); } } /** * Notifies SPI listener when cloud command has been processed. * * @param success Flag of successful command execution. * @param cmdExecId Cloud command execution ID. * @param cmd Cloud command. * @param msg Optional message. */ private void notifySpiListenerOnCommand(boolean success, UUID cmdExecId, GridCloudCommand cmd, @Nullable String msg) { assert cmdExecId != null; assert cmd != null; GridCloudSpiListener tmp = lsnr; if (tmp != null) tmp.onCommand(success, cmdExecId, cmd, msg); } /** * Makes cloud snapshot. * * @return Cloud snapshot. * @throws GridSpiException If any exception occurs. */ private GridCloudSpiSnapshot makeSnapshot() throws GridSpiException { Collection<GridCloudResource> rsrcs = new LinkedList<GridCloudResource>(); for (SecurityGroup grp : getEc2SecurityGroups()) rsrcs.add(createSecurityGroupResource(grp)); Map<String, LinkedList<GridCloudResource>> nodes = new HashMap<String, LinkedList<GridCloudResource>>(); for (GridNode node : getCloudNodes()) { GridCloudResource node0 = createNodeResource(node); LinkedList<GridCloudResource> nodes0 = F.addIfAbsent(nodes, node.<String>attribute(ATTR_EC2_INSTANCE_ID), F.<GridCloudResource>newLinkedList()); assert nodes0 != null; nodes0.add(node0); } Map<String, LinkedList<GridCloudResource>> insts = new HashMap<String, LinkedList<GridCloudResource>>(); for (Instance inst : getEc2Instances()) { GridCloudSpiResourceAdapter inst0 = createInstanceResource(inst); Collection<GridCloudResource> nodes0 = nodes.get(inst0.id()); if (!F.isEmpty(nodes0)) { inst0.setLinks(nodes0); for (GridCloudResource node : nodes0) { ((GridCloudSpiResourceAdapter) node).setLinks(F.<GridCloudResource>asList(inst0)); rsrcs.add(node); } } LinkedList<GridCloudResource> insts0 = F.addIfAbsent(insts, inst.getImageId(), F.<GridCloudResource>newLinkedList()); assert insts0 != null; insts0.add(inst0); rsrcs.add(inst0); } for (Image img : getEc2Images()) { GridCloudSpiResourceAdapter img0 = createImageResource(img); Collection<GridCloudResource> insts0 = insts.get(img0.id()); if (!F.isEmpty(insts0)) { img0.setLinks(insts0); for (GridCloudResource inst : insts0) { Collection<GridCloudResource> links = inst.links(); if (F.isEmpty(links)) links = F.<GridCloudResource>asList(img0); else { links = new LinkedList<GridCloudResource>(links); links.add(img0); } ((GridCloudSpiResourceAdapter) inst).setLinks(links); } } rsrcs.add(img0); } long time = System.currentTimeMillis(); return new GridCloudSpiSnapshotAdapter(UUID.randomUUID(), time, getSpiContext().localNode().id(), rsrcs, F.asMap(CLOUD_API_VER_PARAM, CLOUD_API_VER_VAL, CLOUD_LAST_UPD_TIME_PARAM, String.valueOf(time))); } /** * Gets image resource from EC2 image. * * @param img EC2 image. * @return Image resource. */ private GridCloudSpiResourceAdapter createImageResource(Image img) { assert img != null; Map<String, String> params = new HashMap<String, String>(); params.put(IMG_ARCH, img.getArchitecture()); params.put(IMG_LOC, img.getImageLocation()); params.put(IMG_STATE, img.getState()); params.put(INST_TYPE, img.getImageType()); params.put(IMG_KERNEL_ID, img.getKernelId()); params.put(OWNER_ID, img.getOwnerId()); params.put(INST_PLATFORM, img.getPlatform()); params.put(IMG_RAMDISK_ID, img.getRamdiskId()); params.put(IMG_PUBLIC, String.valueOf(img.isPublic())); params.put(PRODUCT_CODE_IDS, F.concat(F.transform(img.getProductCodes(), new C1<ProductCode, String>() { @Override public String apply(ProductCode e) { return e.getProductCodeId(); } }), VAL_DELIM)); return new GridCloudSpiResourceAdapter(img.getImageId(), CLD_IMAGE, cloudId, params); } /** * Gets instance resource from EC2 instance. * * @param inst EC2 instance. * @return Instance resource. */ private GridCloudSpiResourceAdapter createInstanceResource(Instance inst) { assert inst != null; Map<String, String> params = new HashMap<String, String>(); params.put(INST_STATE_TRANS_REASON, inst.getStateTransitionReason()); params.put(INST_KEY_PAIR_NAME, inst.getKeyName()); params.put(INST_AMI_LAUNCH_IDX, String.valueOf(inst.getAmiLaunchIndex())); params.put(INST_LAUNCH_TIME, String.valueOf(inst.getLaunchTime())); params.put(INST_TYPE, inst.getInstanceType()); params.put(INST_STATE, inst.getState().getName()); params.put(INST_STATE_CODE, String.valueOf(inst.getState().getCode())); params.put(IMG_KERNEL_ID, inst.getKernelId()); params.put(IMG_RAMDISK_ID, inst.getRamdiskId()); params.put(INST_MON_STATE, inst.getMonitoring().getState()); params.put(INST_PLACEMENT, inst.getPlacement().getAvailabilityZone()); params.put(INST_PLATFORM, inst.getPlatform()); params.put(INST_PRIV_DNS, inst.getPrivateDnsName()); params.put(INST_PUB_DNS, inst.getPublicDnsName()); params.put(PRODUCT_CODE_IDS, F.concat(F.transform(inst.getProductCodes(), new C1<ProductCode, String>() { @Override public String apply(ProductCode e) { return e.getProductCodeId(); } }), VAL_DELIM)); return new GridCloudSpiResourceAdapter(inst.getInstanceId(), CLD_INSTANCE, cloudId, params); } /** * Gets security group resource from EC2 security group. * * @param grp EC2 security group. * @return Security group resource. */ private GridCloudResource createSecurityGroupResource(SecurityGroup grp) { assert grp != null; Map<String, String> params = new HashMap<String, String>(); params.put(OWNER_ID, grp.getOwnerId()); params.put(GRP_DESCR, grp.getDescription()); List<IpPermission> perms = grp.getIpPermissions(); int permSize = F.isEmpty(perms) ? 0 : perms.size(); params.put(GRP_IP_PERMS_CNT, String.valueOf(permSize)); for (int i = 0; i < permSize; i++) { IpPermission perm = perms.get(i); StringBuilder buf = new StringBuilder(); buf.append('[').append(IP_PERM_IP_PROTO).append('=').append(perm.getIpProtocol()).append(VAL_DELIM) .append(IP_PERM_FROM_PORT).append('=').append(perm.getFromPort()).append(VAL_DELIM) .append(IP_PERM_TO_PORT).append('=').append(perm.getToPort()).append(VAL_DELIM) .append(IP_PERM_IP_RANGE).append('=').append(perm.getIpRanges()); List<UserIdGroupPair> pairs = perm.getUserIdGroupPairs(); int pairSize = F.isEmpty(pairs) ? 0 : pairs.size(); if (pairSize > 0) { buf.append(VAL_DELIM).append(USER_ID_GRP_PAIR).append("=["); for (int j = 0; j < pairSize; j++) { if (j != 0) buf.append(','); UserIdGroupPair pair = pairs.get(j); buf.append(PAIR).append(j).append("=[").append(PAIR_USER_ID).append('=') .append(pair.getUserId()).append(':').append(PAIR_GRP).append('=') .append(pair.getGroupName()).append(']'); } buf.append(']'); } buf.append(']'); params.put(GRP_IP_PERM + i, buf.toString()); } return new GridCloudSpiResourceAdapter(grp.getGroupName(), CLD_SECURITY_GROUP, cloudId, params); } /** * Gets node resource from grid node. * * @param node Node. * @return Node resource. */ private GridCloudResource createNodeResource(GridNode node) { assert node != null; return new GridCloudSpiResourceAdapter(node.id().toString(), CLD_NODE, cloudId); } /** * Gets EC2 images. * * @return EC2 images. * @throws GridSpiException If any exception occurs. */ private Iterable<Image> getEc2Images() throws GridSpiException { if (imgs == null) { DescribeImagesRequest req = new DescribeImagesRequest(); if (!F.isEmpty(imgIds)) req.setImageIds(imgIds); try { imgs = ec2.describeImages(req).getImages(); if (log.isDebugEnabled()) log.debug("Images initialized: " + imgs); return imgs; } catch (AmazonClientException e) { throw new GridSpiException("Failed to get EC2 images by request: " + req, e); } } else return imgs; } /** * Gets running EC2 instances. * * @param states Optional states of the instances for filtering. * @return EC2 instances. * @throws GridSpiException If any exception occurs. */ private Collection<Instance> getEc2Instances(@Nullable String... states) throws GridSpiException { try { DescribeInstancesResult res; if (!F.isEmpty(states)) { DescribeInstancesRequest req = new DescribeInstancesRequest() .withFilters(new Filter("instance-state-name", Arrays.asList(states))); res = ec2.describeInstances(req); } else res = ec2.describeInstances(); Collection<Instance> insts = new LinkedList<Instance>(); for (Reservation rsrv : res.getReservations()) insts.addAll(rsrv.getInstances()); return insts; } catch (AmazonClientException e) { throw new GridSpiException("Failed to get EC2 instances.", e); } } /** * Gets EC2 security groups. * * @return EC2 security groups. * @throws GridSpiException If any exception occurs. */ private Iterable<SecurityGroup> getEc2SecurityGroups() throws GridSpiException { if (secGrps == null) { try { secGrps = ec2.describeSecurityGroups().getSecurityGroups(); if (log.isDebugEnabled()) log.debug("Security groups initialized: " + secGrps); return secGrps; } catch (AmazonClientException e) { throw new GridSpiException("Failed to get EC2 security groups.", e); } } else return secGrps; } /** * Gets nodes on cloud. * * @return Nodes. */ private Iterable<GridNode> getCloudNodes() { return F.retain(getSpiContext().nodes(), true, new P1<GridNode>() { @Override public boolean apply(GridNode node) { return node.attribute(ATTR_EC2_INSTANCE_ID) != null; } }); } /** {@inheritDoc} */ @Override public String toString() { return S.toString(GridEc2LiteCloudSpi.class, this); } /** * Cloud control thread. */ private class CloudControlThread extends GridSpiThread { /** Creates cloud control thread. */ private CloudControlThread() { super(gridName, "grid-ec2lite-cloud-control", log); } /** {@inheritDoc} */ @Override protected void body() throws InterruptedException { while (!isInterrupted()) { synchronized (mux) { if (cmds.isEmpty()) mux.wait(stateCheckFreq); } checkCloud(); } } } }