org.apache.airavata.gfac.ec2.AmazonInstanceScheduler.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.airavata.gfac.ec2.AmazonInstanceScheduler.java

Source

/*
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package org.apache.airavata.gfac.ec2;

import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.cloudwatch.AmazonCloudWatchClient;
import com.amazonaws.services.cloudwatch.model.Datapoint;
import com.amazonaws.services.cloudwatch.model.Dimension;
import com.amazonaws.services.cloudwatch.model.GetMetricStatisticsRequest;
import com.amazonaws.services.cloudwatch.model.GetMetricStatisticsResult;
import com.amazonaws.services.ec2.AmazonEC2Client;
import com.amazonaws.services.ec2.model.DescribeInstancesResult;
import com.amazonaws.services.ec2.model.Instance;
import com.amazonaws.services.ec2.model.Reservation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.security.NoSuchAlgorithmException;
import java.security.spec.InvalidKeySpecException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

public class AmazonInstanceScheduler {
    private static final Logger log = LoggerFactory.getLogger(AmazonInstanceScheduler.class);

    /* Maximum number of instances that the Scheduler will create*/
    //private static final int MAX_INSTANCE_COUNT = 3;

    /* Maximum number of minutes an instance should be kept alive*/
    public static final int INSTANCE_UP_TIME_THRESHOLD = 60;

    private static volatile AmazonInstanceScheduler scheduler = null;

    private static String imageId = null;

    private static AWSCredentials credential = null;

    private static AmazonEC2Client ec2client = null;

    /* The time interval(minutes) in which the instances will be checked whether they have timed-out*/
    public static final long TERMINATE_THREAD_UPDATE_INTERVAL = 5;

    public static AmazonInstanceScheduler getInstance(String imageId, String accessKey, String secretKey)
            throws IOException, InvalidKeySpecException, NoSuchAlgorithmException {

        if (scheduler == null) {
            synchronized (AmazonInstanceScheduler.class) {
                if (scheduler == null) {
                    new Thread() {
                        @Override
                        public void run() {
                            //noinspection InfiniteLoopStatement
                            while (true) {
                                try {
                                    Thread.sleep(TERMINATE_THREAD_UPDATE_INTERVAL * 60 * 1000);
                                } catch (InterruptedException e) {
                                    // do-nothing
                                }

                                try {
                                    terminateTimedOutAmazonInstances();
                                } catch (Throwable e) {
                                    e.printStackTrace();
                                }
                            }

                        }

                    }.start();

                    scheduler = new AmazonInstanceScheduler();
                }
            }
        }

        AmazonInstanceScheduler.imageId = imageId;
        AmazonInstanceScheduler.credential = new BasicAWSCredentials(accessKey, secretKey);
        AmazonInstanceScheduler.ec2client = new AmazonEC2Client(credential);

        return scheduler;
    }

    /**
     * Returns the amazon instance id of the amazon instance which is having the minimum
     * CPU utilization (out of the already running instances). If the instance which
     * is having the minimum CPU utilization exceeds 80%, ami-id will be returned
     * instead of a an instance id. If a particular running instance's uptime is
     * greater than 55 minutes, that instance will be shut down.
     *
     * @return instance id
     * @throws NoSuchAlgorithmException
     * @throws InvalidKeySpecException
     * @throws IOException
     */
    public String getScheduledAmazonInstance()
            throws NoSuchAlgorithmException, InvalidKeySpecException, IOException {

        SchedulingAlgorithm greedyAglo = new GreedyScheduler();
        return greedyAglo.getScheduledAmazonInstance(ec2client, imageId, credential);
    }

    /**
     * Terminates the Amazon instances that are timed out. Timed out refers to the
     * instances which have been running for more than the INSTANCE_UP_TIME_THRESHOLD.
     */
    private static void terminateTimedOutAmazonInstances() {
        System.out.println("Checking for timed-out instances");
        List<Instance> instanceList = loadInstances(ec2client);
        for (Instance instance : instanceList) {
            String instanceId = instance.getInstanceId();

            long upTime = getInstanceUptime(instance);
            // if the instance up time is greater than the threshold, terminate the instance
            if (upTime > INSTANCE_UP_TIME_THRESHOLD) {
                List<String> requestIds = new ArrayList<String>();
                requestIds.add(instanceId);
                // terminate instance
                System.out.println(
                        "Terminating the instance " + instanceId + " as the up time threshold is exceeded");
                AmazonUtil.terminateInstances(requestIds);
            }
        }

    }

    /**
     * Calculates the instance up time in minutes.
     *
     * @param instance instance to be monitored.
     * @return up time of the instance.
     */
    private static long getInstanceUptime(Instance instance) {
        Date startTime = instance.getLaunchTime();
        Date today = new Date();
        long diff = (today.getTime() - startTime.getTime()) / (1000 * 60);
        System.out.println("Instance launch time   : " + startTime);
        System.out.println("Instance up time (mins): " + diff);
        return diff;
    }

    /**
     * Monitors the CPU Utilization using Amazon Cloud Watch. In order to monitor the instance, Cloud Watch Monitoring
     * should be enabled for the running instance.
     *
     * @param credential EC2 credentials
     * @param instanceId instance id
     * @return average CPU utilization of the instance
     */
    public static double monitorInstance(AWSCredentials credential, String instanceId) {
        try {
            AmazonCloudWatchClient cw = new AmazonCloudWatchClient(credential);

            long offsetInMilliseconds = 1000 * 60 * 60 * 24;
            GetMetricStatisticsRequest request = new GetMetricStatisticsRequest()
                    .withStartTime(new Date(new Date().getTime() - offsetInMilliseconds)).withNamespace("AWS/EC2")
                    .withPeriod(60 * 60)
                    .withDimensions(new Dimension().withName("InstanceId").withValue(instanceId))
                    .withMetricName("CPUUtilization").withStatistics("Average", "Maximum").withEndTime(new Date());
            GetMetricStatisticsResult getMetricStatisticsResult = cw.getMetricStatistics(request);

            double avgCPUUtilization = 0;
            List dataPoint = getMetricStatisticsResult.getDatapoints();
            for (Object aDataPoint : dataPoint) {
                Datapoint dp = (Datapoint) aDataPoint;
                avgCPUUtilization = dp.getAverage();
                log.info(instanceId + " instance's average CPU utilization : " + dp.getAverage());
            }

            return avgCPUUtilization;

        } catch (AmazonServiceException ase) {
            log.error("Caught an AmazonServiceException, which means the request was made  "
                    + "to Amazon EC2, but was rejected with an error response for some reason.");
            log.error("Error Message:    " + ase.getMessage());
            log.error("HTTP Status Code: " + ase.getStatusCode());
            log.error("AWS Error Code:   " + ase.getErrorCode());
            log.error("Error Type:       " + ase.getErrorType());
            log.error("Request ID:       " + ase.getRequestId());

        }
        return 0;
    }

    /**
     * Load instances associated with the given ec2 client
     *
     * @param ec2client ec2 client
     * @return list of instances
     */
    public static List<Instance> loadInstances(AmazonEC2Client ec2client) {
        List<Instance> resultList = new ArrayList<Instance>();
        DescribeInstancesResult describeInstancesResult = ec2client.describeInstances();
        List<Reservation> reservations = describeInstancesResult.getReservations();
        for (Reservation reservation : reservations) {
            for (Instance instance : reservation.getInstances()) {
                System.out.println("instance       : " + instance);
                if ("running".equalsIgnoreCase(instance.getState().getName())) {
                    resultList.add(instance);
                }
            }
        }
        return resultList;
    }

}