Java tutorial
/* * * Copyright 2013 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package com.netflix.ice.basic; import com.amazonaws.services.ec2.AmazonEC2Client; import com.amazonaws.services.ec2.model.*; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.netflix.ice.common.AwsUtils; import com.netflix.ice.common.Poller; import com.netflix.ice.common.TagGroup; import com.netflix.ice.processor.Ec2InstanceReservationPrice; import com.netflix.ice.processor.Ec2InstanceReservationPrice.*; import com.netflix.ice.processor.ProcessorConfig; import com.netflix.ice.processor.ReservationService; import com.netflix.ice.tag.*; import com.netflix.ice.tag.Region; import org.apache.commons.lang.StringUtils; import org.joda.time.DateMidnight; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; import java.util.*; import java.util.concurrent.ConcurrentSkipListMap; public class BasicReservationService extends Poller implements ReservationService { protected Logger logger = LoggerFactory.getLogger(BasicReservationService.class); protected ProcessorConfig config; protected Map<ReservationUtilization, Map<Ec2InstanceReservationPrice.Key, Ec2InstanceReservationPrice>> ec2InstanceReservationPrices; protected Map<ReservationUtilization, Map<TagGroup, List<Reservation>>> reservations; protected ReservationPeriod term; protected ReservationUtilization defaultUtilization; protected Map<ReservationUtilization, File> files; protected Long futureMillis = new DateMidnight().withYearOfCentury(99).getMillis(); protected static Map<String, String> instanceTypes = Maps.newHashMap(); protected static Map<String, String> instanceSizes = Maps.newHashMap(); static { instanceTypes.put("stdResI", "m1"); instanceTypes.put("secgenstdResI", "m3"); instanceTypes.put("uResI", "t1"); instanceTypes.put("hiMemResI", "m2"); instanceTypes.put("hiCPUResI", "c1"); instanceTypes.put("clusterCompResI", "cc1"); instanceTypes.put("clusterHiMemResI", "cr1"); instanceTypes.put("clusterGPUResI", "cg1"); instanceTypes.put("hiIoResI", "hi1"); instanceTypes.put("hiStoreResI", "hs1"); instanceSizes.put("xxxxxxxxl", "8xlarge"); instanceSizes.put("xxxxl", "4xlarge"); instanceSizes.put("xxl", "2xlarge"); instanceSizes.put("xl", "xlarge"); instanceSizes.put("sm", "small"); instanceSizes.put("med", "medium"); instanceSizes.put("lg", "large"); instanceSizes.put("u", "micro"); } public BasicReservationService(ReservationPeriod term, ReservationUtilization defaultUtilization) { this.term = term; this.defaultUtilization = defaultUtilization; ec2InstanceReservationPrices = Maps.newHashMap(); for (ReservationUtilization utilization : ReservationUtilization.values()) { ec2InstanceReservationPrices.put(utilization, new ConcurrentSkipListMap<Ec2InstanceReservationPrice.Key, Ec2InstanceReservationPrice>()); } reservations = Maps.newHashMap(); for (ReservationUtilization utilization : ReservationUtilization.values()) { reservations.put(utilization, Maps.<TagGroup, List<Reservation>>newHashMap()); } } public void init() { this.config = ProcessorConfig.getInstance(); files = Maps.newHashMap(); for (ReservationUtilization utilization : ReservationUtilization.values()) { files.put(utilization, new File(config.localDir, "reservation_prices." + term.name() + "." + utilization.name())); } boolean fileExisted = false; for (ReservationUtilization utilization : ReservationUtilization.values()) { File file = files.get(utilization); AwsUtils.downloadFileIfNotExist(config.workS3BucketName, config.workS3BucketPrefix, file); fileExisted = file.exists(); } if (!fileExisted) { try { pollAPI(); } catch (Exception e) { logger.error("failed to poll reservation prices", e); throw new RuntimeException("failed to poll reservation prices for " + e.getMessage()); } } else { for (ReservationUtilization utilization : ReservationUtilization.values()) { try { File file = files.get(utilization); if (file.exists()) { DataInputStream in = new DataInputStream(new FileInputStream(file)); ec2InstanceReservationPrices.put(utilization, Serializer.deserialize(in)); in.close(); } } catch (Exception e) { throw new RuntimeException("failed to load reservation prices " + e.getMessage()); } } } start(3600, 3600 * 24, true); } @Override protected void poll() throws Exception { logger.info("start polling reservation prices. it might take a while..."); pollAPI(); } private void pollAPI() throws Exception { long currentTime = new DateMidnight().getMillis(); DescribeReservedInstancesOfferingsRequest req = new DescribeReservedInstancesOfferingsRequest().withFilters( new com.amazonaws.services.ec2.model.Filter().withName("marketplace").withValues("false")); String token = null; boolean hasNewPrice = false; AmazonEC2Client ec2Client = new AmazonEC2Client(AwsUtils.awsCredentialsProvider, AwsUtils.clientConfig); for (Region region : Region.getAllRegions()) { ec2Client.setEndpoint("ec2." + region.name + ".amazonaws.com"); do { if (!StringUtils.isEmpty(token)) req.setNextToken(token); DescribeReservedInstancesOfferingsResult offers = ec2Client.describeReservedInstancesOfferings(req); token = offers.getNextToken(); for (ReservedInstancesOffering offer : offers.getReservedInstancesOfferings()) { if (offer.getProductDescription().indexOf("Amazon VPC") >= 0) continue; ReservationUtilization utilization = ReservationUtilization.get(offer.getOfferingType()); Ec2InstanceReservationPrice.ReservationPeriod term = offer.getDuration() / 24 / 3600 > 366 ? Ec2InstanceReservationPrice.ReservationPeriod.threeyear : Ec2InstanceReservationPrice.ReservationPeriod.oneyear; if (term != this.term) continue; double hourly = offer.getUsagePrice(); if (hourly <= 0) { for (RecurringCharge recurringCharge : offer.getRecurringCharges()) { if (recurringCharge.getFrequency().equals("Hourly")) { hourly = recurringCharge.getAmount(); break; } } } UsageType usageType = getUsageType(offer.getInstanceType(), offer.getProductDescription()); hasNewPrice = setPrice(utilization, currentTime, Zone.getZone(offer.getAvailabilityZone()).region, usageType, offer.getFixedPrice(), hourly) || hasNewPrice; logger.info("Setting RI price for " + Zone.getZone(offer.getAvailabilityZone()).region + " " + utilization + " " + usageType + " " + offer.getFixedPrice() + " " + hourly); } } while (!StringUtils.isEmpty(token)); } ec2Client.shutdown(); if (hasNewPrice) { for (ReservationUtilization utilization : files.keySet()) { File file = files.get(utilization); DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); try { Serializer.serialize(out, this.ec2InstanceReservationPrices.get(utilization)); AwsUtils.upload(config.workS3BucketName, config.workS3BucketPrefix, file); } finally { out.close(); } } } } private UsageType getUsageType(String type, String productDescription) { return UsageType.getUsageType(type + InstanceOs.withDescription(productDescription).usageType, Operation.reservedInstancesHeavy, ""); } // private UsageType getUsageType(String type, String size, boolean isWindows) { // type = instanceTypes.get(type); // size = instanceSizes.get(size); // // if (type.equals("cc1") && size.equals("8xlarge")) // type = "cc2"; // return UsageType.getUsageType(type + "." + size + (isWindows ? "." + InstanceOs.windows : ""), Operation.reservedInstances, ""); // } public static class Serializer { public static void serialize(DataOutput out, Map<Ec2InstanceReservationPrice.Key, Ec2InstanceReservationPrice> reservationPrices) throws IOException { out.writeInt(reservationPrices.size()); for (Ec2InstanceReservationPrice.Key key : reservationPrices.keySet()) { Ec2InstanceReservationPrice.Key.Serializer.serialize(out, key); Ec2InstanceReservationPrice.Serializer.serialize(out, reservationPrices.get(key)); } } public static Map<Ec2InstanceReservationPrice.Key, Ec2InstanceReservationPrice> deserialize(DataInput in) throws IOException { int size = in.readInt(); Map<Ec2InstanceReservationPrice.Key, Ec2InstanceReservationPrice> result = new ConcurrentSkipListMap<Ec2InstanceReservationPrice.Key, Ec2InstanceReservationPrice>(); for (int i = 0; i < size; i++) { Ec2InstanceReservationPrice.Key key = Ec2InstanceReservationPrice.Key.Serializer.deserialize(in); Ec2InstanceReservationPrice price = Ec2InstanceReservationPrice.Serializer.deserialize(in); result.put(key, price); } return result; } } private boolean setPrice(ReservationUtilization utilization, long currentTime, Region region, UsageType usageType, double upfront, double hourly) { Ec2InstanceReservationPrice.Key key = new Ec2InstanceReservationPrice.Key(region, usageType); Ec2InstanceReservationPrice reservationPrice = ec2InstanceReservationPrices.get(utilization).get(key); if (reservationPrice == null) { reservationPrice = new Ec2InstanceReservationPrice(); ec2InstanceReservationPrices.get(utilization).put(key, reservationPrice); } Ec2InstanceReservationPrice.Price latestHourly = reservationPrice.hourlyPrice.getCreatePrice(futureMillis); Ec2InstanceReservationPrice.Price latestUpfront = reservationPrice.upfrontPrice .getCreatePrice(futureMillis); if (latestHourly.getListPrice() == null) { latestHourly.setListPrice(hourly); latestUpfront.setListPrice(upfront); //logger.info("setting reservation price for " + usageType + " in " + region + ": " + upfront + " " + hourly); return true; } else if (latestHourly.getListPrice() != hourly || latestUpfront.getListPrice() != upfront) { Ec2InstanceReservationPrice.Price oldHourly = reservationPrice.hourlyPrice.getCreatePrice(currentTime); Ec2InstanceReservationPrice.Price oldUpfront = reservationPrice.upfrontPrice .getCreatePrice(currentTime); oldHourly.setListPrice(latestHourly.getListPrice()); oldUpfront.setListPrice(latestUpfront.getListPrice()); latestHourly.setListPrice(hourly); latestUpfront.setListPrice(upfront); //logger.info("changing reservation price for " + usageType + " in " + region + ": " + upfront + " " + hourly); return true; } else { //logger.info("exisitng reservation price for " + usageType + " in " + region + ": " + upfront + " " + hourly); return false; } } public static class Reservation { final int count; final long start; final long end; final ReservationUtilization utilization; final float fixedPrice; final float usagePrice; public Reservation(int count, long start, long end, ReservationUtilization utilization, float fixedPrice, float usagePrice) { this.count = count; this.start = start; this.end = end; this.utilization = utilization; this.fixedPrice = fixedPrice; this.usagePrice = usagePrice; } } protected double getEc2Tier(long time) { return 0; } public Collection<TagGroup> getTagGroups(ReservationUtilization utilization) { return reservations.get(utilization).keySet(); } public ReservationUtilization getDefaultReservationUtilization() { return defaultUtilization; } public double getLatestHourlyTotalPrice(long time, Region region, UsageType usageType, ReservationUtilization utilization) { Ec2InstanceReservationPrice ec2Price = ec2InstanceReservationPrices.get(utilization) .get(new Ec2InstanceReservationPrice.Key(region, usageType)); double tier = getEc2Tier(time); return ec2Price.hourlyPrice.getPrice(null).getPrice(tier) + ec2Price.upfrontPrice.getPrice(null).getUpfrontAmortized(time, term, tier); } public ReservationInfo getReservation(long time, TagGroup tagGroup, ReservationUtilization utilization) { if (utilization == ReservationUtilization.FIXED) return getFixedReservation(time, tagGroup); double tier = getEc2Tier(time); double upfrontAmortized = 0; double houlyCost = 0; int count = 0; if (this.reservations.get(utilization).containsKey(tagGroup)) { for (Reservation reservation : this.reservations.get(utilization).get(tagGroup)) { if (time >= reservation.start && time < reservation.end) { count += reservation.count; Ec2InstanceReservationPrice.Key key = new Ec2InstanceReservationPrice.Key(tagGroup.region, tagGroup.usageType); Ec2InstanceReservationPrice ec2Price = ec2InstanceReservationPrices.get(utilization).get(key); if (ec2Price != null) { // remove this... upfrontAmortized += reservation.count * ec2Price.upfrontPrice.getPrice(reservation.start) .getUpfrontAmortized(reservation.start, term, tier); houlyCost += reservation.count * ec2Price.hourlyPrice.getPrice(reservation.start).getPrice(tier); } else { logger.error("Not able to find reservation price for " + key); } } } } if (count == 0) { Ec2InstanceReservationPrice.Key key = new Ec2InstanceReservationPrice.Key(tagGroup.region, tagGroup.usageType); Ec2InstanceReservationPrice ec2Price = ec2InstanceReservationPrices.get(utilization).get(key); if (ec2Price != null) { // remove this... upfrontAmortized = ec2Price.upfrontPrice.getPrice(null).getUpfrontAmortized(time, term, tier); houlyCost = ec2Price.hourlyPrice.getPrice(null).getPrice(tier); } } else { upfrontAmortized = upfrontAmortized / count; houlyCost = houlyCost / count; } return new ReservationInfo(count, upfrontAmortized, houlyCost); } private ReservationInfo getFixedReservation(long time, TagGroup tagGroup) { double upfrontAmortized = 0; double houlyCost = 0; int count = 0; if (this.reservations.get(ReservationUtilization.FIXED).containsKey(tagGroup)) { for (Reservation reservation : this.reservations.get(ReservationUtilization.FIXED).get(tagGroup)) { if (time >= reservation.start && time < reservation.end) { count += reservation.count; upfrontAmortized += reservation.count * reservation.fixedPrice / ((reservation.end - reservation.start) / AwsUtils.hourMillis); houlyCost += reservation.count * reservation.usagePrice; } } } if (count > 0) { upfrontAmortized = upfrontAmortized / count; houlyCost = houlyCost / count; } return new ReservationInfo(count, upfrontAmortized, houlyCost); } public void updateEc2Reservations(Map<String, ReservedInstances> reservationsFromApi) { Map<ReservationUtilization, Map<TagGroup, List<Reservation>>> reservationMap = Maps.newTreeMap(); for (ReservationUtilization utilization : ReservationUtilization.values()) { reservationMap.put(utilization, Maps.<TagGroup, List<Reservation>>newHashMap()); } for (String key : reservationsFromApi.keySet()) { ReservedInstances reservedInstances = reservationsFromApi.get(key); if (reservedInstances.getInstanceCount() <= 0) continue; String accountId = key.substring(0, key.indexOf(",")); Account account = config.accountService.getAccountById(accountId); Zone zone = Zone.getZone(reservedInstances.getAvailabilityZone()); if (zone == null) logger.error( "Not able to find zone for reserved instances " + reservedInstances.getAvailabilityZone()); ReservationUtilization utilization = ReservationUtilization.get(reservedInstances.getOfferingType()); long endTime = Math.min(reservedInstances.getEnd().getTime(), reservedInstances.getStart().getTime() + reservedInstances.getDuration() * 1000); if (endTime <= config.startDate.getMillis()) continue; Reservation reservation = new Reservation(reservedInstances.getInstanceCount(), reservedInstances.getStart().getTime(), endTime, utilization, reservedInstances.getFixedPrice(), reservedInstances.getUsagePrice()); String osStr = reservedInstances.getProductDescription(); InstanceOs os = InstanceOs.withDescription(osStr); UsageType usageType = UsageType.getUsageType(reservedInstances.getInstanceType() + os.usageType, "hours"); TagGroup reservationKey = new TagGroup(account, zone.region, zone, Product.ec2_instance, Operation.getReservedInstances(utilization), usageType, null); List<Reservation> reservations = reservationMap.get(utilization).get(reservationKey); if (reservations == null) { reservationMap.get(utilization).put(reservationKey, Lists.<Reservation>newArrayList(reservation)); } else { reservations.add(reservation); } } this.reservations = reservationMap; } }