Java tutorial
/************************************************************************* * (c) Copyright 2016 Hewlett Packard Enterprise Development Company LP * <p> * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; version 3 of the License. * <p> * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * <p> * You should have received a copy of the GNU General Public License * along with this program. If not, see http://www.gnu.org/licenses/. ************************************************************************/ package com.eucalyptus.portal; import com.eucalyptus.auth.AuthContextSupplier; import com.eucalyptus.auth.Permissions; import com.eucalyptus.component.annotation.ComponentNamed; import com.eucalyptus.context.Context; import com.eucalyptus.context.Contexts; import com.eucalyptus.portal.common.model.ViewInstanceUsageReportResponseType; import com.eucalyptus.portal.common.model.ViewInstanceUsageReportType; import com.eucalyptus.portal.common.model.ViewInstanceUsageResult; import com.eucalyptus.portal.common.model.ViewReservedInstanceUtilizationReportResponseType; import com.eucalyptus.portal.common.model.ViewReservedInstanceUtilizationReportType; import com.eucalyptus.portal.common.model.ViewReservedInstanceUtilizationResult; import com.eucalyptus.portal.common.policy.Ec2ReportsPolicySpec; import com.eucalyptus.portal.instanceusage.InstanceHourLog; import com.eucalyptus.portal.instanceusage.InstanceLogs; import com.eucalyptus.util.Exceptions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.log4j.Logger; import org.joda.time.DateTime; import org.joda.time.Days; import org.joda.time.Months; import org.joda.time.Years; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.AbstractMap; import java.util.Calendar; import java.util.Comparator; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collector; import java.util.stream.Collectors; import static com.eucalyptus.util.RestrictedTypes.getIamActionByMessageType; import static java.util.stream.Collectors.*; @SuppressWarnings("unused") @ComponentNamed public class Ec2ReportsService { private static final Logger LOG = Logger.getLogger(Ec2ReportsService.class); public ViewInstanceUsageReportResponseType viewInstanceUsageReport(final ViewInstanceUsageReportType request) throws Ec2ReportsServiceException { final ViewInstanceUsageReportResponseType response = request.getReply(); final Context context = checkAuthorized(); try { final Function<ViewInstanceUsageReportType, Optional<Ec2ReportsServiceException>> requestVerifier = ( req) -> { if (req.getGranularity() == null) return Optional.of(new Ec2ReportsInvalidParameterException("Granularity must be specified")); final String granularity = req.getGranularity().toLowerCase(); if (request.getTimeRangeStart() == null || request.getTimeRangeEnd() == null) return Optional.of( new Ec2ReportsInvalidParameterException("time range start and end must be specified")); if (!Sets.newHashSet("hourly", "hour", "daily", "day", "monthly", "month").contains(granularity)) { return Optional.of(new Ec2ReportsInvalidParameterException( "Can't recognize granularity. Valid values are hourly, daily and monthly")); } if (granularity.equals("hourly") || granularity.equals("hour")) { // AWS: time range up to 7 days when using an hourly data granularity if (Days.daysBetween(new DateTime(request.getTimeRangeStart().getTime()), new DateTime(request.getTimeRangeEnd().getTime())).getDays() > 7) { return Optional.of(new Ec2ReportsInvalidParameterException( "time range is allowed up to 7 days when using an hourly data granularity")); } } else if (granularity.equals("daily") || granularity.equals("day")) { // AWS: time range up to 3 months when using a daily data granularity if (Months.monthsBetween(new DateTime(request.getTimeRangeStart().getTime()), new DateTime(request.getTimeRangeEnd().getTime())).getMonths() > 3) { return Optional.of(new Ec2ReportsInvalidParameterException( "time range is allowed up to 3 months when using a daily data granularity")); } } else { // AWS: time range up to 3 years when using a monthly data granularity. if (Years.yearsBetween(new DateTime(request.getTimeRangeStart().getTime()), new DateTime(request.getTimeRangeEnd().getTime())).getYears() > 3) { return Optional.of(new Ec2ReportsInvalidParameterException( "time range is allowed up to 3 years when using a monthly data granularity")); } } if (request.getGroupBy() != null) { if (request.getGroupBy().getType() == null) return Optional.of(new Ec2ReportsInvalidParameterException( "In group by parameter, type must be specified")); final String groupType = request.getGroupBy().getType().toLowerCase(); final String groupKey = request.getGroupBy().getKey(); if (!Sets.newHashSet("tag", "tags", "instancetype", "instance_type", "platform", "platforms", "availabilityzone", "availability_zone").contains(groupType)) { return Optional.of(new Ec2ReportsInvalidParameterException( "supported types in group by parameter are: tag, instance_type, platform, and availability_zone")); } if ("tag".equals(groupType) || "tags".equals(groupType)) { if (groupKey == null) { return Optional.of(new Ec2ReportsInvalidParameterException( "tag type in group by parameter must also include tag key")); } } } return Optional.empty(); }; final Optional<Ec2ReportsServiceException> error = requestVerifier.apply(request); if (error.isPresent()) { throw error.get(); } final String granularity = request.getGranularity().toLowerCase(); final List<InstanceHourLog> logs = Lists.newArrayList(); if (granularity.equals("hourly") || granularity.equals("hour")) { logs.addAll(InstanceLogs.getInstance().queryHourly(context.getAccountNumber(), request.getTimeRangeStart(), request.getTimeRangeEnd(), request.getFilters())); } else if (granularity.equals("daily") || granularity.equals("day")) { logs.addAll(InstanceLogs.getInstance().queryDaily(context.getAccountNumber(), request.getTimeRangeStart(), request.getTimeRangeEnd(), request.getFilters())); } else { logs.addAll(InstanceLogs.getInstance().queryMonthly(context.getAccountNumber(), request.getTimeRangeStart(), request.getTimeRangeEnd(), request.getFilters())); } Collector<InstanceHourLog, ?, Map<String, List<InstanceHourLog>>> collector = null; Comparator<String> keySorter = String::compareTo; // determines which column appear first if (request.getGroupBy() != null) { final String groupType = request.getGroupBy().getType().toLowerCase(); final String groupKey = request.getGroupBy().getKey(); if ("instancetype".equals(groupType) || "instance_type".equals(groupType)) { collector = groupingBy((log) -> log.getInstanceType(), toList()); keySorter = String::compareTo; // sort by type name is natural } else if ("platform".equals(groupType) || "platforms".equals(groupType)) { collector = groupingBy((log) -> log.getPlatform(), toList()); keySorter = String::compareTo; // linux comes before windows } else if ("availabilityzone".equals(groupType) || "availability_zone".equals(groupType)) { collector = groupingBy((log) -> log.getAvailabilityZone(), toList()); keySorter = String::compareTo; } else if ("tag".equals(groupType) || "tags".equals(groupType)) { // map instanceId to tag value where tag key = group key final Map<String, String> tagValueMap = logs.stream() .collect(groupingBy((log) -> log.getInstanceId(), toList())).entrySet().stream() .map(e -> e.getValue().get(0)) .filter(l -> l.getTags().stream().filter(t -> groupKey.equals(t.getKey())).findAny() .isPresent()) .collect(Collectors.toMap(l -> l.getInstanceId(), l -> l.getTags().stream() .filter(t -> groupKey.equals(t.getKey())).findAny().get().getValue())); logs.stream().forEach(l -> { if (!tagValueMap.containsKey(l.getInstanceId())) tagValueMap.put(l.getInstanceId(), "[UNTAGGED]"); }); collector = groupingBy((log) -> tagValueMap.get(log.getInstanceId()), toList()); keySorter = (s1, s2) -> { if ("[UNTAGGED]".equals(s1) && "[UNTAGGED]".equals(s2)) return 0; else if ("[UNTAGGED]".equals(s1)) return -1; else if ("[UNTAGGED]".equals(s2)) return 1; else return s1.compareTo(s2); }; } else { throw new Ec2ReportsInvalidParameterException( "supported types in group by parameter are: tag, instance_type, platform, and availability_zone"); } } else { collector = groupingBy((log) -> "[INSTANCE HOURS]", toList()); } final Function<Date, AbstractMap.SimpleEntry<Date, Date>> ranger = (logTime) -> { final Calendar start = Calendar.getInstance(); final Calendar end = Calendar.getInstance(); start.setTime(logTime); end.setTime(logTime); if (granularity.equals("hourly") || granularity.equals("hour")) { end.set(Calendar.HOUR_OF_DAY, end.get(Calendar.HOUR_OF_DAY) + 1); } else if (granularity.equals("daily") || granularity.equals("day")) { end.set(Calendar.DAY_OF_MONTH, start.get(Calendar.DAY_OF_MONTH) + 1); start.set(Calendar.HOUR_OF_DAY, 0); end.set(Calendar.HOUR_OF_DAY, 0); } else { end.set(Calendar.MONTH, start.get(Calendar.MONTH) + 1); start.set(Calendar.DAY_OF_MONTH, 1); start.set(Calendar.HOUR_OF_DAY, 0); end.set(Calendar.DAY_OF_MONTH, 1); end.set(Calendar.HOUR_OF_DAY, 0); } for (final int flag : new int[] { Calendar.MINUTE, Calendar.SECOND, Calendar.MILLISECOND }) { start.set(flag, 0); end.set(flag, 0); } return new AbstractMap.SimpleEntry<Date, Date>(start.getTime(), end.getTime()); }; // sum over instance hours whose log_time is the same /* e.g., INPUT: m1.small -> [(i-1d9eb607,2017-03-16 12:00:00), (i-1d9eb607,2017-03-16 13:00:00), (i-fe4a9384,2017-03-16 13:00:00)] OUTPUT: m1.small -> [2017-03-16 12:00:00: 1, 2017-03-16 13:00:00: 2] */ final Map<String, List<InstanceHourLog>> logsByGroupKey = logs.stream().collect(collector); final Map<String, Map<Date, Long>> hoursAtLogTimeByGroupKey = logsByGroupKey.entrySet().stream() .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().stream() .collect(groupingBy(l -> l.getLogTime(), summingLong(l -> l.getHours())) // -> Map<String, Map<Date, Long> ))); // key -> [(log_time, hours)...] final Map<String, List<AbstractMap.SimpleEntry<Date, Long>>> instanceHoursAtLogTime = hoursAtLogTimeByGroupKey .entrySet().stream() .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().entrySet().stream() .map(e1 -> new AbstractMap.SimpleEntry<Date, Long>(e1.getKey(), e1.getValue())) .collect(toList()))); final StringBuilder sb = new StringBuilder(); sb.append("Start Time,End Time"); for (final String key : instanceHoursAtLogTime.keySet().stream().sorted(keySorter).collect(toList())) { sb.append(String.format(",%s", key)); } // fill-in missing logs (with 0 hours) in the table final Set<Date> distinctLogs = instanceHoursAtLogTime.values().stream().flatMap(e -> e.stream()) .map(kv -> kv.getKey()).distinct().collect(toSet()); for (final String groupKey : instanceHoursAtLogTime.keySet()) { final List<AbstractMap.SimpleEntry<Date, Long>> hours = instanceHoursAtLogTime.get(groupKey); final Set<Date> currentLogs = hours.stream().map(kv -> kv.getKey()).collect(toSet()); final List<AbstractMap.SimpleEntry<Date, Long>> normalized = Lists.newArrayList(hours); for (final Date missing : Sets.difference(distinctLogs, currentLogs)) { normalized.add(new AbstractMap.SimpleEntry<Date, Long>(missing, 0L)); } instanceHoursAtLogTime.put(groupKey, normalized); } final DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); final Map<Date, String> lines = Maps.newHashMap(); for (final String groupKey : instanceHoursAtLogTime.keySet().stream().sorted(keySorter) .collect(toList())) { for (final AbstractMap.SimpleEntry<Date, Long> hl : instanceHoursAtLogTime.get(groupKey)) { final Date logTime = hl.getKey(); final Long hours = hl.getValue(); if (!lines.containsKey(logTime)) { lines.put(logTime, String.format("%s,%s,%d", df.format(ranger.apply(logTime).getKey()), df.format(ranger.apply(logTime).getValue()), hours)); } else { lines.put(logTime, String.format("%s,%d", lines.get(logTime), hours)); } } } lines.entrySet().stream().sorted((e1, e2) -> e1.getKey().compareTo(e2.getKey())) // order by log time .map(e -> e.getValue()).forEach(s -> sb.append(String.format("\n%s", s))); final ViewInstanceUsageResult result = new ViewInstanceUsageResult(); result.setUsageReport(sb.toString()); response.setResult(result); } catch (final Ec2ReportsServiceException ex) { throw ex; } catch (final Exception ex) { handleException(ex); } return response; } public ViewReservedInstanceUtilizationReportResponseType viewReservedInstanceUtilizationReport( final ViewReservedInstanceUtilizationReportType request) throws Ec2ReportsServiceException { final ViewReservedInstanceUtilizationReportResponseType response = request.getReply(); final Context context = checkAuthorized(); final ViewReservedInstanceUtilizationResult result = new ViewReservedInstanceUtilizationResult(); result.setUtilizationReport("Start Time, End Time, TAG"); response.setResult(result); return response; } private static Context checkAuthorized() throws Ec2ReportsServiceUnauthorizedException { final Context ctx = Contexts.lookup(); final AuthContextSupplier requestUserSupplier = ctx.getAuthContext(); if (!Permissions.isAuthorized(Ec2ReportsPolicySpec.VENDOR_EC2REPORTS, "", "", null, getIamActionByMessageType(), requestUserSupplier)) { throw new Ec2ReportsServiceUnauthorizedException("UnauthorizedOperation", "You are not authorized to perform this operation."); } return ctx; } /** * Method always throws, signature allows use of "throw handleException ..." */ private static Ec2ReportsServiceException handleException(final Exception e) throws Ec2ReportsServiceException { Exceptions.findAndRethrow(e, Ec2ReportsServiceException.class); LOG.error(e, e); final Ec2ReportsServiceException exception = new Ec2ReportsServiceException("InternalError", String.valueOf(e.getMessage())); if (Contexts.lookup().hasAdministrativePrivileges()) { exception.initCause(e); } throw exception; } public static <T, A, R> Collector<T, A, R> filtering(Predicate<? super T> filter, Collector<T, A, R> downstream) { BiConsumer<A, T> accumulator = downstream.accumulator(); Set<Collector.Characteristics> characteristics = downstream.characteristics(); return Collector.of(downstream.supplier(), (acc, t) -> { if (filter.test(t)) accumulator.accept(acc, t); }, downstream.combiner(), downstream.finisher(), characteristics.toArray(new Collector.Characteristics[0])); }; }