Java tutorial
/* * * Copyright 2013 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package com.netflix.ice.basic; import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient; import com.amazonaws.services.simpleemail.model.RawMessage; import com.amazonaws.services.simpleemail.model.SendRawEmailRequest; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.netflix.ice.common.AwsUtils; import com.netflix.ice.common.ConsolidateType; import com.netflix.ice.common.Poller; import com.netflix.ice.reader.*; import com.netflix.ice.reader.ApplicationGroup; import com.netflix.ice.tag.*; import org.apache.commons.lang.StringUtils; import org.jfree.chart.ChartFactory; import org.jfree.chart.JFreeChart; import org.jfree.chart.axis.NumberAxis; import org.jfree.chart.labels.ItemLabelAnchor; import org.jfree.chart.labels.ItemLabelPosition; import org.jfree.chart.labels.StandardCategoryItemLabelGenerator; import org.jfree.chart.plot.CategoryPlot; import org.jfree.chart.plot.PlotOrientation; import org.jfree.chart.renderer.category.BarRenderer3D; import org.jfree.chart.title.TextTitle; import org.jfree.data.category.DefaultCategoryDataset; import org.jfree.ui.TextAnchor; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Interval; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import javax.activation.DataHandler; import javax.activation.DataSource; import javax.imageio.ImageIO; import javax.mail.BodyPart; import javax.mail.Message; import javax.mail.MessagingException; import javax.mail.Session; import javax.mail.internet.MimeBodyPart; import javax.mail.internet.MimeMessage; import javax.mail.internet.MimeMultipart; import javax.mail.util.ByteArrayDataSource; import java.awt.image.BufferedImage; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.text.FieldPosition; import java.text.NumberFormat; import java.text.ParsePosition; import java.util.*; public class BasicWeeklyCostEmailService extends Poller { private ReaderConfig config; protected final DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy/MM/dd").withZoneUTC(); protected final DateTimeFormatter linkDateFormatter = DateTimeFormat.forPattern("yyyy-MM-dd hha") .withZone(DateTimeZone.UTC); protected final NumberFormat numberFormatter = NumberFormat.getNumberInstance(Locale.US); protected final NumberFormat percentageFormat = NumberFormat.getPercentInstance(); protected final NumberFormat costFormatter; protected int initDelaySec; protected int numWeeks; private String urlPrefix; private ApplicationGroupService applicationGroupService; private String fromEmail; private String bccEmail; private String testEmail; private List<Account> accounts; private List<Region> regions; private List<Product> products; private String headerNote; private String throughputMetrics; public BasicWeeklyCostEmailService(List<Account> accounts, List<Region> regions, List<Product> products, int initDelaySec, int numWeeks, String urlPrefix, ApplicationGroupService applicationGroupService, String fromEmail, String bccEmail, String testEmail) { this.accounts = accounts; this.regions = regions; this.products = products; this.initDelaySec = initDelaySec; this.numWeeks = numWeeks; this.urlPrefix = urlPrefix; this.applicationGroupService = applicationGroupService; this.fromEmail = fromEmail; this.bccEmail = bccEmail; this.testEmail = testEmail; costFormatter = new NumberFormat() { @Override public StringBuffer format(long number, StringBuffer toAppendTo, FieldPosition pos) { return numberFormatter.format(number, toAppendTo, pos).insert(0, "$"); } @Override public StringBuffer format(double number, StringBuffer toAppendTo, FieldPosition pos) { return numberFormatter.format(number, toAppendTo, pos).insert(0, "$"); } @Override public Number parse(String source, ParsePosition parsePosition) { throw new UnsupportedOperationException(); } }; percentageFormat.setMinimumFractionDigits(1); percentageFormat.setMaximumFractionDigits(1); } @Override public void start() { config = ReaderConfig.getInstance(); start(initDelaySec, 7 * 24 * 3600, true); } protected boolean inTest() { return false; } protected String getHeaderNote() { return ""; } protected String getThroughputMetrics() throws Exception { return ""; } protected String getResourceGroupsDisplayName(String product) { if (product.equals(Product.ec2.name)) return "Applications"; else if (product.equals(Product.s3.name)) return "S3 buckets"; else if (product.equals(Product.rds.name)) return "RDS DBs"; else return product + " resource groups"; } @Override protected void poll() throws Exception { trigger(inTest()); } public synchronized void trigger(boolean test) { try { headerNote = getHeaderNote(); throughputMetrics = getThroughputMetrics(); AmazonSimpleEmailServiceClient emailService = AwsUtils.getAmazonSimpleEmailServiceClient(); Map<String, ApplicationGroup> appgroups = applicationGroupService.getApplicationGroups(); Map<String, List<ApplicationGroup>> appgroupsByEmail = collectEmails(appgroups); for (String email : appgroupsByEmail.keySet()) { try { if (!StringUtils.isEmpty(email)) sendEmail(test, emailService, email, appgroupsByEmail.get(email)); } catch (Exception e) { logger.error("error in sending email to " + email, e); } } } catch (Exception e) { logger.error("error sending cost emails", e); } } private static Map<String, List<ApplicationGroup>> collectEmails(Map<String, ApplicationGroup> appgroups) { Map<String, List<ApplicationGroup>> result = Maps.newTreeMap(); for (ApplicationGroup appgroup : appgroups.values()) { if (StringUtils.isEmpty(appgroup.owner)) continue; String email = appgroup.owner.trim().toLowerCase(); List<ApplicationGroup> list = result.get(email); if (list == null) { list = Lists.newArrayList(); result.put(email, list); } list.add(appgroup); } return result; } private File createImage(ApplicationGroup appgroup) throws IOException { Map<String, Double> costs = Maps.newHashMap(); DateTime end = new DateTime(DateTimeZone.UTC).withDayOfWeek(1).withMillisOfDay(0); Interval interval = new Interval(end.minusWeeks(numWeeks), end); for (Product product : products) { List<ResourceGroup> resourceGroups = getResourceGroups(appgroup, product); if (resourceGroups.size() == 0) { continue; } DataManager dataManager = config.managers.getCostManager(product, ConsolidateType.weekly); if (dataManager == null) { continue; } TagLists tagLists = new TagLists(accounts, regions, null, Lists.newArrayList(product), null, null, resourceGroups); Map<Tag, double[]> data = dataManager.getData(interval, tagLists, TagType.Product, AggregateType.none, false); for (Tag tag : data.keySet()) { for (int week = 0; week < numWeeks; week++) { String key = tag + "|" + week; if (costs.containsKey(key)) costs.put(key, data.get(tag)[week] + costs.get(key)); else costs.put(key, data.get(tag)[week]); } } } boolean hasData = false; for (Map.Entry<String, Double> entry : costs.entrySet()) { if (!entry.getKey().contains("monitor") && entry.getValue() != null && entry.getValue() >= 0.1) { hasData = true; break; } } if (!hasData) return null; DefaultCategoryDataset dataset = new DefaultCategoryDataset(); for (Product product : products) { for (int week = 0; week < numWeeks; week++) { String weekStr = String.format("%s - %s week", formatter.print(end.minusWeeks(numWeeks - week)).substring(5), formatter.print(end.minusWeeks(numWeeks - week - 1)).substring(5)); dataset.addValue(costs.get(product + "|" + week), product.name, weekStr); } } JFreeChart chart = ChartFactory.createBarChart3D(appgroup.getDisplayName() + " Weekly AWS Costs", "", "Costs", dataset, PlotOrientation.VERTICAL, true, false, false); CategoryPlot categoryplot = (CategoryPlot) chart.getPlot(); BarRenderer3D renderer = (BarRenderer3D) categoryplot.getRenderer(); renderer.setItemLabelAnchorOffset(10.0); TextTitle title = chart.getTitle(); title.setFont(title.getFont().deriveFont((title.getFont().getSize() - 3))); renderer.setBaseItemLabelGenerator(new StandardCategoryItemLabelGenerator() { public java.lang.String generateLabel(org.jfree.data.category.CategoryDataset dataset, int row, int column) { return costFormatter.format(dataset.getValue(row, column)); } }); renderer.setBaseItemLabelsVisible(true); renderer.setBasePositiveItemLabelPosition( new ItemLabelPosition(ItemLabelAnchor.OUTSIDE12, TextAnchor.BASELINE_CENTER)); NumberAxis numberaxis = (NumberAxis) categoryplot.getRangeAxis(); numberaxis.setNumberFormatOverride(costFormatter); BufferedImage image = chart.createBufferedImage(1200, 400); File outputfile = File.createTempFile("awscost", "png"); ImageIO.write(image, "png", outputfile); return outputfile; } private List<ResourceGroup> getResourceGroups(ApplicationGroup appGroup, Product product) { if (product == Product.monitor) product = Product.ec2; List<List<Product>> products = config.resourceService.getProductsWithResources(); Product productForResource = null; for (List<Product> productList : products) { if (productList.contains(product)) { productForResource = productList.get(0); break; } } if (productForResource == null || appGroup.data.get(productForResource.name) == null) return Lists.newArrayList(); else return ResourceGroup.getResourceGroups(appGroup.data.get(productForResource.name)); } private MimeBodyPart constructEmail(int index, ApplicationGroup appGroup, StringBuilder body) throws IOException, MessagingException { if (index == 0 && !StringUtils.isEmpty(headerNote)) body.append(headerNote); numberFormatter.setMaximumFractionDigits(1); numberFormatter.setMinimumFractionDigits(1); File file = createImage(appGroup); if (file == null) return null; DateTime end = new DateTime(DateTimeZone.UTC).withDayOfWeek(1).withMillisOfDay(0); String link = getLink("area", ConsolidateType.hourly, appGroup, accounts, regions, end.minusWeeks(numWeeks), end); body.append(String.format("<b><h4><a href='%s'>%s</a> Weekly Costs:</h4></b>", link, appGroup.getDisplayName())); body.append("<table style=\"border: 1px solid #DDD; border-collapse: collapse\">"); body.append( "<tr style=\"background-color: whiteSmoke;text-align:center\" ><td style=\"border-left: 1px solid #DDD;\"></td>"); for (int i = 0; i <= accounts.size(); i++) { int cols = i == accounts.size() ? 1 : regions.size(); String accName = i == accounts.size() ? "total" : accounts.get(i).name; body.append(String.format( "<td style=\"border-left: 1px solid #DDD;font-weight: bold;padding: 4px\" colspan='%d'>", cols)) .append(accName).append("</td>"); } body.append("</tr>"); body.append("<tr style=\"background-color: whiteSmoke;text-align:center\" ><td></td>"); for (int i = 0; i < accounts.size(); i++) { boolean first = true; for (Region region : regions) { body.append("<td style=\"font-weight: bold;padding: 4px;" + (first ? "border-left: 1px solid #DDD;" : "") + "\">").append(region.name) .append("</td>"); first = false; } } body.append("<td style=\"border-left: 1px solid #DDD;\"></td></tr>"); Map<String, Double> costs = Maps.newHashMap(); Interval interval = new Interval(end.minusWeeks(numWeeks), end); double[] total = new double[numWeeks]; for (Product product : products) { List<ResourceGroup> resourceGroups = getResourceGroups(appGroup, product); if (resourceGroups.size() == 0) { continue; } DataManager dataManager = config.managers.getCostManager(product, ConsolidateType.weekly); if (dataManager == null) { continue; } for (int i = 0; i < accounts.size(); i++) { List<Account> accountList = Lists.newArrayList(accounts.get(i)); TagLists tagLists = new TagLists(accountList, regions, null, Lists.newArrayList(product), null, null, resourceGroups); Map<Tag, double[]> data = dataManager.getData(interval, tagLists, TagType.Region, AggregateType.none, false); for (Tag tag : data.keySet()) { for (int week = 0; week < numWeeks; week++) { String key = accounts.get(i) + "|" + tag + "|" + week; if (costs.containsKey(key)) costs.put(key, data.get(tag)[week] + costs.get(key)); else costs.put(key, data.get(tag)[week]); total[week] += data.get(tag)[week]; } } } } boolean firstLine = true; DateTime currentWeekEnd = end; for (int week = numWeeks - 1; week >= 0; week--) { String weekStr; if (week == numWeeks - 1) weekStr = "Last week"; else weekStr = (numWeeks - week - 1) + " weeks ago"; String background = week % 2 == 1 ? "background: whiteSmoke;" : ""; body.append(String.format( "<tr style=\"%s\"><td nowrap style=\"border-left: 1px solid #DDD;padding: 4px\">%s (%s - %s)</td>", background, weekStr, formatter.print(currentWeekEnd.minusWeeks(1)).substring(5), formatter.print(currentWeekEnd).substring(5))); for (int i = 0; i < accounts.size(); i++) { Account account = accounts.get(i); for (int j = 0; j < regions.size(); j++) { Region region = regions.get(j); String key = account + "|" + region + "|" + week; double cost = costs.get(key) == null ? 0 : costs.get(key); Double lastCost = week == 0 ? null : costs.get(account + "|" + region + "|" + (week - 1)); link = getLink("column", ConsolidateType.daily, appGroup, Lists.newArrayList(account), Lists.newArrayList(region), currentWeekEnd.minusWeeks(1), currentWeekEnd); body.append(getValueCell(cost, lastCost, link, firstLine)); } } link = getLink("column", ConsolidateType.daily, appGroup, accounts, regions, currentWeekEnd.minusWeeks(1), currentWeekEnd); body.append(getValueCell(total[week], week == 0 ? null : total[week - 1], link, firstLine)); body.append("</tr>"); firstLine = false; currentWeekEnd = currentWeekEnd.minusWeeks(1); } body.append("</table>"); numberFormatter.setMaximumFractionDigits(0); numberFormatter.setMinimumFractionDigits(0); if (!StringUtils.isEmpty(throughputMetrics)) body.append(throughputMetrics); body.append("<br><img src=\"cid:image_cid_" + index + "\"><br>"); for (Map.Entry<String, List<String>> entry : appGroup.data.entrySet()) { String product = entry.getKey(); List<String> selected = entry.getValue(); if (selected == null || selected.size() == 0) continue; link = getLink("area", ConsolidateType.hourly, appGroup, accounts, regions, end.minusWeeks(numWeeks), end); body.append(String.format("<b><h4>%s in <a href='%s'>%s</a>:</h4></b>", getResourceGroupsDisplayName(product), link, appGroup.getDisplayName())); for (String name : selected) body.append(" ").append(name).append("<br>"); } body.append("<hr><br>"); MimeBodyPart mimeBodyPart = new MimeBodyPart(); mimeBodyPart.setFileName(file.getName()); DataSource ds = new ByteArrayDataSource(new FileInputStream(file), "image/png"); mimeBodyPart.setDataHandler(new DataHandler(ds)); mimeBodyPart.setHeader("Content-ID", "<image_cid_" + index + ">"); mimeBodyPart.setHeader("Content-Disposition", "inline"); mimeBodyPart.setDisposition(MimeBodyPart.INLINE); file.delete(); return mimeBodyPart; } private void sendEmail(boolean test, AmazonSimpleEmailServiceClient emailService, String email, List<ApplicationGroup> appGroups) throws IOException, MessagingException { StringBuilder body = new StringBuilder(); body.append( "<html><head><style type=\"text/css\">a:link, a:visited{color:#006DBA;}a:link, a:visited, a:hover {\n" + "text-decoration: none;\n" + "}\n" + "body {\n" + "color: #333;\n" + "}" + "</style></head>"); List<MimeBodyPart> mimeBodyParts = Lists.newArrayList(); int index = 0; String subject = ""; for (ApplicationGroup appGroup : appGroups) { boolean hasData = false; for (String prodName : appGroup.data.keySet()) { if (config.productService.getProductByName(prodName) == null) continue; hasData = appGroup.data.get(prodName) != null && appGroup.data.get(prodName).size() > 0; if (hasData) break; } if (!hasData) continue; try { MimeBodyPart mimeBodyPart = constructEmail(index, appGroup, body); index++; if (mimeBodyPart != null) { mimeBodyParts.add(mimeBodyPart); subject = subject + (subject.length() > 0 ? ", " : "") + appGroup.getDisplayName(); } } catch (Exception e) { logger.error("Error contructing email", e); } } body.append("</html>"); if (mimeBodyParts.size() == 0) return; DateTime end = new DateTime(DateTimeZone.UTC).withDayOfWeek(1).withMillisOfDay(0); subject = String.format("%s Weekly AWS Costs (%s - %s)", subject, formatter.print(end.minusWeeks(1)), formatter.print(end)); String toEmail = test ? testEmail : email; Session session = Session.getInstance(new Properties()); MimeMessage mimeMessage = new MimeMessage(session); mimeMessage.setSubject(subject); mimeMessage.setRecipients(javax.mail.Message.RecipientType.TO, toEmail); if (!test && !StringUtils.isEmpty(bccEmail)) { mimeMessage.addRecipients(Message.RecipientType.BCC, bccEmail); } MimeMultipart mimeMultipart = new MimeMultipart(); BodyPart p = new MimeBodyPart(); p.setContent(body.toString(), "text/html"); mimeMultipart.addBodyPart(p); for (MimeBodyPart mimeBodyPart : mimeBodyParts) mimeMultipart.addBodyPart(mimeBodyPart); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); mimeMessage.setContent(mimeMultipart); mimeMessage.writeTo(outputStream); RawMessage rawMessage = new RawMessage(ByteBuffer.wrap(outputStream.toByteArray())); SendRawEmailRequest rawEmailRequest = new SendRawEmailRequest(rawMessage); rawEmailRequest.setDestinations(Lists.<String>newArrayList(toEmail)); rawEmailRequest.setSource(fromEmail); logger.info("sending email to " + toEmail + " " + body.toString()); emailService.sendRawEmail(rawEmailRequest); } private String getValueCell(double value, Double lastValue, String link, boolean doColor) { Double diffValue = lastValue != null && lastValue != 0 ? (1.0 * value - lastValue) / lastValue : null; String color; if (diffValue == null || !doColor) color = ""; else if (diffValue <= 0) color = "background-color:lightGreen"; else if (diffValue > 0.2) color = "background-color:orangered"; else color = "background-color:orange"; String diff = diffValue != null ? "(" + (value >= lastValue ? "+" : "-") + percentageFormat.format(Math.abs(diffValue)) + ")" : ""; return String.format( "<td nowrap style=\"border-left: 1px solid #DDD;padding: 4px;%s\"><a href=\"%s\">$%s %s</a></td>", color, link, numberFormatter.format(value), diff); } private String getLink(String plotType, ConsolidateType consolidateType, ApplicationGroup appgroup, List<Account> accounts, List<Region> regions, DateTime start, DateTime end) { String link = urlPrefix + appgroup.getLink() + "&plotType=" + plotType + "&consolidate=" + consolidateType + "&start=" + linkDateFormatter.print(start) + "&end=" + linkDateFormatter.print(end) + "&groupBy=ResourceGroup"; if (accounts.size() > 0) link += "&account=" + StringUtils.join(accounts, ","); if (regions.size() > 0) link += "®ion=" + StringUtils.join(regions, ","); link += "&product=" + StringUtils.join(products, ","); return link; } }