com.netflix.ice.processor.BillingFileProcessor.java Source code

Java tutorial

Introduction

Here is the source code for com.netflix.ice.processor.BillingFileProcessor.java

Source

/*
 *
 *  Copyright 2013 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.ice.processor;

import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient;
import com.amazonaws.services.simpleemail.model.*;
import com.csvreader.CsvReader;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.netflix.ice.common.*;
import com.netflix.ice.tag.Account;
import com.netflix.ice.tag.Operation;
import com.netflix.ice.tag.Product;
import com.netflix.ice.tag.Zone;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.Months;
import org.joda.time.Weeks;

import java.io.*;
import java.text.NumberFormat;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

/**
 * Class to process billing files and produce tag, usage, cost output files for reader/UI.
 */
public class BillingFileProcessor extends Poller {

    private static Map<String, Double> ondemandRate = Maps.newHashMap();
    private ProcessorConfig config = ProcessorConfig.getInstance();
    private Long startMilli;
    private Long endMilli;
    private boolean processingMonitor;
    private Map<Product, ReadWriteData> usageDataByProduct;
    private Map<Product, ReadWriteData> costDataByProduct;
    private Double ondemandThreshold;
    private String fromEmail;
    private String alertEmails;
    private String urlPrefix;

    public BillingFileProcessor(String urlPrefix, Double ondemandThreshold, String fromEmail, String alertEmails) {
        this.ondemandThreshold = ondemandThreshold;
        this.fromEmail = fromEmail;
        this.alertEmails = alertEmails;
        this.urlPrefix = urlPrefix;
    }

    @Override
    protected void poll() throws Exception {

        TreeMap<DateTime, List<BillingFile>> filesToProcess = Maps.newTreeMap();
        Map<DateTime, List<BillingFile>> monitorFilesToProcess = Maps.newTreeMap();

        // list the tar.gz file in billing file folder
        for (int i = 0; i < config.billingS3BucketNames.length; i++) {
            String billingS3BucketName = config.billingS3BucketNames[i];
            String billingS3BucketPrefix = config.billingS3BucketPrefixes.length > i
                    ? config.billingS3BucketPrefixes[i]
                    : "";
            String accountId = config.billingAccountIds.length > i ? config.billingAccountIds[i] : "";
            String billingAccessRoleName = config.billingAccessRoleNames.length > i
                    ? config.billingAccessRoleNames[i]
                    : "";
            String billingAccessExternalId = config.billingAccessExternalIds.length > i
                    ? config.billingAccessExternalIds[i]
                    : "";

            logger.info("trying to list objects in billing bucket " + billingS3BucketName
                    + " using assume role, and external id " + billingAccessRoleName + " "
                    + billingAccessExternalId);
            List<S3ObjectSummary> objectSummaries = AwsUtils.listAllObjects(billingS3BucketName,
                    billingS3BucketPrefix, accountId, billingAccessRoleName, billingAccessExternalId);
            logger.info("found " + objectSummaries.size() + " in billing bucket " + billingS3BucketName);
            TreeMap<DateTime, S3ObjectSummary> filesToProcessInOneBucket = Maps.newTreeMap();
            Map<DateTime, S3ObjectSummary> monitorFilesToProcessInOneBucket = Maps.newTreeMap();

            // for each file, download&process if not needed
            for (S3ObjectSummary objectSummary : objectSummaries) {

                String fileKey = objectSummary.getKey();
                DateTime dataTime = AwsUtils.getDateTimeFromFileNameWithTags(fileKey);
                boolean withTags = true;
                if (dataTime == null) {
                    dataTime = AwsUtils.getDateTimeFromFileName(fileKey);
                    withTags = false;
                }

                if (dataTime != null && !dataTime.isBefore(config.startDate)) {
                    if (!filesToProcessInOneBucket.containsKey(dataTime)
                            || withTags && config.resourceService != null
                            || !withTags && config.resourceService == null)
                        filesToProcessInOneBucket.put(dataTime, objectSummary);
                    else
                        logger.info("ignoring file " + objectSummary.getKey());
                } else {
                    logger.info("ignoring file " + objectSummary.getKey());
                }
            }

            for (S3ObjectSummary objectSummary : objectSummaries) {
                String fileKey = objectSummary.getKey();
                DateTime dataTime = AwsUtils.getDateTimeFromFileNameWithMonitoring(fileKey);

                if (dataTime != null && !dataTime.isBefore(config.startDate)) {
                    monitorFilesToProcessInOneBucket.put(dataTime, objectSummary);
                }
            }

            for (DateTime key : filesToProcessInOneBucket.keySet()) {
                List<BillingFile> list = filesToProcess.get(key);
                if (list == null) {
                    list = Lists.newArrayList();
                    filesToProcess.put(key, list);
                }
                list.add(new BillingFile(filesToProcessInOneBucket.get(key), accountId, billingAccessRoleName,
                        billingAccessExternalId, billingS3BucketPrefix));
            }

            for (DateTime key : monitorFilesToProcessInOneBucket.keySet()) {
                List<BillingFile> list = monitorFilesToProcess.get(key);
                if (list == null) {
                    list = Lists.newArrayList();
                    monitorFilesToProcess.put(key, list);
                }
                list.add(new BillingFile(monitorFilesToProcessInOneBucket.get(key), accountId,
                        billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix));
            }
        }

        for (DateTime dataTime : filesToProcess.keySet()) {
            startMilli = endMilli = dataTime.getMillis();
            init();

            boolean hasNewFiles = false;
            boolean hasTags = false;
            long lastProcessed = lastProcessTime(AwsUtils.monthDateFormat.print(dataTime));

            for (BillingFile billingFile : filesToProcess.get(dataTime)) {
                S3ObjectSummary objectSummary = billingFile.s3ObjectSummary;
                if (objectSummary.getLastModified().getTime() < lastProcessed) {
                    logger.info("data has been processed. ignoring " + objectSummary.getKey() + "...");
                    continue;
                }
                hasNewFiles = true;
            }

            if (!hasNewFiles) {
                logger.info("data has been processed. ignoring all files at "
                        + AwsUtils.monthDateFormat.print(dataTime));
                continue;
            }

            long processTime = new DateTime(DateTimeZone.UTC).getMillis();
            for (BillingFile billingFile : filesToProcess.get(dataTime)) {

                S3ObjectSummary objectSummary = billingFile.s3ObjectSummary;
                String fileKey = objectSummary.getKey();

                File file = new File(config.localDir, fileKey.substring(billingFile.prefix.length()));
                logger.info("trying to download " + fileKey + "...");
                boolean downloaded = AwsUtils.downloadFileIfChangedSince(objectSummary.getBucketName(),
                        billingFile.prefix, file, lastProcessed, billingFile.accountId, billingFile.accessRoleName,
                        billingFile.externalId);
                if (downloaded)
                    logger.info("downloaded " + fileKey);
                else {
                    logger.info("file already downloaded " + fileKey + "...");
                }

                logger.info("processing " + fileKey + "...");
                boolean withTags = fileKey.contains("with-resources-and-tags");
                hasTags = hasTags || withTags;
                processingMonitor = false;
                processBillingZipFile(file, withTags);
                logger.info("done processing " + fileKey);
            }

            if (monitorFilesToProcess.get(dataTime) != null) {
                for (BillingFile monitorBillingFile : monitorFilesToProcess.get(dataTime)) {

                    S3ObjectSummary monitorObjectSummary = monitorBillingFile.s3ObjectSummary;
                    if (monitorObjectSummary != null) {
                        String monitorFileKey = monitorObjectSummary.getKey();
                        logger.info("processing " + monitorFileKey + "...");
                        File monitorFile = new File(config.localDir,
                                monitorFileKey.substring(monitorFileKey.lastIndexOf("/") + 1));
                        logger.info("trying to download " + monitorFileKey + "...");
                        boolean downloaded = AwsUtils.downloadFileIfChangedSince(
                                monitorObjectSummary.getBucketName(), monitorBillingFile.prefix, monitorFile,
                                lastProcessed, monitorBillingFile.accountId, monitorBillingFile.accessRoleName,
                                monitorBillingFile.externalId);
                        if (downloaded)
                            logger.info("downloaded " + monitorFile);
                        else
                            logger.warn(monitorFile + "already downloaded...");
                        FileInputStream in = new FileInputStream(monitorFile);
                        try {
                            processingMonitor = true;
                            processBillingFile(monitorFile.getName(), in, true);
                        } catch (Exception e) {
                            logger.error("Error processing " + monitorFile, e);
                        } finally {
                            in.close();
                        }
                    }
                }
            }

            if (dataTime.equals(filesToProcess.lastKey())) {
                int hours = (int) ((endMilli - startMilli) / 3600000L);
                logger.info("cut hours to " + hours);
                cutData(hours);
            }

            // now get reservation capacity to calculate upfront and un-used cost
            for (Ec2InstanceReservationPrice.ReservationUtilization utilization : Ec2InstanceReservationPrice.ReservationUtilization
                    .values())
                processReservations(utilization);

            if (hasTags && config.resourceService != null)
                config.resourceService.commit();

            logger.info("archiving results for " + dataTime + "...");
            archive();
            logger.info("done archiving " + dataTime);

            updateProcessTime(AwsUtils.monthDateFormat.print(dataTime), processTime);
            if (dataTime.equals(filesToProcess.lastKey())) {
                sendOndemandCostAlert();
            }
        }

        logger.info("AWS usage processed.");
    }

    private void borrow(int i, long time, Map<TagGroup, Double> usageMap, Map<TagGroup, Double> costMap,
            List<Account> fromAccounts, TagGroup tagGroup,
            Ec2InstanceReservationPrice.ReservationUtilization utilization, boolean forBonus) {

        Double existing = usageMap.get(tagGroup);

        if (existing != null && config.accountService.externalMappingExist(tagGroup.account, tagGroup.zone)
                && fromAccounts != null) {

            for (Account from : fromAccounts) {
                if (existing <= 0)
                    break;

                TagGroup unusedTagGroup = new TagGroup(from, tagGroup.region, tagGroup.zone, tagGroup.product,
                        Operation.getUnusedInstances(utilization), tagGroup.usageType, null);
                Double unused = usageMap.get(unusedTagGroup);

                if (unused != null && unused > 0) {
                    double hourlyCost = costMap.get(unusedTagGroup) / unused;

                    double reservedBorrowed = Math.min(existing, unused);
                    double reservedUnused = unused - reservedBorrowed;

                    existing -= reservedBorrowed;

                    TagGroup borrowedTagGroup = new TagGroup(tagGroup.account, tagGroup.region, tagGroup.zone,
                            tagGroup.product, Operation.getBorrowedInstances(utilization), tagGroup.usageType,
                            null);
                    TagGroup lentTagGroup = new TagGroup(from, tagGroup.region, tagGroup.zone, tagGroup.product,
                            Operation.getLentInstances(utilization), tagGroup.usageType, null);

                    Double existingLent = usageMap.get(lentTagGroup);
                    double reservedLent = existingLent == null ? reservedBorrowed : reservedBorrowed + existingLent;
                    Double existingBorrowed = usageMap.get(borrowedTagGroup);
                    reservedBorrowed = existingBorrowed == null ? reservedBorrowed
                            : reservedBorrowed + existingBorrowed;

                    usageMap.put(borrowedTagGroup, reservedBorrowed);
                    costMap.put(borrowedTagGroup, reservedBorrowed * hourlyCost);
                    usageMap.put(lentTagGroup, reservedLent);
                    costMap.put(lentTagGroup, reservedLent * hourlyCost);
                    usageMap.put(tagGroup, existing);
                    costMap.put(tagGroup, existing * hourlyCost);

                    usageMap.put(unusedTagGroup, reservedUnused);
                    costMap.put(unusedTagGroup, reservedUnused * hourlyCost);
                }
            }
        }

        // the rest is bonus
        if (existing != null && existing > 0 && !forBonus) {
            ReservationService.ReservationInfo reservation = config.reservationService.getReservation(time,
                    tagGroup, utilization);
            TagGroup bonusTagGroup = new TagGroup(tagGroup.account, tagGroup.region, tagGroup.zone,
                    tagGroup.product, Operation.getBonusReservedInstances(utilization), tagGroup.usageType, null);
            usageMap.put(bonusTagGroup, existing);
            costMap.put(bonusTagGroup, existing * reservation.reservationHourlyCost);

            usageMap.remove(tagGroup);
            costMap.remove(tagGroup);
        }
    }

    private void processReservations(Ec2InstanceReservationPrice.ReservationUtilization utilization) {

        if (config.reservationService.getTagGroups(utilization).size() == 0)
            return;

        ReadWriteData usageData = usageDataByProduct.get(null);
        ReadWriteData costData = costDataByProduct.get(null);

        Map<Account, List<Account>> reservationAccounts = config.accountService.getReservationAccounts();
        Set<Account> reservationOwners = reservationAccounts.keySet();
        Map<Account, List<Account>> reservationBorrowers = Maps.newHashMap();
        for (Account account : reservationAccounts.keySet()) {
            List<Account> list = reservationAccounts.get(account);
            for (Account borrowingAccount : list) {
                if (borrowingAccount.name.equals(account.name))
                    continue;
                List<Account> from = reservationBorrowers.get(borrowingAccount);
                if (from == null) {
                    from = Lists.newArrayList();
                    reservationBorrowers.put(borrowingAccount, from);
                }
                from.add(account);
            }
        }

        // first mark owner accounts
        Set<TagGroup> toMarkOwners = Sets.newTreeSet();
        for (TagGroup tagGroup : config.reservationService.getTagGroups(utilization)) {

            for (int i = 0; i < usageData.getNum(); i++) {

                Map<TagGroup, Double> usageMap = usageData.getData(i);
                Map<TagGroup, Double> costMap = costData.getData(i);

                Double existing = usageMap.get(tagGroup);
                double value = existing == null ? 0 : existing;
                ReservationService.ReservationInfo reservation = config.reservationService
                        .getReservation(startMilli + i * AwsUtils.hourMillis, tagGroup, utilization);
                double reservedUsed = Math.min(value, reservation.capacity);
                double reservedUnused = reservation.capacity - reservedUsed;
                double bonusReserved = value > reservation.capacity ? value - reservation.capacity : 0;

                if (reservedUsed > 0 || existing != null) {
                    usageMap.put(tagGroup, reservedUsed);
                    costMap.put(tagGroup, reservedUsed * reservation.reservationHourlyCost);
                }

                if (reservedUnused > 0) {
                    TagGroup unusedTagGroup = new TagGroup(tagGroup.account, tagGroup.region, tagGroup.zone,
                            tagGroup.product, Operation.getUnusedInstances(utilization), tagGroup.usageType, null);
                    usageMap.put(unusedTagGroup, reservedUnused);
                    costMap.put(unusedTagGroup, reservedUnused * reservation.reservationHourlyCost);
                }

                if (bonusReserved > 0) {
                    TagGroup bonusTagGroup = new TagGroup(tagGroup.account, tagGroup.region, tagGroup.zone,
                            tagGroup.product, Operation.getBonusReservedInstances(utilization), tagGroup.usageType,
                            null);
                    usageMap.put(bonusTagGroup, bonusReserved);
                    costMap.put(bonusTagGroup, bonusReserved * reservation.reservationHourlyCost);
                }

                if (reservation.capacity > 0) {
                    TagGroup upfrontTagGroup = new TagGroup(tagGroup.account, tagGroup.region, tagGroup.zone,
                            tagGroup.product, Operation.getUpfrontAmortized(utilization), tagGroup.usageType, null);
                    costMap.put(upfrontTagGroup, reservation.capacity * reservation.upfrontAmortized);
                }
            }

            toMarkOwners.add(new TagGroup(tagGroup.account, tagGroup.region, tagGroup.zone, tagGroup.product,
                    Operation.getReservedInstances(utilization), tagGroup.usageType, null));
        }

        // now mark borrowing accounts
        Set<TagGroup> toMarkBorrowing = Sets.newTreeSet();
        for (TagGroup tagGroup : usageData.getTagGroups()) {
            if (tagGroup.resourceGroup == null && tagGroup.product == Product.ec2_instance
                    && (tagGroup.operation == Operation.getReservedInstances(utilization)
                            && !toMarkOwners.contains(tagGroup)
                            || tagGroup.operation == Operation.getBonusReservedInstances(utilization))) {

                toMarkBorrowing.add(tagGroup);
            }
        }
        for (TagGroup tagGroup : toMarkBorrowing) {
            for (int i = 0; i < usageData.getNum(); i++) {

                Map<TagGroup, Double> usageMap = usageData.getData(i);
                Map<TagGroup, Double> costMap = costData.getData(i);

                borrow(i, startMilli + i * AwsUtils.hourMillis, usageMap, costMap,
                        reservationBorrowers.get(tagGroup.account), tagGroup, utilization,
                        reservationOwners.contains(tagGroup.account));
            }
        }
    }

    private void cutData(int hours) {
        for (ReadWriteData data : usageDataByProduct.values()) {
            data.cutData(hours);
        }
        for (ReadWriteData data : costDataByProduct.values()) {
            data.cutData(hours);
        }
    }

    private void archive() throws Exception {

        logger.info("archiving tag data...");

        for (Product product : costDataByProduct.keySet()) {
            TagGroupWriter writer = new TagGroupWriter(product == null ? "all" : product.name);
            writer.archive(startMilli, costDataByProduct.get(product).getTagGroups());
        }

        logger.info("archiving summary data...");

        archiveSummary(usageDataByProduct, "usage_");
        archiveSummary(costDataByProduct, "cost_");

        logger.info("archiving hourly data...");

        archiveHourly(usageDataByProduct, "usage_");
        archiveHourly(costDataByProduct, "cost_");

        logger.info("archiving data done.");
    }

    private void archiveHourly(Map<Product, ReadWriteData> dataMap, String prefix) throws Exception {
        DateTime monthDateTime = new DateTime(startMilli, DateTimeZone.UTC);
        for (Product product : dataMap.keySet()) {
            String prodName = product == null ? "all" : product.name;
            DataWriter writer = new DataWriter(
                    prefix + "hourly_" + prodName + "_" + AwsUtils.monthDateFormat.print(monthDateTime), false);
            writer.archive(dataMap.get(product));
        }
    }

    private void addValue(List<Map<TagGroup, Double>> list, int index, TagGroup tagGroup, double v) {
        Map<TagGroup, Double> map = ReadWriteData.getCreateData(list, index);
        Double existedV = map.get(tagGroup);
        map.put(tagGroup, existedV == null ? v : existedV + v);
    }

    private void archiveSummary(Map<Product, ReadWriteData> dataMap, String prefix) throws Exception {

        DateTime monthDateTime = new DateTime(startMilli, DateTimeZone.UTC);

        for (Product product : dataMap.keySet()) {

            String prodName = product == null ? "all" : product.name;
            ReadWriteData data = dataMap.get(product);
            Collection<TagGroup> tagGroups = data.getTagGroups();

            // init daily, weekly and monthly
            List<Map<TagGroup, Double>> daily = Lists.newArrayList();
            List<Map<TagGroup, Double>> weekly = Lists.newArrayList();
            List<Map<TagGroup, Double>> monthly = Lists.newArrayList();

            // get last month data
            ReadWriteData lastMonthData = new DataWriter(prefix + "hourly_" + prodName + "_"
                    + AwsUtils.monthDateFormat.print(monthDateTime.minusMonths(1)), true).getData();

            // aggregate to daily, weekly and monthly
            int dayOfWeek = monthDateTime.getDayOfWeek();
            int daysFromLastMonth = dayOfWeek - 1;
            int lastMonthNumHours = monthDateTime.minusMonths(1).dayOfMonth().getMaximumValue() * 24;
            for (int hour = 0 - daysFromLastMonth * 24; hour < data.getNum(); hour++) {
                if (hour < 0) {
                    // handle data from last month, add to weekly
                    Map<TagGroup, Double> prevData = lastMonthData.getData(lastMonthNumHours + hour);
                    for (TagGroup tagGroup : tagGroups) {
                        Double v = prevData.get(tagGroup);
                        if (v != null && v != 0) {
                            addValue(weekly, 0, tagGroup, v);
                        }
                    }
                } else {
                    // this month, add to weekly, monthly and daily
                    Map<TagGroup, Double> map = data.getData(hour);

                    for (TagGroup tagGroup : tagGroups) {
                        Double v = map.get(tagGroup);
                        if (v != null && v != 0) {
                            addValue(monthly, 0, tagGroup, v);
                            addValue(daily, hour / 24, tagGroup, v);
                            addValue(weekly, (hour + daysFromLastMonth * 24) / 24 / 7, tagGroup, v);
                        }
                    }
                }
            }

            // archive daily
            int year = monthDateTime.getYear();
            DataWriter writer = new DataWriter(prefix + "daily_" + prodName + "_" + year, true);
            ReadWriteData dailyData = writer.getData();
            dailyData.setData(daily, monthDateTime.getDayOfYear() - 1, false);
            writer.archive();

            // archive monthly
            writer = new DataWriter(prefix + "monthly_" + prodName, true);
            ReadWriteData monthlyData = writer.getData();
            monthlyData.setData(monthly, Months.monthsBetween(config.startDate, monthDateTime).getMonths(), false);
            writer.archive();

            // archive weekly
            writer = new DataWriter(prefix + "weekly_" + prodName, true);
            ReadWriteData weeklyData = writer.getData();
            DateTime weekStart = monthDateTime.withDayOfWeek(1);
            int index;
            if (!weekStart.isAfter(config.startDate))
                index = 0;
            else
                index = Weeks.weeksBetween(config.startDate, weekStart).getWeeks()
                        + (config.startDate.dayOfWeek() == weekStart.dayOfWeek() ? 0 : 1);
            weeklyData.setData(weekly, index, true);
            writer.archive();
        }
    }

    private void init() {
        usageDataByProduct = new HashMap<Product, ReadWriteData>();
        costDataByProduct = new HashMap<Product, ReadWriteData>();
        usageDataByProduct.put(null, new ReadWriteData());
        costDataByProduct.put(null, new ReadWriteData());
    }

    private void processBillingZipFile(File file, boolean withTags) throws IOException {

        InputStream input = new FileInputStream(file);
        ZipInputStream zipInput;

        zipInput = new ZipInputStream(input);

        try {
            ZipEntry entry;
            while ((entry = zipInput.getNextEntry()) != null) {
                if (entry.isDirectory())
                    continue;

                processBillingFile(entry.getName(), zipInput, withTags);
            }
        } catch (IOException e) {
            if (e.getMessage().equals("Stream closed"))
                logger.info("reached end of file.");
            else
                logger.error("Error processing " + file, e);
        } finally {
            try {
                zipInput.close();
            } catch (IOException e) {
                logger.error("Error closing " + file, e);
            }
            try {
                input.close();
            } catch (IOException e1) {
                logger.error("Cannot close input for " + file, e1);
            }
        }
    }

    private void processBillingFile(String fileName, InputStream tempIn, boolean withTags) {

        CsvReader reader = new CsvReader(new InputStreamReader(tempIn), ',');

        long lineNumber = 0;
        List<String[]> delayedItems = Lists.newArrayList();
        try {
            reader.readRecord();
            String[] headers = reader.getValues();

            config.lineItemProcessor.initIndexes(withTags, headers);

            while (reader.readRecord()) {
                String[] items = reader.getValues();
                try {
                    processOneLine(delayedItems, items);
                } catch (Exception e) {
                    logger.error(StringUtils.join(items, ","), e);
                }
                lineNumber++;

                if (lineNumber % 500000 == 0) {
                    logger.info("processed " + lineNumber + " lines...");
                }
                //                if (lineNumber == 40000000) {//100000000      //
                //                    break;
                //                }
            }
        } catch (IOException e) {
            logger.error("Error processing " + fileName + " at line " + lineNumber, e);
        } finally {
            try {
                reader.close();
            } catch (Exception e) {
                logger.error("Cannot close BufferedReader...", e);
            }
        }

        for (String[] items : delayedItems) {
            processOneLine(null, items);
        }
    }

    private void processOneLine(List<String[]> delayedItems, String[] items) {

        LineItemProcessor.Result result = config.lineItemProcessor.process(startMilli, delayedItems == null, config,
                items, usageDataByProduct, costDataByProduct, ondemandRate);

        if (result == LineItemProcessor.Result.delay) {
            delayedItems.add(items);
        } else if (result == LineItemProcessor.Result.hourly && !processingMonitor) {
            endMilli = Math.max(endMilli, config.lineItemProcessor.getEndMillis(items));
        }
    }

    private Map<Long, Map<Ec2InstanceReservationPrice.Key, Double>> getOndemandCosts(long fromMillis) {
        Map<Long, Map<Ec2InstanceReservationPrice.Key, Double>> ondemandCostsByHour = Maps.newHashMap();
        ReadWriteData costs = costDataByProduct.get(null);

        Collection<TagGroup> tagGroups = costs.getTagGroups();
        for (int i = 0; i < costs.getNum(); i++) {
            Long millis = startMilli + i * AwsUtils.hourMillis;
            if (millis < fromMillis)
                continue;

            Map<Ec2InstanceReservationPrice.Key, Double> ondemandCosts = Maps.newHashMap();
            ondemandCostsByHour.put(millis, ondemandCosts);

            Map<TagGroup, Double> data = costs.getData(i);
            for (TagGroup tagGroup : tagGroups) {
                if (tagGroup.product == Product.ec2_instance && tagGroup.operation == Operation.ondemandInstances
                        && data.get(tagGroup) != null) {
                    Ec2InstanceReservationPrice.Key key = new Ec2InstanceReservationPrice.Key(tagGroup.region,
                            tagGroup.usageType);
                    if (ondemandCosts.get(key) != null)
                        ondemandCosts.put(key, data.get(tagGroup) + ondemandCosts.get(key));
                    else
                        ondemandCosts.put(key, data.get(tagGroup));
                }
            }
        }

        return ondemandCostsByHour;
    }

    private void updateLastMillis(long millis, String filename) {
        AmazonS3Client s3Client = AwsUtils.getAmazonS3Client();
        s3Client.putObject(config.workS3BucketName, config.workS3BucketPrefix + filename,
                IOUtils.toInputStream(millis + ""), new ObjectMetadata());
    }

    private Long getLastMillis(String filename) {
        AmazonS3Client s3Client = AwsUtils.getAmazonS3Client();
        InputStream in = null;
        try {
            in = s3Client.getObject(config.workS3BucketName, config.workS3BucketPrefix + filename)
                    .getObjectContent();
            return Long.parseLong(IOUtils.toString(in));
        } catch (Exception e) {
            logger.error("Error reading from file " + filename, e);
            return 0L;
        } finally {
            if (in != null)
                try {
                    in.close();
                } catch (Exception e) {
                }
        }
    }

    private Long lastProcessTime(String timeStr) {
        return getLastMillis("lastProcessMillis_" + timeStr);
    }

    private void updateProcessTime(String timeStr, long millis) {
        updateLastMillis(millis, "lastProcessMillis_" + timeStr);
    }

    private Long lastAlertMillis() {
        return getLastMillis("ondemandAlertMillis");
    }

    private void updateLastAlertMillis(Long millis) {
        updateLastMillis(millis, "ondemandAlertMillis");
    }

    private void sendOndemandCostAlert() {

        if (ondemandThreshold == null || StringUtils.isEmpty(fromEmail) || StringUtils.isEmpty(alertEmails)
                || endMilli < lastAlertMillis() + AwsUtils.hourMillis * 24)
            return;

        Map<Long, Map<Ec2InstanceReservationPrice.Key, Double>> ondemandCosts = getOndemandCosts(
                lastAlertMillis() + AwsUtils.hourMillis);
        Long maxHour = null;
        double maxTotal = ondemandThreshold;

        for (Long hour : ondemandCosts.keySet()) {
            double total = 0;
            for (Double value : ondemandCosts.get(hour).values())
                total += value;

            if (total > maxTotal) {
                maxHour = hour;
                maxTotal = total;
            }
        }

        if (maxHour != null) {
            NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.US);
            String subject = String.format("Alert: Ondemand cost per hour reached $%s at %s",
                    numberFormat.format(maxTotal), AwsUtils.dateFormatter.print(maxHour));
            StringBuilder body = new StringBuilder();
            body.append(String.format("Total ondemand cost $%s at %s:<br><br>", numberFormat.format(maxTotal),
                    AwsUtils.dateFormatter.print(maxHour)));
            TreeMap<Double, String> costs = Maps.newTreeMap();
            for (Map.Entry<Ec2InstanceReservationPrice.Key, Double> entry : ondemandCosts.get(maxHour).entrySet()) {
                costs.put(entry.getValue(), entry.getKey().region + " " + entry.getKey().usageType + ": ");
            }
            for (Double cost : costs.descendingKeySet()) {
                if (cost > 0)
                    body.append(costs.get(cost)).append("$" + numberFormat.format(cost)).append("<br>");
            }
            body.append("<br>Please go to <a href=\"" + urlPrefix
                    + "dashboard/reservation#usage_cost=cost&groupBy=UsageType&product=ec2_instance&operation=OndemandInstances\">Ice</a> for details.");
            SendEmailRequest request = new SendEmailRequest();
            request.withSource(fromEmail);
            List<String> emails = Lists.newArrayList(alertEmails.split(","));
            request.withDestination(new Destination(emails));
            request.withMessage(
                    new Message(new Content(subject), new Body().withHtml(new Content(body.toString()))));

            AmazonSimpleEmailServiceClient emailService = AwsUtils.getAmazonSimpleEmailServiceClient();
            try {
                emailService.sendEmail(request);
                updateLastAlertMillis(endMilli);
                logger.info("updateLastAlertMillis " + endMilli);
            } catch (Exception e) {
                logger.error("Error in sending alert emails", e);
            }
        }
    }

    private class BillingFile {
        final S3ObjectSummary s3ObjectSummary;
        final String accountId;
        final String accessRoleName;
        final String externalId;
        final String prefix;

        BillingFile(S3ObjectSummary s3ObjectSummary, String accountId, String accessRoleName, String externalId,
                String prefix) {
            this.s3ObjectSummary = s3ObjectSummary;
            this.accountId = accountId;
            this.accessRoleName = accessRoleName;
            this.externalId = externalId;
            this.prefix = prefix;
        }
    }
}