io.ecarf.core.utils.UsageParser.java Source code

Java tutorial

Introduction

Here is the source code for io.ecarf.core.utils.UsageParser.java

Source

/**
 * The contents of this file may be used under the terms of the Apache License, Version 2.0
 * in which case, the provisions of the Apache License Version 2.0 are applicable instead of those above.
 *
 * Copyright 2014, Ecarf.io
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.ecarf.core.utils;

import io.cloudex.cloud.impl.google.compute.GoogleMetaData;
import io.cloudex.framework.cloud.entities.StorageObject;
import io.ecarf.core.cloud.impl.google.EcarfGoogleCloudServiceImpl;

import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.lang.StringUtils;

/**
 * 
 * Parse Google Compute Engine Usage CSV files
 * Report Date  MeasurementId   Quantity    Unit    Resource URI    ResourceId  Location
 * @author Omer Dawelbeit (omerio)
 *
 */
public class UsageParser {

    private static final String USAGE_PREFIX = "usage_gce_";

    private static final String MEASURE_PREFIX = "com.google.cloud/services/compute-engine/";

    private static final String VM = "Vmimage";

    private static final String NETWORK = "Network";
    private static final String DISK = "StoragePdCapacity";
    private static final String DISK_SSD = "StoragePdSsd";
    private static final String IMAGE = "StorageImage";
    private static final String CONTAINER_ENGINE_VM = "Licensed";

    // 10 minutes minimum billable time
    private static final int MINUTE = 60;
    private static final int MIN_BILL = 10 * MINUTE;
    private static final int HOUR = MINUTE * MINUTE;

    private EcarfGoogleCloudServiceImpl service;

    private Set<String> files = new HashSet<>();

    private Set<String> measurementIds = new HashSet<>();

    private Set<String> vms = new HashSet<>();

    private Map<String, Usage> usages = new TreeMap<>();

    private int numberOfVms;

    public UsageParser(String folder) throws Exception {

        super();

        boolean remote = folder.startsWith(GoogleMetaData.CLOUD_STORAGE_PREFIX);

        if (remote) {

            String bucket = StringUtils.remove(folder, GoogleMetaData.CLOUD_STORAGE_PREFIX);
            this.setUp();
            List<StorageObject> objects = this.service.listCloudStorageObjects(bucket);

            for (StorageObject object : objects) {
                String name = object.getName();
                if (name.endsWith(Constants.DOT_LOG)) {
                    String localFile = FilenameUtils.getLocalFilePath(name);

                    service.downloadObjectFromCloudStorage(name, localFile, bucket);

                    this.files.add(localFile);
                }
            }

        } else {
            // local file
            DirectoryStream.Filter<Path> filter = new DirectoryStream.Filter<Path>() {
                public boolean accept(Path file) throws IOException {

                    String filename = file.toString();
                    return filename.contains(USAGE_PREFIX);
                }
            };

            Path dir = Paths.get(folder);
            try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, filter)) {
                for (Path path : stream) {
                    this.files.add(path.toString());
                }
            }
        }
    }

    public void setUp() throws Exception {
        this.service = new EcarfGoogleCloudServiceImpl();
        TestUtils.prepare(service);
    }

    public void parse() throws FileNotFoundException, IOException {

        for (String file : files) {

            try (BufferedReader reader = new BufferedReader(new FileReader(file), Constants.GZIP_BUF_SIZE);) {

                Iterable<CSVRecord> records = CSVFormat.DEFAULT.withHeader().withSkipHeaderRecord().parse(reader);

                for (CSVRecord record : records) {
                    String[] values = record.values();

                    String measurement = StringUtils.remove(values[1], MEASURE_PREFIX);
                    this.measurementIds.add(measurement);

                    if (!measurement.contains(CONTAINER_ENGINE_VM)) {

                        if (measurement.contains(VM)) {
                            this.numberOfVms++;

                            this.vms.add(values[4]);
                        }

                        Usage usage = this.usages.get(measurement);

                        if (usage == null) {
                            usage = new Usage();
                            this.usages.put(measurement, usage);
                        }

                        long value = Long.parseLong(values[2]);

                        usage.raw += value;

                        if (measurement.contains(VM)) {

                            long adjusted = value;
                            // minimum billable is 10 minutes for VMs
                            if (adjusted < MIN_BILL) {
                                adjusted = MIN_BILL;
                            }

                            // round up value to the nearest minute
                            adjusted = (long) (MINUTE * Math.ceil(adjusted / 60.0));

                            usage.value += adjusted;

                            // hourly based billing
                            adjusted = value;
                            if (adjusted < HOUR) {
                                adjusted = HOUR;

                            } else {
                                adjusted = (long) (HOUR * Math.ceil(adjusted / 3600.0));
                            }
                            usage.adjusted += adjusted;

                        }

                    } else {
                        // container engine vms
                        System.out.println(StringUtils.join(values, ','));
                    }

                }
            }
        }

        for (String measureId : this.measurementIds) {
            System.out.println(measureId);
        }

        System.out.println("Total number of VMs: " + this.numberOfVms);

        System.out.println(this.vms);
        System.out.println(this.vms.size());

        for (Entry<String, Usage> entry : this.usages.entrySet()) {
            Usage usage = entry.getValue();
            System.out.println(entry.getKey() + ',' + usage.raw + ',' + usage.value + ',' + usage.adjusted);
        }
    }

    /**
     * 
     * @param args
     * @throws Exception
     */
    public static void main(String[] args) throws Exception {
        if (args.length != 1) {
            System.out.println("Usage: UsageParser gs://<bucket> or LogParser /path/to/directory");
            System.exit(-1);
        }

        String path = args[0];

        UsageParser parser = new UsageParser(path);
        parser.parse();

    }

    public class Usage {
        long raw;
        long value;
        long adjusted;
    }

}