com.oneops.ops.dao.PerfDataAccessor.java Source code

Java tutorial

Introduction

Here is the source code for com.oneops.ops.dao.PerfDataAccessor.java

Source

/*******************************************************************************
 *
 *   Copyright 2015 Walmart, Inc.
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 *
 *******************************************************************************/
package com.oneops.ops.dao;

import static me.prettyprint.hector.api.factory.HFactory.createKeyspace;
import static me.prettyprint.hector.api.factory.HFactory.createMutator;

import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.*;

import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;

import com.oneops.cassandra.ClusterBootstrap;
import com.oneops.ops.PerfDataRequest;
import com.oneops.ops.PerfHeader;
import com.oneops.sensor.events.PerfEvent;
import com.oneops.sensor.events.PerfEventPayload;

import me.prettyprint.cassandra.model.BasicColumnFamilyDefinition;
import me.prettyprint.cassandra.model.ConfigurableConsistencyLevel;
import me.prettyprint.cassandra.serializers.BytesArraySerializer;
import me.prettyprint.cassandra.serializers.DoubleSerializer;
import me.prettyprint.cassandra.serializers.LongSerializer;
import me.prettyprint.cassandra.serializers.StringSerializer;
import me.prettyprint.cassandra.service.ThriftCfDef;
import me.prettyprint.cassandra.service.ThriftKsDef;
import me.prettyprint.hector.api.Cluster;
import me.prettyprint.hector.api.HConsistencyLevel;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.*;
import me.prettyprint.hector.api.ddl.ColumnType;
import me.prettyprint.hector.api.ddl.ComparatorType;
import me.prettyprint.hector.api.exceptions.HInvalidRequestException;
import me.prettyprint.hector.api.exceptions.HectorException;
import me.prettyprint.hector.api.factory.HFactory;
import me.prettyprint.hector.api.mutation.Mutator;
import me.prettyprint.hector.api.query.*;

/**
 * PerfDataAccessor - encapsulates cassandra data access for performance.
 * <p>
 * data clients - write: PerfSink uses process() and constructor or  property setters for cassandra connection.
 *                 read: web.PerfController uses getPerfData() getLogData() ;
 */
@Component
public class PerfDataAccessor implements CassandraConstants {

    private String clusterName;
    private String keyspaceName;
    private String hostName;
    private boolean isTestMode = false;

    private static final StringSerializer stringSerializer = StringSerializer.get();
    private static final BytesArraySerializer bytesSerializer = BytesArraySerializer.get();
    private static final LongSerializer longSerializer = LongSerializer.get();
    private static final DoubleSerializer doubleSerializer = DoubleSerializer.get();

    protected Cluster cluster;
    protected Keyspace keyspace;

    private static final String DATA_CF = "data";
    private static final String HEADER_SCF = "header";
    protected static final String CI_METRIC_CF = "ci_metric";
    private static final String CHART_SCF = "chart";
    private static final String[] buckets = { "1m", "5m", "15m", "1h", "6h", "1d" };

    private static Logger logger = Logger.getLogger(PerfDataAccessor.class);
    private static PerfHeaderDao phd = null;
    private ClusterBootstrap cb;

    public static int MAX_EXCEPTION_COUNT = 10;
    // Default time seconds for hector to connect to cassandra.
    private static final int TIMEOUT_IN_SECONDS = Integer
            .valueOf(System.getProperty("dao.timeToWaitForCassandra", "7"));

    /**
     * Sets the cluster name.
     *
     * @param name the new cluster name
     */
    public void setClusterName(String name) {
        clusterName = name;
    }

    /**
     * Sets the keyspace name.
     *
     * @param name the new keyspace name
     */
    public void setKeyspaceName(String name) {
        keyspaceName = name;
    }

    /**
     * Sets the cluster bootstrap
     *
     * @param cb ClusterBootstrap
     */
    public void setClusterBootstrap(ClusterBootstrap cb) {
        this.cb = cb;
    }

    private String connectToCluster() {
        logger.info("Initializing PerfDataAccessor cluster...");
        // No of connections and timeouts are configured as part of system property
        cluster = cb.getCluster(clusterName);
        logger.info("Connected to cluster : " + clusterName);
        return "Ready!";
    }

    /**
     * Inits the DAOs/connections
     */
    public void init() {
        logger.info("PerfDataAccessor: " + ":" + clusterName + ":" + keyspaceName);
        ExecutorService executor = Executors.newSingleThreadExecutor();
        Future<String> future = executor.submit(this::connectToCluster);

        try {
            logger.info("Started connecting.. with timeOut " + TIMEOUT_IN_SECONDS);
            logger.info(future.get(TIMEOUT_IN_SECONDS, TimeUnit.SECONDS));
            logger.info("Finished connecting!");

        } catch (TimeoutException e) {
            logger.error("no cassandra hosts available - shutting down");
            throw new HectorException("TimeOut occured in getting the cassandra connection");
        } catch (InterruptedException e) {
            e.printStackTrace();
        } catch (ExecutionException e) {
            e.printStackTrace();
        }

        executor.shutdownNow();
        initCluster();
    }

    public void initSync() {
        logger.info("PerfDataAccessor - initSync: " + ":" + clusterName + ":" + keyspaceName);
        connectToCluster();
        initCluster();
    }

    private void initCluster() {
        if (cluster.getConnectionManager().getActivePools().isEmpty()) {
            logger.error("no cassandra hosts available - shutting down");
            throw new HectorException("no cassandra hosts available ");
        } else {
            logger.info("hosts: " + cluster.getConnectionManager().getHosts().toString());
            logger.info("downed hosts: " + cluster.getConnectionManager().getDownedHosts().toString());
        }

        try {
            cluster.addKeyspace(
                    new ThriftKsDef(keyspaceName, "org.apache.cassandra.locator.SimpleStrategy", 1, null));
            logger.info("adding keyspace: " + keyspaceName);
        } catch (HInvalidRequestException e) {
            logger.info(" adding " + keyspaceName + " keyspace: " + e.getMessage());
        }

        ConfigurableConsistencyLevel cl = new ConfigurableConsistencyLevel();
        cl.setDefaultWriteConsistencyLevel(HConsistencyLevel.ONE);
        cl.setDefaultReadConsistencyLevel(HConsistencyLevel.ONE);
        keyspace = createKeyspace(keyspaceName, cluster, cl);

        BasicColumnFamilyDefinition cfo = new BasicColumnFamilyDefinition();

        // Add sharded data cf
        for (String bucket : buckets) {
            cfo = new BasicColumnFamilyDefinition();
            cfo.setColumnType(ColumnType.STANDARD);
            String bucketCf = DATA_CF + "_" + bucket;
            if (isTestMode)
                bucketCf += "_test";

            cfo.setName(bucketCf);
            cfo.setComparatorType(ComparatorType.BYTESTYPE);
            cfo.setKeyspaceName(keyspaceName);

            try {
                cluster.addColumnFamily(new ThriftCfDef((cfo)));
                logger.info("adding cf: " + bucketCf);
            } catch (HInvalidRequestException e) {
                logger.info("adding " + bucketCf + " " + e.getMessage());
            }
        }

        // header
        cfo = new BasicColumnFamilyDefinition();
        cfo.setColumnType(ColumnType.SUPER);
        String headerCfName = HEADER_SCF;
        if (isTestMode)
            headerCfName += "_test";
        cfo.setName(headerCfName);
        cfo.setComparatorType(ComparatorType.BYTESTYPE);
        cfo.setSubComparatorType(ComparatorType.BYTESTYPE);
        cfo.setKeyspaceName(keyspaceName);

        try {
            cluster.addColumnFamily(new ThriftCfDef((cfo)));
            logger.info("adding cf: " + headerCfName);
        } catch (HInvalidRequestException e) {
            logger.info("adding " + headerCfName + " " + e.getMessage());
        }

        // chart
        cfo = new BasicColumnFamilyDefinition();
        cfo.setColumnType(ColumnType.SUPER);
        cfo.setName(CHART_SCF);
        cfo.setComparatorType(ComparatorType.ASCIITYPE);
        cfo.setSubComparatorType(ComparatorType.ASCIITYPE);
        cfo.setKeyspaceName(keyspaceName);

        try {
            cluster.addColumnFamily(new ThriftCfDef((cfo)));
            logger.info("adding cf: " + CHART_SCF);
        } catch (HInvalidRequestException e) {
            logger.info("adding " + CHART_SCF + " " + e.getMessage());
        }

        phd = new PerfHeaderDao();
        phd.setClusterBootstrap(cb);
        phd.setClusterName(clusterName);
        phd.setKeyspaceName(keyspaceName);
        phd.init();

        try {
            hostName = InetAddress.getLocalHost().getHostName();
        } catch (UnknownHostException e) {
            try {
                hostName = InetAddress.getLocalHost().getHostAddress();
            } catch (UnknownHostException e1) {
                logger.error("could not get hostname or ip...fail");
                //Wrapping it with Hector exception as it causes Cassandra Connection Failures
                throw new HectorException("could not get hostname or ip...fail");
            }
        }
    }

    private String getRraByStat(String stat, int reqStep) {
        return stat + "-" + getShard(reqStep);
    }

    private String getShard(int reqStep) {
        String rra = null;
        if (reqStep < 300)
            rra = "1m";
        else if (reqStep < 900)
            rra = "5m";
        else if (reqStep < 3600)
            rra = "15m";
        else if (reqStep < 21600)
            rra = "1h";
        else if (reqStep < 86000)
            rra = "6h";
        else
            rra = "1d";
        return rra;
    }

    private int alignRraStep(int reqStep) {
        int step = 60;
        // 1 day
        if (reqStep > 86000) {
            step = 86400;
            // 6hr
        } else if (reqStep > 21599) {
            step = 21600;
            // 1hr
        } else if (reqStep > 3599) {
            step = 3600;
            // 15min
        } else if (reqStep > 899) {
            step = 900;
            // 5min
        } else if (reqStep > 299) {
            step = 300;
        }
        return step;
    }

    /**
     * Gets the perf data series. Contains tmp code for conversion of data cf to sharded data cfs
     *
     * @param req the request object
     * @return the perf data series
     */
    public String getPerfDataSeries(PerfDataRequest req) {

        Long start = Long.valueOf(req.getStart());
        Long end = Long.valueOf(req.getEnd());
        int maxColumns = (int) (end - start);
        StringBuilder jsonOut = new StringBuilder("");

        try {
            long startTime = System.currentTimeMillis();
            String stat = "rra-average";
            if (req.getStat_function() != null) {
                stat = "rra-" + req.getStat_function();
            }

            String rra = getRraByStat(stat, req.getStep());
            int step = alignRraStep(req.getStep());
            List<byte[]> keys = new ArrayList<byte[]>();
            StringBuilder sb = new StringBuilder("");
            long adjustedStart = start - start % step;
            String dataCF = DATA_CF + "_" + getShard(step);
            if (isTestMode)
                dataCF += "_test";

            MultigetSliceQuery<byte[], Long, Double> multigetSliceQuery = HFactory
                    .createMultigetSliceQuery(keyspace, bytesSerializer, longSerializer, doubleSerializer);
            multigetSliceQuery.setColumnFamily(dataCF);
            keys = new ArrayList<>();
            sb = new StringBuilder("");
            for (int i = 0; i < req.getMetrics().length; i++) {
                String metricDs = req.getMetrics()[i];
                String key = Long.valueOf(req.getCi_id()).toString() + ":" + metricDs + ":" + rra;
                keys.add(key.getBytes());
                sb.append(" " + key);
            }
            multigetSliceQuery.setKeys(keys);
            multigetSliceQuery.setRange(adjustedStart, end, false, maxColumns);

            logger.info("start:" + start + " end:" + end + " for: " + sb);
            long cassStart = System.currentTimeMillis();

            QueryResult<Rows<byte[], Long, Double>> result = multigetSliceQuery.execute();
            Rows<byte[], Long, Double> rows = result.get();

            long cassEnd = System.currentTimeMillis();
            long cassDuration = cassEnd - cassStart;

            // put the by-metric results into 1 csv-like table
            // (time,metric1,metric2,etc)
            // ... should find faster way to do this, but still 10x faster than
            // gwt DataTable serialization
            int rowCount = 0;
            int totalSampleCount = 0;
            HashMap<String, HashMap<Long, Double>> resultMap = new HashMap<String, HashMap<Long, Double>>();

            for (Row<byte[], Long, Double> row : rows) {

                String rowKey = new String(row.getKey());
                HashMap<Long, Double> results = null;
                if (resultMap.containsKey(rowKey)) {
                    results = resultMap.get(rowKey);
                } else {
                    results = new HashMap<>();
                    resultMap.put(rowKey, results);
                }

                List<HColumn<Long, Double>> cols = row.getColumnSlice().getColumns();
                Iterator<HColumn<Long, Double>> listIter = cols.listIterator();

                while (listIter.hasNext()) {
                    HColumn<Long, Double> c = (HColumn<Long, Double>) listIter.next();
                    results.put(c.getName(), c.getValue());
                }
            }

            for (String rowKey : resultMap.keySet()) {

                HashMap<Long, Double> results = resultMap.get(rowKey);

                if (rowCount > 0) {
                    jsonOut.append(",\n");
                }

                String[] keyParts = rowKey.split(":");
                String ciId = keyParts[0];
                String metric = keyParts[1] + ":" + keyParts[2];

                jsonOut.append("{ \"header\":{\"ci_id\":" + ciId + ", \"metric\":\"" + metric + "\", \"step\":"
                        + step + ", \"start\":" + adjustedStart + "},\n ");
                jsonOut.append("\"data\":[");

                long currentBucket = adjustedStart;
                int sampleCount = 0;
                int emptyCount = 0;

                SortedSet<Long> sortedKeys = new TreeSet<>(results.keySet());

                for (long sampleBucket : sortedKeys) {
                    double value = results.get(sampleBucket);
                    if (sampleBucket != currentBucket) {
                        while (sampleBucket > currentBucket) {
                            if (sampleCount > 0) {
                                jsonOut.append(",");
                            }
                            jsonOut.append("null");
                            currentBucket += step;
                            emptyCount++;
                            sampleCount++;
                        }
                    }

                    if (sampleCount > 0) {
                        jsonOut.append(",");
                    }
                    jsonOut.append((Math.round(value * 1000.0) / 1000.0));

                    currentBucket += step;
                    totalSampleCount++;
                    sampleCount++;
                }

                jsonOut.append("]}");
                rowCount++;
                logger.debug("got samples:" + sampleCount + " gaps:" + emptyCount);
            }
            long endTime = System.currentTimeMillis();
            long duration = endTime - startTime;

            logger.debug("getPerfData took: " + duration + " ms (cass query: " + cassDuration + " ms) returning: "
                    + totalSampleCount + " rows of " + rowCount + " metrics");

        } catch (HectorException he) {
            he.printStackTrace();
        }
        return jsonOut.toString();
    }

    public void reportMetricCounts() {
        int rowCount = 0;
        int totalColCount = 0;
        int pageSize = 1000;

        RangeSlicesQuery<String, Long, Double> query = HFactory
                .createRangeSlicesQuery(keyspace, stringSerializer, longSerializer, doubleSerializer)
                .setColumnFamily(DATA_CF).setReturnKeysOnly().setRowCount(pageSize);

        String lastKey = null;

        while (true) {
            query.setKeys(lastKey, null);

            QueryResult<OrderedRows<String, Long, Double>> result = query.execute();
            OrderedRows<String, Long, Double> orderedRows = result.get();
            Iterator<Row<String, Long, Double>> rowsIterator = orderedRows.iterator();

            // we'll skip this first one, since it is the same as the last one
            // from previous time we executed
            if (lastKey != null && rowsIterator != null)
                rowsIterator.next();

            // each row / rra
            while (rowsIterator.hasNext()) {
                Row<String, Long, Double> row = rowsIterator.next();
                // count values
                CountQuery<byte[], Long> cq = HFactory.createCountQuery(keyspace, bytesSerializer, longSerializer)
                        .setColumnFamily(DATA_CF).setKey(row.getKey().getBytes())
                        .setRange(1L, null, Integer.MAX_VALUE);

                QueryResult<Integer> r = cq.execute();
                logger.info(row.getKey() + ": " + r.get());

                rowCount++;
                totalColCount += r.get();
                lastKey = row.getKey();
            }
            logger.info("rows: " + rowCount + " cols: " + totalColCount);

            if (orderedRows.getCount() < pageSize)
                break;

        }

    }

    public void purgeMetrics(long time) {
        purgeMetrics(time, "1m");
    }

    public void purgeMetrics(long time, String bucket) {

        int rowCount = 0;
        int totalColCount = 0;
        int totalColDeleted = 0;

        Long start = 1L;
        Long end = time;

        // safeguard not to delete anything in past week
        long now = System.currentTimeMillis() / 1000;
        logger.info("       now: " + now);
        logger.info("startEpoch: " + time);

        if (time + (60 * 60 * 24 * 7) > now) {
            logger.error("input time too soon - cannot be within past week");
            return;
        }

        int maxColumns = (int) (end - start);
        int pageSize = 1000;
        String lastKey = null;

        Mutator<byte[]> mutator = HFactory.createMutator(keyspace, bytesSerializer);

        RangeSlicesQuery<String, Long, Double> query = HFactory
                .createRangeSlicesQuery(keyspace, stringSerializer, longSerializer, doubleSerializer)
                .setColumnFamily(DATA_CF).setReturnKeysOnly().setRowCount(pageSize);

        while (true) {
            query.setKeys(lastKey, null);

            QueryResult<OrderedRows<String, Long, Double>> result = query.execute();
            OrderedRows<String, Long, Double> orderedRows = result.get();
            Iterator<Row<String, Long, Double>> rowsIterator = orderedRows.iterator();

            // we'll skip this first one, since it is the same as the last one
            // from previous time we executed
            if (lastKey != null && rowsIterator != null)
                rowsIterator.next();

            while (rowsIterator.hasNext()) {
                Row<String, Long, Double> row = rowsIterator.next();

                if (!row.getKey().endsWith("-" + bucket)) {
                    continue;
                }

                rowCount++;
                lastKey = row.getKey();

                List<byte[]> keys = new ArrayList<byte[]>();
                keys.add(row.getKey().getBytes());

                MultigetSliceQuery<byte[], Long, Double> multigetSliceQuery = HFactory
                        .createMultigetSliceQuery(keyspace, bytesSerializer, longSerializer, doubleSerializer)
                        .setColumnFamily(DATA_CF).setKeys(keys).setRange(start, end, false, maxColumns);

                QueryResult<Rows<byte[], Long, Double>> colResult = multigetSliceQuery.execute();
                Rows<byte[], Long, Double> rows = colResult.get();

                int sampleCount = 0;
                int deletedCount = 0;
                for (Row<byte[], Long, Double> rowResult : rows) {

                    List<HColumn<Long, Double>> cols = rowResult.getColumnSlice().getColumns();
                    Iterator<HColumn<Long, Double>> listIter = cols.listIterator();

                    while (listIter.hasNext()) {
                        HColumn<Long, Double> c = (HColumn<Long, Double>) listIter.next();

                        if (c.getName() < time) {
                            mutator.addDeletion(row.getKey().getBytes(), DATA_CF, c.getName(), longSerializer);
                            deletedCount++;
                        }
                        sampleCount++;
                    }

                    totalColDeleted += deletedCount;
                    totalColCount += sampleCount;

                    mutator.execute();
                }

                logger.info(row.getKey() + ": " + sampleCount + " deleted: " + deletedCount);
                if (rows.getCount() < pageSize)
                    break;

            }
            logger.info("rows: " + rowCount + " cols: " + totalColCount + " deleted: " + totalColDeleted);

            if (orderedRows.getCount() < pageSize)
                break;

        }

    }

    public List<HSuperColumn<String, String, String>> getChart(String key) {
        SuperSliceQuery<String, String, String, String> q = HFactory.createSuperSliceQuery(keyspace,
                stringSerializer, stringSerializer, stringSerializer, stringSerializer);
        q.setColumnFamily(CHART).setKey(key).setRange(null, null, false, 100);

        // execute query and get result list
        QueryResult<SuperSlice<String, String, String>> result = q.execute();
        return result.get().getSuperColumns();
    }

    public void insert(Mutator<String> mutator, String key, Map<String, String> columnsMap, String columnFamily,
            String superColumn) {
        List<HColumn<String, String>> columns = new ArrayList<HColumn<String, String>>();
        for (String name : columnsMap.keySet()) {
            HFactory.createStringColumn(name, columnsMap.get(name));
        }
        mutator.insert(key, columnFamily, HFactory.createSuperColumn(superColumn, columns, stringSerializer,
                stringSerializer, stringSerializer));
    }

    public Mutator<String> newMutator() {
        return createMutator(keyspace, stringSerializer);
    }

    public void execute(Mutator<String> mutator) {
        mutator.execute();
    }

    public void writeSampleToHeaderAndBuckets(String key, long endTime, PerfHeader header, Map<String, Double> data,
            HashMap<String, PerfEvent> perfEventMap) throws IOException {

        StringBuilder pendingKeys = new StringBuilder("");
        Mutator<byte[]> mutator = createMutator(keyspace, bytesSerializer);

        phd.putHeader(key, header, mutator);
        // write the buckets / archives
        for (String dsRraTime : data.keySet()) {
            // only supporting avg due to volume
            if (!dsRraTime.contains("rra-average"))
                continue;

            String[] dsRraTimeParts = dsRraTime.split("::");
            String dsRra = dsRraTimeParts[0];
            String bucketKey = key + ":" + dsRra;
            long bucketEndTime = Long.parseLong(dsRraTimeParts[1]);

            Double cdpValue = Math.round(data.get(dsRraTime).doubleValue() * 1000.0) / 1000.0;
            if (dsRra.endsWith(LOGBUCKET)) {
                logger.info("write " + bucketKey + " : " + cdpValue);
            }
            String shard = dsRra.substring(dsRra.length() - 3).replace("-", "");
            int ttl = getTTL(shard);
            HColumn<Long, Double> column = createDataColumn(bucketEndTime, cdpValue.doubleValue());
            column.setTtl(ttl);

            String dataCF = DATA_CF + "_" + shard;
            if (isTestMode)
                dataCF += "_test";

            mutator.addInsertion(bucketKey.getBytes(), dataCF, column);
            pendingKeys.append(" ," + bucketKey);

            // send the consolidated perf event to sensor
            PerfEvent pe = null;
            String[] rraParts = dsRra.split("-");
            String eventBucket = rraParts[rraParts.length - 1];
            if (perfEventMap.containsKey(eventBucket)) {
                pe = perfEventMap.get(eventBucket);
            } else {
                pe = setEventBucket(perfEventMap, eventBucket);
            }

            String ds = rraParts[0].replace(":rra", "");
            String rraType = rraParts[1];

            if (rraType.equalsIgnoreCase(AVERAGE)) {
                pe.getMetrics().addAvg(ds, cdpValue);
            } else if (rraType.equalsIgnoreCase(COUNT)) {
                pe.getMetrics().addCount(ds, cdpValue);
            } else if (rraType.equalsIgnoreCase(MAX)) {
                pe.getMetrics().addMax(ds, cdpValue);
            } else if (rraType.equalsIgnoreCase(MIN)) {
                pe.getMetrics().addMin(ds, cdpValue);
            } else if (rraType.equalsIgnoreCase(SUM)) {
                pe.getMetrics().addSum(ds, cdpValue);
            }

        }

        logger.debug("write keys:" + pendingKeys);

        // perform the insert/updates
        mutator.execute();
    }

    private String translateBucket(String bucket) {
        String parts[] = bucket.split("-");
        if (parts[1].equals("avg"))
            return "rra-average-" + parts[0];
        else if (parts[1].equals("min"))
            return "rra-min-" + parts[0];
        else if (parts[1].equals("max"))
            return "rra-max-" + parts[0];

        return "";
    }

    public void writeBucket(PerfEvent perfEvent) throws IOException {
        Mutator<byte[]> mutator = createMutator(keyspace, bytesSerializer);

        String columnKey = perfEvent.getCiId() + ":" + perfEvent.getGrouping();
        String aggregate = translateBucket(perfEvent.getBucket());
        String shard = aggregate.substring(aggregate.length() - 3).replace("-", "");
        int ttl = getTTL(shard);
        String dataCF = DATA_CF + "_" + shard;
        if (isTestMode)
            dataCF += "_test";

        if (perfEvent.getMetrics().getAvg() != null) {
            for (String key : perfEvent.getMetrics().getAvg().keySet()) {
                Double value = perfEvent.getMetrics().getAvg().get(key);
                String bucketKey = columnKey + ":" + key + ":" + aggregate;
                long bucketEndTime = perfEvent.getTimestamp();

                logger.debug("write " + dataCF + ' ' + bucketKey + " " + bucketEndTime + ":" + value);
                HColumn<Long, Double> column = createDataColumn(bucketEndTime, value.doubleValue());
                column.setTtl(ttl);
                mutator.addInsertion(bucketKey.getBytes(), dataCF, column);
            }
        }

        if (perfEvent.getMetrics().getMin() != null) {
            for (String key : perfEvent.getMetrics().getMin().keySet()) {
                Double value = perfEvent.getMetrics().getMin().get(key);
                String bucketKey = columnKey + ":" + key + ":" + aggregate;
                long bucketEndTime = perfEvent.getTimestamp();

                logger.debug("write " + dataCF + ' ' + bucketKey + " " + bucketEndTime + ":" + value);
                HColumn<Long, Double> column = createDataColumn(bucketEndTime, value.doubleValue());
                column.setTtl(ttl);
                mutator.addInsertion(bucketKey.getBytes(), dataCF, column);
            }
        }

        if (perfEvent.getMetrics().getMax() != null) {
            for (String key : perfEvent.getMetrics().getMax().keySet()) {
                Double value = perfEvent.getMetrics().getMax().get(key);
                String bucketKey = columnKey + ":" + key + ":" + aggregate;
                long bucketEndTime = perfEvent.getTimestamp();

                logger.debug("write " + dataCF + ' ' + bucketKey + " " + bucketEndTime + ":" + value);
                HColumn<Long, Double> column = createDataColumn(bucketEndTime, value.doubleValue());
                column.setTtl(ttl);
                mutator.addInsertion(bucketKey.getBytes(), dataCF, column);
            }
        }

        // perform the insert/updates
        mutator.execute();
    }

    private int getTTL(String dsRra) {
        // 1m - 2d
        // 5m - 10d
        // 15m- 30d
        // 1h - 120d
        // 6h - 720d
        // 1d - 2880d
        int ttl = 86400 * 2;
        if (dsRra.equals("1m")) {
            return ttl;
        } else if (dsRra.equals("5m")) {
            ttl = 86400 * 10;
        } else if (dsRra.equals("15m")) {
            ttl = 86400 * 30;
        } else if (dsRra.equals("1h")) {
            ttl = 86400 * 120;
        } else if (dsRra.equals("6h")) {
            ttl = 86400 * 720;
        } else {
            ttl = 86400 * 2880;
        }
        return ttl;
    }

    private PerfEvent setEventBucket(HashMap<String, PerfEvent> perfEventMap, String bucketName) {
        PerfEvent pe = new PerfEvent();
        PerfEvent base = perfEventMap.get("1m");
        pe.setBucket(bucketName);

        pe.setCiId(base.getCiId());
        pe.setGrouping(base.getGrouping());
        pe.setManifestId(base.getManifestId());
        pe.setSource(base.getSource());
        pe.setTimestamp(base.getTimestamp());

        PerfEventPayload load = new PerfEventPayload();
        load.setAvg(new HashMap<>());
        load.setMax(new HashMap<>());
        load.setMin(new HashMap<>());
        load.setSum(new HashMap<>());
        load.setCount(new HashMap<>());
        pe.setMetrics(load);
        perfEventMap.put(bucketName, pe);
        return pe;
    }

    /**
     * sets the long timestamp, double value of a given metric-instance:rra
     */
    private HColumn<Long, Double> createDataColumn(long name, double value) {
        return HFactory.createColumn(name, value, longSerializer, doubleSerializer);
    }

    /**
     * Gets the perf data table.
     *
     * @param req the req
     * @return the perf data table
     */
    @SuppressWarnings("unchecked")
    public String getPerfDataTable(PerfDataRequest req) {

        Long start = Long.valueOf(req.getStart());
        Long end = Long.valueOf(req.getEnd());
        int maxColumns = (int) (end - start);
        String jsonOut = "";

        try {
            long startTime = System.currentTimeMillis();
            String stat = "rra-average";
            if (req.getStat_function() != null) {
                stat = "rra-" + req.getStat_function();
            }

            String rra = getRraByStat(stat, req.getStep());
            int step = alignRraStep(req.getStep());

            MultigetSliceQuery<byte[], Long, Double> multigetSliceQuery = HFactory
                    .createMultigetSliceQuery(keyspace, bytesSerializer, longSerializer, doubleSerializer);
            multigetSliceQuery.setColumnFamily(DATA_CF);
            List<byte[]> keys = new ArrayList<>();
            StringBuilder keyString = new StringBuilder("");
            for (int i = 0; i < req.getMetrics().length; i++) {
                String metricDs = req.getMetrics()[i];
                String key = Long.valueOf(req.getCi_id()).toString() + ":" + metricDs + ":" + rra;
                keys.add(key.getBytes());
                keyString.append(" ").append(key);
            }
            multigetSliceQuery.setKeys(keys);
            long adjustedStart = bucketize(start, step);
            multigetSliceQuery.setRange(adjustedStart, end, false, maxColumns);

            multigetSliceQuery.setRange(start, end, false, maxColumns);

            logger.debug("start:" + start + " end:" + end + " for: " + keyString);
            long cassStart = System.currentTimeMillis();

            QueryResult<Rows<byte[], Long, Double>> result = multigetSliceQuery.execute();
            Rows<byte[], Long, Double> rows = result.get();

            long cassEnd = System.currentTimeMillis();
            long cassDuration = cassEnd - cassStart;

            Map<String, Object> valMap = new HashMap<>();
            Map<Long, Integer> timeMap = new TreeMap<>();
            List<String> rowKeys = new ArrayList<>();

            // put the by-metric results into 1 csv-like table
            // (time,metric1,metric2,etc)
            // ... should find faster way to do this, but still 10x faster than
            // gwt DataTable serialization
            int rowCount = 0;
            for (Row<byte[], Long, Double> row : rows) {

                String rowKey = new String(row.getKey());
                Map<Long, Double> colMap = new HashMap<>();
                valMap.put(rowKey, colMap);
                rowKeys.add(rowKey);
                jsonOut += ",\"" + rowKey + "\"";

                List<HColumn<Long, Double>> cols = row.getColumnSlice().getColumns();
                Iterator<HColumn<Long, Double>> listIter = cols.listIterator();
                while (listIter.hasNext()) {
                    HColumn<Long, Double> c = (HColumn<Long, Double>) listIter.next();
                    colMap.put(c.getName(), c.getValue());
                    timeMap.put(c.getName(), 1);
                }
                rowCount++;
            }
            jsonOut += "]\n";

            int resultRowCount = 0;
            for (Long time : timeMap.keySet()) {
                jsonOut += ",[" + time;

                for (int i = 0; i < rowCount; i++) {
                    String rowKey = rowKeys.get(i);
                    Map<Long, Double> colMap = (Map<Long, Double>) valMap.get(rowKey);
                    Double val = colMap.get(time);
                    jsonOut += ",";
                    // round to 1000ths ; client doesn't need to process all the
                    // 10 decimal point resolution
                    if (val != null) {
                        jsonOut += Math.round(val * 1000.0) / 1000.0;
                    }
                }
                jsonOut += "]\n";
                resultRowCount++;
            }

            long endTime = System.currentTimeMillis();
            long duration = endTime - startTime;

            logger.info("getPerfData took: " + duration + " ms (cass query: " + cassDuration + " ms) returning: "
                    + resultRowCount + " rows of " + rowCount + " metrics");

        } catch (HectorException he) {
            he.printStackTrace();
        }

        return jsonOut;
    }

    /**
     * Bucketize.
     *
     * @param timestamp the timestamp
     * @param step      the step
     * @return the long
     */
    public static long bucketize(long timestamp, long step) {
        return timestamp - timestamp % step;
    }

    public PerfHeader getHeader(String columnKey) {
        return phd.getHeader(columnKey);
    }

    public String getHostName() {
        return hostName;
    }

    public boolean isTestMode() {
        return isTestMode;
    }

    public void setTestMode(boolean isTestMode) {
        this.isTestMode = isTestMode;
    }

}