co.cask.cdap.gateway.handlers.metrics.MetricsHandlerTestRun.java Source code

Java tutorial

Introduction

Here is the source code for co.cask.cdap.gateway.handlers.metrics.MetricsHandlerTestRun.java

Source

/*
 * Copyright  2015 Cask Data, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package co.cask.cdap.gateway.handlers.metrics;

import co.cask.cdap.api.metrics.MetricDeleteQuery;
import co.cask.cdap.api.metrics.MetricType;
import co.cask.cdap.api.metrics.MetricValues;
import co.cask.cdap.api.metrics.Metrics;
import co.cask.cdap.api.metrics.MetricsContext;
import co.cask.cdap.app.metrics.MapReduceMetrics;
import co.cask.cdap.app.metrics.ProgramUserMetrics;
import co.cask.cdap.proto.MetricQueryResult;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.http.HttpResponse;
import org.apache.http.util.EntityUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

import java.lang.reflect.Type;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

/**
 * Search available contexts and metrics tests
 */
public class MetricsHandlerTestRun extends MetricsSuiteTestBase {
    private static final Gson GSON = new Gson();

    private static final List<String> FLOW_TAGS_HUMAN = ImmutableList.of("namespace", "app", "flow", "flowlet");

    private static long emitTs;

    @Before
    public void setup() throws Exception {
        setupMetrics();
    }

    private static void setupMetrics() throws Exception {
        // Adding metrics for app "WordCount1" in namespace "myspace", "WCount1" in "yourspace"
        MetricsContext collector = collectionService
                .getContext(getFlowletContext("myspace", "WordCount1", "WordCounter", "run1", "splitter"));
        collector.increment("reads", 1);
        collector.increment("writes", 1);
        collector = collectionService
                .getContext(getFlowletContext("yourspace", "WCount1", "WordCounter", "run1", "splitter"));
        collector.increment("reads", 1);
        collector = collectionService
                .getContext(getFlowletContext("yourspace", "WCount1", "WCounter", "run1", "splitter"));
        emitTs = System.currentTimeMillis();
        // we want to emit in two different seconds
        // todo : figure out why we need this
        TimeUnit.SECONDS.sleep(1);
        collector.increment("reads", 1);
        TimeUnit.MILLISECONDS.sleep(2000);
        collector.increment("reads", 2);

        collector = collectionService
                .getContext(getFlowletContext("yourspace", "WCount1", "WCounter", "run1", "counter"));
        collector.increment("reads", 1);
        collector = collectionService.getContext(getMapReduceTaskContext("yourspace", "WCount1", "ClassicWordCount",
                MapReduceMetrics.TaskType.Mapper, "run1", "task1"));
        collector.increment("reads", 1);
        collector = collectionService.getContext(getMapReduceTaskContext("yourspace", "WCount1", "ClassicWordCount",
                MapReduceMetrics.TaskType.Reducer, "run1", "task2"));
        collector.increment("reads", 1);
        collector = collectionService
                .getContext(getFlowletContext("myspace", "WordCount1", "WordCounter", "run1", "splitter"));
        collector.increment("reads", 1);
        collector.increment("writes", 1);

        collector = collectionService
                .getContext(getFlowletContext("myspace", "WordCount1", "WordCounter", "run1", "collector"));
        collector.increment("aa", 1);
        collector.increment("zz", 1);
        collector.increment("ab", 1);

        collector = collectionService
                .getContext(getWorkerContext("yourspace", "WCount1", "WorkerWordCount", "run1", "task1"));

        collector.increment("workerreads", 5);
        collector.increment("workerwrites", 6);

        collector = collectionService
                .getContext(getWorkerContext("yourspace", "WCount1", "WorkerWordCount", "run2", "task1"));

        collector.increment("workerreads", 5);
        collector.increment("workerwrites", 6);

        // also: user metrics
        Metrics userMetrics = new ProgramUserMetrics(collectionService
                .getContext(getFlowletContext("myspace", "WordCount1", "WordCounter", "run1", "splitter")));
        userMetrics.count("reads", 1);
        userMetrics.count("writes", 2);

        collector = collectionService.getContext(new HashMap<String, String>());
        collector.increment("resources.total.storage", 10);

        // need a better way to do this
        TimeUnit.SECONDS.sleep(2);
    }

    private List<Map<String, String>> getSearchResultExpected(String... searchExpectations) {
        // iterate search results, add them as name-value map to the result list and return the list
        List<Map<String, String>> result = Lists.newArrayList();
        for (int i = 0; i < searchExpectations.length; i += 2) {
            result.add(ImmutableMap.of("name", searchExpectations[i], "value", searchExpectations[i + 1]));
        }
        return result;
    }

    @Test
    public void testSearchWithTags() throws Exception {
        // empty context
        verifySearchResultWithTags("/v3/metrics/search?target=tag",
                getSearchResultExpected("namespace", "myspace", "namespace", "yourspace", "namespace", "system"));

        // WordCount is in myspace, WCount in yourspace
        verifySearchResultWithTags("/v3/metrics/search?target=tag&tag=namespace:myspace",
                getSearchResultExpected("app", "WordCount1"));
        verifySearchResultWithTags("/v3/metrics/search?target=tag&tag=namespace:yourspace",
                getSearchResultExpected("app", "WCount1"));

        verifySearchResultWithTags("/v3/metrics/search?target=tag&tag=namespace:yourspace",
                getSearchResultExpected("app", "WCount1"));

        // WordCount should be found in myspace, not in yourspace
        verifySearchResultWithTags("/v3/metrics/search?target=tag&tag=namespace:myspace&tag=app:WordCount1",
                getSearchResultExpected("flow", "WordCounter"));

        verifySearchResultWithTags("/v3/metrics/search?target=tag&tag=namespace:yourspace&tag=app:WordCount1",
                getSearchResultExpected());

        // WCount should be found in yourspace, not in myspace
        verifySearchResultWithTags("/v3/metrics/search?target=tag&tag=namespace:yourspace&tag=app:WCount1",
                getSearchResultExpected("flow", "WCounter", "flow", "WordCounter", "mapreduce", "ClassicWordCount",
                        "worker", "WorkerWordCount"));

        verifySearchResultWithTags("/v3/metrics/search?target=tag&tag=namespace:myspace&tag=app:WCount1",
                getSearchResultExpected());

        // verify other metrics for WCount app
        verifySearchResultWithTags("/v3/metrics/search?target=tag&tag=namespace:yourspace&tag=app:WCount1"
                + "&tag=mapreduce:ClassicWordCount", getSearchResultExpected("run", "run1"));

        verifySearchResultWithTags(
                "/v3/metrics/search?target=tag&tag=namespace:yourspace&tag=app:WCount1"
                        + "&tag=mapreduce:ClassicWordCount&tag=run:run1",
                getSearchResultExpected("tasktype", "m", "tasktype", "r"));

        verifySearchResultWithTags(
                "/v3/metrics/search?target=tag&tag=namespace:yourspace&tag=app:WCount1"
                        + "&tag=mapreduce:ClassicWordCount&tag=run:run1&tag=tasktype:m",
                getSearchResultExpected("instance", "task1"));

        verifySearchResultWithTags(
                "/v3/metrics/search?target=tag&tag=namespace:yourspace&tag=app:WCount1"
                        + "&tag=mapreduce:ClassicWordCount&tag=run:run1&tag=tasktype:m&tag=instance:task1",
                getSearchResultExpected());

        verifySearchResultWithTags("/v3/metrics/search?target=tag&tag=namespace:yourspace&tag=app:WCount1"
                + "&tag=worker:WorkerWordCount", getSearchResultExpected("run", "run1", "run", "run2"));
        verifySearchResultWithTags("/v3/metrics/search?target=tag&tag=namespace:yourspace&tag=app:WCount1"
                + "&tag=worker:WorkerWordCount&tag=run:run1", getSearchResultExpected("instance", "task1"));

        // verify "*"

        verifySearchResultWithTags("/v3/metrics/search?target=tag&tag=namespace:*",
                getSearchResultExpected("app", "WordCount1", "app", "WCount1", "component", "metrics.processor"));

        verifySearchResultWithTags("/v3/metrics/search?target=tag&tag=namespace:*&tag=app:*",
                getSearchResultExpected("flow", "WCounter", "flow", "WordCounter", "mapreduce", "ClassicWordCount",
                        "worker", "WorkerWordCount"));

        verifySearchResultWithTags("/v3/metrics/search?target=tag&tag=namespace:*&tag=app:*&tag=flow:*",
                getSearchResultExpected("run", "run1"));
    }

    @Test
    public void testAggregateQueryBatch() throws Exception {

        QueryRequestFormat query1 = new QueryRequestFormat(
                getTagsMap("namespace", "yourspace", "app", "WCount1", "flow", "WCounter", "flowlet", "splitter"),
                ImmutableList.of("system.reads"), ImmutableList.<String>of(), ImmutableMap.<String, String>of());

        // empty time range should default to aggregate=true
        QueryRequestFormat query2 = new QueryRequestFormat(
                getTagsMap("namespace", "yourspace", "app", "WCount1", "flow", "WCounter", "flowlet", "counter"),
                ImmutableList.of("system.reads"), ImmutableList.<String>of(), ImmutableMap.of("aggregate", "true"));

        QueryRequestFormat query3 = new QueryRequestFormat(
                getTagsMap("namespace", "yourspace", "app", "WCount1", "flow", "WCounter", "flowlet", "*"),
                ImmutableList.of("system.reads"), ImmutableList.<String>of(), ImmutableMap.of("aggregate", "true"));

        QueryRequestFormat query4 = new QueryRequestFormat(
                ImmutableMap.of("namespace", "myspace", "app", "WordCount1", "flow", "WordCounter", "flowlet",
                        "splitter"),
                ImmutableList.of("system.reads", "system.writes"), ImmutableList.<String>of(),
                ImmutableMap.of("aggregate", "true"));

        // test batching of multiple queries

        String resolution = Integer.MAX_VALUE + "s";

        ImmutableMap<String, QueryResult> expected = ImmutableMap.of("testQuery1", new QueryResult(
                ImmutableList.of(new TimeSeriesSummary(ImmutableMap.<String, String>of(), "system.reads", 1, 3)),
                resolution), "testQuery2",
                new QueryResult(
                        ImmutableList
                                .of(new TimeSeriesSummary(ImmutableMap.<String, String>of(), "system.reads", 1, 1)),
                        resolution));

        batchTest(ImmutableMap.of("testQuery1", query1, "testQuery2", query2), expected);

        // test batching of multiple queries, with one query having multiple metrics to query

        expected = ImmutableMap.of("testQuery3",
                new QueryResult(ImmutableList
                        .of(new TimeSeriesSummary(ImmutableMap.<String, String>of(), "system.reads", 1, 4)),
                        resolution),
                "testQuery4",
                new QueryResult(
                        ImmutableList.of(
                                new TimeSeriesSummary(ImmutableMap.<String, String>of(), "system.reads", 1, 2),
                                new TimeSeriesSummary(ImmutableMap.<String, String>of(), "system.writes", 1, 2)),
                        resolution));

        batchTest(ImmutableMap.of("testQuery3", query3, "testQuery4", query4), expected);
    }

    @Test
    public void testInvalidRequest() throws Exception {
        // test invalid request - query without any query Params and body content
        HttpResponse response = doPost("/v3/metrics/query", null);
        Assert.assertEquals(400, response.getStatusLine().getStatusCode());

        // batch query with empty metrics list
        QueryRequestFormat invalidQuery = new QueryRequestFormat(
                ImmutableMap.of("namespace", "myspace", "app", "WordCount1", "flow", "WordCounter", "flowlet",
                        "splitter"),
                ImmutableList.<String>of(), ImmutableList.<String>of(), ImmutableMap.of("aggregate", "true"));

        response = doPost("/v3/metrics/query", GSON.toJson(ImmutableMap.of("invalid", invalidQuery)));
        Assert.assertEquals(400, response.getStatusLine().getStatusCode());

        // test invalid request - query without any metric Params
        response = doPost("/v3/metrics/query?context=namespace.default", null);
        Assert.assertEquals(400, response.getStatusLine().getStatusCode());
        response = doPost("/v3/metrics/query?tag=namespace.default", null);
        Assert.assertEquals(400, response.getStatusLine().getStatusCode());
    }

    private Map<String, String> getTagsMap(String... entries) {
        Map<String, String> tagsMap = Maps.newHashMap();
        for (int i = 0; i < entries.length; i += 2) {
            tagsMap.put(entries[i], entries[i + 1]);
        }
        return tagsMap;
    }

    /**
     * Helper class to construct json for MetricQueryRequest for batch queries
     */
    private class QueryRequestFormat {
        Map<String, String> tags;
        List<String> metrics;
        List<String> groupBy;
        Map<String, String> timeRange;

        QueryRequestFormat(Map<String, String> tags, List<String> metrics, List<String> groupBy,
                Map<String, String> timeRange) {
            this.tags = tags;
            this.metrics = metrics;
            this.groupBy = groupBy;
            this.timeRange = timeRange;
        }
    }

    private class QueryResult {
        private ImmutableList<TimeSeriesSummary> expectedList;
        private String expectedResolution;

        public QueryResult(ImmutableList<TimeSeriesSummary> list, String resolution) {
            expectedList = list;
            expectedResolution = resolution;
        }

        public String getExpectedResolution() {
            return expectedResolution;
        }

        public ImmutableList<TimeSeriesSummary> getExpectedList() {
            return expectedList;
        }

    }

    @Test
    public void testTimeRangeQueryBatch() throws Exception {
        // note: times are in seconds, hence "divide by 1000";
        long start = (emitTs - 60 * 1000) / 1000;
        long end = (emitTs + 60 * 1000) / 1000;
        int count = 120;

        QueryRequestFormat query1 = new QueryRequestFormat(
                getTagsMap("namespace", "yourspace", "app", "WCount1", "flow", "WCounter", "flowlet", "splitter"),
                ImmutableList.of("system.reads"), ImmutableList.<String>of(),
                ImmutableMap.of("start", String.valueOf(start), "end", String.valueOf(end)));

        QueryRequestFormat query2 = new QueryRequestFormat(
                getTagsMap("namespace", "yourspace", "app", "WCount1", "flow", "WCounter"),
                ImmutableList.of("system.reads"), ImmutableList.of("flowlet"),
                ImmutableMap.of("start", String.valueOf(start), "count", String.valueOf(count)));

        ImmutableMap<String, QueryResult> expected = ImmutableMap.of("timeRangeQuery1",
                new QueryResult(ImmutableList
                        .of(new TimeSeriesSummary(ImmutableMap.<String, String>of(), "system.reads", 2, 3)), "1s"),
                "timeRangeQuery2",
                new QueryResult(ImmutableList.of(
                        new TimeSeriesSummary(ImmutableMap.of("flowlet", "counter"), "system.reads", 1, 1),
                        new TimeSeriesSummary(ImmutableMap.of("flowlet", "splitter"), "system.reads", 2, 3)),
                        "1s"));

        Map<String, QueryRequestFormat> batchQueries = ImmutableMap.of("timeRangeQuery1", query1, "timeRangeQuery2",
                query2);
        batchTest(batchQueries, expected);
    }

    @Test
    public void testMultipleMetricsSingleContext() throws Exception {
        verifyAggregateQueryResult(
                "/v3/metrics/query?tag=namespace:myspace&tag=app:WordCount1&tag=flow:WordCounter&tag=flowlet:splitter"
                        + "&metric=system.reads&metric=system.writes&aggregate=true",
                ImmutableList.of(2L, 2L));

        long start = (emitTs - 60 * 1000) / 1000;
        long end = (emitTs + 300 * 1000) / 1000;
        verifyRangeQueryResult(
                "/v3/metrics/query?tag=namespace:myspace&tag=app:WordCount1&tag=flow:WordCounter&tag=flowlet:collector"
                        + "&metric=system.aa&metric=system.ab&metric=system.zz&start=" + start + "&end=" + end,
                ImmutableList.of(1L, 1L, 1L), ImmutableList.of(1L, 1L, 1L));
    }

    @Test
    public void testQueryMetricsWithTags() throws Exception {
        //aggregate result, in the right namespace
        verifyAggregateQueryResult("/v3/metrics/query?" + getTags("yourspace", "WCount1", "WCounter", "splitter")
                + "&metric=system.reads&aggregate=true", 3);
        verifyAggregateQueryResult("/v3/metrics/query?" + getTags("yourspace", "WCount1", "WCounter", "counter")
                + "&metric=system.reads&aggregate=true", 1);

        verifyAggregateQueryResult("/v3/metrics/query?" + getTags("yourspace", "WCount1", "WCounter", "*")
                + "&metric=system.reads&aggregate=true", 4);

        verifyAggregateQueryResult("/v3/metrics/query?" + getTags("yourspace", "WCount1", "WCounter")
                + "&metric=system.reads&aggregate=true", 4);

        // aggregate result, in the wrong namespace
        verifyEmptyQueryResult("/v3/metrics/query?" + getTags("myspace", "WCount1", "WCounter", "splitter")
                + "&metric=system.reads&aggregate=true");

        // time range
        // now-60s, now+60s
        verifyRangeQueryResult("/v3/metrics/query?" + getTags("yourspace", "WCount1", "WCounter", "splitter")
                + "&metric=system.reads&start=now%2D60s&end=now%2B60s", 2, 3);

        // note: times are in seconds, hence "divide by 1000";
        long start = (emitTs - 60 * 1000) / 1000;
        long end = (emitTs + 60 * 1000) / 1000;
        verifyRangeQueryResult("/v3/metrics/query?" + getTags("yourspace", "WCount1", "WCounter", "splitter")
                + "&metric=system.reads&start=" + start + "&end=" + end, 2, 3);
        // range query, in the wrong namespace
        verifyEmptyQueryResult("/v3/metrics/query?" + getTags("myspace", "WCount1", "WCounter", "splitter")
                + "&metric=system.reads&start=" + start + "&end=" + end);

        List<TimeSeriesResult> groupByResult = ImmutableList.of(
                new TimeSeriesResult(ImmutableMap.of("flowlet", "counter"), 1),
                new TimeSeriesResult(ImmutableMap.of("flowlet", "splitter"), 3));

        verifyGroupByResult("/v3/metrics/query?" + getTags("yourspace", "WCount1", "WCounter")
                + "&metric=system.reads&groupBy=flowlet&start=" + start + "&end=" + end, groupByResult);

        groupByResult = ImmutableList.of(
                new TimeSeriesResult(ImmutableMap.of("namespace", "myspace", "flowlet", "splitter"), 2),
                new TimeSeriesResult(ImmutableMap.of("namespace", "yourspace", "flowlet", "counter"), 1),
                new TimeSeriesResult(ImmutableMap.of("namespace", "yourspace", "flowlet", "splitter"), 4));

        verifyGroupByResult("/v3/metrics/query?metric=system.reads" + "&groupBy=namespace&groupBy=flowlet&start="
                + start + "&end=" + end, groupByResult);
    }

    @Test
    public void testInterpolate() throws Exception {
        long start = System.currentTimeMillis() / 1000;
        long end = start + 3;
        Map<String, String> sliceBy = getFlowletContext("interspace", "WordCount1", "WordCounter", "run1",
                "splitter");
        MetricValues value = new MetricValues(sliceBy, "reads", start, 100, MetricType.COUNTER);
        metricStore.add(value);

        value = new MetricValues(sliceBy, "reads", end, 400, MetricType.COUNTER);
        metricStore.add(value);

        verifyRangeQueryResult("/v3/metrics/query?" + getTags("interspace", "WordCount1", "WordCounter", "splitter")
                + "&metric=system.reads&interpolate=step&start=" + start + "&end=" + end, 4, 700);

        verifyRangeQueryResult("/v3/metrics/query?" + getTags("interspace", "WordCount1", "WordCounter", "splitter")
                + "&metric=system.reads&interpolate=linear&start=" + start + "&end=" + end, 4, 1000);

        // delete the added metrics for testing interpolator
        MetricDeleteQuery deleteQuery = new MetricDeleteQuery(start, end, sliceBy);
        metricStore.delete(deleteQuery);
    }

    @Test
    public void testResolutionInResponse() throws Exception {
        long start = 1;

        String url = "/v3/metrics/query?" + getTags("resolutions", "WordCount1", "WordCounter", "splitter")
                + "&metric=system.reads&resolution=auto&start=" + (start - 1) + "&end=" + (start + 36000);
        MetricQueryResult queryResult = post(url, MetricQueryResult.class);
        Assert.assertEquals("3600s", queryResult.getResolution());

        url = "/v3/metrics/query?" + getTags("resolutions", "WordCount1", "WordCounter", "splitter")
                + "&metric=system.reads&resolution=1m&start=" + (start - 1) + "&end=" + (start + 36000);
        queryResult = post(url, MetricQueryResult.class);
        Assert.assertEquals("60s", queryResult.getResolution());

        // Have an aggregate query and ensure that its resolution is INT_MAX
        url = "/v3/metrics/query?" + getTags("WordCount1", "WordCounter", "splitter") + "&metric=system.reads";
        queryResult = post(url, MetricQueryResult.class);
        Assert.assertEquals(Integer.MAX_VALUE + "s", queryResult.getResolution());

    }

    @Test
    public void testAutoResolutions() throws Exception {
        long start = 1;
        Map<String, String> sliceBy = getFlowletContext("resolutions", "WordCount1", "WordCounter", "run1",
                "splitter");

        // 1 second
        metricStore.add(new MetricValues(sliceBy, "reads", start, 1, MetricType.COUNTER));
        // 30 second
        metricStore.add(new MetricValues(sliceBy, "reads", start + 30, 1, MetricType.COUNTER));
        // 1 minute
        metricStore.add(new MetricValues(sliceBy, "reads", start + 60, 1, MetricType.COUNTER));
        // 10 minutes
        metricStore.add(new MetricValues(sliceBy, "reads", start + 600, 1, MetricType.COUNTER));
        // 1 hour
        metricStore.add(new MetricValues(sliceBy, "reads", start + 3600, 1, MetricType.COUNTER));
        // 10 hour
        metricStore.add(new MetricValues(sliceBy, "reads", start + 36000, 1, MetricType.COUNTER));

        // seconds
        verifyRangeQueryResult(
                "/v3/metrics/query?" + getTags("resolutions", "WordCount1", "WordCounter", "splitter")
                        + "&metric=system.reads&resolution=auto&start=" + start + "&end=" + (start + 600),
                4, 4);

        // minutes
        verifyRangeQueryResult(
                "/v3/metrics/query?" + getTags("resolutions", "WordCount1", "WordCounter", "splitter")
                        + "&metric=system.reads&resolution=auto&start=" + (start - 1) + "&end=" + (start + 600),
                3, 4);

        // minutes
        verifyRangeQueryResult(
                "/v3/metrics/query?" + getTags("resolutions", "WordCount1", "WordCounter", "splitter")
                        + "&metric=system.reads&resolution=auto&start=" + (start - 1) + "&end=" + (start + 3600),
                4, 5);

        // hours
        verifyRangeQueryResult(
                "/v3/metrics/query?" + getTags("resolutions", "WordCount1", "WordCounter", "splitter")
                        + "&metric=system.reads&resolution=auto&start=" + (start - 1) + "&end=" + (start + 36000),
                3, 6);

        // delete the added metrics for testing auto resolutions
        MetricDeleteQuery deleteQuery = new MetricDeleteQuery(start, (start + 36000), sliceBy);
        metricStore.delete(deleteQuery);
    }

    private void verifyGroupByResult(String url, List<TimeSeriesResult> groupByResult) throws Exception {
        MetricQueryResult result = post(url, MetricQueryResult.class);
        Assert.assertEquals(groupByResult.size(), result.getSeries().length);
        for (MetricQueryResult.TimeSeries timeSeries : result.getSeries()) {
            boolean timeSeriesMatchFound = false;
            for (TimeSeriesResult expectedTs : groupByResult) {
                if (expectedTs.getTagValues().equals(ImmutableMap.copyOf(timeSeries.getGrouping()))) {
                    assertTimeValues(expectedTs, timeSeries);
                    timeSeriesMatchFound = true;
                }
            }
            Assert.assertTrue(timeSeriesMatchFound);
        }
    }

    private void assertTimeValues(TimeSeriesResult expected, MetricQueryResult.TimeSeries actual) {
        long expectedValue = expected.getTimeSeriesSum();
        for (MetricQueryResult.TimeValue timeValue : actual.getData()) {
            expectedValue -= timeValue.getValue();
        }
        Assert.assertEquals(0, expectedValue);
    }

    private void verifyEmptyQueryResult(String url) throws Exception {
        MetricQueryResult queryResult = post(url, MetricQueryResult.class);
        Assert.assertEquals(0, queryResult.getSeries().length);
    }

    private String getTags(String... tags) {
        String result = "";
        for (int i = 0; i < tags.length; i++) {
            result += "&tag=" + FLOW_TAGS_HUMAN.get(i) + ":" + tags[i];
        }
        return result;
    }

    @Test
    public void testSearchMetricsWithTags() throws Exception {
        // metrics in myspace
        verifySearchMetricResult(
                "/v3/metrics/search?target=metric&tag=namespace:myspace&tag=app:WordCount1"
                        + "&tag=flow:WordCounter&tag=dataset:*&tag=run:run1&tag=flowlet:splitter",
                ImmutableList.of("system.reads", "system.writes", "user.reads", "user.writes"));

        verifySearchMetricResult(
                "/v3/metrics/search?target=metric&tag=namespace:myspace&tag=app:WordCount1"
                        + "&tag=flow:WordCounter&tag=dataset:*&tag=run:run1&tag=flowlet:collector",
                ImmutableList.of("system.aa", "system.ab", "system.zz"));

        verifySearchMetricResult(
                "/v3/metrics/search?target=metric&tag=namespace:myspace&tag=app:WordCount1"
                        + "&tag=flow:WordCounter&tag=dataset:*&tag=run:run1",
                ImmutableList.of("system.aa", "system.ab", "system.reads", "system.writes", "system.zz",
                        "user.reads", "user.writes"));

        // wrong namespace
        verifySearchMetricResult(
                "/v3/metrics/search?target=metric&tag=namespace:yourspace&tag=app:WordCount1"
                        + "&tag=flow:WordCounter&tag=dataset:*&tag=run:run1&tag=flowlet:splitter",
                ImmutableList.<String>of());

        // metrics in yourspace
        verifySearchMetricResult(
                "/v3/metrics/search?target=metric&tag=namespace:yourspace&tag=app:WCount1"
                        + "&tag=flow:WCounter&tag=dataset:*&tag=run:run1&tag=flowlet:splitter",
                ImmutableList.of("system.reads"));

        verifySearchMetricResult("/v3/metrics/search?target=metric&tag=namespace:yourspace",
                ImmutableList.of("system.reads", "system.workerreads", "system.workerwrites"));

        // wrong namespace
        verifySearchMetricResult(
                "/v3/metrics/search?target=metric&tag=namespace:myspace&tag=app:WCount1"
                        + "&tag=flow:WCounter&tag=dataset:*&tag=run:run1&tag=flowlet:splitter",
                ImmutableList.<String>of());

        // verify "*"
        verifySearchMetricResult(
                "/v3/metrics/search?target=metric&tag=namespace:myspace&tag=app:WordCount1"
                        + "&tag=flow:WordCounter&tag=dataset:*&tag=run:run1&tag=flowlet:*",
                ImmutableList.of("system.aa", "system.ab", "system.reads", "system.writes", "system.zz",
                        "user.reads", "user.writes"));
        verifySearchMetricResult(
                "/v3/metrics/search?target=metric&tag=namespace:myspace&tag=app:WordCount1"
                        + "&tag=flow:*&tag=dataset:*&tag=run:run1",
                ImmutableList.of("system.aa", "system.ab", "system.reads", "system.writes", "system.zz",
                        "user.reads", "user.writes"));
    }

    /**
     * Used to test time range queries when requests are batched
     */
    class TimeSeriesSummary {
        Map<String, String> grouping;
        String metricName;
        long numPoints;
        long totalSum;

        public TimeSeriesSummary(Map<String, String> grouping, String metricName, long numPoints, long totalSum) {
            this.grouping = grouping;
            this.metricName = metricName;
            this.numPoints = numPoints;
            this.totalSum = totalSum;
        }

        public long getTotalSum() {
            return totalSum;
        }

        public long getNumPoints() {
            return numPoints;
        }

        public String getMetricName() {
            return metricName;
        }

        public Map<String, String> getGrouping() {
            return grouping;
        }
    }

    private void batchTest(Map<String, QueryRequestFormat> jsonBatch, ImmutableMap<String, QueryResult> expected)
            throws Exception {
        String url = "/v3/metrics/query";
        Map<String, MetricQueryResult> results = post(url, GSON.toJson(jsonBatch),
                new TypeToken<Map<String, MetricQueryResult>>() {
                }.getType());

        // check we have all the keys
        Assert.assertEquals(expected.keySet(), results.keySet());
        for (Map.Entry<String, MetricQueryResult> entry : results.entrySet()) {
            ImmutableList<TimeSeriesSummary> expectedTimeSeriesSummary = expected.get(entry.getKey())
                    .getExpectedList();
            MetricQueryResult actualQueryResult = entry.getValue();
            compareQueryResults(expectedTimeSeriesSummary, actualQueryResult);
            Assert.assertEquals(expected.get(entry.getKey()).getExpectedResolution(),
                    actualQueryResult.getResolution());
        }
    }

    private void compareQueryResults(ImmutableList<TimeSeriesSummary> expected, MetricQueryResult actual) {

        MetricQueryResult.TimeSeries[] actualTimeSeries = actual.getSeries();
        for (MetricQueryResult.TimeSeries actualTimeSery : actualTimeSeries) {
            TimeSeriesSummary expectedTimeSeriesSummary = findExpectedQueryResult(expected, actualTimeSery);
            Assert.assertNotNull(expectedTimeSeriesSummary);
            MetricQueryResult.TimeValue[] values = actualTimeSery.getData();
            long numPoints = 0;
            long totalSum = 0;
            for (MetricQueryResult.TimeValue tv : values) {
                numPoints++;
                totalSum += tv.getValue();
            }
            Assert.assertEquals(expectedTimeSeriesSummary.getNumPoints(), numPoints);
            Assert.assertEquals(expectedTimeSeriesSummary.getTotalSum(), totalSum);
        }
    }

    private TimeSeriesSummary findExpectedQueryResult(ImmutableList<TimeSeriesSummary> expected,
            MetricQueryResult.TimeSeries actualTimeSeries) {
        for (TimeSeriesSummary result : expected) {
            if (result.getGrouping().equals(actualTimeSeries.getGrouping())
                    && result.getMetricName().equals(actualTimeSeries.getMetricName())) {
                return result;
            }
        }

        return null;
    }

    private void verifyAggregateQueryResult(String url, List<Long> expectedValue) throws Exception {
        MetricQueryResult queryResult = post(url, MetricQueryResult.class);
        for (int i = 0; i < queryResult.getSeries().length; i++) {
            Assert.assertEquals(expectedValue.get(i), (Long) queryResult.getSeries()[i].getData()[0].getValue());
        }
    }

    private void verifyRangeQueryResult(String url, List<Long> expectedPoints, List<Long> expectedSum)
            throws Exception {
        MetricQueryResult queryResult = post(url, MetricQueryResult.class);
        for (int i = 0; i < queryResult.getSeries().length; i++) {
            verifyTimeSeries(queryResult.getSeries()[i], expectedPoints.get(i), expectedSum.get(i));
        }
    }

    @Test
    public void testResultLimit() throws Exception {
        long start = 1;
        Map<String, String> sliceBy = getFlowletContext("resolutions", "WordCount1", "WordCounter", "run1",
                "splitter");

        // 1 second
        metricStore.add(new MetricValues(sliceBy, "reads", start, 1, MetricType.COUNTER));
        // 30 second
        metricStore.add(new MetricValues(sliceBy, "reads", start + 30, 1, MetricType.COUNTER));
        // 1 minute
        metricStore.add(new MetricValues(sliceBy, "reads", start + 60, 1, MetricType.COUNTER));
        // 10 minutes
        metricStore.add(new MetricValues(sliceBy, "reads", start + 600, 1, MetricType.COUNTER));
        // 1 hour
        metricStore.add(new MetricValues(sliceBy, "reads", start + 3600, 1, MetricType.COUNTER));

        // count is one record
        verifyRangeQueryResult(
                "/v3/metrics/query?" + getTags("resolutions", "WordCount1", "WordCounter", "splitter")
                        + "&metric=system.reads&resolution=auto&count=1&start=" + start + "&end=" + (start + 600),
                1, 1);

        // count is greater than data points in time-range
        verifyRangeQueryResult(
                "/v3/metrics/query?" + getTags("resolutions", "WordCount1", "WordCounter", "splitter")
                        + "&metric=system.reads&resolution=auto&count=6&start=" + start + "&end=" + (start + 600),
                4, 4);

        // count is less than data points in time-range
        verifyRangeQueryResult("/v3/metrics/query?"
                + getTags("resolutions", "WordCount1", "WordCounter", "splitter")
                + "&metric=system.reads&resolution=auto&count=2&start=" + (start - 1) + "&end=" + (start + 3600), 2,
                3);
    }

    private void verifyAggregateQueryResult(String url, long expectedValue) throws Exception {
        // todo : can refactor this to test only the new tag name queries once we deprecate queryParam using context.
        MetricQueryResult queryResult = post(url, MetricQueryResult.class);
        Assert.assertEquals(expectedValue, queryResult.getSeries()[0].getData()[0].getValue());
    }

    private void verifyRangeQueryResult(String url, long nonZeroPointsCount, long expectedSum) throws Exception {
        MetricQueryResult queryResult = post(url, MetricQueryResult.class);
        verifyTimeSeries(queryResult.getSeries()[0], nonZeroPointsCount, expectedSum);
    }

    private void verifyTimeSeries(MetricQueryResult.TimeSeries timeSeries, long nonZeroPointsCount,
            long expectedSum) {
        MetricQueryResult.TimeValue[] data = timeSeries.getData();

        for (MetricQueryResult.TimeValue point : data) {
            if (point.getValue() != 0) {
                nonZeroPointsCount--;
                expectedSum -= point.getValue();
            }
        }
        Assert.assertEquals(0, nonZeroPointsCount);
        Assert.assertEquals(0, expectedSum);
    }

    private <T> T post(String url, Type type) throws Exception {
        return post(url, null, type);
    }

    private <T> T post(String url, String body, Type type) throws Exception {
        HttpResponse response = doPost(url, body);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        return GSON.fromJson(EntityUtils.toString(response.getEntity(), Charsets.UTF_8), type);
    }

    private void verifySearchMetricResult(String url, List<String> expectedValues) throws Exception {
        HttpResponse response = doPost(url, null);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        String result = EntityUtils.toString(response.getEntity());
        List<String> reply = GSON.fromJson(result, new TypeToken<List<String>>() {
        }.getType());
        reply = removeMetricsSystemMetrics(reply);
        Assert.assertEquals(expectedValues.size(), reply.size());
        for (int i = 0; i < expectedValues.size(); i++) {
            Assert.assertEquals(expectedValues.get(i), reply.get(i));
        }
    }

    private List<String> removeMetricsSystemMetrics(List<String> reply) {
        List<String> result = Lists.newArrayList();
        for (String metric : reply) {
            if (!metric.startsWith("system.metrics") && !metric.startsWith("user.metrics")) {
                result.add(metric);
            }
        }
        return result;
    }

    private void verifySearchResultWithTags(String url, List<Map<String, String>> expectedValues) throws Exception {
        HttpResponse response = doPost(url, null);
        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
        String result = EntityUtils.toString(response.getEntity(), Charsets.UTF_8);
        List<Map<String, String>> reply = GSON.fromJson(result, new TypeToken<List<Map<String, String>>>() {
        }.getType());
        Assert.assertTrue(reply.containsAll(expectedValues) && expectedValues.containsAll(reply));
    }

    class TimeSeriesResult {
        public Map<String, String> getTagValues() {
            return tagValues;
        }

        public long getTimeSeriesSum() {
            return timeSeriesSum;
        }

        Map<String, String> tagValues;
        long timeSeriesSum;

        TimeSeriesResult(Map<String, String> tagValues, long timeSeriesSum) {
            this.tagValues = tagValues;
            this.timeSeriesSum = timeSeriesSum;
        }
    }

}