Java tutorial
/* * Copyright 2014-2015 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.cdap.metrics.query; import co.cask.cdap.api.dataset.lib.cube.AggregationFunction; import co.cask.cdap.api.dataset.lib.cube.Interpolator; import co.cask.cdap.api.dataset.lib.cube.Interpolators; import co.cask.cdap.api.metrics.MetricDataQuery; import co.cask.cdap.api.metrics.MetricDeleteQuery; import co.cask.cdap.common.conf.Constants; import co.cask.cdap.common.utils.TimeMathParser; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import org.apache.commons.lang.CharEncoding; import org.jboss.netty.handler.codec.http.QueryStringDecoder; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URLDecoder; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; /** * For parsing metrics REST request. */ final class MetricQueryParser { private static final String COUNT = "count"; private static final String START_TIME = "start"; private static final String RESOLUTION = "resolution"; private static final String END_TIME = "end"; private static final String RUN_ID = "runs"; private static final String INTERPOLATE = "interpolate"; private static final String STEP_INTERPOLATOR = "step"; private static final String LINEAR_INTERPOLATOR = "linear"; private static final String MAX_INTERPOLATE_GAP = "maxInterpolateGap"; private static final String TRANSACTION_METRICS_CONTEXT = "transactions"; private static final String AUTO_RESOLUTION = "auto"; public enum PathType { APPS, DATASETS, STREAMS, CLUSTER, SERVICES, SPARK } // we need to duplicate that as we don't have dependency on app-fabric public enum ProgramType { FLOWS("f", Constants.Metrics.Tag.FLOW), MAPREDUCE("b", Constants.Metrics.Tag.MAPREDUCE), HANDLERS("h", Constants.Metrics.Tag.HANDLER), SERVICES("u", Constants.Metrics.Tag.SERVICE), SPARK("s", Constants.Metrics.Tag.SPARK); private final String code; private final String tagName; private ProgramType(String code, String tagName) { this.code = code; this.tagName = tagName; } public String getCode() { return code; } public String getTagName() { return tagName; } } private enum MapReduceType { MAPPERS("m"), REDUCERS("r"); private final String id; private MapReduceType(String id) { this.id = id; } private String getId() { return id; } } enum Resolution { SECOND(1), MINUTE(60), HOUR(3600); private int resolution; private Resolution(int resolution) { this.resolution = resolution; } public int getResolution() { return resolution; } } public enum MetricsScope { SYSTEM, USER } private static String urlDecode(String str) { try { return URLDecoder.decode(str, CharEncoding.UTF_8); } catch (UnsupportedEncodingException e) { throw new IllegalArgumentException("unsupported encoding in path element", e); } } /** * Given a full metrics path like '/v2/metrics/system/apps/collect.events', strip the preceding version and * metrics to return 'system/apps/collect.events', representing the context and metric, which can then be * parsed by this parser. * * @param path request path. * @return request path stripped of version and metrics. */ static String stripVersionAndMetricsFromPath(String path) { // +8 for "/metrics" int startPos = Constants.Gateway.API_VERSION_3.length() + 8; return path.substring(startPos, path.length()); } static MetricDeleteQuery parseDelete(URI requestURI, String metricPrefix) throws MetricsPathException { MetricDataQueryBuilder builder = new MetricDataQueryBuilder(); parseContext(requestURI.getPath(), builder); builder.setStartTs(0); builder.setEndTs(Integer.MAX_VALUE - 1); builder.setMetricName(metricPrefix); MetricDataQuery query = builder.build(); return new MetricDeleteQuery(query.getStartTs(), query.getEndTs(), query.getMetrics().keySet(), query.getSliceByTags()); } static MetricDataQuery parse(URI requestURI) throws MetricsPathException { MetricDataQueryBuilder builder = new MetricDataQueryBuilder(); // metric will be at the end. String uriPath = requestURI.getRawPath(); int index = uriPath.lastIndexOf("/"); builder.setMetricName(urlDecode(uriPath.substring(index + 1))); // strip the metric from the end of the path if (index != -1) { String strippedPath = uriPath.substring(0, index); if (strippedPath.startsWith("/system/cluster")) { builder.setSliceByTagValues( ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, Constants.SYSTEM_NAMESPACE)); builder.setScope("system"); } else if (strippedPath.startsWith("/system/transactions")) { builder.setSliceByTagValues(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, Constants.SYSTEM_NAMESPACE, Constants.Metrics.Tag.COMPONENT, TRANSACTION_METRICS_CONTEXT)); builder.setScope("system"); } else { parseContext(strippedPath, builder); } } else { builder.setSliceByTagValues(Maps.<String, String>newHashMap()); } parseQueryString(requestURI, builder); return builder.build(); } /** * Parse the context path, setting the relevant context fields in the builder. * Context starts after the scope and looks something like: * system/apps/{app-id}/{program-type}/{program-id}/{component-type}/{component-id} */ static void parseContext(String path, MetricDataQueryBuilder builder) throws MetricsPathException { Map<String, String> tagValues = Maps.newHashMap(); Iterator<String> pathParts = Splitter.on('/').omitEmptyStrings().split(path).iterator(); // everything if (!pathParts.hasNext()) { builder.setSliceByTagValues(tagValues); return; } // scope is the first part of the path String scopeStr = pathParts.next(); try { // we do conversion to validate value builder.setScope(MetricsScope.valueOf(scopeStr.toUpperCase()).toString().toLowerCase()); } catch (IllegalArgumentException e) { throw new MetricsPathException("invalid scope: " + scopeStr); } // streams, datasets, apps, or nothing. if (!pathParts.hasNext()) { builder.setSliceByTagValues(tagValues); return; } // apps, streams, or datasets String pathTypeStr = pathParts.next(); PathType pathType; try { pathType = PathType.valueOf(pathTypeStr.toUpperCase()); } catch (IllegalArgumentException e) { throw new MetricsPathException("invalid type: " + pathTypeStr); } switch (pathType) { case APPS: // Note: If v3 APIs use this class, we may have to get namespaceId from higher up tagValues.put(Constants.Metrics.Tag.NAMESPACE, Constants.DEFAULT_NAMESPACE); tagValues.put(Constants.Metrics.Tag.APP, urlDecode(pathParts.next())); parseSubContext(pathParts, tagValues); break; case STREAMS: // Note: If v3 APIs use this class, we may have to get namespaceId from higher up tagValues.put(Constants.Metrics.Tag.NAMESPACE, Constants.DEFAULT_NAMESPACE); if (!pathParts.hasNext()) { throw new MetricsPathException("'streams' must be followed by a stream name"); } tagValues.put(Constants.Metrics.Tag.STREAM, urlDecode(pathParts.next())); break; case DATASETS: // Note: If v3 APIs use this class, we may have to get namespaceId from higher up tagValues.put(Constants.Metrics.Tag.NAMESPACE, Constants.DEFAULT_NAMESPACE); if (!pathParts.hasNext()) { throw new MetricsPathException("'datasets' must be followed by a dataset name"); } tagValues.put(Constants.Metrics.Tag.DATASET, urlDecode(pathParts.next())); // path can be /metric/scope/datasets/{dataset}/apps/... if (pathParts.hasNext()) { if (!pathParts.next().equals("apps")) { throw new MetricsPathException("expecting 'apps' after stream or dataset name"); } tagValues.put(Constants.Metrics.Tag.APP, urlDecode(pathParts.next())); parseSubContext(pathParts, tagValues); } break; case SERVICES: // Note: If v3 APIs use this class, we may have to get namespaceId from higher up tagValues.put(Constants.Metrics.Tag.NAMESPACE, Constants.SYSTEM_NAMESPACE); parseSystemService(pathParts, tagValues); break; } if (pathParts.hasNext()) { throw new MetricsPathException("path contains too many elements: " + path); } builder.setSliceByTagValues(tagValues); } private static void parseSystemService(Iterator<String> pathParts, Map<String, String> tagValues) throws MetricsPathException { if (!pathParts.hasNext()) { throw new MetricsPathException("'services must be followed by a service name"); } tagValues.put(Constants.Metrics.Tag.COMPONENT, urlDecode(pathParts.next())); if (!pathParts.hasNext()) { return; } // skipping "/handlers" String next = pathParts.next(); if (!"handlers".equals(next)) { throw new MetricsPathException("'handlers must be followed by a service name"); } tagValues.put(Constants.Metrics.Tag.HANDLER, urlDecode(pathParts.next())); if (!pathParts.hasNext()) { return; } // skipping "/runs" next = pathParts.next(); if (RUN_ID.equals(next)) { tagValues.put(Constants.Metrics.Tag.RUN_ID, urlDecode(pathParts.next())); if (!pathParts.hasNext()) { return; } // skipping "/methods" pathParts.next(); } tagValues.put(Constants.Metrics.Tag.METHOD, urlDecode(pathParts.next())); } /** * pathParts should look like {app-id}/{program-type}/{program-id}/{component-type}/{component-id}. */ static void parseSubContext(Iterator<String> pathParts, Map<String, String> tagValues) throws MetricsPathException { if (!pathParts.hasNext()) { return; } // request-type: flows, mapreduce or handlers or services(user) String pathProgramTypeStr = pathParts.next(); ProgramType programType; try { programType = ProgramType.valueOf(pathProgramTypeStr.toUpperCase()); } catch (IllegalArgumentException e) { throw new MetricsPathException("invalid program type: " + pathProgramTypeStr); } if (pathParts.hasNext()) { tagValues.put(programType.getTagName(), pathParts.next()); } else { // given program type, match any type name under the type tagValues.put(programType.getTagName(), null); } if (!pathParts.hasNext()) { return; } switch (programType) { case MAPREDUCE: String mrTypeStr = pathParts.next(); if (mrTypeStr.equals(RUN_ID)) { parseRunId(pathParts, tagValues); if (pathParts.hasNext()) { mrTypeStr = pathParts.next(); } else { return; } } MapReduceType mrType; try { mrType = MapReduceType.valueOf(mrTypeStr.toUpperCase()); } catch (IllegalArgumentException e) { throw new MetricsPathException( "invalid mapreduce component: " + mrTypeStr + ". must be 'mappers' or 'reducers'."); } tagValues.put(Constants.Metrics.Tag.MR_TASK_TYPE, mrType.getId()); break; case FLOWS: buildFlowletContext(pathParts, tagValues); break; case HANDLERS: buildComponentTypeContext(pathParts, tagValues, "methods", "handler", Constants.Metrics.Tag.METHOD); break; case SERVICES: buildComponentTypeContext(pathParts, tagValues, "handlers", "service", Constants.Metrics.Tag.HANDLER); break; case SPARK: if (pathParts.hasNext()) { if (pathParts.next().equals(RUN_ID)) { parseRunId(pathParts, tagValues); } } break; } if (pathParts.hasNext()) { throw new MetricsPathException("path contains too many elements"); } } private static void buildComponentTypeContext(Iterator<String> pathParts, Map<String, String> tagValues, String componentType, String requestType, String componentTagName) throws MetricsPathException { String nextPath = pathParts.next(); if (nextPath.equals(RUN_ID)) { tagValues.put(Constants.Metrics.Tag.RUN_ID, pathParts.next()); if (pathParts.hasNext()) { nextPath = pathParts.next(); } else { return; } } if (!nextPath.equals(componentType)) { String exception = String.format("Expecting '%s' after the %s name ", componentType, requestType.substring(0, requestType.length() - 1)); throw new MetricsPathException(exception); } if (!pathParts.hasNext()) { String exception = String.format("'%s' must be followed by a %s name ", componentType, componentType.substring(0, componentType.length() - 1)); throw new MetricsPathException(exception); } tagValues.put(componentTagName, urlDecode(pathParts.next())); } private static void parseRunId(Iterator<String> pathParts, Map<String, String> tagValues) throws MetricsPathException { if (!pathParts.hasNext()) { throw new MetricsPathException("expecting " + RUN_ID + " value after the identifier runs in path"); } tagValues.put(Constants.Metrics.Tag.RUN_ID, pathParts.next()); } /** * At this point, pathParts should look like flowlets/{flowlet-id}/queues/{queue-id}, with queues being optional. */ private static void buildFlowletContext(Iterator<String> pathParts, Map<String, String> tagValues) throws MetricsPathException { buildComponentTypeContext(pathParts, tagValues, "flowlets", "flows", Constants.Metrics.Tag.FLOWLET); if (pathParts.hasNext()) { if (!pathParts.next().equals("queues")) { throw new MetricsPathException("expecting 'queues' after the flowlet name"); } if (!pathParts.hasNext()) { throw new MetricsPathException("'queues' must be followed by a queue name"); } tagValues.put(Constants.Metrics.Tag.FLOWLET_QUEUE, urlDecode(pathParts.next())); } } /** * From the query string determine the query type, time range and related parameters. */ public static void parseQueryString(URI requestURI, MetricDataQueryBuilder builder) throws MetricsPathException { Map<String, List<String>> queryParams = new QueryStringDecoder(requestURI).getParameters(); parseTimeseries(queryParams, builder); } private static boolean isAutoResolution(Map<String, List<String>> queryParams) { return queryParams.get(RESOLUTION).get(0).equals(AUTO_RESOLUTION); } private static void parseTimeseries(Map<String, List<String>> queryParams, MetricDataQueryBuilder builder) { int count; long startTime; long endTime; int resolution; long now = TimeUnit.SECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS); if (queryParams.containsKey(RESOLUTION) && !isAutoResolution(queryParams)) { resolution = TimeMathParser.resolutionInSeconds(queryParams.get(RESOLUTION).get(0)); if (!((resolution == 3600) || (resolution == 60) || (resolution == 1))) { throw new IllegalArgumentException("Resolution interval not supported, only 1 second, 1 minute and " + "1 hour resolutions are supported currently"); } } else { // if resolution is not provided set default 1 resolution = 1; } if (queryParams.containsKey(START_TIME) && queryParams.containsKey(END_TIME)) { startTime = TimeMathParser.parseTimeInSeconds(now, queryParams.get(START_TIME).get(0)); endTime = TimeMathParser.parseTimeInSeconds(now, queryParams.get(END_TIME).get(0)); if (queryParams.containsKey(RESOLUTION)) { if (isAutoResolution(queryParams)) { // auto determine resolution, based on time difference. Resolution autoResolution = getResolution(endTime - startTime); resolution = autoResolution.getResolution(); } } else { resolution = Resolution.SECOND.getResolution(); } if (queryParams.containsKey(COUNT)) { count = Integer.parseInt(queryParams.get(COUNT).get(0)); } else { count = (int) (((endTime / resolution * resolution) - (startTime / resolution * resolution)) / resolution + 1); } } else if (queryParams.containsKey(COUNT)) { count = Integer.parseInt(queryParams.get(COUNT).get(0)); // both start and end times are inclusive, which is the reason for the +-1. if (queryParams.containsKey(START_TIME)) { startTime = TimeMathParser.parseTimeInSeconds(now, queryParams.get(START_TIME).get(0)); endTime = startTime + (count * resolution) - resolution; } else if (queryParams.containsKey(END_TIME)) { endTime = TimeMathParser.parseTimeInSeconds(now, queryParams.get(END_TIME).get(0)); startTime = endTime - (count * resolution) + resolution; } else { // if only count is specified, assume the current time is desired as the end. endTime = now - Constants.Metrics.Query.QUERY_SECOND_DELAY; startTime = endTime - (count * resolution) + resolution; } } else { startTime = 0; endTime = 0; count = 1; // max int means aggregate "all time totals" resolution = Integer.MAX_VALUE; } builder.setStartTs(startTime); builder.setEndTs(endTime); builder.setLimit(count); builder.setResolution(resolution); setInterpolator(queryParams, builder); } static Resolution getResolution(long difference) { if (difference > Constants.Metrics.Query.MAX_HOUR_RESOLUTION_QUERY_INTERVAL) { return Resolution.HOUR; } else if (difference > Constants.Metrics.Query.MAX_MINUTE_RESOLUTION_QUERY_INTERVAL) { return Resolution.MINUTE; } else { return Resolution.SECOND; } } private static void setInterpolator(Map<String, List<String>> queryParams, MetricDataQueryBuilder builder) { Interpolator interpolator = null; if (queryParams.containsKey(INTERPOLATE)) { String interpolatorType = queryParams.get(INTERPOLATE).get(0); // timeLimit used in case there is a big gap in the data and we don't want to interpolate points. // the limit defines how big the gap has to be in seconds before we just say they're all zeroes. long timeLimit = queryParams.containsKey(MAX_INTERPOLATE_GAP) ? Long.parseLong(queryParams.get(MAX_INTERPOLATE_GAP).get(0)) : Long.MAX_VALUE; if (STEP_INTERPOLATOR.equals(interpolatorType)) { interpolator = new Interpolators.Step(timeLimit); } else if (LINEAR_INTERPOLATOR.equals(interpolatorType)) { interpolator = new Interpolators.Linear(timeLimit); } } builder.setInterpolator(interpolator); } static class MetricDataQueryBuilder { private long startTs; private long endTs; private int resolution; private String scope; private String metricName; // todo: should be aggregation? e.g. also support min/max, etc. private Map<String, String> sliceByTagValues; private int limit; private Interpolator interpolator; public void setStartTs(long startTs) { this.startTs = startTs; } public void setEndTs(long endTs) { this.endTs = endTs; } public void setResolution(int resolution) { this.resolution = resolution; } public void setMetricName(String metricName) { this.metricName = metricName; } public void setScope(String scope) { this.scope = scope; } public void setSliceByTagValues(Map<String, String> sliceByTagValues) { this.sliceByTagValues = sliceByTagValues; } public void setLimit(int limit) { this.limit = limit; } public MetricDataQuery build() { Map<String, AggregationFunction> metrics = ImmutableMap.of(scope + "." + metricName, AggregationFunction.SUM); return new MetricDataQuery(startTs, endTs, resolution, limit, metrics, sliceByTagValues, new ArrayList<String>(), interpolator); } public void setInterpolator(Interpolator interpolator) { this.interpolator = interpolator; } } }