Java tutorial
/** * Copyright 2014 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.flipkart.foxtrot.core.querystore.actions; import com.flipkart.foxtrot.common.ActionResponse; import com.flipkart.foxtrot.common.Period; import com.flipkart.foxtrot.common.query.Filter; import com.flipkart.foxtrot.common.query.FilterCombinerType; import com.flipkart.foxtrot.common.query.datetime.LastFilter; import com.flipkart.foxtrot.common.query.general.InFilter; import com.flipkart.foxtrot.common.trend.TrendRequest; import com.flipkart.foxtrot.common.trend.TrendResponse; import com.flipkart.foxtrot.common.util.CollectionUtils; import com.flipkart.foxtrot.core.cache.CacheManager; import com.flipkart.foxtrot.core.common.Action; import com.flipkart.foxtrot.core.datastore.DataStore; import com.flipkart.foxtrot.core.exception.FoxtrotException; import com.flipkart.foxtrot.core.exception.FoxtrotExceptions; import com.flipkart.foxtrot.core.exception.MalformedQueryException; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.google.common.collect.Lists; import com.yammer.dropwizard.util.Duration; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.TreeMap; /** * User: Santanu Sinha (santanu.sinha@flipkart.com) * Date: 30/03/14 * Time: 10:27 PM */ @AnalyticsProvider(opcode = "trend", request = TrendRequest.class, response = TrendResponse.class, cacheable = true) public class TrendAction extends Action<TrendRequest> { public TrendAction(TrendRequest parameter, TableMetadataManager tableMetadataManager, DataStore dataStore, QueryStore queryStore, ElasticsearchConnection connection, String cacheToken, CacheManager cacheManager) { super(parameter, tableMetadataManager, dataStore, queryStore, connection, cacheToken, cacheManager); } @Override protected void preprocess() { getParameter().setTable(ElasticsearchUtils.getValidTableName(getParameter().getTable())); if (null != getParameter().getValues() && getParameter().getValues().size() != 0) { List<Object> values = (List) getParameter().getValues(); Filter filter = new InFilter(getParameter().getField(), values); getParameter().getFilters().add(filter); } } @Override public String getMetricKey() { return getParameter().getTable(); } @Override protected String getRequestCacheKey() { TrendRequest query = getParameter(); long filterHashKey = 0L; if (query.getFilters() != null) { for (Filter filter : query.getFilters()) { filterHashKey += 31 * filter.hashCode(); } } if (query.getValues() != null) { for (String value : query.getValues()) { filterHashKey += 31 * value.hashCode(); } } if (null != query.getUniqueCountOn()) { filterHashKey += 31 * query.getUniqueCountOn().hashCode(); } filterHashKey += 31 * query.getPeriod().name().hashCode(); filterHashKey += 31 * query.getTimestamp().hashCode(); filterHashKey += 31 * (query.getField() != null ? query.getField().hashCode() : "FIELD".hashCode()); return String.format("%s-%s-%s-%d", query.getTable(), query.getField(), query.getPeriod(), filterHashKey); } @Override public void validateImpl(TrendRequest parameter) throws MalformedQueryException { List<String> validationErrors = Lists.newArrayList(); if (CollectionUtils.isNullOrEmpty(parameter.getTable())) { validationErrors.add("table name cannot be null or empty"); } if (CollectionUtils.isNullOrEmpty(parameter.getField())) { validationErrors.add("field name cannot be null or empty"); } if (CollectionUtils.isNullOrEmpty(parameter.getTimestamp())) { validationErrors.add("timestamp field cannot be null or empty"); } if (parameter.getPeriod() == null) { validationErrors.add(String.format("specify time period (%s)", StringUtils.join(Period.values()))); } if (parameter.getUniqueCountOn() != null && parameter.getUniqueCountOn().isEmpty()) { validationErrors.add("unique field cannot be empty (can be null)"); } if (!CollectionUtils.isNullOrEmpty(validationErrors)) { throw FoxtrotExceptions.createMalformedQueryException(parameter, validationErrors); } } @Override public ActionResponse execute(TrendRequest parameter) throws FoxtrotException { SearchRequestBuilder searchRequestBuilder; try { AbstractAggregationBuilder aggregationBuilder = buildAggregation(parameter); searchRequestBuilder = getConnection().getClient() .prepareSearch(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) .setIndicesOptions(Utils.indicesOptions()) .setQuery(new ElasticSearchQueryGenerator(FilterCombinerType.and) .genFilter(parameter.getFilters())) .setSearchType(SearchType.COUNT).addAggregation(aggregationBuilder); } catch (Exception e) { throw FoxtrotExceptions.queryCreationException(parameter, e); } try { SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); Aggregations aggregations = searchResponse.getAggregations(); if (aggregations != null) { return buildResponse(parameter, aggregations); } else { return new TrendResponse(Collections.<String, List<TrendResponse.Count>>emptyMap()); } } catch (ElasticsearchException e) { throw FoxtrotExceptions.createQueryExecutionException(parameter, e); } } @Override protected Filter getDefaultTimeSpan() { LastFilter lastFilter = new LastFilter(); lastFilter.setField("_timestamp"); lastFilter.setDuration(Duration.days(1)); return lastFilter; } private AbstractAggregationBuilder buildAggregation(TrendRequest request) { DateHistogram.Interval interval = Utils.getHistogramInterval(request.getPeriod()); String field = request.getField(); DateHistogramBuilder histogramBuilder = Utils.buildDateHistogramAggregation(request.getTimestamp(), interval); if (!CollectionUtils.isNullOrEmpty(getParameter().getUniqueCountOn())) { histogramBuilder.subAggregation(Utils.buildCardinalityAggregation(getParameter().getUniqueCountOn())); } return AggregationBuilders.terms(Utils.sanitizeFieldForAggregation(field)).field(field).size(0) .subAggregation(histogramBuilder); } private TrendResponse buildResponse(TrendRequest request, Aggregations aggregations) { String field = request.getField(); Map<String, List<TrendResponse.Count>> trendCounts = new TreeMap<>(); Terms terms = aggregations.get(Utils.sanitizeFieldForAggregation(field)); for (Terms.Bucket bucket : terms.getBuckets()) { final String key = bucket.getKeyAsText().string(); List<TrendResponse.Count> counts = Lists.newArrayList(); Aggregations subAggregations = bucket.getAggregations(); Histogram histogram = subAggregations.get(Utils.getDateHistogramKey(request.getTimestamp())); for (Histogram.Bucket histogramBucket : histogram.getBuckets()) { if (!CollectionUtils.isNullOrEmpty(getParameter().getUniqueCountOn())) { String uniqueCountKey = Utils.sanitizeFieldForAggregation(getParameter().getUniqueCountOn()); Cardinality cardinality = histogramBucket.getAggregations().get(uniqueCountKey); counts.add(new TrendResponse.Count(histogramBucket.getKeyAsNumber(), cardinality.getValue())); } else { counts.add(new TrendResponse.Count(histogramBucket.getKeyAsNumber(), histogramBucket.getDocCount())); } } trendCounts.put(key, counts); } return new TrendResponse(trendCounts); } }