Java tutorial
/* * SonarQube * Copyright (C) 2009-2017 SonarSource SA * mailto:info AT sonarsource DOT com * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 3 of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ package org.sonar.server.issue.index; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.TimeZone; import java.util.regex.Pattern; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; import org.elasticsearch.search.aggregations.metrics.min.Min; import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder; import org.joda.time.Duration; import org.sonar.api.issue.Issue; import org.sonar.api.resources.Scopes; import org.sonar.api.utils.DateUtils; import org.sonar.api.utils.System2; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.component.ComponentDto; import org.sonar.db.organization.OrganizationDto; import org.sonar.server.es.EsClient; import org.sonar.server.es.EsUtils; import org.sonar.server.es.SearchOptions; import org.sonar.server.es.SearchResult; import org.sonar.server.es.Sorting; import org.sonar.server.es.StickyFacetBuilder; import org.sonar.server.issue.IssueQuery; import org.sonar.server.permission.index.AuthorizationTypeSupport; import org.sonar.server.user.UserSession; import org.sonar.server.view.index.ViewIndexDefinition; import static java.lang.String.format; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.existsQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.index.query.QueryBuilders.termsQuery; import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars; import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_ORGANIZATION_UUID; import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE; import static org.sonarqube.ws.client.issue.IssuesWsParameters.DEPRECATED_FACET_MODE_DEBT; import static org.sonarqube.ws.client.issue.IssuesWsParameters.DEPRECATED_PARAM_ACTION_PLANS; import static org.sonarqube.ws.client.issue.IssuesWsParameters.FACET_ASSIGNED_TO_ME; import static org.sonarqube.ws.client.issue.IssuesWsParameters.FACET_MODE_EFFORT; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_ASSIGNEES; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_AUTHORS; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_CREATED_AT; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_DIRECTORIES; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_FILE_UUIDS; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_LANGUAGES; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_MODULE_UUIDS; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_PROJECT_UUIDS; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_REPORTERS; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_RESOLUTIONS; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_RULES; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_SEVERITIES; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_STATUSES; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_TAGS; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_TYPES; /** * The unique entry-point to interact with Elasticsearch index "issues". * All the requests are listed here. */ public class IssueIndex { private static final String SUBSTRING_MATCH_REGEXP = ".*%s.*"; public static final List<String> SUPPORTED_FACETS = ImmutableList.of(PARAM_SEVERITIES, PARAM_STATUSES, PARAM_RESOLUTIONS, DEPRECATED_PARAM_ACTION_PLANS, PARAM_PROJECT_UUIDS, PARAM_RULES, PARAM_ASSIGNEES, FACET_ASSIGNED_TO_ME, PARAM_REPORTERS, PARAM_AUTHORS, PARAM_MODULE_UUIDS, PARAM_FILE_UUIDS, PARAM_DIRECTORIES, PARAM_LANGUAGES, PARAM_TAGS, PARAM_TYPES, PARAM_CREATED_AT); // TODO to be documented // TODO move to Facets ? private static final String FACET_SUFFIX_MISSING = "_missing"; private static final String IS_ASSIGNED_FILTER = "__isAssigned"; private static final SumBuilder EFFORT_AGGREGATION = AggregationBuilders.sum(FACET_MODE_EFFORT) .field(IssueIndexDefinition.FIELD_ISSUE_EFFORT); private static final Order EFFORT_AGGREGATION_ORDER = Order.aggregation(FACET_MODE_EFFORT, false); private static final int DEFAULT_FACET_SIZE = 15; private static final Duration TWENTY_DAYS = Duration.standardDays(20L); private static final Duration TWENTY_WEEKS = Duration.standardDays(20L * 7L); private static final Duration TWENTY_MONTHS = Duration.standardDays(20L * 30L); public static final String AGGREGATION_NAME_FOR_TAGS = "tags__issues"; private final Sorting sorting; private final EsClient client; private final System2 system; private final UserSession userSession; private final AuthorizationTypeSupport authorizationTypeSupport; public IssueIndex(EsClient client, System2 system, UserSession userSession, AuthorizationTypeSupport authorizationTypeSupport) { this.client = client; this.system = system; this.userSession = userSession; this.authorizationTypeSupport = authorizationTypeSupport; this.sorting = new Sorting(); this.sorting.add(IssueQuery.SORT_BY_ASSIGNEE, IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE); this.sorting.add(IssueQuery.SORT_BY_STATUS, IssueIndexDefinition.FIELD_ISSUE_STATUS); this.sorting.add(IssueQuery.SORT_BY_SEVERITY, IssueIndexDefinition.FIELD_ISSUE_SEVERITY_VALUE); this.sorting.add(IssueQuery.SORT_BY_CREATION_DATE, IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT); this.sorting.add(IssueQuery.SORT_BY_UPDATE_DATE, IssueIndexDefinition.FIELD_ISSUE_FUNC_UPDATED_AT); this.sorting.add(IssueQuery.SORT_BY_CLOSE_DATE, IssueIndexDefinition.FIELD_ISSUE_FUNC_CLOSED_AT); this.sorting.add(IssueQuery.SORT_BY_FILE_LINE, IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID); this.sorting.add(IssueQuery.SORT_BY_FILE_LINE, IssueIndexDefinition.FIELD_ISSUE_FILE_PATH); this.sorting.add(IssueQuery.SORT_BY_FILE_LINE, IssueIndexDefinition.FIELD_ISSUE_LINE); this.sorting.add(IssueQuery.SORT_BY_FILE_LINE, IssueIndexDefinition.FIELD_ISSUE_SEVERITY_VALUE).reverse(); this.sorting.add(IssueQuery.SORT_BY_FILE_LINE, IssueIndexDefinition.FIELD_ISSUE_KEY); // by default order by created date, project, file, line and issue key (in order to be deterministic when same ms) this.sorting.addDefault(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT).reverse(); this.sorting.addDefault(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID); this.sorting.addDefault(IssueIndexDefinition.FIELD_ISSUE_FILE_PATH); this.sorting.addDefault(IssueIndexDefinition.FIELD_ISSUE_LINE); this.sorting.addDefault(IssueIndexDefinition.FIELD_ISSUE_KEY); } public SearchResult<IssueDoc> search(IssueQuery query, SearchOptions options) { SearchRequestBuilder requestBuilder = client.prepareSearch(INDEX_TYPE_ISSUE); configureSorting(query, requestBuilder); configurePagination(options, requestBuilder); configureRouting(query, options, requestBuilder); QueryBuilder esQuery = matchAllQuery(); BoolQueryBuilder esFilter = boolQuery(); Map<String, QueryBuilder> filters = createFilters(query); for (QueryBuilder filter : filters.values()) { if (filter != null) { esFilter.must(filter); } } if (esFilter.hasClauses()) { requestBuilder.setQuery(boolQuery().must(esQuery).filter(esFilter)); } else { requestBuilder.setQuery(esQuery); } configureStickyFacets(query, options, filters, esQuery, requestBuilder); SearchResponse response = requestBuilder.get(); return new SearchResult<>(response, IssueDoc::new); } /** * Optimization - do not send ES request to all shards when scope is restricted * to a set of projects. Because project UUID is used for routing, the request * can be sent to only the shards containing the specified projects. * Note that sticky facets may involve all projects, so this optimization must be * disabled when facets are enabled. */ private static void configureRouting(IssueQuery query, SearchOptions options, SearchRequestBuilder requestBuilder) { Collection<String> uuids = query.projectUuids(); if (!uuids.isEmpty() && options.getFacets().isEmpty()) { requestBuilder.setRouting(uuids.toArray(new String[uuids.size()])); } } private void configureSorting(IssueQuery query, SearchRequestBuilder esRequest) { String sortField = query.sort(); if (sortField != null) { boolean asc = BooleanUtils.isTrue(query.asc()); sorting.fill(esRequest, sortField, asc); } else { sorting.fillDefault(esRequest); } } private static void configurePagination(SearchOptions options, SearchRequestBuilder esSearch) { esSearch.setFrom(options.getOffset()).setSize(options.getLimit()); } private Map<String, QueryBuilder> createFilters(IssueQuery query) { Map<String, QueryBuilder> filters = new HashMap<>(); filters.put("__authorization", createAuthorizationFilter(query.checkAuthorization())); // Issue is assigned Filter if (BooleanUtils.isTrue(query.assigned())) { filters.put(IS_ASSIGNED_FILTER, existsQuery(IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE)); } else if (BooleanUtils.isFalse(query.assigned())) { filters.put(IS_ASSIGNED_FILTER, boolQuery().mustNot(existsQuery(IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE))); } // Issue is Resolved Filter String isResolved = "__isResolved"; if (BooleanUtils.isTrue(query.resolved())) { filters.put(isResolved, existsQuery(IssueIndexDefinition.FIELD_ISSUE_RESOLUTION)); } else if (BooleanUtils.isFalse(query.resolved())) { filters.put(isResolved, boolQuery().mustNot(existsQuery(IssueIndexDefinition.FIELD_ISSUE_RESOLUTION))); } // Field Filters filters.put(IssueIndexDefinition.FIELD_ISSUE_KEY, createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_KEY, query.issueKeys())); filters.put(IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE, createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE, query.assignees())); addComponentRelatedFilters(query, filters); filters.put(IssueIndexDefinition.FIELD_ISSUE_LANGUAGE, createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_LANGUAGE, query.languages())); filters.put(IssueIndexDefinition.FIELD_ISSUE_TAGS, createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_TAGS, query.tags())); filters.put(IssueIndexDefinition.FIELD_ISSUE_TYPE, createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_TYPE, query.types())); filters.put(IssueIndexDefinition.FIELD_ISSUE_RESOLUTION, createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_RESOLUTION, query.resolutions())); filters.put(IssueIndexDefinition.FIELD_ISSUE_AUTHOR_LOGIN, createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_AUTHOR_LOGIN, query.authors())); filters.put(IssueIndexDefinition.FIELD_ISSUE_RULE_KEY, createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_RULE_KEY, query.rules())); filters.put(IssueIndexDefinition.FIELD_ISSUE_SEVERITY, createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_SEVERITY, query.severities())); filters.put(IssueIndexDefinition.FIELD_ISSUE_STATUS, createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_STATUS, query.statuses())); filters.put(IssueIndexDefinition.FIELD_ISSUE_ORGANIZATION_UUID, createTermFilter(IssueIndexDefinition.FIELD_ISSUE_ORGANIZATION_UUID, query.organizationUuid())); addDatesFilter(filters, query); return filters; } private static void addComponentRelatedFilters(IssueQuery query, Map<String, QueryBuilder> filters) { QueryBuilder viewFilter = createViewFilter(query.viewUuids()); QueryBuilder componentFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, query.componentUuids()); QueryBuilder projectFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, query.projectUuids()); QueryBuilder moduleRootFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_MODULE_PATH, query.moduleRootUuids()); QueryBuilder moduleFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_MODULE_UUID, query.moduleUuids()); QueryBuilder directoryFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_DIRECTORY_PATH, query.directories()); QueryBuilder fileFilter = createTermsFilter(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, query.fileUuids()); if (BooleanUtils.isTrue(query.onComponentOnly())) { filters.put(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, componentFilter); } else { filters.put("__view", viewFilter); filters.put(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, projectFilter); filters.put("__module", moduleRootFilter); filters.put(IssueIndexDefinition.FIELD_ISSUE_MODULE_UUID, moduleFilter); filters.put(IssueIndexDefinition.FIELD_ISSUE_DIRECTORY_PATH, directoryFilter); if (fileFilter != null) { filters.put(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, fileFilter); } else { filters.put(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, componentFilter); } } } @CheckForNull private static QueryBuilder createViewFilter(Collection<String> viewUuids) { if (viewUuids.isEmpty()) { return null; } BoolQueryBuilder viewsFilter = boolQuery(); for (String viewUuid : viewUuids) { viewsFilter.should(QueryBuilders.termsLookupQuery(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID) .lookupIndex(ViewIndexDefinition.INDEX_TYPE_VIEW.getIndex()) .lookupType(ViewIndexDefinition.INDEX_TYPE_VIEW.getType()).lookupId(viewUuid) .lookupPath(ViewIndexDefinition.FIELD_PROJECTS)); } return viewsFilter; } private QueryBuilder createAuthorizationFilter(boolean checkAuthorization) { if (checkAuthorization) { return authorizationTypeSupport.createQueryFilter(); } return matchAllQuery(); } private void addDatesFilter(Map<String, QueryBuilder> filters, IssueQuery query) { Date createdAfter = query.createdAfter(); Date createdBefore = query.createdBefore(); validateCreationDateBounds(createdBefore, createdAfter); if (createdAfter != null) { filters.put("__createdAfter", QueryBuilders.rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT).gte(createdAfter)); } if (createdBefore != null) { filters.put("__createdBefore", QueryBuilders.rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT).lt(createdBefore)); } Date createdAt = query.createdAt(); if (createdAt != null) { filters.put("__createdAt", termQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT, createdAt)); } } private void validateCreationDateBounds(@Nullable Date createdBefore, @Nullable Date createdAfter) { Preconditions.checkArgument(createdAfter == null || createdAfter.before(new Date(system.now())), "Start bound cannot be in the future"); Preconditions.checkArgument( createdAfter == null || createdBefore == null || createdAfter.before(createdBefore), "Start bound cannot be larger or equal to end bound"); } private void configureStickyFacets(IssueQuery query, SearchOptions options, Map<String, QueryBuilder> filters, QueryBuilder esQuery, SearchRequestBuilder esSearch) { if (!options.getFacets().isEmpty()) { StickyFacetBuilder stickyFacetBuilder = newStickyFacetBuilder(query, filters, esQuery); // Execute Term aggregations addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, PARAM_SEVERITIES, IssueIndexDefinition.FIELD_ISSUE_SEVERITY); addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, PARAM_STATUSES, IssueIndexDefinition.FIELD_ISSUE_STATUS); addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, PARAM_PROJECT_UUIDS, IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, query.projectUuids().toArray()); addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, PARAM_MODULE_UUIDS, IssueIndexDefinition.FIELD_ISSUE_MODULE_UUID, query.moduleUuids().toArray()); addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, PARAM_DIRECTORIES, IssueIndexDefinition.FIELD_ISSUE_DIRECTORY_PATH, query.directories().toArray()); addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, PARAM_FILE_UUIDS, IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, query.fileUuids().toArray()); addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, PARAM_LANGUAGES, IssueIndexDefinition.FIELD_ISSUE_LANGUAGE, query.languages().toArray()); addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, PARAM_RULES, IssueIndexDefinition.FIELD_ISSUE_RULE_KEY, query.rules().toArray()); addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, PARAM_AUTHORS, IssueIndexDefinition.FIELD_ISSUE_AUTHOR_LOGIN, query.authors().toArray()); if (options.getFacets().contains(PARAM_TAGS)) { esSearch.addAggregation(stickyFacetBuilder.buildStickyFacet(IssueIndexDefinition.FIELD_ISSUE_TAGS, PARAM_TAGS, query.tags().toArray())); } if (options.getFacets().contains(PARAM_TYPES)) { esSearch.addAggregation(stickyFacetBuilder.buildStickyFacet(IssueIndexDefinition.FIELD_ISSUE_TYPE, PARAM_TYPES, query.types().toArray())); } if (options.getFacets().contains(PARAM_RESOLUTIONS)) { esSearch.addAggregation(createResolutionFacet(query, filters, esQuery)); } if (options.getFacets().contains(PARAM_ASSIGNEES)) { esSearch.addAggregation(createAssigneesFacet(query, filters, esQuery)); } addAssignedToMeFacetIfNeeded(esSearch, options, query, filters, esQuery); if (options.getFacets().contains(PARAM_CREATED_AT)) { getCreatedAtFacet(query, filters, esQuery).ifPresent(esSearch::addAggregation); } } if (hasQueryEffortFacet(query)) { esSearch.addAggregation(EFFORT_AGGREGATION); } } private static StickyFacetBuilder newStickyFacetBuilder(IssueQuery query, Map<String, QueryBuilder> filters, QueryBuilder esQuery) { if (hasQueryEffortFacet(query)) { return new StickyFacetBuilder(esQuery, filters, EFFORT_AGGREGATION, EFFORT_AGGREGATION_ORDER); } return new StickyFacetBuilder(esQuery, filters); } private static void addSimpleStickyFacetIfNeeded(SearchOptions options, StickyFacetBuilder stickyFacetBuilder, SearchRequestBuilder esSearch, String facetName, String fieldName, Object... selectedValues) { if (options.getFacets().contains(facetName)) { esSearch.addAggregation( stickyFacetBuilder.buildStickyFacet(fieldName, facetName, DEFAULT_FACET_SIZE, selectedValues)); } } private static AggregationBuilder addEffortAggregationIfNeeded(IssueQuery query, AggregationBuilder aggregation) { if (hasQueryEffortFacet(query)) { aggregation.subAggregation(EFFORT_AGGREGATION); } return aggregation; } private static boolean hasQueryEffortFacet(IssueQuery query) { return FACET_MODE_EFFORT.equals(query.facetMode()) || DEPRECATED_FACET_MODE_DEBT.equals(query.facetMode()); } private Optional<AggregationBuilder> getCreatedAtFacet(IssueQuery query, Map<String, QueryBuilder> filters, QueryBuilder esQuery) { long startTime; Date createdAfter = query.createdAfter(); if (createdAfter == null) { Optional<Long> minDate = getMinCreatedAt(filters, esQuery); if (!minDate.isPresent()) { return Optional.empty(); } startTime = minDate.get(); } else { startTime = createdAfter.getTime(); } Date createdBefore = query.createdBefore(); long endTime = createdBefore == null ? system.now() : createdBefore.getTime(); Duration timeSpan = new Duration(startTime, endTime); DateHistogramInterval bucketSize = DateHistogramInterval.YEAR; if (timeSpan.isShorterThan(TWENTY_DAYS)) { bucketSize = DateHistogramInterval.DAY; } else if (timeSpan.isShorterThan(TWENTY_WEEKS)) { bucketSize = DateHistogramInterval.WEEK; } else if (timeSpan.isShorterThan(TWENTY_MONTHS)) { bucketSize = DateHistogramInterval.MONTH; } // from GMT to server TZ int offsetInSeconds = -system.getDefaultTimeZone().getRawOffset() / 1_000; AggregationBuilder dateHistogram = AggregationBuilders.dateHistogram(PARAM_CREATED_AT) .field(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT).interval(bucketSize).minDocCount(0L) .format(DateUtils.DATETIME_FORMAT).timeZone(TimeZone.getTimeZone("GMT").getID()) .offset(offsetInSeconds + "s") // ES dateHistogram bounds are inclusive while createdBefore parameter is exclusive .extendedBounds(startTime, endTime - 1_000L); dateHistogram = addEffortAggregationIfNeeded(query, dateHistogram); return Optional.of(dateHistogram); } private Optional<Long> getMinCreatedAt(Map<String, QueryBuilder> filters, QueryBuilder esQuery) { String facetNameAndField = IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT; SearchRequestBuilder esRequest = client.prepareSearch(INDEX_TYPE_ISSUE).setSize(0); BoolQueryBuilder esFilter = boolQuery(); filters.values().stream().filter(Objects::nonNull).forEach(esFilter::must); if (esFilter.hasClauses()) { esRequest.setQuery(QueryBuilders.filteredQuery(esQuery, esFilter)); } else { esRequest.setQuery(esQuery); } esRequest.addAggregation(AggregationBuilders.min(facetNameAndField).field(facetNameAndField)); Min minValue = esRequest.get().getAggregations().get(facetNameAndField); Double actualValue = minValue.getValue(); if (actualValue.isInfinite()) { return Optional.empty(); } return Optional.of(actualValue.longValue()); } private static AggregationBuilder createAssigneesFacet(IssueQuery query, Map<String, QueryBuilder> filters, QueryBuilder queryBuilder) { String fieldName = IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE; String facetName = PARAM_ASSIGNEES; // Same as in super.stickyFacetBuilder Map<String, QueryBuilder> assigneeFilters = Maps.newHashMap(filters); assigneeFilters.remove(IS_ASSIGNED_FILTER); assigneeFilters.remove(fieldName); StickyFacetBuilder assigneeFacetBuilder = newStickyFacetBuilder(query, assigneeFilters, queryBuilder); BoolQueryBuilder facetFilter = assigneeFacetBuilder.getStickyFacetFilter(fieldName); FilterAggregationBuilder facetTopAggregation = assigneeFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, DEFAULT_FACET_SIZE); Collection<String> assigneesEscaped = escapeValuesForFacetInclusion(query.assignees()); if (!assigneesEscaped.isEmpty()) { facetTopAggregation = assigneeFacetBuilder.addSelectedItemsToFacet(fieldName, facetName, facetTopAggregation, t -> t, assigneesEscaped.toArray()); } // Add missing facet for unassigned issues facetTopAggregation.subAggregation(addEffortAggregationIfNeeded(query, AggregationBuilders.missing(facetName + FACET_SUFFIX_MISSING).field(fieldName))); return AggregationBuilders.global(facetName).subAggregation(facetTopAggregation); } private static Collection<String> escapeValuesForFacetInclusion(@Nullable Collection<String> values) { if (values == null) { return Collections.emptyList(); } return values.stream().map(Pattern::quote).collect(MoreCollectors.toArrayList(values.size())); } private void addAssignedToMeFacetIfNeeded(SearchRequestBuilder builder, SearchOptions options, IssueQuery query, Map<String, QueryBuilder> filters, QueryBuilder queryBuilder) { String login = userSession.getLogin(); if (!options.getFacets().contains(FACET_ASSIGNED_TO_ME) || StringUtils.isEmpty(login)) { return; } String fieldName = IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE; String facetName = FACET_ASSIGNED_TO_ME; // Same as in super.stickyFacetBuilder StickyFacetBuilder assignedToMeFacetBuilder = newStickyFacetBuilder(query, filters, queryBuilder); BoolQueryBuilder facetFilter = assignedToMeFacetBuilder.getStickyFacetFilter(IS_ASSIGNED_FILTER, fieldName); FilterAggregationBuilder facetTopAggregation = AggregationBuilders.filter(facetName + "__filter") .filter(facetFilter).subAggregation(addEffortAggregationIfNeeded(query, AggregationBuilders .terms(facetName + "__terms").field(fieldName).include(escapeSpecialRegexChars(login)))); builder.addAggregation(AggregationBuilders.global(facetName).subAggregation(facetTopAggregation)); } private static AggregationBuilder createResolutionFacet(IssueQuery query, Map<String, QueryBuilder> filters, QueryBuilder esQuery) { String fieldName = IssueIndexDefinition.FIELD_ISSUE_RESOLUTION; String facetName = PARAM_RESOLUTIONS; // Same as in super.stickyFacetBuilder Map<String, QueryBuilder> resolutionFilters = Maps.newHashMap(filters); resolutionFilters.remove("__isResolved"); resolutionFilters.remove(fieldName); StickyFacetBuilder assigneeFacetBuilder = newStickyFacetBuilder(query, resolutionFilters, esQuery); BoolQueryBuilder facetFilter = assigneeFacetBuilder.getStickyFacetFilter(fieldName); FilterAggregationBuilder facetTopAggregation = assigneeFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, DEFAULT_FACET_SIZE); facetTopAggregation = assigneeFacetBuilder.addSelectedItemsToFacet(fieldName, facetName, facetTopAggregation, t -> t); // Add missing facet for unresolved issues facetTopAggregation.subAggregation(addEffortAggregationIfNeeded(query, AggregationBuilders.missing(facetName + FACET_SUFFIX_MISSING).field(fieldName))); return AggregationBuilders.global(facetName).subAggregation(facetTopAggregation); } @CheckForNull private static QueryBuilder createTermsFilter(String field, Collection<?> values) { return values.isEmpty() ? null : termsQuery(field, values); } @CheckForNull private static QueryBuilder createTermFilter(String field, @Nullable String value) { return value == null ? null : termQuery(field, value); } public List<String> listTags(OrganizationDto organization, @Nullable String textQuery, int maxNumberOfTags) { SearchRequestBuilder requestBuilder = client.prepareSearch(INDEX_TYPE_ISSUE) .setQuery(boolQuery().filter(termQuery(FIELD_ISSUE_ORGANIZATION_UUID, organization.getUuid()))) .setSize(0); TermsBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME_FOR_TAGS) .field(IssueIndexDefinition.FIELD_ISSUE_TAGS).size(maxNumberOfTags).order(Terms.Order.term(true)) .minDocCount(1L); if (textQuery != null) { String escapedTextQuery = escapeSpecialRegexChars(textQuery); termsAggregation.include(format(SUBSTRING_MATCH_REGEXP, escapedTextQuery)); } requestBuilder.addAggregation(termsAggregation); SearchResponse searchResponse = requestBuilder.get(); Terms issuesResult = searchResponse.getAggregations().get(AGGREGATION_NAME_FOR_TAGS); return EsUtils.termsKeys(issuesResult); } public Map<String, Long> countTags(IssueQuery query, int maxNumberOfTags) { Terms terms = listTermsMatching(IssueIndexDefinition.FIELD_ISSUE_TAGS, query, null, Terms.Order.count(false), maxNumberOfTags); return EsUtils.termsToMap(terms); } public List<String> listAuthors(IssueQuery query, @Nullable String textQuery, int maxNumberOfAuthors) { Terms terms = listTermsMatching(IssueIndexDefinition.FIELD_ISSUE_AUTHOR_LOGIN, query, textQuery, Terms.Order.term(true), maxNumberOfAuthors); return EsUtils.termsKeys(terms); } private Terms listTermsMatching(String fieldName, IssueQuery query, @Nullable String textQuery, Terms.Order termsOrder, int maxNumberOfTags) { SearchRequestBuilder requestBuilder = client.prepareSearch(INDEX_TYPE_ISSUE) // Avoids returning search hits .setSize(0); requestBuilder.setQuery(boolQuery().must(QueryBuilders.matchAllQuery()).filter(createBoolFilter(query))); TermsBuilder aggreg = AggregationBuilders.terms("_ref").field(fieldName).size(maxNumberOfTags) .order(termsOrder).minDocCount(1L); if (textQuery != null) { aggreg.include(format(SUBSTRING_MATCH_REGEXP, escapeSpecialRegexChars(textQuery))); } SearchResponse searchResponse = requestBuilder.addAggregation(aggreg).get(); return searchResponse.getAggregations().get("_ref"); } private BoolQueryBuilder createBoolFilter(IssueQuery query) { BoolQueryBuilder boolQuery = boolQuery(); for (QueryBuilder filter : createFilters(query).values()) { if (filter != null) { boolQuery.must(filter); } } return boolQuery; } /** * Return non closed issues for a given project, module, or file. Other kind of components are not allowed. * Only fields needed for the batch are returned. */ public Iterator<IssueDoc> selectIssuesForBatch(ComponentDto component) { BoolQueryBuilder filter = boolQuery().must(createAuthorizationFilter(true)) .mustNot(termsQuery(IssueIndexDefinition.FIELD_ISSUE_STATUS, Issue.STATUS_CLOSED)); switch (component.scope()) { case Scopes.PROJECT: filter.must(termsQuery(IssueIndexDefinition.FIELD_ISSUE_MODULE_PATH, component.uuid())); break; case Scopes.FILE: filter.must(termsQuery(IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID, component.uuid())); break; default: throw new IllegalStateException(format("Component of scope '%s' is not allowed", component.scope())); } SearchRequestBuilder requestBuilder = client.prepareSearch(INDEX_TYPE_ISSUE).setSearchType(SearchType.SCAN) .setScroll(TimeValue.timeValueMinutes(EsUtils.SCROLL_TIME_IN_MINUTES)).setSize(10_000) .setFetchSource(new String[] { IssueIndexDefinition.FIELD_ISSUE_KEY, IssueIndexDefinition.FIELD_ISSUE_RULE_KEY, IssueIndexDefinition.FIELD_ISSUE_MODULE_UUID, IssueIndexDefinition.FIELD_ISSUE_FILE_PATH, IssueIndexDefinition.FIELD_ISSUE_SEVERITY, IssueIndexDefinition.FIELD_ISSUE_MANUAL_SEVERITY, IssueIndexDefinition.FIELD_ISSUE_RESOLUTION, IssueIndexDefinition.FIELD_ISSUE_STATUS, IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE, IssueIndexDefinition.FIELD_ISSUE_LINE, IssueIndexDefinition.FIELD_ISSUE_MESSAGE, IssueIndexDefinition.FIELD_ISSUE_CHECKSUM, IssueIndexDefinition.FIELD_ISSUE_TYPE, IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT }, null) .setQuery(boolQuery().must(matchAllQuery()).filter(filter)); SearchResponse response = requestBuilder.get(); return EsUtils.scroll(client, response.getScrollId(), IssueDoc::new); } }