Java tutorial
/* * Copyright 2017 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package stroom.statistics.server.sql; import com.google.common.base.Preconditions; import org.apache.commons.pool2.BasePooledObjectFactory; import org.apache.commons.pool2.PooledObject; import org.apache.commons.pool2.impl.DefaultPooledObject; import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import stroom.datasource.api.v2.DataSourceField; import stroom.entity.shared.Period; import stroom.entity.shared.Range; import stroom.node.server.StroomPropertyService; import stroom.query.common.v2.DateExpressionParser; import stroom.query.api.v2.ExpressionItem; import stroom.query.api.v2.ExpressionOperator; import stroom.query.api.v2.ExpressionTerm; import stroom.query.api.v2.SearchRequest; import stroom.statistics.server.sql.datasource.StatisticStoreCache; import stroom.statistics.server.sql.datasource.StatisticStoreValidator; import stroom.statistics.server.sql.exception.StatisticsEventValidationException; import stroom.statistics.server.sql.rollup.RollUpBitMask; import stroom.statistics.server.sql.rollup.RolledUpStatisticEvent; import stroom.statistics.server.sql.search.CountStatisticDataPoint; import stroom.statistics.server.sql.search.FilterTermsTree; import stroom.statistics.server.sql.search.FilterTermsTreeBuilder; import stroom.statistics.server.sql.search.FindEventCriteria; import stroom.statistics.server.sql.search.StatisticDataPoint; import stroom.statistics.server.sql.search.StatisticDataSet; import stroom.statistics.server.sql.search.ValueStatisticDataPoint; import stroom.statistics.shared.StatisticStore; import stroom.statistics.shared.StatisticStoreEntity; import stroom.statistics.shared.StatisticType; import stroom.statistics.shared.common.CustomRollUpMask; import stroom.statistics.shared.common.StatisticRollUpType; import stroom.util.shared.ModelStringUtil; import stroom.util.spring.StroomFrequencySchedule; import javax.inject.Inject; import javax.inject.Named; import javax.sql.DataSource; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Component public class SQLStatisticEventStore implements Statistics { public static final Logger LOGGER = LoggerFactory.getLogger(SQLStatisticEventStore.class); static final String PROP_KEY_SQL_SEARCH_MAX_RESULTS = "stroom.statistics.sql.search.maxResults"; private static final List<ExpressionTerm.Condition> SUPPORTED_DATE_CONDITIONS = Arrays .asList(ExpressionTerm.Condition.BETWEEN); private static final int DEFAULT_POOL_SIZE = 10; private static final long DEFAULT_SIZE_THRESHOLD = 1000000L; private static final Set<String> BLACK_LISTED_INDEX_FIELDS = Collections.emptySet(); /** * Keep half the time out our SQL insert threshold */ private static final long DEFAULT_AGE_MS_THRESHOLD = TimeUnit.MINUTES.toMillis(5); // @formatter:off private static final String STAT_QUERY_SKELETON = "" + "select " + "K." + SQLStatisticNames.NAME + ", " + "V." + SQLStatisticNames.PRECISION + ", " + "V." + SQLStatisticNames.TIME_MS + ", " + "V." + SQLStatisticNames.VALUE_TYPE + ", " + "V." + SQLStatisticNames.VALUE + ", " + "V." + SQLStatisticNames.COUNT + " " + "FROM " + SQLStatisticNames.SQL_STATISTIC_KEY_TABLE_NAME + " K " + "JOIN " + SQLStatisticNames.SQL_STATISTIC_VALUE_TABLE_NAME + " V ON (K." + SQLStatisticNames.ID + " = V." + SQLStatisticNames.SQL_STATISTIC_KEY_FOREIGN_KEY + ") " + "WHERE K." + SQLStatisticNames.NAME + " LIKE ? " + "AND K." + SQLStatisticNames.NAME + " REGEXP ? " + "AND V." + SQLStatisticNames.TIME_MS + " >= ? " + "AND V." + SQLStatisticNames.TIME_MS + " < ?"; // @formatter:on private final StatisticStoreValidator statisticsDataSourceValidator; private final StatisticStoreCache statisticsDataSourceCache; private final SQLStatisticCache statisticCache; private final DataSource statisticsDataSource; private final StroomPropertyService propertyService; /** * SQL for testing querying the stat/tag names * <p> * create table test (name varchar(255)) ENGINE=InnoDB DEFAULT * CHARSET=latin1; * <p> * insert into test values ('StatName1'); * <p> * insert into test values ('StatName2Tag1Val1Tag2Val2'); * <p> * insert into test values ('StatName2Tag2Val2Tag1Val1'); * <p> * select * from test where name REGEXP '^StatName1(|$)'; * <p> * select * from test where name REGEXP 'Tag1Val1(|$)'; */ private long poolAgeMsThreshold = DEFAULT_AGE_MS_THRESHOLD; private long aggregatorSizeThreshold = DEFAULT_SIZE_THRESHOLD; private int poolSize = DEFAULT_POOL_SIZE; private GenericObjectPool<SQLStatisticAggregateMap> objectPool; @Inject SQLStatisticEventStore(final StatisticStoreValidator statisticsDataSourceValidator, final StatisticStoreCache statisticsDataSourceCache, final SQLStatisticCache statisticCache, @Named("statisticsDataSource") final DataSource statisticsDataSource, final StroomPropertyService propertyService) { this.statisticsDataSourceValidator = statisticsDataSourceValidator; this.propertyService = propertyService; this.statisticsDataSourceCache = statisticsDataSourceCache; this.statisticCache = statisticCache; this.statisticsDataSource = statisticsDataSource; initPool(getObjectPoolConfig()); } public SQLStatisticEventStore(final int poolSize, final long aggregatorSizeThreshold, final long poolAgeMsThreshold, final StatisticStoreValidator statisticsDataSourceValidator, final StatisticStoreCache statisticsDataSourceCache, final SQLStatisticCache statisticCache, final DataSource statisticsDataSource, final StroomPropertyService propertyService) { this.statisticsDataSourceValidator = statisticsDataSourceValidator; this.statisticsDataSourceCache = statisticsDataSourceCache; this.statisticCache = statisticCache; this.statisticsDataSource = statisticsDataSource; this.aggregatorSizeThreshold = aggregatorSizeThreshold; this.poolAgeMsThreshold = poolAgeMsThreshold; this.poolSize = poolSize; this.propertyService = propertyService; initPool(getObjectPoolConfig()); } protected static FindEventCriteria buildCriteria(final SearchRequest searchRequest, final StatisticStoreEntity dataSource) { LOGGER.trace("buildCriteria called for statistic {}", dataSource.getName()); // Get the current time in millis since epoch. final long nowEpochMilli = System.currentTimeMillis(); // object looks a bit like this // AND // Date Time between 2014-10-22T23:00:00.000Z,2014-10-23T23:00:00.000Z final ExpressionOperator topLevelExpressionOperator = searchRequest.getQuery().getExpression(); if (topLevelExpressionOperator == null || topLevelExpressionOperator.getOp() == null) { throw new IllegalArgumentException( "The top level operator for the query must be one of [" + ExpressionOperator.Op.values() + "]"); } final List<ExpressionItem> childExpressions = topLevelExpressionOperator.getChildren(); int validDateTermsFound = 0; int dateTermsFound = 0; // Identify the date term in the search criteria. Currently we must have // a exactly one BETWEEN operator on the // datetime // field to be able to search. This is because of the way the search in // hbase is done, ie. by start/stop row // key. // It may be possible to expand the capability to make multiple searches // but that is currently not in place ExpressionTerm dateTerm = null; if (childExpressions != null) { for (final ExpressionItem expressionItem : childExpressions) { if (expressionItem instanceof ExpressionTerm) { final ExpressionTerm expressionTerm = (ExpressionTerm) expressionItem; if (expressionTerm.getField() == null) { throw new IllegalArgumentException("Expression term does not have a field specified"); } if (expressionTerm.getField().equals(StatisticStoreEntity.FIELD_NAME_DATE_TIME)) { dateTermsFound++; if (SUPPORTED_DATE_CONDITIONS.contains(expressionTerm.getCondition())) { dateTerm = expressionTerm; validDateTermsFound++; } } } else if (expressionItem instanceof ExpressionOperator) { if (((ExpressionOperator) expressionItem).getOp() == null) { throw new IllegalArgumentException( "An operator in the query is missing a type, it should be one of " + ExpressionOperator.Op.values()); } } } } // ensure we have a date term if (dateTermsFound != 1 || validDateTermsFound != 1) { throw new UnsupportedOperationException( "Search queries on the statistic store must contain one term using the '" + StatisticStoreEntity.FIELD_NAME_DATE_TIME + "' field with one of the following condtitions [" + SUPPORTED_DATE_CONDITIONS.toString() + "]. Please amend the query"); } // ensure the value field is not used in the query terms if (contains(searchRequest.getQuery().getExpression(), StatisticStoreEntity.FIELD_NAME_VALUE)) { throw new UnsupportedOperationException( "Search queries containing the field '" + StatisticStoreEntity.FIELD_NAME_VALUE + "' are not supported. Please remove it from the query"); } // if we have got here then we have a single BETWEEN date term, so parse // it. final Range<Long> range = extractRange(dateTerm, searchRequest.getDateTimeLocale(), nowEpochMilli); final List<ExpressionTerm> termNodesInFilter = new ArrayList<>(); findAllTermNodes(topLevelExpressionOperator, termNodesInFilter); final Set<String> rolledUpFieldNames = new HashSet<>(); for (final ExpressionTerm term : termNodesInFilter) { // add any fields that use the roll up marker to the black list. If // somebody has said user=* then we do not // want that in the filter as it will slow it down. The fact that // they have said user=* means it will use // the statistic name appropriate for that rollup, meaning the // filtering is built into the stat name. if (term.getValue().equals(RollUpBitMask.ROLL_UP_TAG_VALUE)) { rolledUpFieldNames.add(term.getField()); } } if (!rolledUpFieldNames.isEmpty()) { if (dataSource.getRollUpType().equals(StatisticRollUpType.NONE)) { throw new UnsupportedOperationException( "Query contains rolled up terms but the Statistic Data Source does not support any roll-ups"); } else if (dataSource.getRollUpType().equals(StatisticRollUpType.CUSTOM)) { if (!dataSource.isRollUpCombinationSupported(rolledUpFieldNames)) { throw new UnsupportedOperationException(String.format( "The query contains a combination of rolled up fields %s that is not in the list of custom roll-ups for the statistic data source", rolledUpFieldNames)); } } } // Date Time is handled spearately to the the filter tree so ignore it // in the conversion final Set<String> blackListedFieldNames = new HashSet<>(); blackListedFieldNames.addAll(rolledUpFieldNames); blackListedFieldNames.add(StatisticStoreEntity.FIELD_NAME_DATE_TIME); final FilterTermsTree filterTermsTree = FilterTermsTreeBuilder .convertExpresionItemsTree(topLevelExpressionOperator, blackListedFieldNames); final FindEventCriteria criteria = FindEventCriteria.instance(new Period(range.getFrom(), range.getTo()), dataSource.getName(), filterTermsTree, rolledUpFieldNames); if (LOGGER.isInfoEnabled()) { LOGGER.info("Searching statistics store with criteria: {}", criteria.toString()); } return criteria; } /** * Recursive method to populates the passed list with all enabled * {@link ExpressionTerm} nodes found in the tree. */ public static void findAllTermNodes(final ExpressionItem node, final List<ExpressionTerm> termsFound) { // Don't go any further down this branch if this node is disabled. if (node.enabled()) { if (node instanceof ExpressionTerm) { final ExpressionTerm termNode = (ExpressionTerm) node; termsFound.add(termNode); } else if (node instanceof ExpressionOperator) { for (final ExpressionItem childNode : ((ExpressionOperator) node).getChildren()) { findAllTermNodes(childNode, termsFound); } } } } public static boolean contains(final ExpressionItem expressionItem, final String fieldToFind) { boolean hasBeenFound = false; if (expressionItem instanceof ExpressionOperator) { if (((ExpressionOperator) expressionItem).getChildren() != null) { for (final ExpressionItem item : ((ExpressionOperator) expressionItem).getChildren()) { hasBeenFound = contains(item, fieldToFind); if (hasBeenFound) { break; } } } } else { if (((ExpressionTerm) expressionItem).getField() != null) { hasBeenFound = ((ExpressionTerm) expressionItem).getField().equals(fieldToFind); } } return hasBeenFound; } // TODO could go futher up the chain so is store agnostic public static RolledUpStatisticEvent generateTagRollUps(final StatisticEvent event, final StatisticStoreEntity statisticsDataSource) { RolledUpStatisticEvent rolledUpStatisticEvent = null; final int eventTagListSize = event.getTagList().size(); final StatisticRollUpType rollUpType = statisticsDataSource.getRollUpType(); if (eventTagListSize == 0 || StatisticRollUpType.NONE.equals(rollUpType)) { rolledUpStatisticEvent = new RolledUpStatisticEvent(event); } else if (StatisticRollUpType.ALL.equals(rollUpType)) { final List<List<StatisticTag>> tagListPerms = generateStatisticTagPerms(event.getTagList(), RollUpBitMask.getRollUpPermutationsAsBooleans(eventTagListSize)); // wrap the original event along with the perms list rolledUpStatisticEvent = new RolledUpStatisticEvent(event, tagListPerms); } else if (StatisticRollUpType.CUSTOM.equals(rollUpType)) { final Set<List<Boolean>> perms = new HashSet<>(); for (final CustomRollUpMask mask : statisticsDataSource.getStatisticDataSourceDataObject() .getCustomRollUpMasks()) { final RollUpBitMask rollUpBitMask = RollUpBitMask.fromTagPositions(mask.getRolledUpTagPositions()); perms.add(rollUpBitMask.getBooleanMask(eventTagListSize)); } final List<List<StatisticTag>> tagListPerms = generateStatisticTagPerms(event.getTagList(), perms); rolledUpStatisticEvent = new RolledUpStatisticEvent(event, tagListPerms); } return rolledUpStatisticEvent; } private static Range<Long> extractRange(final ExpressionTerm dateTerm, final String timeZoneId, final long nowEpochMilli) { final String[] dateArr = dateTerm.getValue().split(","); if (dateArr.length != 2) { throw new RuntimeException("DateTime term is not a valid format, term: " + dateTerm.toString()); } long rangeFrom = DateExpressionParser.parse(dateArr[0], timeZoneId, nowEpochMilli) .map(time -> time.toInstant().toEpochMilli()).orElse(0L); // add one to make it exclusive long rangeTo = DateExpressionParser.parse(dateArr[1], timeZoneId, nowEpochMilli) .map(time -> time.toInstant().toEpochMilli() + 1).orElse(Long.MAX_VALUE); final Range<Long> range = new Range<>(rangeFrom, rangeTo); return range; } private static long parseDateTime(final String type, final String value, final String timeZoneId, final long nowEpochMilli) { final ZonedDateTime dateTime; try { dateTime = DateExpressionParser.parse(value, timeZoneId, nowEpochMilli).get(); } catch (final Exception e) { throw new RuntimeException("DateTime term has an invalid '" + type + "' value of '" + value + "'"); } if (dateTime == null) { throw new RuntimeException("DateTime term has an invalid '" + type + "' value of '" + value + "'"); } return dateTime.toInstant().toEpochMilli(); } private static List<List<StatisticTag>> generateStatisticTagPerms(final List<StatisticTag> eventTags, final Set<List<Boolean>> perms) { final List<List<StatisticTag>> tagListPerms = new ArrayList<>(); final int eventTagListSize = eventTags.size(); for (final List<Boolean> perm : perms) { final List<StatisticTag> tags = new ArrayList<>(); for (int i = 0; i < eventTagListSize; i++) { if (perm.get(i).booleanValue() == true) { // true means a rolled up tag so create a new tag with the // rolled up marker tags.add(new StatisticTag(eventTags.get(i).getTag(), RollUpBitMask.ROLL_UP_TAG_VALUE)); } else { // false means not rolled up so use the existing tag's value tags.add(eventTags.get(i)); } } tagListPerms.add(tags); } return tagListPerms; } /** * TODO: This is a bit simplistic as a user could create a filter that said * user=user1 AND user='*' which makes no sense. At the moment we would * assume that the user tag is being rolled up so user=user1 would never be * found in the data and thus would return no data. */ public static RollUpBitMask buildRollUpBitMaskFromCriteria(final FindEventCriteria criteria, final StatisticStoreEntity statisticsDataSource) { final Set<String> rolledUpTagsFound = criteria.getRolledUpFieldNames(); final RollUpBitMask result; if (rolledUpTagsFound.size() > 0) { final List<Integer> rollUpTagPositionList = new ArrayList<>(); for (final String tag : rolledUpTagsFound) { final Integer position = statisticsDataSource.getPositionInFieldList(tag); if (position == null) { throw new RuntimeException(String.format("No field position found for tag %s", tag)); } rollUpTagPositionList.add(position); } result = RollUpBitMask.fromTagPositions(rollUpTagPositionList); } else { result = RollUpBitMask.ZERO_MASK; } return result; } // public static boolean isDataStoreEnabled(final String engineName, final StroomPropertyService propertyService) { // final String enabledEngines = propertyService // .getProperty(CommonStatisticConstants.STROOM_STATISTIC_ENGINES_PROPERTY_NAME); // // if (LOGGER.isDebugEnabled()) { // LOGGER.debug("{} property value: {}", CommonStatisticConstants.STROOM_STATISTIC_ENGINES_PROPERTY_NAME, // enabledEngines); // } // // boolean result = false; // // if (enabledEngines != null) { // for (final String engine : enabledEngines.split(",")) { // if (engine.equals(engineName)) { // result = true; // } // } // } // return result; // } protected Set<String> getIndexFieldBlackList() { return BLACK_LISTED_INDEX_FIELDS; } private GenericObjectPoolConfig getObjectPoolConfig() { final GenericObjectPoolConfig config = new GenericObjectPoolConfig(); // Max number of idle items .... same as our pool size config.setMaxIdle(poolSize); // Pool size config.setMaxTotal(poolSize); // Returns the minimum amount of time an object may sit idle in the pool // before it is eligible for eviction by the idle object evictor // Here if it is idle for 10 min's it will simply return It will also // return by validateObject if it is simple more than 10min old config.setMinEvictableIdleTimeMillis(poolAgeMsThreshold); // Check for idle objects never .... we will do this with task sytstem config.setTimeBetweenEvictionRunsMillis(0); // Must cause other threads to block to wait for a object config.setBlockWhenExhausted(true); config.setJmxEnabled(false); // Check item on just before returning to pool config.setTestOnReturn(true); return config; } @StroomFrequencySchedule("1m") public void evict() { LOGGER.debug("evict"); try { objectPool.evict(); } catch (final Exception ex) { LOGGER.error("evict", ex); } } private void initPool(final GenericObjectPoolConfig config) { objectPool = new GenericObjectPool<>(new ObjectFactory(), config); } /** * Get the threshold datetime (ms) for processing a statistic * * @return The threshold in ms since unix epoch */ private Long getEventProcessingThresholdMs() { final String eventProcessingThresholdStr = propertyService .getProperty(SQLStatisticConstants.PROP_KEY_STATS_MAX_PROCESSING_AGE); if (StringUtils.hasText(eventProcessingThresholdStr)) { final long duration = ModelStringUtil.parseDurationString(eventProcessingThresholdStr); return Long.valueOf(System.currentTimeMillis() - duration); } else { return null; } } private boolean isStatisticEventInsideProcessingThreshold(final StatisticEvent statisticEvent, final Long optionalEventProcessingThresholdMs) { return statisticEvent .getTimeMs() > (optionalEventProcessingThresholdMs != null ? optionalEventProcessingThresholdMs : 0); } public SQLStatisticAggregateMap createAggregateMap() { return new SQLStatisticAggregateMap(); } public void destroyAggregateMap(final SQLStatisticAggregateMap map) { LOGGER.debug("destroyAggregateMap - Flushing map size={}", map.size()); statisticCache.add(map); } @Override public void putEvents(final List<StatisticEvent> statisticEvents, final StatisticStore statisticStore) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("putEvents - count={}", statisticEvents.size()); } final Long optionalEventProcessingThresholdMs = getEventProcessingThresholdMs(); final StatisticStoreEntity entity = (StatisticStoreEntity) Preconditions.checkNotNull(statisticStore); // validate the first stat in the batch to check we have a statistic // data source for it. if (validateStatisticDataSource(statisticEvents.iterator().next(), entity) == false) { // no StatisticsDataSource entity so don't record the stat as we // will have no way of querying the stat throw new RuntimeException( String.format("Invalid or missing statistic datasource with name %s", entity.getName())); } try { final SQLStatisticAggregateMap statisticAggregateMap = objectPool.borrowObject(); try { for (final StatisticEvent statisticEvent : statisticEvents) { // Only process a stat if it is inside the processing // threshold if (isStatisticEventInsideProcessingThreshold(statisticEvent, optionalEventProcessingThresholdMs)) { final RolledUpStatisticEvent rolledUpStatisticEvent = generateTagRollUps(statisticEvent, entity); statisticAggregateMap.addRolledUpEvent(rolledUpStatisticEvent, entity.getPrecision()); } } } finally { objectPool.returnObject(statisticAggregateMap); } } catch (final StatisticsEventValidationException seve) { throw new RuntimeException(seve.getMessage(), seve); } catch (final Exception ex) { LOGGER.error("putEvent()", ex); } } @Override public void putEvent(final StatisticEvent statisticEvent, final StatisticStore statisticStore) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("putEvent - count=1"); } final StatisticStoreEntity entity = (StatisticStoreEntity) statisticStore; // validate the first stat in the batch to check we have a statistic // data source for it. if (validateStatisticDataSource(statisticEvent, entity) == false) { // no StatisticsDataSource entity so don't record the stat as we // will have no way of querying the stat throw new RuntimeException( String.format("Invalid or missing statistic datasource with name %s", entity.getName())); } // Only process a stat if it is inside the processing threshold if (isStatisticEventInsideProcessingThreshold(statisticEvent, getEventProcessingThresholdMs())) { final RolledUpStatisticEvent rolledUpStatisticEvent = generateTagRollUps(statisticEvent, entity); try { final SQLStatisticAggregateMap statisticAggregateMap = objectPool.borrowObject(); try { statisticAggregateMap.addRolledUpEvent(rolledUpStatisticEvent, entity.getPrecision()); } finally { objectPool.returnObject(statisticAggregateMap); } } catch (final StatisticsEventValidationException seve) { throw new RuntimeException(seve.getMessage(), seve); } catch (final Exception ex) { LOGGER.error("putEvent()", ex); throw new RuntimeException(String.format("Exception adding statistics to the aggregateMap"), ex); } } } public StatisticDataSet searchStatisticsData(final SearchRequest searchRequest, final StatisticStoreEntity dataSource) { final FindEventCriteria criteria = buildCriteria(searchRequest, dataSource); return performStatisticQuery(dataSource, criteria); } @Override public List<String> getValuesByTag(final String tagName) { throw new UnsupportedOperationException("Code waiting to be written"); } @Override public List<String> getValuesByTagAndPartialValue(final String tagName, final String partialValue) { throw new UnsupportedOperationException("Code waiting to be written"); } @Override public void flushAllEvents() { throw new UnsupportedOperationException("Code waiting to be written"); } @Override public String toString() { return "numActive=" + objectPool.getNumActive() + ", numIdle=" + objectPool.getNumIdle(); } public int getNumActive() { return objectPool.getNumActive(); } public int getNumIdle() { return objectPool.getNumIdle(); } private StatisticDataSet performStatisticQuery(final StatisticStoreEntity dataSource, final FindEventCriteria criteria) { final Set<StatisticDataPoint> dataPoints = new HashSet<>(); // TODO need to fingure out how we get the precision final StatisticDataSet statisticDataSet = new StatisticDataSet(dataSource.getName(), dataSource.getStatisticType(), 1000L, dataPoints); try (Connection connection = statisticsDataSource.getConnection()) { try (PreparedStatement ps = buildSearchPreparedStatement(dataSource, criteria, connection)) { try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { final StatisticType statisticType = StatisticType.PRIMITIVE_VALUE_CONVERTER .fromPrimitiveValue(rs.getByte(SQLStatisticNames.VALUE_TYPE)); final List<StatisticTag> statisticTags = extractStatisticTagsFromColumn( rs.getString(SQLStatisticNames.NAME)); final long timeMs = rs.getLong(SQLStatisticNames.TIME_MS); // the precision in the table represents the number of zeros // of millisecond precision, e.g. // 6=1,000,000ms final long precisionMs = (long) Math.pow(10, rs.getInt(SQLStatisticNames.PRECISION)); StatisticDataPoint statisticDataPoint; if (StatisticType.COUNT.equals(statisticType)) { statisticDataPoint = new CountStatisticDataPoint(timeMs, precisionMs, statisticTags, rs.getLong(SQLStatisticNames.COUNT)); } else { final double aggregatedValue = rs.getDouble(SQLStatisticNames.VALUE); final long count = rs.getLong(SQLStatisticNames.COUNT); // the aggregateValue is sum of all values against that // key/time. We therefore need to get the // average using the count column final double averagedValue = count != 0 ? (aggregatedValue / count) : 0; // min/max are not supported by SQL stats so use -1 statisticDataPoint = new ValueStatisticDataPoint(timeMs, precisionMs, statisticTags, count, averagedValue); } statisticDataSet.addDataPoint(statisticDataPoint); } } } } catch (final SQLException sqlEx) { LOGGER.error("performStatisticQuery failed", sqlEx); throw new RuntimeException("performStatisticQuery failed", sqlEx); } return statisticDataSet; } /** * @param columnValue The value from the STAT_KEY.NAME column which could be of the * form 'StatName' or 'StatNameTag1Tag1Val1Tag2Tag2Val1' * @return A list of {@link StatisticTag} objects built from the tag/value * token pairs in the string or an empty list if there are none. */ private List<StatisticTag> extractStatisticTagsFromColumn(final String columnValue) { final String[] tokens = columnValue.split(SQLStatisticConstants.NAME_SEPARATOR); final List<StatisticTag> statisticTags = new ArrayList<>(); if (tokens.length == 1) { // no separators so there are no tags } else if (tokens.length % 2 == 0) { throw new RuntimeException( String.format("Expecting an odd number of tokens, columnValue: %s", columnValue)); } else { // stat name will be at pos 0 so start at 1 for (int i = 1; i < tokens.length; i++) { final String tag = tokens[i++]; String value = tokens[i]; if (value.equals(SQLStatisticConstants.NULL_VALUE_STRING)) { value = null; } final StatisticTag statisticTag = new StatisticTag(tag, value); statisticTags.add(statisticTag); } } return statisticTags; } private PreparedStatement buildSearchPreparedStatement(final StatisticStoreEntity dataSource, final FindEventCriteria criteria, final Connection connection) throws SQLException { final RollUpBitMask rollUpBitMask = SQLStatisticEventStore.buildRollUpBitMaskFromCriteria(criteria, dataSource); final String statNameWithMask = dataSource.getName() + rollUpBitMask.asHexString(); final List<String> bindVariables = new ArrayList<>(); String sqlQuery = STAT_QUERY_SKELETON + " "; final String whereClause = SQLTagValueWhereClauseConverter .buildTagValueWhereClause(criteria.getFilterTermsTree(), bindVariables); if (whereClause != null && whereClause.length() != 0) { sqlQuery += " AND " + whereClause; } final int maxResults = propertyService.getIntProperty(PROP_KEY_SQL_SEARCH_MAX_RESULTS, 100000); sqlQuery += " LIMIT " + maxResults; LOGGER.debug("Search query: {}", sqlQuery); final PreparedStatement ps = connection.prepareStatement(sqlQuery); int position = 1; // do a like on the name first so we can hit the index before doing the // slow regex matches ps.setString(position++, statNameWithMask + "%"); // regex to match on the stat name which is always at the start of the // string and either has a ps.setString(position++, "^" + statNameWithMask + "(" + SQLStatisticConstants.NAME_SEPARATOR + "|$)"); // set the start/end dates ps.setLong(position++, criteria.getPeriod().getFromMs()); ps.setLong(position++, criteria.getPeriod().getToMs()); for (final String bindVariable : bindVariables) { ps.setString(position++, bindVariable); } return ps; } @Override public void putEvent(final StatisticEvent statisticEvent) { final StatisticStoreEntity statisticsDataSource = getStatisticsDataSource(statisticEvent.getName()); putEvent(statisticEvent, statisticsDataSource); } @Override public void putEvents(final List<StatisticEvent> statisticEvents) { statisticEvents.stream().collect(Collectors.groupingBy(StatisticEvent::getName, Collectors.toList())) .values().forEach(this::putBatch); } /** * @param eventsBatch A batch of {@link StatisticEvent} all with the same statistic name */ private void putBatch(final List<StatisticEvent> eventsBatch) { if (eventsBatch.size() > 0) { final StatisticEvent firstEventInBatch = eventsBatch.get(0); final String statName = firstEventInBatch.getName(); final StatisticStoreEntity statisticsDataSource = getStatisticsDataSource(statName); if (statisticsDataSource == null) { throw new RuntimeException(String.format("No statistic data source exists for name %s", statName)); } putEvents(eventsBatch, statisticsDataSource); eventsBatch.clear(); } } protected boolean validateStatisticDataSource(final StatisticEvent statisticEvent, final StatisticStoreEntity statisticsDataSource) { if (statisticsDataSourceValidator != null) { return statisticsDataSourceValidator.validateStatisticDataSource(statisticEvent.getName(), statisticEvent.getType(), statisticsDataSource); } else { // no validator has been supplied so return true return true; } } protected StatisticStoreEntity getStatisticsDataSource(final String statisticName) { return statisticsDataSourceCache.getStatisticsDataSource(statisticName); } public List<DataSourceField> getSupportedFields(final List<DataSourceField> indexFields) { final Set<String> blackList = getIndexFieldBlackList(); if (blackList.size() == 0) { // nothing blacklisted so just return the standard list from the // data source return indexFields; } else { // construct an anonymous class instance that will filter out black // listed index fields, as supplied by the // sub-class final List<DataSourceField> supportedIndexFields = new ArrayList<>(); indexFields.stream().filter(indexField -> !blackList.contains(indexField.getName())) .forEach(supportedIndexFields::add); return supportedIndexFields; } } // public boolean isDataStoreEnabled() { // return SQLStatisticEventStore.isDataStoreEnabled(getEngineName(), propertyService); // } public List<Set<Integer>> getFieldPositionsForBitMasks(final List<Short> maskValues) { if (maskValues != null) { final List<Set<Integer>> tagPosPermsList = new ArrayList<>(); for (final Short maskValue : maskValues) { tagPosPermsList.add(RollUpBitMask.fromShort(maskValue).getTagPositions()); } return tagPosPermsList; } else { return Collections.emptyList(); } } private class ObjectFactory extends BasePooledObjectFactory<SQLStatisticAggregateMap> { @Override public SQLStatisticAggregateMap create() throws Exception { return createAggregateMap(); } @Override public PooledObject<SQLStatisticAggregateMap> wrap(final SQLStatisticAggregateMap obj) { return new DefaultPooledObject<>(obj); } @Override public void destroyObject(final PooledObject<SQLStatisticAggregateMap> p) throws Exception { super.destroyObject(p); destroyAggregateMap(p.getObject()); } /** * Should we give this item back to the pool */ @Override public boolean validateObject(final PooledObject<SQLStatisticAggregateMap> p) { if (p.getObject().size() >= aggregatorSizeThreshold) { return false; } final long age = System.currentTimeMillis() - p.getCreateTime(); if (age > poolAgeMsThreshold) { return false; } return super.validateObject(p); } } }