Java tutorial
/***************************************************************************** * * Copyright (C) Zenoss, Inc. 2010, 2014, all rights reserved. * * This content is made available according to terms specified in * License.zenoss under the directory where your Zenoss product is installed. * ****************************************************************************/ package org.zenoss.zep.index.impl; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.protobuf.ProtocolMessageEnum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.zenoss.protobufs.util.Util.TimestampRange; import org.zenoss.protobufs.zep.Zep.EventDetailItem.EventDetailType; import org.zenoss.protobufs.zep.Zep.EventDetailFilter; import org.zenoss.protobufs.zep.Zep.NumberRange; import org.zenoss.protobufs.zep.Zep.EventFilter; import org.zenoss.protobufs.zep.Zep.EventTagFilter; import org.zenoss.protobufs.zep.Zep.FilterOperator; import org.zenoss.zep.ZepException; import org.zenoss.zep.utils.IpRange; import org.zenoss.zep.utils.IpUtils; import java.net.InetAddress; import java.util.*; import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.zenoss.zep.index.impl.IndexConstants.*; public abstract class BaseQueryBuilder<B extends BaseQueryBuilder<B>> { protected static final Logger logger = LoggerFactory.getLogger(BaseQueryBuilder.class); public static enum Occur { MUST, SHOULD, MUST_NOT } public static enum FieldType { DATE_RANGE, ENUM_NUMBER, FULL_TEXT, IDENTIFIER, IP_ADDRESS_SUBSTRING, IP_ADDRESS, IP_ADDRESS_RANGE, NUMERIC_RANGE, PATH, TERM, WILDCARD } protected final Occur occur; protected final Set<B> subClauses; private final Map<FieldType, Map<String, Set<?>>> fieldsValues; protected BaseQueryBuilder(Occur occur) { this.occur = occur; subClauses = Sets.newHashSet(); fieldsValues = Maps.newEnumMap(FieldType.class); for (FieldType type : FieldType.values()) fieldsValues.put(type, new HashMap<String, Set<?>>()); } protected abstract B subBuilder(Occur occur); protected final synchronized <T> Set<T> getFieldSet(FieldType type, String fieldName) { Set<?> values = fieldsValues.get(type).get(fieldName); if (values == null) values = Sets.newHashSet(); fieldsValues.get(type).put(fieldName, values); return (Set<T>) values; } protected final <T> void addFieldValues(FieldType type, String fieldName, Collection<T> values) { addFieldValues(type, fieldName, values, FilterOperator.OR); } protected final <T> void addFieldValues(FieldType type, String fieldName, Collection<T> values, FilterOperator op) { if (values == null || values.isEmpty()) return; if (op != FilterOperator.AND && op != FilterOperator.OR) throw new UnsupportedOperationException("Unexpected op: " + op); switch (occur) { case SHOULD: if (op == FilterOperator.AND) { B sub = subBuilder(Occur.MUST); sub.getFieldSet(type, fieldName).addAll(values); subClauses.add(sub); } else getFieldSet(type, fieldName).addAll(values); break; case MUST: if (op == FilterOperator.AND) getFieldSet(type, fieldName).addAll(values); else { B sub = subBuilder(Occur.SHOULD); sub.getFieldSet(type, fieldName).addAll(values); subClauses.add(sub); } break; case MUST_NOT: if (op == FilterOperator.AND) { B sub = subBuilder(Occur.SHOULD); sub.getFieldSet(type, fieldName).addAll(values); subClauses.add(sub); } else getFieldSet(type, fieldName).addAll(values); break; default: throw new UnsupportedOperationException("Unexpected occur: " + occur); } } protected final Set<Entry<String, Set<?>>> fieldsValues(FieldType type) { return fieldsValues.get(type).entrySet(); } public final void addFilter(EventFilter filter) throws ZepException { addNumberRangeFields(FIELD_COUNT, filter.getCountRangeList()); addWildcardFields(FIELD_CURRENT_USER_NAME, filter.getCurrentUserNameList()); addIdentifierFields(FIELD_ELEMENT_IDENTIFIER, filter.getElementIdentifierList()); addIdentifierFields(FIELD_ELEMENT_TITLE, filter.getElementTitleList()); addIdentifierFields(FIELD_ELEMENT_SUB_IDENTIFIER, filter.getElementSubIdentifierList()); addIdentifierFields(FIELD_ELEMENT_SUB_TITLE, filter.getElementSubTitleList()); addWildcardFields(FIELD_FINGERPRINT, filter.getFingerprintList()); addFullTextFields(FIELD_SUMMARY, filter.getEventSummaryList()); addFullTextFields(FIELD_MESSAGE, filter.getMessageList()); addTimestampRanges(FIELD_FIRST_SEEN_TIME, filter.getFirstSeenList()); addTimestampRanges(FIELD_LAST_SEEN_TIME, filter.getLastSeenList()); addTimestampRanges(FIELD_STATUS_CHANGE_TIME, filter.getStatusChangeList()); addTimestampRanges(FIELD_UPDATE_TIME, filter.getUpdateTimeList()); addFieldOfEnumNumbers(FIELD_STATUS, filter.getStatusList()); addFieldOfEnumNumbers(FIELD_SEVERITY, filter.getSeverityList()); addWildcardFields(FIELD_AGENT, filter.getAgentList()); addWildcardFields(FIELD_MONITOR, filter.getMonitorList()); addWildcardFields(FIELD_EVENT_KEY, filter.getEventKeyList()); addWildcardFields(FIELD_EVENT_CLASS_KEY, filter.getEventClassKeyList()); addWildcardFields(FIELD_EVENT_GROUP, filter.getEventGroupList()); addPathFields(FIELD_EVENT_CLASS, filter.getEventClassList()); for (EventTagFilter tagFilter : filter.getTagFilterList()) { addTermFields(FIELD_TAGS, tagFilter.getTagUuidsList(), tagFilter.getOp()); } addWildcardFields(FIELD_UUID, filter.getUuidList()); addDetails(filter.getDetailsList()); for (EventFilter subFilter : filter.getSubfilterList()) { B sub = subBuilder(occur); sub.addFilter(subFilter); subClauses.add(sub); } } protected final void addTermField(String fieldName, String value) { getFieldSet(FieldType.TERM, fieldName).add(value); } protected final void addTermFields(String fieldName, Collection<String> values, FilterOperator op) { addFieldValues(FieldType.TERM, fieldName, values, op); } /** * Special case queries for event classes. Queries that begin with a slash and end * with a slash or an asterisk result in a starts-with query on the non-analyzed * field name. Queries that begin with a slash and end with anything else are an * exact match on the non-analyzed field name. Otherwise, the query is run through * the analyzer and substring matching is performed. * * @param fieldName Analyzed field name. * @param paths Queries to search on. */ protected final void addPathFields(String fieldName, Collection<String> paths) { addFieldValues(FieldType.PATH, fieldName, paths); } protected final void addFieldOfEnumNumbers(String fieldName, Collection<? extends ProtocolMessageEnum> numbers) { addFieldValues(FieldType.ENUM_NUMBER, fieldName, numbers); } protected final void addFullTextFields(String fieldName, List<String> values) { addFieldValues(FieldType.FULL_TEXT, fieldName, values); } public void addRange(String fieldName, Long from, Long to) { getFieldSet(FieldType.NUMERIC_RANGE, fieldName).add(new Range<Long>(from, to)); } protected final void addTimestampRanges(String fieldName, Collection<TimestampRange> ranges) { addFieldValues(FieldType.DATE_RANGE, fieldName, ranges); } protected final void addNumberRangeFields(String fieldName, Collection<NumberRange> ranges) { final List<Range<Long>> countRangeList = Lists.newArrayList(); for (NumberRange range : ranges) { countRangeList.add(new Range<Long>(range.hasFrom() ? range.getFrom() : null, range.hasTo() ? range.getTo() : null)); } addFieldValues(FieldType.NUMERIC_RANGE, fieldName, countRangeList); } protected final <T extends Number & Comparable<T>> void addNumericRangeField(String fieldName, Range<T> range) { getFieldSet(FieldType.NUMERIC_RANGE, fieldName).add(range); } /** * Special case queries for identifier fields. Queries that are enclosed in quotes result in * an exact query for the string in the non-analyzed field. Queries that end in an asterisk * result in a prefix query in the non-analyzed field. Queries of a length less than * the {@link IndexConstants#MIN_NGRAM_SIZE} are converted to prefix queries on the * non-analyzed field. All other queries are sent to the NGram analyzed field for efficient * substring matches. * * @param fieldName Analyzed field name. * @param values Queries to search on. */ protected final void addIdentifierFields(String fieldName, Collection<String> values) { addFieldValues(FieldType.IDENTIFIER, fieldName, values); } protected final void addIpAddressSubstringField(String fieldName, String value) { getFieldSet(FieldType.IP_ADDRESS_SUBSTRING, fieldName).add(value); } protected final void addIpAddressField(String fieldName, InetAddress value) { getFieldSet(FieldType.IP_ADDRESS, fieldName).add(value); } protected final void addIpAddressRangeField(String fieldName, IpRange value) { getFieldSet(FieldType.IP_ADDRESS_RANGE, fieldName).add(value); } protected final void addWildcardFields(String fieldName, Collection<String> values) { addFieldValues(FieldType.WILDCARD, fieldName, values); } private interface StringToNumber<T extends Number & Comparable<T>> { T convert(String s); } private final StringToNumber<Integer> INTEGER_CONVERTER = new StringToNumber<Integer>() { @Override public Integer convert(String s) { return (s == null || s.isEmpty()) ? null : Integer.valueOf(s); } }; private final StringToNumber<Float> FLOAT_CONVERTER = new StringToNumber<Float>() { @Override public Float convert(String s) { return (s == null || s.isEmpty()) ? null : Float.valueOf(s); } }; private final StringToNumber<Long> LONG_CONVERTER = new StringToNumber<Long>() { @Override public Long convert(String s) { return (s == null || s.isEmpty()) ? null : Long.valueOf(s); } }; private final StringToNumber<Double> DOUBLE_CONVERTER = new StringToNumber<Double>() { @Override public Double convert(String s) { return (s == null || s.isEmpty()) ? null : Double.valueOf(s); } }; public static class Range<T extends Comparable<T>> implements Comparable<Range<T>> { public final T from, to; public Range(T from, T to) { if (from != null && to != null && from.compareTo(to) > 0) throw new IllegalArgumentException("inverted range"); this.from = from; this.to = to; } public String toString() { StringBuilder sb = new StringBuilder(); sb.append('('); sb.append(from); sb.append('-'); sb.append(to); sb.append(')'); return sb.toString(); } @Override public int hashCode() { int result = from != null ? from.hashCode() : 0; result = 31 * result + (to != null ? to.hashCode() : 0); return result; } @Override public boolean equals(Object o) { if (this == o) return true; if (o instanceof Range) { Range that = (Range) o; return ((this.from == null) ? (that.from == null) : this.from.equals(that.from)) && ((this.to == null) ? (that.to == null) : this.to.equals(that.to)); } return false; } @Override public int compareTo(Range<T> that) { if (this.from == null) { if (that.from == null) { if (this.to == null) { if (that.to == null) return 0; else return 1; } else { if (that.to == null) return -1; else return this.to.compareTo(that.to); } } else { return -1; } } else { if (that.from == null) return 1; else { int i = this.from.compareTo(that.from); if (i != 0) return i; if (this.to == null) { if (that.to == null) return 0; else return 1; } else { if (that.to == null) return -1; else return this.to.compareTo(that.to); } } } } /** @return null if ranges cannot be merged (because they don't overlap and are not adjacent) */ public Range<T> merge(Range<T> that) { if (this.to != null && that.from != null && this.to.compareTo(that.from) < 0) { Object o = this.to; if ((o instanceof Long) || (o instanceof Integer)) { if (((Number) this.to).longValue() + 1 == ((Number) that.from).longValue()) { return new Range<T>(this.from, that.to); } } return null; } else if (that.to != null && this.from != null && that.to.compareTo(this.from) < 0) { Object o = this.to; if ((o instanceof Long) || (o instanceof Integer)) { if (((Number) that.to).longValue() + 1 == ((Number) this.from).longValue()) { return new Range<T>(that.from, this.to); } } return null; } return new Range<T>( this.from == null || that.from == null ? null : this.from.compareTo(that.from) <= 0 ? this.from : that.from, this.to == null || that.to == null ? null : this.to.compareTo(that.to) >= 0 ? this.to : that.to); } } private static <T extends Number & Comparable<T>> Range<T> parseNumericValue(String value, StringToNumber<T> converter) throws ZepException { if (value.isEmpty()) { throw new ZepException("Empty numeric value"); } final String left, right; int colonIndex = value.indexOf(':'); // any ':' means this is a range of some kind. if (colonIndex == -1) { left = right = value; } else { left = value.substring(0, colonIndex); right = value.substring(colonIndex + 1); } return new Range<T>(converter.convert(left), converter.convert(right)); } protected static String nonAnalyzed(String fieldName) { String result = NON_ANALYZED.get(fieldName); if (result != null) return result; return fieldName + SORT_SUFFIX; } protected static String unquote(String str) { final int len = str.length(); String unquoted = str; if (len >= 2 && str.charAt(0) == '"' && str.charAt(len - 1) == '"') { unquoted = str.substring(1, len - 1); } return unquoted; } protected static final Pattern LEADING_ZEROS = Pattern.compile("0+(.*)"); protected static String removeLeadingZeros(String original) { String ret = original; final Matcher matcher = LEADING_ZEROS.matcher(original); if (matcher.matches()) { final String remaining = matcher.group(1); if (remaining.isEmpty()) { ret = "0"; } else { // Preserve leading zero if next character is non numeric final char firstChar = Character.toLowerCase(remaining.charAt(0)); if ((firstChar < '0' || firstChar > '9') && (firstChar < 'a' || firstChar > 'f')) { ret = "0" + remaining; } else { ret = remaining; } } } return ret; } private void addDetails(Collection<EventDetailFilter> filters) throws ZepException { if (!filters.isEmpty()) { for (EventDetailFilter edf : filters) { B sub = subBuilder(edf.getOp().equals(FilterOperator.OR) ? Occur.SHOULD : Occur.MUST); final String key = getDetailKey(edf.getKey()); if (key == null) { throw new ZepException("Event detail is not indexed: " + edf.getKey()); } EventDetailType detailType = getDetailType(edf.getKey()); switch (detailType) { case STRING: sub.addWildcardFields(key, edf.getValueList()); break; case INTEGER: for (String val : edf.getValueList()) sub.addNumericRangeField(key, parseNumericValue(val, INTEGER_CONVERTER)); break; case FLOAT: for (String val : edf.getValueList()) sub.addNumericRangeField(key, parseNumericValue(val, FLOAT_CONVERTER)); break; case LONG: for (String val : edf.getValueList()) sub.addNumericRangeField(key, parseNumericValue(val, LONG_CONVERTER)); break; case DOUBLE: for (String val : edf.getValueList()) sub.addNumericRangeField(key, parseNumericValue(val, DOUBLE_CONVERTER)); break; case PATH: sub.addPathFields(key, edf.getValueList()); break; case IP_ADDRESS: for (String val : edf.getValueList()) { val = unquote(val); try { // Try to parse as IP range IpRange range = IpUtils.parseRange(val); if (range.getFrom().equals(range.getTo())) sub.addIpAddressField(key, range.getFrom()); else sub.addIpAddressRangeField(key, range); } catch (IllegalArgumentException e) { // Didn't match IP range - try performing a substring match sub.addIpAddressSubstringField(key, val); logger.warn(e.getLocalizedMessage()); } } break; default: throw new ZepException("Unsupported detail type: " + detailType); } if (!sub.isEmpty()) { this.subClauses.add(sub); } } } } protected abstract EventDetailType getDetailType(String key) throws ZepException; protected abstract String getDetailKey(String key) throws ZepException; public String toString() { StringBuilder sb = new StringBuilder(); sb.append(getClass().getSimpleName()); sb.append("["); sb.append(occur); sb.append(" match : "); boolean any = false; for (FieldType ft : FieldType.values()) { Map<String, Set<?>> fieldValues = fieldsValues.get(ft); if (fieldValues == null || fieldValues.isEmpty()) continue; if (any) sb.append(", "); sb.append(ft); sb.append(": "); sb.append(fieldValues); any = true; } if (!subClauses.isEmpty()) { if (any) sb.append(", "); sb.append("SUB_CLAUSES: "); sb.append(subClauses); any = true; } sb.append("]"); return sb.toString(); } protected boolean isEmpty() { for (Map<String, Set<?>> fieldValues : fieldsValues.values()) if (!fieldValues.isEmpty()) return false; for (B sub : subClauses) { if (!sub.isEmpty()) return false; } return true; } }