Java tutorial
// This file is part of OpenTSDB. // Copyright (C) 2013 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 2.1 of the License, or (at your // option) any later version. This program is distributed in the hope that it // will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty // of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser // General Public License for more details. You should have received a copy // of the GNU Lesser General Public License along with this program. If not, // see <http://www.gnu.org/licenses/>. package net.opentsdb.tree; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.regex.Pattern; import org.hbase.async.Bytes; import org.hbase.async.DeleteRequest; import org.hbase.async.GetRequest; import org.hbase.async.HBaseException; import org.hbase.async.KeyValue; import org.hbase.async.PutRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import net.opentsdb.core.TSDB; import net.opentsdb.utils.JSON; import net.opentsdb.utils.JSONException; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.stumbleupon.async.Callback; import com.stumbleupon.async.Deferred; /** * Represents single rule in a set of rules for a given tree. Each rule is * uniquely identified by: * <ul><li>tree_id - The ID of the tree to which the rule belongs</li> * <li>level - Outer processing order where the rule resides. Lower values are * processed first. Starts at 0.</li> * <li>order - Inner processing order within a given level. Lower values are * processed first. Starts at 0.</li></ul> * Each rule is stored as an individual column so that they can be modified * individually. RPC calls can also bulk replace rule sets. * @since 2.0 */ @JsonIgnoreProperties(ignoreUnknown = true) @JsonAutoDetect(fieldVisibility = Visibility.PUBLIC_ONLY) public final class TreeRule { /** Types of tree rules */ public enum TreeRuleType { METRIC, /** A simple metric rule */ METRIC_CUSTOM, /** Matches on UID Meta custom field */ TAGK, /** Matches on a tagk name */ TAGK_CUSTOM, /** Matches on a UID Meta custom field */ TAGV_CUSTOM /** Matches on a UID Meta custom field */ } private static final Logger LOG = LoggerFactory.getLogger(TreeRule.class); /** Charset used to convert Strings to byte arrays and back. */ private static final Charset CHARSET = Charset.forName("ISO-8859-1"); /** ASCII Rule prefix. Qualifier is tree_rule:<level>:<order> */ private static final byte[] RULE_PREFIX = "tree_rule:".getBytes(CHARSET); /** Type of rule */ @JsonDeserialize(using = JSON.TreeRuleTypeDeserializer.class) private TreeRuleType type = null; /** Name of the field to match on if applicable */ private String field = ""; /** Name of the custom field to match on, the key */ private String custom_field = ""; /** User supplied regular expression before parsing */ private String regex = ""; /** Separation character or string */ private String separator = ""; /** An optional description of the rule */ private String description = ""; /** Optional notes about the rule */ private String notes = ""; /** Optional group index for extracting from regex matches */ private int regex_group_idx = 0; /** Optioanl display format override */ private String display_format = ""; /** Required level where the rule resides */ private int level = 0; /** Required order where the rule resides */ private int order = 0; /** The tree this rule belongs to */ private int tree_id = 0; /** Compiled regex pattern, compiled after processing */ private Pattern compiled_regex = null; /** Tracks fields that have changed by the user to avoid overwrites */ private final HashMap<String, Boolean> changed = new HashMap<String, Boolean>(); /** * Default constructor necessary for de/serialization */ public TreeRule() { initializeChangedMap(); } /** * Constructor initializes the tree ID * @param tree_id The tree this rule belongs to */ public TreeRule(final int tree_id) { this.tree_id = tree_id; initializeChangedMap(); } /** * Copy constructor that creates a completely independent copy of the original * object * @param original The original object to copy from * @throws PatternSyntaxException if the regex is invalid */ public TreeRule(final TreeRule original) { custom_field = original.custom_field; description = original.description; display_format = original.display_format; field = original.field; level = original.level; notes = original.notes; order = original.order; regex_group_idx = original.regex_group_idx; separator = original.separator; tree_id = original.tree_id; type = original.type; setRegex(original.regex); initializeChangedMap(); } /** * Copies changed fields from the incoming rule to the local rule * @param rule The rule to copy from * @param overwrite Whether or not to replace all fields in the local object * @return True if there were changes, false if everything was identical */ public boolean copyChanges(final TreeRule rule, final boolean overwrite) { if (rule == null) { throw new IllegalArgumentException("Cannot copy a null rule"); } if (tree_id != rule.tree_id) { throw new IllegalArgumentException("Tree IDs do not match"); } if (level != rule.level) { throw new IllegalArgumentException("Levels do not match"); } if (order != rule.order) { throw new IllegalArgumentException("Orders do not match"); } if (overwrite || (rule.changed.get("type") && type != rule.type)) { type = rule.type; changed.put("type", true); } if (overwrite || (rule.changed.get("field") && !field.equals(rule.field))) { field = rule.field; changed.put("field", true); } if (overwrite || (rule.changed.get("custom_field") && !custom_field.equals(rule.custom_field))) { custom_field = rule.custom_field; changed.put("custom_field", true); } if (overwrite || (rule.changed.get("regex") && !regex.equals(rule.regex))) { // validate and compile via the setter setRegex(rule.regex); } if (overwrite || (rule.changed.get("separator") && !separator.equals(rule.separator))) { separator = rule.separator; changed.put("separator", true); } if (overwrite || (rule.changed.get("description") && !description.equals(rule.description))) { description = rule.description; changed.put("description", true); } if (overwrite || (rule.changed.get("notes") && !notes.equals(rule.notes))) { notes = rule.notes; changed.put("notes", true); } if (overwrite || (rule.changed.get("regex_group_idx") && regex_group_idx != rule.regex_group_idx)) { regex_group_idx = rule.regex_group_idx; changed.put("regex_group_idx", true); } if (overwrite || (rule.changed.get("display_format") && !display_format.equals(rule.display_format))) { display_format = rule.display_format; changed.put("display_format", true); } for (boolean has_changes : changed.values()) { if (has_changes) { return true; } } return false; } /** @return the rule ID as [tree_id:level:order] */ @Override public String toString() { return "[" + tree_id + ":" + level + ":" + order + ":" + type + "]"; } /** * Attempts to write the rule to storage via CompareAndSet, merging changes * with an existing rule. * <b>Note:</b> If the local object didn't have any fields set by the caller * or there weren't any changes, then the data will not be written and an * exception will be thrown. * <b>Note:</b> This method also validates the rule, making sure that proper * combinations of data exist before writing to storage. * @param tsdb The TSDB to use for storage access * @param overwrite When the RPC method is PUT, will overwrite all user * accessible fields * @return True if the CAS call succeeded, false if the stored data was * modified in flight. This should be retried if that happens. * @throws HBaseException if there was an issue * @throws IllegalArgumentException if parsing failed or the tree ID was * invalid or validation failed * @throws IllegalStateException if the data hasn't changed. This is OK! * @throws JSONException if the object could not be serialized */ public Deferred<Boolean> syncToStorage(final TSDB tsdb, final boolean overwrite) { if (tree_id < 1 || tree_id > 65535) { throw new IllegalArgumentException("Invalid Tree ID"); } // if there aren't any changes, save time and bandwidth by not writing to // storage boolean has_changes = false; for (Map.Entry<String, Boolean> entry : changed.entrySet()) { if (entry.getValue()) { has_changes = true; break; } } if (!has_changes) { LOG.trace(this + " does not have changes, skipping sync to storage"); throw new IllegalStateException("No changes detected in the rule"); } /** * Executes the CAS after retrieving existing rule from storage, if it * exists. */ final class StoreCB implements Callback<Deferred<Boolean>, TreeRule> { final TreeRule local_rule; public StoreCB(final TreeRule local_rule) { this.local_rule = local_rule; } /** * @return True if the CAS was successful, false if not */ @Override public Deferred<Boolean> call(final TreeRule fetched_rule) { TreeRule stored_rule = fetched_rule; final byte[] original_rule = stored_rule == null ? new byte[0] : JSON.serializeToBytes(stored_rule); if (stored_rule == null) { stored_rule = local_rule; } else { if (!stored_rule.copyChanges(local_rule, overwrite)) { LOG.debug(this + " does not have changes, skipping sync to storage"); throw new IllegalStateException("No changes detected in the rule"); } } // reset the local change map so we don't keep writing on subsequent // requests initializeChangedMap(); // validate before storing stored_rule.validateRule(); final PutRequest put = new PutRequest(tsdb.treeTable(), Tree.idToBytes(tree_id), Tree.TREE_FAMILY(), getQualifier(level, order), JSON.serializeToBytes(stored_rule)); return tsdb.getClient().compareAndSet(put, original_rule); } } // start the callback chain by fetching from storage return fetchRule(tsdb, tree_id, level, order).addCallbackDeferring(new StoreCB(this)); } /** * Parses a rule from the given column. Used by the Tree class when scanning * a row for rules. * @param column The column to parse * @return A valid TreeRule object if parsed successfully * @throws IllegalArgumentException if the column was empty * @throws JSONException if the object could not be serialized */ public static TreeRule parseFromStorage(final KeyValue column) { if (column.value() == null) { throw new IllegalArgumentException("Tree rule column value was null"); } final TreeRule rule = JSON.parseToObject(column.value(), TreeRule.class); rule.initializeChangedMap(); return rule; } /** * Attempts to retrieve the specified tree rule from storage. * @param tsdb The TSDB to use for storage access * @param tree_id ID of the tree the rule belongs to * @param level Level where the rule resides * @param order Order where the rule resides * @return A TreeRule object if found, null if it does not exist * @throws HBaseException if there was an issue * @throws IllegalArgumentException if the one of the required parameters was * missing * @throws JSONException if the object could not be serialized */ public static Deferred<TreeRule> fetchRule(final TSDB tsdb, final int tree_id, final int level, final int order) { if (tree_id < 1 || tree_id > 65535) { throw new IllegalArgumentException("Invalid Tree ID"); } if (level < 0) { throw new IllegalArgumentException("Invalid rule level"); } if (order < 0) { throw new IllegalArgumentException("Invalid rule order"); } // fetch the whole row final GetRequest get = new GetRequest(tsdb.treeTable(), Tree.idToBytes(tree_id)); get.family(Tree.TREE_FAMILY()); get.qualifier(getQualifier(level, order)); /** * Called after fetching to parse the results */ final class FetchCB implements Callback<Deferred<TreeRule>, ArrayList<KeyValue>> { @Override public Deferred<TreeRule> call(final ArrayList<KeyValue> row) { if (row == null || row.isEmpty()) { return Deferred.fromResult(null); } return Deferred.fromResult(parseFromStorage(row.get(0))); } } return tsdb.getClient().get(get).addCallbackDeferring(new FetchCB()); } /** * Attempts to delete the specified rule from storage * @param tsdb The TSDB to use for storage access * @param tree_id ID of the tree the rule belongs to * @param level Level where the rule resides * @param order Order where the rule resides * @return A deferred without meaning. The response may be null and should * only be used to track completion. * @throws HBaseException if there was an issue * @throws IllegalArgumentException if the one of the required parameters was * missing */ public static Deferred<Object> deleteRule(final TSDB tsdb, final int tree_id, final int level, final int order) { if (tree_id < 1 || tree_id > 65535) { throw new IllegalArgumentException("Invalid Tree ID"); } if (level < 0) { throw new IllegalArgumentException("Invalid rule level"); } if (order < 0) { throw new IllegalArgumentException("Invalid rule order"); } final DeleteRequest delete = new DeleteRequest(tsdb.treeTable(), Tree.idToBytes(tree_id), Tree.TREE_FAMILY(), getQualifier(level, order)); return tsdb.getClient().delete(delete); } /** * Attempts to delete all rules belonging to the given tree. * @param tsdb The TSDB to use for storage access * @param tree_id ID of the tree the rules belongs to * @return A deferred to wait on for completion. The value has no meaning and * may be null. * @throws HBaseException if there was an issue * @throws IllegalArgumentException if the one of the required parameters was * missing */ public static Deferred<Object> deleteAllRules(final TSDB tsdb, final int tree_id) { if (tree_id < 1 || tree_id > 65535) { throw new IllegalArgumentException("Invalid Tree ID"); } // fetch the whole row final GetRequest get = new GetRequest(tsdb.treeTable(), Tree.idToBytes(tree_id)); get.family(Tree.TREE_FAMILY()); /** * Called after fetching the requested row. If the row is empty, we just * return, otherwise we compile a list of qualifiers to delete and submit * a single delete request to storage. */ final class GetCB implements Callback<Deferred<Object>, ArrayList<KeyValue>> { @Override public Deferred<Object> call(final ArrayList<KeyValue> row) throws Exception { if (row == null || row.isEmpty()) { return Deferred.fromResult(null); } final ArrayList<byte[]> qualifiers = new ArrayList<byte[]>(row.size()); for (KeyValue column : row) { if (column.qualifier().length > RULE_PREFIX.length && Bytes.memcmp(RULE_PREFIX, column.qualifier(), 0, RULE_PREFIX.length) == 0) { qualifiers.add(column.qualifier()); } } final DeleteRequest delete = new DeleteRequest(tsdb.treeTable(), Tree.idToBytes(tree_id), Tree.TREE_FAMILY(), qualifiers.toArray(new byte[qualifiers.size()][])); return tsdb.getClient().delete(delete); } } return tsdb.getClient().get(get).addCallbackDeferring(new GetCB()); } /** * Parses a string into a rule type enumerator * @param type The string to parse * @return The type enumerator * @throws IllegalArgumentException if the type was empty or invalid */ public static TreeRuleType stringToType(final String type) { if (type == null || type.isEmpty()) { throw new IllegalArgumentException("Rule type was empty"); } else if (type.toLowerCase().equals("metric")) { return TreeRuleType.METRIC; } else if (type.toLowerCase().equals("metric_custom")) { return TreeRuleType.METRIC_CUSTOM; } else if (type.toLowerCase().equals("tagk")) { return TreeRuleType.TAGK; } else if (type.toLowerCase().equals("tagk_custom")) { return TreeRuleType.TAGK_CUSTOM; } else if (type.toLowerCase().equals("tagv_custom")) { return TreeRuleType.TAGV_CUSTOM; } else { throw new IllegalArgumentException("Unrecognized rule type"); } } /** @return The configured rule column prefix */ public static byte[] RULE_PREFIX() { return RULE_PREFIX; } /** * Completes the column qualifier given a level and order using the configured * prefix * @param level The level of the rule * @param order The order of the rule * @return A byte array with the column qualifier */ public static byte[] getQualifier(final int level, final int order) { final byte[] suffix = (level + ":" + order).getBytes(CHARSET); final byte[] qualifier = new byte[RULE_PREFIX.length + suffix.length]; System.arraycopy(RULE_PREFIX, 0, qualifier, 0, RULE_PREFIX.length); System.arraycopy(suffix, 0, qualifier, RULE_PREFIX.length, suffix.length); return qualifier; } /** * Sets or resets the changed map flags */ private void initializeChangedMap() { // set changed flags changed.put("type", false); changed.put("field", false); changed.put("custom_field", false); changed.put("regex", false); changed.put("separator", false); changed.put("description", false); changed.put("notes", false); changed.put("regex_group_idx", false); changed.put("display_format", false); changed.put("level", false); changed.put("order", false); // tree_id can't change } /** * Checks that the local rule has valid data, i.e. that for different types * of rules, the proper parameters exist. For example, a {@code TAGV_CUSTOM} * rule must have a valid {@code field} parameter set. * @throws IllegalArgumentException if an invalid combination of parameters * is provided */ private void validateRule() { if (type == null) { throw new IllegalArgumentException("Missing rule type"); } switch (type) { case METRIC: // nothing to validate break; case METRIC_CUSTOM: case TAGK_CUSTOM: case TAGV_CUSTOM: if (field == null || field.isEmpty()) { throw new IllegalArgumentException("Missing field name required for " + type + " rule"); } if (custom_field == null || custom_field.isEmpty()) { throw new IllegalArgumentException("Missing custom field name required for " + type + " rule"); } break; case TAGK: if (field == null || field.isEmpty()) { throw new IllegalArgumentException("Missing field name required for " + type + " rule"); } break; default: throw new IllegalArgumentException("Invalid rule type"); } if ((regex != null || !regex.isEmpty()) && regex_group_idx < 0) { throw new IllegalArgumentException("Invalid regex group index. Cannot be less than 0"); } } // GETTERS AND SETTERS ---------------------------- /** @return the type of rule*/ public TreeRuleType getType() { return type; } /** @return the name of the field to match on */ public String getField() { return field; } /** @return the custom_field if matching */ public String getCustomField() { return custom_field; } /** @return the user supplied, uncompiled regex */ public String getRegex() { return regex; } /** @return an optional separator*/ public String getSeparator() { return separator; } /** @return the description of the rule*/ public String getDescription() { return description; } /** @return the notes */ public String getNotes() { return notes; } /** @return the regex_group_idx if using regex group extraction */ public int getRegexGroupIdx() { return regex_group_idx; } /** @return the display_format */ public String getDisplayFormat() { return display_format; } /** @return the level where the rule resides*/ public int getLevel() { return level; } /** @return the order of rule processing within a level */ public int getOrder() { return order; } /** @return the tree_id */ public int getTreeId() { return tree_id; } /** @return the compiled_regex */ @JsonIgnore public Pattern getCompiledRegex() { return compiled_regex; } /** @param type The type of rule */ public void setType(TreeRuleType type) { if (this.type != type) { changed.put("type", true); this.type = type; } } /** @param field The field name for matching */ public void setField(String field) { if (!this.field.equals(field)) { changed.put("field", true); this.field = field; } } /** @param custom_field The custom field name to set if matching */ public void setCustomField(String custom_field) { if (!this.custom_field.equals(custom_field)) { changed.put("custom_field", true); this.custom_field = custom_field; } } /** * @param regex Stores AND compiles the regex string for use in processing * @throws PatternSyntaxException if the regex is invalid */ public void setRegex(String regex) { if (!this.regex.equals(regex)) { changed.put("regex", true); this.regex = regex; if (regex != null && !regex.isEmpty()) { this.compiled_regex = Pattern.compile(regex); } else { this.compiled_regex = null; } } } /** @param separator A character or string to separate on */ public void setSeparator(String separator) { if (!this.separator.equals(separator)) { changed.put("separator", true); this.separator = separator; } } /** @param description A brief description of the rule */ public void setDescription(String description) { if (!this.description.equals(description)) { changed.put("description", true); this.description = description; } } /** @param notes Optional detailed notes about the rule */ public void setNotes(String notes) { if (!this.notes.equals(notes)) { changed.put("notes", true); this.notes = notes; } } /** @param regex_group_idx An optional index (start at 0) to use for regex * group extraction. Must be a positive value. */ public void setRegexGroupIdx(int regex_group_idx) { if (this.regex_group_idx != regex_group_idx) { changed.put("regex_group_idx", true); this.regex_group_idx = regex_group_idx; } } /** @param display_format Optional format string to alter the display name */ public void setDisplayFormat(String display_format) { if (!this.display_format.equals(display_format)) { changed.put("display_format", true); this.display_format = display_format; } } /** @param level The top level processing order. Must be 0 or greater * @throws IllegalArgumentException if the level was negative */ public void setLevel(int level) { if (level < 0) { throw new IllegalArgumentException("Negative levels are not allowed"); } if (this.level != level) { changed.put("level", true); this.level = level; } } /** @param order The order of processing within a level. * Must be 0 or greater * @throws IllegalArgumentException if the order was negative */ public void setOrder(int order) { if (level < 0) { throw new IllegalArgumentException("Negative orders are not allowed"); } if (this.order != order) { changed.put("order", true); this.order = order; } } /** @param tree_id The tree_id to set */ public void setTreeId(int tree_id) { this.tree_id = tree_id; } }