Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.ambari.server.controller.utilities; import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.ambari.server.controller.internal.PropertyInfo; import org.apache.ambari.server.controller.internal.RequestImpl; import org.apache.ambari.server.controller.jmx.JMXPropertyProvider; import org.apache.ambari.server.controller.spi.PageRequest; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.spi.SortRequest; import org.apache.ambari.server.controller.spi.TemporalInfo; import org.apache.ambari.server.state.stack.Metric; import org.apache.commons.lang.StringUtils; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; /** * Utility class that provides Property helper methods. */ public class PropertyHelper { private static final String PROPERTIES_FILE = "properties.json"; private static final String GANGLIA_PROPERTIES_FILE = "ganglia_properties.json"; private static final String SQLSERVER_PROPERTIES_FILE = "sqlserver_properties.json"; private static final String JMX_PROPERTIES_FILE = "jmx_properties.json"; private static final String KEY_PROPERTIES_FILE = "key_properties.json"; private static final char EXTERNAL_PATH_SEP = '/'; /** * Aggregate functions implicitly supported by the Metrics Service */ public static final List<String> AGGREGATE_FUNCTION_IDENTIFIERS = Arrays.asList("._sum", "._max", "._min", "._avg", "._rate"); private static final List<Resource.InternalType> REPORT_METRIC_RESOURCES = Arrays .asList(Resource.InternalType.Cluster, Resource.InternalType.Host); private static final Map<Resource.InternalType, Set<String>> PROPERTY_IDS = readPropertyIds(PROPERTIES_FILE); private static final Map<Resource.InternalType, Map<String, Map<String, PropertyInfo>>> JMX_PROPERTY_IDS = readPropertyProviderIds( JMX_PROPERTIES_FILE); private static final Map<Resource.InternalType, Map<String, Map<String, PropertyInfo>>> GANGLIA_PROPERTY_IDS = readPropertyProviderIds( GANGLIA_PROPERTIES_FILE); private static final Map<Resource.InternalType, Map<String, Map<String, PropertyInfo>>> SQLSERVER_PROPERTY_IDS = readPropertyProviderIds( SQLSERVER_PROPERTIES_FILE); private static final Map<Resource.InternalType, Map<Resource.Type, String>> KEY_PROPERTY_IDS = readKeyPropertyIds( KEY_PROPERTIES_FILE); // Suffixes to add for Namenode rpc metrics prefixes private static final Map<String, List<String>> RPC_METRIC_SUFFIXES = new HashMap<>(); /** * Regular expression to check for replacement arguments (e.g. $1) in a property id. */ private static final Pattern CHECK_FOR_METRIC_ARGUMENTS_REGEX = Pattern.compile(".*\\$\\d+.*"); /** * This regular expression will match on every {@code /} in a given string * that is not inside of quotes. The following string would be tokenized like * so: * <p/> * {@code foo/$1.substring(5)/bar/$2.replaceAll(\"/a/b//c///\")/baz} * <ul> * <li>foo</li> * <li>$1.substring(5)</li> * <li>bar</li> * <li>$2.replaceAll(\"/a/b//c///\")</li> * <li>baz</li> * </ul> * * This is necessary to be able to properly tokenize a property with {@code /} * while also ensuring we don't match on {@code /} that appears inside of * quotes. */ private static final Pattern METRIC_CATEGORY_TOKENIZE_REGEX = Pattern .compile("/+(?=([^\"\\\\\\\\]*(\\\\\\\\.|\"([^\"\\\\\\\\]*\\\\\\\\.)*[^\"\\\\\\\\]*\"))*[^\"]*$)"); static { RPC_METRIC_SUFFIXES.put("metrics/rpc/", Arrays.asList("client", "datanode", "healthcheck")); RPC_METRIC_SUFFIXES.put("metrics/rpcdetailed/", Arrays.asList("client", "datanode", "healthcheck")); } public static String getPropertyId(String category, String name) { String propertyId = (category == null || category.isEmpty()) ? name : (name == null || name.isEmpty()) ? category : category + EXTERNAL_PATH_SEP + name; if (propertyId.endsWith("/")) { propertyId = propertyId.substring(0, propertyId.length() - 1); } return propertyId; } public static Set<String> getPropertyIds(Resource.Type resourceType) { Set<String> propertyIds = PROPERTY_IDS.get(resourceType.getInternalType()); return propertyIds == null ? Collections.<String>emptySet() : propertyIds; } /** * Extract the set of property ids from a component PropertyInfo map. * * @param componentPropertyInfoMap the map * * @return the set of property ids */ public static Set<String> getPropertyIds(Map<String, Map<String, PropertyInfo>> componentPropertyInfoMap) { Set<String> propertyIds = new HashSet<String>(); for (Map.Entry<String, Map<String, PropertyInfo>> entry : componentPropertyInfoMap.entrySet()) { propertyIds.addAll(entry.getValue().keySet()); } return propertyIds; } public static Map<String, Map<String, PropertyInfo>> getMetricPropertyIds(Resource.Type resourceType) { return GANGLIA_PROPERTY_IDS.get(resourceType.getInternalType()); } public static Map<String, Map<String, PropertyInfo>> getSQLServerPropertyIds(Resource.Type resourceType) { return SQLSERVER_PROPERTY_IDS.get(resourceType.getInternalType()); } public static Map<String, Map<String, PropertyInfo>> getJMXPropertyIds(Resource.Type resourceType) { return JMX_PROPERTY_IDS.get(resourceType.getInternalType()); } public static Map<Resource.Type, String> getKeyPropertyIds(Resource.Type resourceType) { return KEY_PROPERTY_IDS.get(resourceType.getInternalType()); } /** * Helper to get a property name from a string. * * @param absProperty the fully qualified property * * @return the property name */ public static String getPropertyName(String absProperty) { int lastPathSep = absProperty.lastIndexOf(EXTERNAL_PATH_SEP); return lastPathSep == -1 ? absProperty : absProperty.substring(lastPathSep + 1); } /** * Gets the parent category from a given property string. This method is used * in many places by many different consumers. In general, it will check to * see if the property contains arguments. If not, then a simple * {@link String#substring(int, int)} is used along with * {@link #EXTERNAL_PATH_SEP}. * <p/> * In the event that a property contains a $d parameter, it will attempt to * strip out any embedded methods in the various tokens. For example, if a * property of {@code foo/$1.substring(5)/bar/$2.substring(1)/baz} is given, * the expected recursive categories would be: * <ul> * <li>foo</li> * <li>foo/$1</li> * <li>foo/$1/bar</li> * <li>foo/$1/bar</li> * <li>foo/$1/bar/$2</li> * </ul> * * {@code foo/$1.substring(5)/bar} is incorrect as a category. * * @param property * the fully qualified property * * @return the property category; null if there is no category */ public static String getPropertyCategory(String property) { int lastPathSep = -1; if (!containsArguments(property)) { lastPathSep = property.lastIndexOf(EXTERNAL_PATH_SEP); return lastPathSep == -1 ? null : property.substring(0, lastPathSep); } // attempt to split the property into its parts String[] tokens = METRIC_CATEGORY_TOKENIZE_REGEX.split(property); if (null == tokens || tokens.length == 0) { return null; } StringBuilder categoryBuilder = new StringBuilder(); for (int i = 0; i < tokens.length - 1; i++) { String token = tokens[i]; // if the token contains arguments, turn $1.method() into $1, if (containsArguments(token)) { int methodIndex = token.indexOf('.'); if (methodIndex != -1) { // normally knowing where $1.method() is would be enough, but some // properties may omit the / (like FLUME) so the property would look // like /$1.method()FailureCount instead of /$1.method()/FailureCount int parensIndex = token.lastIndexOf(')'); if (parensIndex < token.length() - 1) { // cut out $1 String temp = token.substring(0, methodIndex); // append FailureCount temp += token.substring(parensIndex + 1); // $1.method()FailureCount -> $1FailureCount token = temp; } else { token = token.substring(0, methodIndex); } } } // only append a / if this is not the last part of the parent category categoryBuilder.append(token); if (i < tokens.length - 2) { categoryBuilder.append('/'); } } String category = categoryBuilder.toString(); return category; } /** * Get the set of categories for the given property ids. * * @param propertyIds the property ids * * @return the set of categories */ public static Set<String> getCategories(Set<String> propertyIds) { Set<String> categories = new HashSet<String>(); for (String property : propertyIds) { String category = PropertyHelper.getPropertyCategory(property); while (category != null) { categories.add(category); category = PropertyHelper.getPropertyCategory(category); } } return categories; } /** * Check if the given property id or one of its parent category ids is contained * in the given set of property ids. * * @param propertyIds the set of property ids * @param propertyId the property id * * @return true if the given property id of one of its parent category ids is * contained in the given set of property ids */ public static boolean containsProperty(Set<String> propertyIds, String propertyId) { if (propertyIds.contains(propertyId)) { return true; } String category = PropertyHelper.getPropertyCategory(propertyId); while (category != null) { if (propertyIds.contains(category)) { return true; } category = PropertyHelper.getPropertyCategory(category); } return false; } /** * Check to see if the given property id contains replacement arguments (e.g. $1) * * @param propertyId the property id to check * * @return true if the given property id contains any replacement arguments */ public static boolean containsArguments(String propertyId) { if (!propertyId.contains("$")) { return false; } Matcher matcher = CHECK_FOR_METRIC_ARGUMENTS_REGEX.matcher(propertyId); return matcher.find(); } /** * Get all of the property ids associated with the given request. * * @param request the request * * @return the associated properties */ public static Set<String> getAssociatedPropertyIds(Request request) { Set<String> ids = request.getPropertyIds(); if (ids != null) { ids = new HashSet<String>(ids); } else { ids = new HashSet<String>(); } Set<Map<String, Object>> properties = request.getProperties(); if (properties != null) { for (Map<String, Object> propertyMap : properties) { ids.addAll(propertyMap.keySet()); } } return ids; } /** * Get a map of all the property values keyed by property id for the given resource. * * @param resource the resource * * @return the map of properties for the given resource */ public static Map<String, Object> getProperties(Resource resource) { Map<String, Object> properties = new HashMap<String, Object>(); Map<String, Map<String, Object>> categories = resource.getPropertiesMap(); for (Map.Entry<String, Map<String, Object>> categoryEntry : categories.entrySet()) { for (Map.Entry<String, Object> propertyEntry : categoryEntry.getValue().entrySet()) { properties.put(getPropertyId(categoryEntry.getKey(), propertyEntry.getKey()), propertyEntry.getValue()); } } return properties; } /** * Factory method to create a create request from the given set of property maps. * Each map contains the properties to be used to create a resource. Multiple maps in the * set should result in multiple creates. * * @param properties resource properties associated with the request; may be null * @param requestInfoProperties request specific properties; may be null */ public static Request getCreateRequest(Set<Map<String, Object>> properties, Map<String, String> requestInfoProperties) { return new RequestImpl(null, properties, requestInfoProperties, null); } /** * Factory method to create a read request from the given set of property ids. The set of * property ids represents the properties of interest for the query. * * @param propertyIds the property ids associated with the request; may be null */ public static Request getReadRequest(Set<String> propertyIds) { return PropertyHelper.getReadRequest(propertyIds, null); } /** * Factory method to create a read request from the given set of property ids. The set of * property ids represents the properties of interest for the query. * * @param propertyIds the property ids associated with the request; may be null * @param mapTemporalInfo the temporal info */ public static Request getReadRequest(Set<String> propertyIds, Map<String, TemporalInfo> mapTemporalInfo) { return PropertyHelper.getReadRequest(propertyIds, null, mapTemporalInfo, null, null); } /** * Factory method to create a read request from the given set of property ids. * The set of property ids represents the properties of interest for the * query. * * @param propertyIds * the property ids associated with the request; may be null * @param requestInfoProperties * request info properties * @param mapTemporalInfo * the temporal info * @param pageRequest * an optional page request, or {@code null} for none. * @param sortRequest * an optional sort request, or {@code null} for none. */ public static Request getReadRequest(Set<String> propertyIds, Map<String, String> requestInfoProperties, Map<String, TemporalInfo> mapTemporalInfo, PageRequest pageRequest, SortRequest sortRequest) { return new RequestImpl(propertyIds, null, requestInfoProperties, mapTemporalInfo, sortRequest, pageRequest); } /** * Factory method to create a read request from the given set of property ids. The set of * property ids represents the properties of interest for the query. * * @param propertyIds the property ids associated with the request; may be null */ public static Request getReadRequest(String... propertyIds) { return PropertyHelper.getReadRequest(new HashSet<String>(Arrays.asList(propertyIds))); } /** * Factory method to create an update request from the given map of properties. * The properties values in the given map are used to update the resource. * * @param properties resource properties associated with the request; may be null * @param requestInfoProperties request specific properties; may be null */ public static Request getUpdateRequest(Map<String, Object> properties, Map<String, String> requestInfoProperties) { return new RequestImpl(null, Collections.singleton(properties), requestInfoProperties, null); } private static Map<Resource.InternalType, Map<String, Map<String, PropertyInfo>>> readPropertyProviderIds( String filename) { ObjectMapper mapper = new ObjectMapper(); try { Map<Resource.InternalType, Map<String, Map<String, Metric>>> resourceMetricMap = mapper.readValue( ClassLoader.getSystemResourceAsStream(filename), new TypeReference<Map<Resource.InternalType, Map<String, Map<String, Metric>>>>() { }); Map<Resource.InternalType, Map<String, Map<String, PropertyInfo>>> resourceMetrics = new HashMap<Resource.InternalType, Map<String, Map<String, PropertyInfo>>>(); for (Map.Entry<Resource.InternalType, Map<String, Map<String, Metric>>> resourceEntry : resourceMetricMap .entrySet()) { Map<String, Map<String, PropertyInfo>> componentMetrics = new HashMap<String, Map<String, PropertyInfo>>(); for (Map.Entry<String, Map<String, Metric>> componentEntry : resourceEntry.getValue().entrySet()) { Map<String, PropertyInfo> metrics = new HashMap<String, PropertyInfo>(); for (Map.Entry<String, Metric> metricEntry : componentEntry.getValue().entrySet()) { String property = metricEntry.getKey(); Metric metric = metricEntry.getValue(); PropertyInfo propertyInfo = new PropertyInfo(metric.getMetric(), metric.isTemporal(), metric.isPointInTime()); propertyInfo.setAmsId(metric.getAmsId()); propertyInfo.setAmsHostMetric(metric.isAmsHostMetric()); propertyInfo.setUnit(metric.getUnit()); metrics.put(property, propertyInfo); } componentMetrics.put(componentEntry.getKey(), metrics); } if (REPORT_METRIC_RESOURCES.contains(resourceEntry.getKey())) { updateMetricsWithAggregateFunctionSupport(componentMetrics); } resourceMetrics.put(resourceEntry.getKey(), componentMetrics); } return resourceMetrics; } catch (IOException e) { throw new IllegalStateException("Can't read properties file " + filename, e); } } private static Map<Resource.InternalType, Set<String>> readPropertyIds(String filename) { ObjectMapper mapper = new ObjectMapper(); try { return mapper.readValue(ClassLoader.getSystemResourceAsStream(filename), new TypeReference<Map<Resource.InternalType, Set<String>>>() { }); } catch (IOException e) { throw new IllegalStateException("Can't read properties file " + filename, e); } } private static Map<Resource.InternalType, Map<Resource.Type, String>> readKeyPropertyIds(String filename) { ObjectMapper mapper = new ObjectMapper(); try { Map<Resource.InternalType, Map<Resource.InternalType, String>> map = mapper.readValue( ClassLoader.getSystemResourceAsStream(filename), new TypeReference<Map<Resource.InternalType, Map<Resource.InternalType, String>>>() { }); Map<Resource.InternalType, Map<Resource.Type, String>> returnMap = new HashMap<Resource.InternalType, Map<Resource.Type, String>>(); // convert inner maps from InternalType to Type for (Map.Entry<Resource.InternalType, Map<Resource.InternalType, String>> entry : map.entrySet()) { Map<Resource.Type, String> innerMap = new HashMap<Resource.Type, String>(); for (Map.Entry<Resource.InternalType, String> entry1 : entry.getValue().entrySet()) { innerMap.put(Resource.Type.values()[entry1.getKey().ordinal()], entry1.getValue()); } returnMap.put(entry.getKey(), innerMap); } return returnMap; } catch (IOException e) { throw new IllegalStateException("Can't read properties file " + filename, e); } } protected static class Metric { private String metric; private boolean pointInTime; private boolean temporal; private String amsId; private boolean amsHostMetric; private String unit = "unitless"; private Metric() { } protected Metric(String metric, boolean pointInTime, boolean temporal, String unit) { this.metric = metric; this.pointInTime = pointInTime; this.temporal = temporal; this.unit = unit; } public String getMetric() { return metric; } public void setMetric(String metric) { this.metric = metric; } public boolean isPointInTime() { return pointInTime; } public void setPointInTime(boolean pointInTime) { this.pointInTime = pointInTime; } public boolean isTemporal() { return temporal; } public void setTemporal(boolean temporal) { this.temporal = temporal; } public String getAmsId() { return amsId; } public void setAmsId(String amsId) { this.amsId = amsId; } public boolean isAmsHostMetric() { return amsHostMetric; } public void setAmsHostMetric(boolean amsHostMetric) { this.amsHostMetric = amsHostMetric; } public void setUnit(String unit) { this.unit = unit; } public String getUnit() { return unit; } } /** * This method adds supported propertyInfo for component metrics with * aggregate function ids. API calls with multiple aggregate functions * applied to a single metric need this support. * * Currently this support is added only for Component & Report metrics. * This can be easily extended by making the call from the appropriate * sub class of: @AMSPropertyProvider. * */ public static void updateMetricsWithAggregateFunctionSupport(Map<String, Map<String, PropertyInfo>> metrics) { if (metrics == null || metrics.isEmpty()) { return; } // For every component for (Map.Entry<String, Map<String, PropertyInfo>> metricInfoEntry : metrics.entrySet()) { Map<String, PropertyInfo> metricInfo = metricInfoEntry.getValue(); Map<String, PropertyInfo> aggregateMetrics = new HashMap<String, PropertyInfo>(); // For every metric for (Map.Entry<String, PropertyInfo> metricEntry : metricInfo.entrySet()) { // For each aggregate function id for (String identifierToAdd : AGGREGATE_FUNCTION_IDENTIFIERS) { String metricInfoKey = metricEntry.getKey() + identifierToAdd; // This disallows metric key suffix of the form "._sum._sum" for // the sake of avoiding duplicates if (metricInfo.containsKey(metricInfoKey)) { continue; } PropertyInfo propertyInfo = metricEntry.getValue(); PropertyInfo metricInfoValue = new PropertyInfo(propertyInfo.getPropertyId() + identifierToAdd, propertyInfo.isTemporal(), propertyInfo.isPointInTime()); metricInfoValue.setAmsHostMetric(propertyInfo.isAmsHostMetric()); metricInfoValue.setAmsId(!StringUtils.isEmpty(propertyInfo.getAmsId()) ? propertyInfo.getAmsId() + identifierToAdd : null); metricInfoValue.setUnit(propertyInfo.getUnit()); aggregateMetrics.put(metricInfoKey, metricInfoValue); } } metricInfo.putAll(aggregateMetrics); } } /** * Check if property ends with a trailing suffix of function id. */ public static boolean hasAggregateFunctionSuffix(String propertyId) { for (String aggregateFunctionId : AGGREGATE_FUNCTION_IDENTIFIERS) { if (propertyId.endsWith(aggregateFunctionId)) { return true; } } return false; } /** * Special handle rpc port tags added to metric names for HDFS Namenode * * Returns the replacement definitions */ public static Map<String, org.apache.ambari.server.state.stack.Metric> processRpcMetricDefinition( String metricType, String componentName, String propertyId, org.apache.ambari.server.state.stack.Metric metric) { Map<String, org.apache.ambari.server.state.stack.Metric> replacementMap = null; if (componentName.equalsIgnoreCase("NAMENODE")) { for (Map.Entry<String, List<String>> entry : RPC_METRIC_SUFFIXES.entrySet()) { String prefix = entry.getKey(); if (propertyId.startsWith(prefix)) { replacementMap = new HashMap<>(); for (String suffix : entry.getValue()) { String newMetricName; if ("jmx".equals(metricType)) { newMetricName = insertTagIntoCategoty(suffix, metric.getName()); } else { newMetricName = insertTagInToMetricName(suffix, metric.getName(), prefix); } org.apache.ambari.server.state.stack.Metric newMetric = new org.apache.ambari.server.state.stack.Metric( newMetricName, metric.isPointInTime(), metric.isTemporal(), metric.isAmsHostMetric(), metric.getUnit()); replacementMap.put(insertTagInToMetricName(suffix, propertyId, prefix), newMetric); } } } } return replacementMap; } /** * Returns tag inserted metric name after the prefix. * @param tag E.g.: client * @param metricName : rpc.rpc.CallQueueLength Or metrics/rpc/CallQueueLen * @param prefix : rpc.rpc * @return rpc.rpc.client.CallQueueLength Or metrics/rpc/client/CallQueueLen */ static String insertTagInToMetricName(String tag, String metricName, String prefix) { String sepExpr = "\\."; String seperator = "."; if (metricName.indexOf(EXTERNAL_PATH_SEP) != -1) { sepExpr = Character.toString(EXTERNAL_PATH_SEP); seperator = sepExpr; } String prefixSep = prefix.contains(".") ? "\\." : "" + EXTERNAL_PATH_SEP; // Remove separator if any if (prefix.substring(prefix.length() - 1).equals(prefixSep)) { prefix = prefix.substring(0, prefix.length() - 1); } int pos = prefix.split(prefixSep).length - 1; String[] parts = metricName.split(sepExpr); StringBuilder sb = new StringBuilder(); for (int i = 0; i < parts.length; i++) { sb.append(parts[i]); if (i < parts.length - 1) { sb.append(seperator); } if (i == pos) { // append the tag sb.append(tag); sb.append(seperator); } } return sb.toString(); } static String insertTagIntoCategoty(String tag, String metricName) { int pos = metricName.lastIndexOf('.'); return metricName.substring(0, pos) + ",tag=" + tag + metricName.substring(pos); } }