Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.ambari.scom; import org.apache.ambari.server.controller.internal.AbstractPropertyProvider; import org.apache.ambari.server.controller.internal.PropertyInfo; import org.apache.ambari.server.controller.jdbc.ConnectionFactory; import org.apache.ambari.server.controller.spi.Predicate; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.controller.spi.TemporalInfo; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Serializable; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.text.NumberFormat; import java.text.ParsePosition; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; /** * SQL based property/metrics provider required for ambari-scom. */ public class SQLPropertyProvider extends AbstractPropertyProvider { private final HostInfoProvider hostProvider; private final String clusterNamePropertyId; private final String hostNamePropertyId; private final String componentNamePropertyId; private final String serviceNamePropertyId; private final ConnectionFactory connectionFactory; // ----- Constants --------------------------------------------------------- private static final String GET_METRICS_STATEMENT = "SELECT s.RecordTypeContext, s.RecordTypeName, s.TagPairs, s.NodeName, s.ServiceName, mn.Name AS MetricName, s.RecordTimeStamp, mp.MetricValue\n" + "FROM HadoopMetrics.dbo.MetricPair mp\n" + " INNER JOIN (\n" + " SELECT mr.RecordID AS RecordID, mr.RecordTimeStamp AS RecordTimeStamp, rt.Context AS RecordTypeContext, rt.Name AS RecordTypeName, ts.TagPairs AS TagPairs, nd.Name AS NodeName, sr.Name AS ServiceName\n" + " FROM HadoopMetrics.dbo.MetricRecord mr\n" + " INNER JOIN HadoopMetrics.dbo.RecordType rt ON (mr.RecordTypeId = rt.RecordTypeId)\n" + " INNER JOIN HadoopMetrics.dbo.TagSet ts ON (mr.TagSetID = ts.TagSetID)\n" + " INNER JOIN HadoopMetrics.dbo.Node nd ON (mr.NodeID = nd.NodeID)\n" + " INNER JOIN HadoopMetrics.dbo.Service sr ON (mr.ServiceID = sr.ServiceID)\n" + " WHERE rt.Context in (%s)\n" + " AND rt.Name in (%s)\n" + " AND (ts.TagPairs LIKE %s)\n" + " AND (nd.Name in (%s))\n" + " AND (sr.Name in (%s))\n" + " AND mr.RecordTimestamp >= %d\n" + " AND mr.RecordTimestamp <= %d\n" + " ) s ON (mp.RecordID = s.RecordID)\n" + " INNER JOIN HadoopMetrics.dbo.MetricName mn ON (mp.MetricID = mn.MetricID)\n" + "WHERE (mn.Name in (%s))"; protected final static Logger LOG = LoggerFactory.getLogger(SQLPropertyProvider.class); // ----- Constructors ------------------------------------------------------ public SQLPropertyProvider(Map<String, Map<String, PropertyInfo>> componentPropertyInfoMap, HostInfoProvider hostProvider, String clusterNamePropertyId, String hostNamePropertyId, String componentNamePropertyId, String serviceNamePropertyId, ConnectionFactory connectionFactory) { super(componentPropertyInfoMap); this.hostProvider = hostProvider; this.clusterNamePropertyId = clusterNamePropertyId; this.hostNamePropertyId = hostNamePropertyId; this.componentNamePropertyId = componentNamePropertyId; this.serviceNamePropertyId = serviceNamePropertyId; this.connectionFactory = connectionFactory; } // ----- PropertyProvider -------------------------------------------------- @Override public Set<Resource> populateResources(Set<Resource> resources, Request request, Predicate predicate) throws SystemException { Set<Resource> keepers = new HashSet<Resource>(); try { Connection connection = connectionFactory.getConnection(); try { Statement statement = connection.createStatement(); try { for (Resource resource : resources) { if (populateResource(resource, request, predicate, statement)) { keepers.add(resource); } } } finally { statement.close(); } } finally { connection.close(); } } catch (SQLException e) { if (LOG.isErrorEnabled()) { LOG.error("Error during populateResources call."); LOG.debug("Error during populateResources call : caught exception", e); } } return keepers; } // ----- helper methods ---------------------------------------------------- // Populate the given resource private boolean populateResource(Resource resource, Request request, Predicate predicate, Statement statement) throws SystemException { Set<String> ids = getRequestPropertyIds(request, predicate); if (ids.isEmpty()) { // no properties requested ... nothing to do. return true; } String componentName = (String) resource.getPropertyValue(componentNamePropertyId); String serviceName = (String) resource.getPropertyValue(serviceNamePropertyId); if (getComponentMetrics().get(componentName) == null) { // no metrics defined for the given component ... nothing to do. return true; } String clusterName = (String) resource.getPropertyValue(clusterNamePropertyId); String hostName = getHost(resource, clusterName, componentName); if (hostName == null) { throw new SystemException("Unable to get metrics. No host name for " + componentName, null); } Set<MetricDefinition> metricsDefinitionSet = new HashSet<MetricDefinition>(); for (String id : ids) { Map<String, PropertyInfo> propertyInfoMap = getPropertyInfoMap(componentName, id); for (Map.Entry<String, PropertyInfo> entry : propertyInfoMap.entrySet()) { String propertyKey = entry.getKey(); PropertyInfo propertyInfo = entry.getValue(); if (containsArguments(propertyKey)) { propertyInfo = updatePropertyInfo(propertyKey, id, propertyInfo); } String propertyId = propertyInfo.getPropertyId(); TemporalInfo temporalInfo = request.getTemporalInfo(id); if ((propertyInfo.isPointInTime() && temporalInfo == null) || (propertyInfo.isTemporal() && temporalInfo != null)) { long startTime; long endTime; if (temporalInfo != null) { Long endTimeSeconds = temporalInfo.getEndTime(); endTime = endTimeSeconds != -1 ? endTimeSeconds * 1000 : Long.MAX_VALUE; startTime = temporalInfo.getStartTime() * 1000; } else { startTime = 0L; endTime = Long.MAX_VALUE; } String category = ""; String recordTypeContext = ""; String recordTypeName = ""; String metricName = ""; String tagPairsPattern = ""; int dotIndex = propertyId.lastIndexOf('.'); if (dotIndex != -1) { category = propertyId.substring(0, dotIndex); metricName = propertyId.substring(dotIndex + 1); } String[] parts = category.split("\\."); if (parts.length >= 2) { recordTypeContext = parts[0]; recordTypeName = parts[1]; if (containsArguments(propertyKey) && parts.length > 2) { tagPairsPattern = StringUtils.join(Arrays.copyOfRange(parts, 2, parts.length), "."); } metricsDefinitionSet.add(new MetricDefinition(startTime, endTime, recordTypeContext, recordTypeName, tagPairsPattern, metricName, serviceName != null && serviceName.toLowerCase().equals("hbase") ? serviceName.toLowerCase() : componentName.toLowerCase(), hostName, propertyKey, id, temporalInfo)); } else { if (LOG.isWarnEnabled()) { LOG.warn("Can't get metrics for " + id + " : " + propertyId); } } } } } Map<MetricDefinition, List<DataPoint>> results = getMetric(metricsDefinitionSet, statement); for (MetricDefinition metricDefinition : metricsDefinitionSet) { List<DataPoint> dataPoints = results.containsKey(metricDefinition) ? results.get(metricDefinition) : new ArrayList<DataPoint>(); TemporalInfo temporalInfo = metricDefinition.getTemporalInfo(); String propertyKey = metricDefinition.getPropertyKey(); String requestedPropertyKey = metricDefinition.getRequestedPropertyKey(); if (dataPoints != null) { if (temporalInfo == null) { // return the value of the last data point int length = dataPoints.size(); Serializable value = length > 0 ? dataPoints.get(length - 1).getValue() : 0; resource.setProperty(propertyKey, value); } else { Number[][] dp = new Number[dataPoints.size()][2]; for (int i = 0; i < dp.length; i++) { dp[i][0] = dataPoints.get(i).getValue(); dp[i][1] = dataPoints.get(i).getTimestamp() / 1000; } if (containsArguments(propertyKey)) { resource.setProperty(requestedPropertyKey, dp); } else { resource.setProperty(propertyKey, dp); } } } } return true; } // get a metric from a sql connection private Map<MetricDefinition, List<DataPoint>> getMetric(Set<MetricDefinition> metricDefinitionSet, Statement statement) throws SystemException { Map<MetricDefinition, List<DataPoint>> results = new HashMap<MetricDefinition, List<DataPoint>>(); try { StringBuilder query = new StringBuilder(); Set<String> recordTypeContexts = new HashSet<String>(); Set<String> recordTypeNamess = new HashSet<String>(); Set<String> tagPairsPatterns = new HashSet<String>(); Set<String> nodeNames = new HashSet<String>(); Set<String> serviceNames = new HashSet<String>(); Set<String> metricNames = new HashSet<String>(); long startTime = 0, endTime = 0; for (MetricDefinition metricDefinition : metricDefinitionSet) { if (metricDefinition.getRecordTypeContext() == null || metricDefinition.getRecordTypeName() == null || metricDefinition.getNodeName() == null) { continue; } recordTypeContexts.add(metricDefinition.getRecordTypeContext()); recordTypeNamess.add(metricDefinition.getRecordTypeName()); tagPairsPatterns.add(metricDefinition.getTagPairsPattern()); nodeNames.add(metricDefinition.getNodeName()); serviceNames.add(metricDefinition.getServiceName()); metricNames.add(metricDefinition.getMetricName()); startTime = metricDefinition.getStartTime(); endTime = metricDefinition.getEndTime(); } for (String tagPairsPattern : tagPairsPatterns) { if (query.length() != 0) { query.append("\nUNION\n"); } query.append(String.format(GET_METRICS_STATEMENT, "'" + StringUtils.join(recordTypeContexts, "','") + "'", "'" + StringUtils.join(recordTypeNamess, "','") + "'", "'%" + tagPairsPattern + "%'", "'" + StringUtils.join(nodeNames, "','") + "'", "'" + StringUtils.join(serviceNames, "','") + "'", startTime, endTime, "'" + StringUtils.join(metricNames, "','") + "'")); } ResultSet rs = null; if (query.length() != 0) { rs = statement.executeQuery(query.toString()); } if (rs != null) { //(RecordTimeStamp bigint, MetricValue NVARCHAR(512)) while (rs.next()) { MetricDefinition metricDefinition = new MetricDefinition(rs.getString("RecordTypeContext"), rs.getString("RecordTypeName"), rs.getString("TagPairs"), rs.getString("MetricName"), rs.getString("ServiceName"), rs.getString("NodeName")); ParsePosition parsePosition = new ParsePosition(0); NumberFormat numberFormat = NumberFormat.getInstance(); Number parsedNumber = numberFormat.parse(rs.getNString("MetricValue"), parsePosition); if (results.containsKey(metricDefinition)) { results.get(metricDefinition) .add(new DataPoint(rs.getLong("RecordTimeStamp"), parsedNumber)); } else { List<DataPoint> dataPoints = new ArrayList<DataPoint>(); dataPoints.add(new DataPoint(rs.getLong("RecordTimeStamp"), parsedNumber)); results.put(metricDefinition, dataPoints); } } } } catch (SQLException e) { throw new SystemException("Error during getMetric call : caught exception - ", e); } return results; } // get the hostname for a given resource private String getHost(Resource resource, String clusterName, String componentName) throws SystemException { return hostNamePropertyId == null ? hostProvider.getHostName(clusterName, componentName) : hostProvider.getHostName((String) resource.getPropertyValue(hostNamePropertyId)); } // ----- inner class : DataPoint ------------------------------------------- /** * Structure to hold a single datapoint (value/timestamp pair) retrieved from the db. */ private static class DataPoint { private final long timestamp; private final Number value; // ----- Constructor ------------------------------------------------- /** * Construct a data point from the given value and timestamp. * * @param timestamp the timestamp * @param value the value */ private DataPoint(long timestamp, Number value) { this.timestamp = timestamp; this.value = value; } // ----- DataPoint --------------------------------------------------- /** * Get the timestamp value. * * @return the timestamp */ public long getTimestamp() { return timestamp; } /** * Get the value. * * @return the value */ public Number getValue() { return value; } // ----- Object overrides -------------------------------------------- @Override public String toString() { return "{" + value + " : " + timestamp + "}"; } } private class MetricDefinition { long startTime; long endTime; String recordTypeContext; String recordTypeName; String tagPairsPattern; String metricName; String serviceName; String nodeName; String propertyKey; String requestedPropertyKey; TemporalInfo temporalInfo; private MetricDefinition(long startTime, long endTime, String recordTypeContext, String recordTypeName, String tagPairsPattern, String metricName, String serviceName, String nodeName, String propertyKey, String requestedPropertyKey, TemporalInfo temporalInfo) { this.startTime = startTime; this.endTime = endTime; this.recordTypeContext = recordTypeContext; this.recordTypeName = recordTypeName; this.tagPairsPattern = tagPairsPattern; this.metricName = metricName; this.serviceName = serviceName; this.nodeName = nodeName; this.propertyKey = propertyKey; this.requestedPropertyKey = requestedPropertyKey; this.temporalInfo = temporalInfo; } private MetricDefinition(String recordTypeContext, String recordTypeName, String tagPairsPattern, String metricName, String serviceName, String nodeName) { this.recordTypeContext = recordTypeContext; this.recordTypeName = recordTypeName; this.tagPairsPattern = tagPairsPattern; this.metricName = metricName; this.serviceName = serviceName; this.nodeName = nodeName; } public long getStartTime() { return startTime; } public void setStartTime(long startTime) { this.startTime = startTime; } public long getEndTime() { return endTime; } public void setEndTime(long endTime) { this.endTime = endTime; } public String getRecordTypeContext() { return recordTypeContext; } public void setRecordTypeContext(String recordTypeContext) { this.recordTypeContext = recordTypeContext; } public String getRecordTypeName() { return recordTypeName; } public void setRecordTypeName(String recordTypeName) { this.recordTypeName = recordTypeName; } public String getTagPairsPattern() { return tagPairsPattern; } public void getTagPairsPattern(String tagPairsPattern) { this.tagPairsPattern = tagPairsPattern; } public String getMetricName() { return metricName; } public void setMetricName(String metricName) { this.metricName = metricName; } public String getServiceName() { return serviceName; } public void setServiceName(String serviceName) { this.serviceName = serviceName; } public String getNodeName() { return nodeName; } public void setNodeName(String nodeName) { this.nodeName = nodeName; } public String getPropertyKey() { return propertyKey; } public void setPropertyKey(String propertyKey) { this.propertyKey = propertyKey; } public String getRequestedPropertyKey() { return requestedPropertyKey; } public void setRequestedPropertyKey(String requestedPropertyKey) { this.requestedPropertyKey = requestedPropertyKey; } public TemporalInfo getTemporalInfo() { return temporalInfo; } public void setTemporalInfo(TemporalInfo temporalInfo) { this.temporalInfo = temporalInfo; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; MetricDefinition that = (MetricDefinition) o; if (metricName != null ? !metricName.equals(that.metricName) : that.metricName != null) return false; if (nodeName != null ? !nodeName.equalsIgnoreCase(that.nodeName) : that.nodeName != null) return false; if (recordTypeContext != null ? !recordTypeContext.equals(that.recordTypeContext) : that.recordTypeContext != null) return false; if (recordTypeName != null ? !recordTypeName.equals(that.recordTypeName) : that.recordTypeName != null) return false; if (serviceName != null ? !serviceName.equals(that.serviceName) : that.serviceName != null) return false; if (tagPairsPattern != null ? !(tagPairsPattern.contains(that.tagPairsPattern) || that.tagPairsPattern.contains(tagPairsPattern)) : that.tagPairsPattern != null) return false; return true; } @Override public int hashCode() { int result = recordTypeContext != null ? recordTypeContext.hashCode() : 0; result = 31 * result + (recordTypeName != null ? recordTypeName.hashCode() : 0); result = 31 * result + (metricName != null ? metricName.hashCode() : 0); result = 31 * result + (serviceName != null ? serviceName.hashCode() : 0); result = 31 * result + (nodeName != null ? nodeName.toLowerCase().hashCode() : 0); return result; } } }