Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.ambari.server.state; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; import com.google.common.collect.Maps; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.ConfigurationRequest; import org.apache.ambari.server.orm.dao.ClusterDAO; import org.apache.ambari.server.orm.entities.ClusterConfigEntity; import org.apache.ambari.server.state.PropertyInfo.PropertyType; import org.apache.ambari.server.state.configgroup.ConfigGroup; import org.apache.ambari.server.upgrade.UpgradeCatalog170; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.inject.Inject; import com.google.inject.Singleton; import com.google.inject.persist.Transactional; /** * Helper class that works with config traversals. */ @Singleton public class ConfigHelper { private Clusters clusters = null; private AmbariMetaInfo ambariMetaInfo = null; private ClusterDAO clusterDAO = null; private static final String DELETED = "DELETED_"; public static final String CLUSTER_DEFAULT_TAG = "tag"; private final boolean STALE_CONFIGS_CACHE_ENABLED; private final int STALE_CONFIGS_CACHE_EXPIRATION_TIME = 300; private final Cache<ServiceComponentHost, Boolean> staleConfigsCache; private static final Logger LOG = LoggerFactory.getLogger(ConfigHelper.class); /** * List of property prefixes and names. Please keep in alphabetical order. */ public static final String HBASE_SITE = "hbase-site"; public static final String HDFS_SITE = "hdfs-site"; public static final String HIVE_SITE = "hive-site"; public static final String YARN_SITE = "yarn-site"; public static final String HTTP_ONLY = "HTTP_ONLY"; public static final String HTTPS_ONLY = "HTTPS_ONLY"; @Inject public ConfigHelper(Clusters c, AmbariMetaInfo metaInfo, Configuration configuration, ClusterDAO clusterDAO) { clusters = c; ambariMetaInfo = metaInfo; this.clusterDAO = clusterDAO; STALE_CONFIGS_CACHE_ENABLED = configuration.isStaleConfigCacheEnabled(); staleConfigsCache = CacheBuilder.newBuilder() .expireAfterWrite(STALE_CONFIGS_CACHE_EXPIRATION_TIME, TimeUnit.SECONDS).build(); } /** * Gets the desired tags for a cluster and host * * @param cluster the cluster * @param hostName the host name * @return a map of tag type to tag names with overrides * @throws AmbariException */ public Map<String, Map<String, String>> getEffectiveDesiredTags(Cluster cluster, String hostName) throws AmbariException { Host host = (hostName == null) ? null : clusters.getHost(hostName); Map<String, HostConfig> desiredHostConfigs = (host == null) ? null : host.getDesiredHostConfigs(cluster); return getEffectiveDesiredTags(cluster, desiredHostConfigs); } /** * Gets the desired tags for a cluster and overrides for a host * * @param cluster the cluster * @param hostConfigOverrides the host overrides applied using config groups * @return a map of tag type to tag names with overrides */ private Map<String, Map<String, String>> getEffectiveDesiredTags(Cluster cluster, Map<String, HostConfig> hostConfigOverrides) { Map<String, DesiredConfig> clusterDesired = (cluster == null) ? new HashMap<String, DesiredConfig>() : cluster.getDesiredConfigs(); Map<String, Map<String, String>> resolved = new TreeMap<String, Map<String, String>>(); // Do not use host component config mappings. Instead, the rules are: // 1) Use the cluster desired config // 2) override (1) with config-group overrides for (Entry<String, DesiredConfig> clusterEntry : clusterDesired.entrySet()) { String type = clusterEntry.getKey(); String tag = clusterEntry.getValue().getTag(); // 1) start with cluster config Config config = cluster.getConfig(type, tag); if (null == config) { continue; } Map<String, String> tags = new LinkedHashMap<String, String>(); tags.put(CLUSTER_DEFAULT_TAG, config.getTag()); // AMBARI-3672. Only consider Config groups for override tags // tags -> (configGroupId, versionTag) if (hostConfigOverrides != null) { HostConfig hostConfig = hostConfigOverrides.get(config.getType()); if (hostConfig != null) { for (Entry<Long, String> tagEntry : hostConfig.getConfigGroupOverrides().entrySet()) { tags.put(tagEntry.getKey().toString(), tagEntry.getValue()); } } } resolved.put(type, tags); } return resolved; } /** * Get all config properties for a cluster given a set of configType to * versionTags map. This helper method merges all the override tags with a * the properties from parent cluster config properties * * @param cluster * @param desiredTags * @return {type : {key, value}} */ public Map<String, Map<String, String>> getEffectiveConfigProperties(Cluster cluster, Map<String, Map<String, String>> desiredTags) { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); if (desiredTags != null) { for (Entry<String, Map<String, String>> entry : desiredTags.entrySet()) { String type = entry.getKey(); Map<String, String> propertyMap = properties.get(type); if (propertyMap == null) { propertyMap = new HashMap<String, String>(); } Map<String, String> tags = new HashMap<String, String>(entry.getValue()); String clusterTag = tags.get(CLUSTER_DEFAULT_TAG); // Overrides is only supported if the config type exists at cluster // level if (clusterTag != null) { Config config = cluster.getConfig(type, clusterTag); if (config != null) { propertyMap.putAll(config.getProperties()); } tags.remove(CLUSTER_DEFAULT_TAG); // Now merge overrides for (Entry<String, String> overrideEntry : tags.entrySet()) { Config overrideConfig = cluster.getConfig(type, overrideEntry.getValue()); if (overrideConfig != null) { propertyMap = getMergedConfig(propertyMap, overrideConfig.getProperties()); } } } properties.put(type, propertyMap); } } return properties; } /** * Get all config attributes for a cluster given a set of configType to * versionTags map. This helper method merges all the override tags with a * the attributes from parent cluster config properties * * @param cluster * @param desiredTags * @return {type : {attribute : {property, attributeValue}} */ public Map<String, Map<String, Map<String, String>>> getEffectiveConfigAttributes(Cluster cluster, Map<String, Map<String, String>> desiredTags) { Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String, Map<String, Map<String, String>>>(); if (desiredTags != null) { for (Entry<String, Map<String, String>> entry : desiredTags.entrySet()) { String type = entry.getKey(); Map<String, Map<String, String>> attributesMap = null; Map<String, String> tags = new HashMap<String, String>(entry.getValue()); String clusterTag = tags.get(CLUSTER_DEFAULT_TAG); if (clusterTag != null) { Config config = cluster.getConfig(type, clusterTag); if (config != null) { attributesMap = new TreeMap<String, Map<String, String>>(); cloneAttributesMap(config.getPropertiesAttributes(), attributesMap); } tags.remove(CLUSTER_DEFAULT_TAG); } for (Entry<String, String> overrideEntry : tags.entrySet()) { Config overrideConfig = cluster.getConfig(type, overrideEntry.getValue()); overrideAttributes(overrideConfig, attributesMap); } if (attributesMap != null) { attributes.put(type, attributesMap); } } } return attributes; } /** * Merge override with original, if original property doesn't exist, * add it to the properties * * @param persistedClusterConfig * @param override * @return */ public Map<String, String> getMergedConfig(Map<String, String> persistedClusterConfig, Map<String, String> override) { Map<String, String> finalConfig = new HashMap<String, String>(persistedClusterConfig); if (override != null && override.size() > 0) { for (Entry<String, String> entry : override.entrySet()) { Boolean deleted = 0 == entry.getKey().indexOf(DELETED); String nameToUse = deleted ? entry.getKey().substring(DELETED.length()) : entry.getKey(); if (finalConfig.containsKey(nameToUse)) { finalConfig.remove(nameToUse); } if (!deleted) { finalConfig.put(nameToUse, entry.getValue()); } } } return finalConfig; } /** * Merge override attributes with original ones. * If overrideConfig#getPropertiesAttributes does not contain occurrence of override for any of * properties from overrideConfig#getProperties then persisted attribute should be removed. */ public Map<String, Map<String, String>> overrideAttributes(Config overrideConfig, Map<String, Map<String, String>> persistedAttributes) { if (overrideConfig != null && persistedAttributes != null) { Map<String, Map<String, String>> overrideAttributes = overrideConfig.getPropertiesAttributes(); if (overrideAttributes != null) { cloneAttributesMap(overrideAttributes, persistedAttributes); Map<String, String> overrideProperties = overrideConfig.getProperties(); if (overrideProperties != null) { Set<String> overriddenProperties = overrideProperties.keySet(); for (String overriddenProperty : overriddenProperties) { for (Entry<String, Map<String, String>> persistedAttribute : persistedAttributes .entrySet()) { String attributeName = persistedAttribute.getKey(); Map<String, String> persistedAttributeValues = persistedAttribute.getValue(); Map<String, String> overrideAttributeValues = overrideAttributes.get(attributeName); if (overrideAttributeValues == null || !overrideAttributeValues.containsKey(overriddenProperty)) { persistedAttributeValues.remove(overriddenProperty); } } } } } } return persistedAttributes; } public void cloneAttributesMap(Map<String, Map<String, String>> sourceAttributesMap, Map<String, Map<String, String>> targetAttributesMap) { if (sourceAttributesMap != null && targetAttributesMap != null) { for (Entry<String, Map<String, String>> attributesEntry : sourceAttributesMap.entrySet()) { String attributeName = attributesEntry.getKey(); if (!targetAttributesMap.containsKey(attributeName)) { targetAttributesMap.put(attributeName, new TreeMap<String, String>()); } for (Entry<String, String> attributesValue : attributesEntry.getValue().entrySet()) { targetAttributesMap.get(attributeName).put(attributesValue.getKey(), attributesValue.getValue()); } } } } public void applyCustomConfig(Map<String, Map<String, String>> configurations, String type, String name, String value, Boolean deleted) { if (!configurations.containsKey(type)) { configurations.put(type, new HashMap<String, String>()); } String nameToUse = deleted ? DELETED + name : name; Map<String, String> properties = configurations.get(type); if (properties.containsKey(nameToUse)) { properties.remove(nameToUse); } properties.put(nameToUse, value); } /** * The purpose of this method is to determine if a {@link ServiceComponentHost}'s * known actual configs are different than what is set on the cluster (the desired). * The following logic is applied: * <ul> * <li>Desired type does not exist on the SCH (actual) * <ul> * <li>Type does not exist on the stack: <code>false</code></li> * <li>Type exists on the stack: <code>true</code> if the config key is on the stack. * otherwise <code>false</code></li> * </ul> * </li> * <li> Desired type exists for the SCH * <ul> * <li>Desired tags already set for the SCH (actual): <code>false</code></li> * <li>Desired tags DO NOT match SCH: <code>true</code> if the changed keys * exist on the stack, otherwise <code>false</code></li> * </ul> * </li> * </ul> * * @param @ServiceComponentHost * @return <code>true</code> if the actual configs are stale */ public boolean isStaleConfigs(ServiceComponentHost sch) throws AmbariException { Boolean stale = null; if (STALE_CONFIGS_CACHE_ENABLED) { stale = staleConfigsCache.getIfPresent(sch); } if (stale == null) { stale = calculateIsStaleConfigs(sch); staleConfigsCache.put(sch, stale); } return stale; } /** * Invalidates cached isStale values for hostname * * @param hostname */ public void invalidateStaleConfigsCache(String hostname) { try { for (Cluster cluster : clusters.getClustersForHost(hostname)) { for (ServiceComponentHost sch : cluster.getServiceComponentHosts(hostname)) { invalidateStaleConfigsCache(sch); } } } catch (AmbariException e) { LOG.warn("Unable to find clusters for host " + hostname); } } /** * Invalidates isStale cache */ public void invalidateStaleConfigsCache() { staleConfigsCache.invalidateAll(); } /** * Invalidates cached isStale value for sch * * @param sch */ public void invalidateStaleConfigsCache(ServiceComponentHost sch) { staleConfigsCache.invalidate(sch); } /** * Remove configs by type * * @param type config Type */ @Transactional public void removeConfigsByType(Cluster cluster, String type) { Set<String> globalVersions = cluster.getConfigsByType(type).keySet(); for (String version : globalVersions) { ClusterConfigEntity clusterConfigEntity = clusterDAO.findConfig(cluster.getClusterId(), type, version); clusterDAO.removeConfig(clusterConfigEntity); } } /** * Gets all the config dictionary where property with the given name is present in stack definitions * * @param stackId * @param propertyName */ public Set<String> findConfigTypesByPropertyName(StackId stackId, String propertyName, String clusterName) throws AmbariException { StackInfo stack = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion()); Set<String> result = new HashSet<String>(); for (Service service : clusters.getCluster(clusterName).getServices().values()) { Set<PropertyInfo> stackProperties = ambariMetaInfo.getServiceProperties(stack.getName(), stack.getVersion(), service.getName()); Set<PropertyInfo> stackLevelProperties = ambariMetaInfo.getStackProperties(stack.getName(), stack.getVersion()); stackProperties.addAll(stackLevelProperties); for (PropertyInfo stackProperty : stackProperties) { if (stackProperty.getName().equals(propertyName)) { String configType = fileNameToConfigType(stackProperty.getFilename()); result.add(configType); } } } return result; } public Set<String> getPropertyValuesWithPropertyType(StackId stackId, PropertyType propertyType, Cluster cluster) throws AmbariException { StackInfo stack = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion()); Set<String> result = new HashSet<String>(); for (Service service : cluster.getServices().values()) { Set<PropertyInfo> serviceProperties = ambariMetaInfo.getServiceProperties(stack.getName(), stack.getVersion(), service.getName()); for (PropertyInfo serviceProperty : serviceProperties) { if (serviceProperty.getPropertyTypes().contains(propertyType)) { String stackPropertyConfigType = fileNameToConfigType(serviceProperty.getFilename()); try { result.add(cluster.getDesiredConfigByType(stackPropertyConfigType).getProperties() .get(serviceProperty.getName())); } catch (Exception ex) { } } } } Set<PropertyInfo> stackProperties = ambariMetaInfo.getStackProperties(stack.getName(), stack.getVersion()); for (PropertyInfo stackProperty : stackProperties) { if (stackProperty.getPropertyTypes().contains(propertyType)) { String stackPropertyConfigType = fileNameToConfigType(stackProperty.getFilename()); result.add(cluster.getDesiredConfigByType(stackPropertyConfigType).getProperties() .get(stackProperty.getName())); } } return result; } public String getPropertyValueFromStackDefinitions(Cluster cluster, String configType, String propertyName) throws AmbariException { StackId stackId = cluster.getCurrentStackVersion(); StackInfo stack = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion()); for (ServiceInfo serviceInfo : stack.getServices()) { Set<PropertyInfo> serviceProperties = ambariMetaInfo.getServiceProperties(stack.getName(), stack.getVersion(), serviceInfo.getName()); Set<PropertyInfo> stackProperties = ambariMetaInfo.getStackProperties(stack.getName(), stack.getVersion()); serviceProperties.addAll(stackProperties); for (PropertyInfo stackProperty : serviceProperties) { String stackPropertyConfigType = fileNameToConfigType(stackProperty.getFilename()); if (stackProperty.getName().equals(propertyName) && stackPropertyConfigType.equals(configType)) { return stackProperty.getValue(); } } } return null; } /** * Gets the configuration value referenced by the specified placeholder from * the cluster configuration. This will take a configuration placeholder such * as {{hdfs-site/foo}} and return the value of {@code foo} defined in * {@code hdfs-site}. * * @param cluster the cluster to use when rendering the placeholder value (not * {@code null}). * @param placeholder the placeholder value, such as {{hdfs-site/foobar}} (not * {@code null} ) * @return the configuration value, or {@code null} if none. * @throws AmbariException if there was a problem parsing the placeholder or retrieving the * referenced value. */ public String getPlaceholderValueFromDesiredConfigurations(Cluster cluster, String placeholder) { // remove the {{ and }} from the placholder if (placeholder.startsWith("{{") && placeholder.endsWith("}}")) { placeholder = placeholder.substring(2, placeholder.length() - 2).trim(); } // break up hdfs-site/foobar into hdfs-site and foobar int delimiterPosition = placeholder.indexOf("/"); if (delimiterPosition < 0) { return placeholder; } String configType = placeholder.substring(0, delimiterPosition); String propertyName = placeholder.substring(delimiterPosition + 1, placeholder.length()); // return the value if it exists, otherwise return the placeholder String value = getValueFromDesiredConfigurations(cluster, configType, propertyName); return value != null ? value : placeholder; } public String getValueFromDesiredConfigurations(Cluster cluster, String configType, String propertyName) { Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs(); DesiredConfig desiredConfig = desiredConfigs.get(configType); Config config = cluster.getConfig(configType, desiredConfig.getTag()); Map<String, String> configurationProperties = config.getProperties(); if (null != configurationProperties) { String value = configurationProperties.get(propertyName); if (null != value) { return value; } } return null; } public ServiceInfo getPropertyOwnerService(Cluster cluster, String configType, String propertyName) throws AmbariException { StackId stackId = cluster.getCurrentStackVersion(); StackInfo stack = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion()); for (ServiceInfo serviceInfo : stack.getServices()) { Set<PropertyInfo> serviceProperties = ambariMetaInfo.getServiceProperties(stack.getName(), stack.getVersion(), serviceInfo.getName()); for (PropertyInfo stackProperty : serviceProperties) { String stackPropertyConfigType = fileNameToConfigType(stackProperty.getFilename()); if (stackProperty.getName().equals(propertyName) && stackPropertyConfigType.equals(configType)) { return serviceInfo; } } } return null; } public Set<PropertyInfo> getServiceProperties(Cluster cluster, String serviceName) throws AmbariException { // The original implementation of this method is to return all properties regardless of whether // they should be excluded or not. By setting removeExcluded to false in the method invocation // below, no attempt will be made to remove properties that exist in excluded types. return getServiceProperties(cluster.getCurrentStackVersion(), serviceName, false); } /** * Retrieves a Set of PropertyInfo objects containing the relevant properties for the requested * service. * <p/> * If <code>removeExcluded</code> is <code>true</code>, the service's excluded configuration types * are used to prune off PropertyInfos that should be ignored; else if <code>false</code>, all * PropertyInfos will be returned. * * @param stackId a StackId declaring the relevant stack * @param serviceName a String containing the requested service's name * @param removeExcluded a boolean value indicating whether to remove properties from excluded * configuration types (<code>true</code>) or return the complete set of properties regardless of exclusions (<code>false</code>) * @return a Set of PropertyInfo objects for the requested service * @throws AmbariException if the requested stack or the requested service is not found */ public Set<PropertyInfo> getServiceProperties(StackId stackId, String serviceName, boolean removeExcluded) throws AmbariException { ServiceInfo service = ambariMetaInfo.getService(stackId.getStackName(), stackId.getStackVersion(), serviceName); Set<PropertyInfo> properties = new HashSet<PropertyInfo>(service.getProperties()); if (removeExcluded) { Set<String> excludedConfigTypes = service.getExcludedConfigTypes(); // excludedConfigTypes can be null since org.apache.ambari.server.state.ServiceInfo.setExcludedConfigTypes() // allows for null values if ((excludedConfigTypes != null) && !excludedConfigTypes.isEmpty()) { // Iterate through the set of found PropertyInfo instances and remove ones that should be // excluded. Iterator<PropertyInfo> iterator = properties.iterator(); while (iterator.hasNext()) { PropertyInfo propertyInfo = iterator.next(); // If the config type for the current PropertyInfo is containing within an excluded type, // remove it from the set of properties being returned if (excludedConfigTypes .contains(ConfigHelper.fileNameToConfigType(propertyInfo.getFilename()))) { iterator.remove(); } } } } return properties; } public Set<PropertyInfo> getStackProperties(Cluster cluster) throws AmbariException { StackId stackId = cluster.getCurrentStackVersion(); StackInfo stack = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion()); return ambariMetaInfo.getStackProperties(stack.getName(), stack.getVersion()); } /** * A helper method to create a new {@link Config} for a given configuration * type and updates to the current values, if any. This method will perform the following tasks: * <ul> * <li>Marge the specified updates with the properties of the current version of the * configuration</li> * <li>Create a {@link Config} in the cluster for the specified type. This * will have the proper versions and tags set automatically.</li> * <li>Set the cluster's {@link DesiredConfig} to the new configuration</li> * <li>Create an entry in the configuration history with a note and username.</li> * <ul> * * @param cluster * @param controller * @param configType * @param updates * @param authenticatedUserName * @param serviceVersionNote * @throws AmbariException */ public void updateConfigType(Cluster cluster, AmbariManagementController controller, String configType, Map<String, String> updates, String authenticatedUserName, String serviceVersionNote) throws AmbariException { if ((configType != null) && (updates != null) && !updates.isEmpty()) { Config oldConfig = cluster.getDesiredConfigByType(configType); Map<String, String> oldConfigProperties; Map<String, String> properties = new HashMap<String, String>(); if (oldConfig == null) { oldConfigProperties = null; } else { oldConfigProperties = oldConfig.getProperties(); if (oldConfigProperties != null) { properties.putAll(oldConfig.getProperties()); } } properties.putAll(updates); if ((oldConfigProperties == null) || !Maps.difference(oldConfigProperties, properties).areEqual()) { createConfigType(cluster, controller, configType, properties, authenticatedUserName, serviceVersionNote); } } } /** * A helper method to create a new {@link Config} for a given configuration * type. This method will perform the following tasks: * <ul> * <li>Create a {@link Config} in the cluster for the specified type. This * will have the proper versions and tags set automatically.</li> * <li>Set the cluster's {@link DesiredConfig} to the new configuration</li> * <li>Create an entry in the configuration history with a note and username.</li> * <ul> * * @param cluster * @param controller * @param configType * @param properties * @param authenticatedUserName * @param serviceVersionNote * @throws AmbariException */ public void createConfigType(Cluster cluster, AmbariManagementController controller, String configType, Map<String, String> properties, String authenticatedUserName, String serviceVersionNote) throws AmbariException { String tag = "version1"; if (cluster.getConfigsByType(configType) != null) { tag = "version" + System.currentTimeMillis(); } // update the configuration ConfigurationRequest configurationRequest = new ConfigurationRequest(); configurationRequest.setClusterName(cluster.getClusterName()); configurationRequest.setVersionTag(tag); configurationRequest.setType(configType); configurationRequest.setProperties(properties); configurationRequest.setServiceConfigVersionNote(serviceVersionNote); controller.createConfiguration(configurationRequest); // create the configuration history entry Config baseConfig = cluster.getConfig(configurationRequest.getType(), configurationRequest.getVersionTag()); if (baseConfig != null) { cluster.addDesiredConfig(authenticatedUserName, Collections.singleton(baseConfig), serviceVersionNote); } } /** * Since global configs are deprecated since 1.7.0, but still supported. * We should automatically map any globals used, to *-env dictionaries. * * @param configurations map of configurations keyed by type */ public void moveDeprecatedGlobals(StackId stackId, Map<String, Map<String, String>> configurations, String clusterName) { Map<String, String> globalConfigurations = new HashMap<String, String>(); if (configurations.get(Configuration.GLOBAL_CONFIG_TAG) == null || configurations.get(Configuration.GLOBAL_CONFIG_TAG).size() == 0) { return; } globalConfigurations.putAll(configurations.get(Configuration.GLOBAL_CONFIG_TAG)); if (globalConfigurations != null && globalConfigurations.size() != 0) { LOG.warn("Global configurations are deprecated, " + "please use *-env"); } for (Map.Entry<String, String> property : globalConfigurations.entrySet()) { String propertyName = property.getKey(); String propertyValue = property.getValue(); Set<String> newConfigTypes = null; try { newConfigTypes = findConfigTypesByPropertyName(stackId, propertyName, clusterName); } catch (AmbariException e) { LOG.error("Exception while getting configurations from the stacks", e); return; } newConfigTypes.remove(Configuration.GLOBAL_CONFIG_TAG); String newConfigType = null; if (newConfigTypes.size() > 0) { newConfigType = newConfigTypes.iterator().next(); } else { newConfigType = UpgradeCatalog170.getAdditionalMappingGlobalToEnv().get(propertyName); } if (newConfigType == null) { LOG.warn("Cannot find where to map " + propertyName + " from " + Configuration.GLOBAL_CONFIG_TAG + " (value=" + propertyValue + ")"); continue; } LOG.info("Mapping config " + propertyName + " from " + Configuration.GLOBAL_CONFIG_TAG + " to " + newConfigType + " (value=" + propertyValue + ")"); configurations.get(Configuration.GLOBAL_CONFIG_TAG).remove(propertyName); if (!configurations.containsKey(newConfigType)) { configurations.put(newConfigType, new HashMap<String, String>()); } configurations.get(newConfigType).put(propertyName, propertyValue); } if (configurations.get(Configuration.GLOBAL_CONFIG_TAG).size() == 0) { configurations.remove(Configuration.GLOBAL_CONFIG_TAG); } } private boolean calculateIsStaleConfigs(ServiceComponentHost sch) throws AmbariException { if (sch.isRestartRequired()) { return true; } Map<String, HostConfig> actual = sch.getActualConfigs(); if (null == actual || actual.isEmpty()) { return false; } Cluster cluster = clusters.getClusterById(sch.getClusterId()); StackId stackId = cluster.getDesiredStackVersion(); Map<String, Map<String, String>> desired = getEffectiveDesiredTags(cluster, sch.getHostName()); ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(), stackId.getStackVersion(), sch.getServiceName()); ComponentInfo componentInfo = serviceInfo.getComponentByName(sch.getServiceComponentName()); // Configs are considered stale when: // - desired type DOES NOT exist in actual // --- desired type DOES NOT exist in stack: not_stale // --- desired type DOES exist in stack: check stack for any key: stale // - desired type DOES exist in actual // --- desired tags DO match actual tags: not_stale // --- desired tags DO NOT match actual tags // ---- merge values, determine changed keys, check stack: stale boolean stale = false; Iterator<Entry<String, Map<String, String>>> it = desired.entrySet().iterator(); while (it.hasNext() && !stale) { Entry<String, Map<String, String>> desiredEntry = it.next(); String type = desiredEntry.getKey(); Map<String, String> tags = desiredEntry.getValue(); if (!actual.containsKey(type)) { // desired is set, but actual is not if (!serviceInfo.hasConfigDependency(type)) { stale = componentInfo != null && componentInfo.hasConfigType(type); } else if (type.equals(Configuration.GLOBAL_CONFIG_TAG)) { // find out if the keys are stale by first checking the target service, // then all services Collection<String> keys = mergeKeyNames(cluster, type, tags.values()); if (serviceInfo.hasDependencyAndPropertyFor(type, keys) || !hasPropertyFor(stackId, type, keys)) { stale = true; } } else { stale = true; } } else { // desired and actual both define the type HostConfig hc = actual.get(type); Map<String, String> actualTags = buildTags(hc); if (!isTagChanged(tags, actualTags, hasGroupSpecificConfigsForType(cluster, sch.getHostName(), type))) { stale = false; } else if (type.equals(Configuration.GLOBAL_CONFIG_TAG)) { // tags are changed, need to find out what has changed, // and if it applies // to the service Collection<String> changed = findChangedKeys(cluster, type, tags.values(), actualTags.values()); if (serviceInfo.hasDependencyAndPropertyFor(type, changed)) { stale = true; } } else { stale = serviceInfo.hasConfigDependency(type) || componentInfo.hasConfigType(type); } } } return stale; } /** * Determines if the hostname has group specific configs for the type specified * * @param cluster * @param hostname of the host to look for * @param type the type to look for (e.g. flume-conf) * @return <code>true</code> if the hostname has group specific configuration for the type */ private boolean hasGroupSpecificConfigsForType(Cluster cluster, String hostname, String type) { try { Map<Long, ConfigGroup> configGroups = cluster.getConfigGroupsByHostname(hostname); if (configGroups != null && !configGroups.isEmpty()) { for (ConfigGroup configGroup : configGroups.values()) { Config config = configGroup.getConfigurations().get(type); if (config != null) { return true; } } } } catch (AmbariException ambariException) { LOG.warn("Could not determine group configuration for host. Details: " + ambariException.getMessage()); } return false; } /** * @return <code>true</code> if any service on the stack defines a property * for the type. */ private boolean hasPropertyFor(StackId stack, String type, Collection<String> keys) throws AmbariException { for (ServiceInfo svc : ambariMetaInfo.getServices(stack.getStackName(), stack.getStackVersion()).values()) { if (svc.hasDependencyAndPropertyFor(type, keys)) { return true; } } return false; } /** * @return the keys that have changed values */ private Collection<String> findChangedKeys(Cluster cluster, String type, Collection<String> desiredTags, Collection<String> actualTags) { Map<String, String> desiredValues = new HashMap<String, String>(); Map<String, String> actualValues = new HashMap<String, String>(); for (String tag : desiredTags) { Config config = cluster.getConfig(type, tag); if (null != config) { desiredValues.putAll(config.getProperties()); } } for (String tag : actualTags) { Config config = cluster.getConfig(type, tag); if (null != config) { actualValues.putAll(config.getProperties()); } } List<String> keys = new ArrayList<String>(); for (Entry<String, String> entry : desiredValues.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); if (!actualValues.containsKey(key)) { keys.add(key); } else if (!actualValues.get(key).equals(value)) { keys.add(key); } } return keys; } /** * @return the map of tags for a desired config */ private Map<String, String> buildTags(HostConfig hc) { Map<String, String> map = new LinkedHashMap<String, String>(); map.put(CLUSTER_DEFAULT_TAG, hc.getDefaultVersionTag()); if (hc.getConfigGroupOverrides() != null) { for (Entry<Long, String> entry : hc.getConfigGroupOverrides().entrySet()) { map.put(entry.getKey().toString(), entry.getValue()); } } return map; } /** * @return true if the tags are different in any way, even if not-specified */ private boolean isTagChanged(Map<String, String> desiredTags, Map<String, String> actualTags, boolean groupSpecificConfigs) { if (!actualTags.get(CLUSTER_DEFAULT_TAG).equals(desiredTags.get(CLUSTER_DEFAULT_TAG)) && !groupSpecificConfigs) { return true; } // if the host has group specific configs for type we should ignore the cluster level configs and compare specifics if (groupSpecificConfigs) { actualTags.remove(CLUSTER_DEFAULT_TAG); desiredTags.remove(CLUSTER_DEFAULT_TAG); } Set<String> desiredSet = new HashSet<String>(desiredTags.values()); Set<String> actualSet = new HashSet<String>(actualTags.values()); // Both desired and actual should be exactly the same return !desiredSet.equals(actualSet); } /** * @return the list of combined config property names */ private Collection<String> mergeKeyNames(Cluster cluster, String type, Collection<String> tags) { Set<String> names = new HashSet<String>(); for (String tag : tags) { Config config = cluster.getConfig(type, tag); if (null != config) { names.addAll(config.getProperties().keySet()); } } return names; } public static String fileNameToConfigType(String filename) { int extIndex = filename.indexOf(AmbariMetaInfo.SERVICE_CONFIG_FILE_NAME_POSTFIX); return filename.substring(0, extIndex); } }