Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.ambari.server.upgrade; import java.io.StringWriter; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.regex.Matcher; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.orm.DBAccessor; import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo; import org.apache.ambari.server.orm.dao.AlertDefinitionDAO; import org.apache.ambari.server.orm.dao.ArtifactDAO; import org.apache.ambari.server.orm.dao.ClusterDAO; import org.apache.ambari.server.orm.dao.ClusterVersionDAO; import org.apache.ambari.server.orm.dao.DaoUtils; import org.apache.ambari.server.orm.dao.HostVersionDAO; import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; import org.apache.ambari.server.orm.dao.StackDAO; import org.apache.ambari.server.orm.dao.UpgradeDAO; import org.apache.ambari.server.orm.entities.AlertDefinitionEntity; import org.apache.ambari.server.orm.entities.ArtifactEntity; import org.apache.ambari.server.orm.entities.ClusterEntity; import org.apache.ambari.server.orm.entities.ClusterVersionEntity; import org.apache.ambari.server.orm.entities.HostEntity; import org.apache.ambari.server.orm.entities.HostVersionEntity; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.orm.entities.UpgradeEntity; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; import org.apache.ambari.server.state.RepositoryVersionState; import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.state.alert.SourceType; import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor; import org.apache.ambari.server.state.kerberos.KerberosDescriptor; import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory; import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor; import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor; import org.apache.ambari.server.state.stack.upgrade.Direction; import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper; import org.apache.ambari.server.state.stack.upgrade.UpgradeType; import org.apache.ambari.server.utils.VersionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.persist.Transactional; /** * Upgrade catalog for version 2.2.0. */ public class UpgradeCatalog220 extends AbstractUpgradeCatalog { private static final String UPGRADE_TABLE = "upgrade"; private static final String STORM_SITE = "storm-site"; private static final String HDFS_SITE_CONFIG = "hdfs-site"; private static final String TOPOLOGY_CONFIG = "topology"; private static final String KAFKA_BROKER = "kafka-broker"; private static final String KAFKA_ENV_CONFIG = "kafka-env"; private static final String KAFKA_ENV_CONTENT_KERBEROS_PARAMS = "export KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}"; private static final String AMS_ENV = "ams-env"; private static final String AMS_HBASE_ENV = "ams-hbase-env"; private static final String AMS_SITE = "ams-site"; private static final String AMS_HBASE_SITE = "ams-hbase-site"; private static final String AMS_HBASE_SITE_ZK_TIMEOUT_PROPERTY = "zookeeper.session.timeout.localHBaseCluster"; private static final String AMS_HBASE_SITE_NORMALIZER_ENABLED_PROPERTY = "hbase.normalizer.enabled"; private static final String AMS_HBASE_SITE_NORMALIZER_PERIOD_PROPERTY = "hbase.normalizer.period"; private static final String AMS_HBASE_SITE_NORMALIZER_CLASS_PROPERTY = "hbase.master.normalizer.class"; private static final String TIMELINE_METRICS_HBASE_FIFO_COMPACTION_ENABLED = "timeline.metrics.hbase.fifo.compaction.enabled"; private static final String HBASE_ENV_CONFIG = "hbase-env"; private static final String FLUME_ENV_CONFIG = "flume-env"; private static final String HIVE_SITE_CONFIG = "hive-site"; private static final String HIVE_ENV_CONFIG = "hive-env"; private static final String RANGER_ENV_CONFIG = "ranger-env"; private static final String RANGER_UGSYNC_SITE_CONFIG = "ranger-ugsync-site"; private static final String ZOOKEEPER_LOG4J_CONFIG = "zookeeper-log4j"; private static final String NIMBS_MONITOR_FREQ_SECS_PROPERTY = "nimbus.monitor.freq.secs"; private static final String STORM_METRICS_REPORTER = "metrics.reporter.register"; private static final String HIVE_SERVER2_OPERATION_LOG_LOCATION_PROPERTY = "hive.server2.logging.operation.log.location"; private static final String HADOOP_ENV_CONFIG = "hadoop-env"; private static final String CONTENT_PROPERTY = "content"; private static final String HADOOP_ENV_CONTENT_TO_APPEND = "\n{% if is_datanode_max_locked_memory_set %}\n" + "# Fix temporary bug, when ulimit from conf files is not picked up, without full relogin. \n" + "# Makes sense to fix only when runing DN as root \n" + "if [ \"$command\" == \"datanode\" ] && [ \"$EUID\" -eq 0 ] && [ -n \"$HADOOP_SECURE_DN_USER\" ]; then\n" + " ulimit -l {{datanode_max_locked_memory}}\n" + "fi\n" + "{% endif %}\n"; private static final String DOWNGRADE_ALLOWED_COLUMN = "downgrade_allowed"; private static final String UPGRADE_SKIP_FAILURE_COLUMN = "skip_failures"; private static final String UPGRADE_SKIP_SC_FAILURE_COLUMN = "skip_sc_failures"; public static final String UPGRADE_PACKAGE_COL = "upgrade_package"; public static final String UPGRADE_TYPE_COL = "upgrade_type"; public static final String REPO_VERSION_TABLE = "repo_version"; private static final String HOST_ROLE_COMMAND_TABLE = "host_role_command"; private static final String HOST_ID_COL = "host_id"; private static final String KERBEROS_DESCRIPTOR_TABLE = "kerberos_descriptor"; private static final String KERBEROS_DESCRIPTOR_NAME_COLUMN = "kerberos_descriptor_name"; private static final String KERBEROS_DESCRIPTOR_COLUMN = "kerberos_descriptor"; private static final String RANGER_HDFS_PLUGIN_ENABLED_PROPERTY = "ranger-hdfs-plugin-enabled"; private static final String RANGER_HIVE_PLUGIN_ENABLED_PROPERTY = "ranger-hive-plugin-enabled"; private static final String RANGER_HBASE_PLUGIN_ENABLED_PROPERTY = "ranger-hbase-plugin-enabled"; private static final String RANGER_STORM_PLUGIN_ENABLED_PROPERTY = "ranger-storm-plugin-enabled"; private static final String RANGER_KNOX_PLUGIN_ENABLED_PROPERTY = "ranger-knox-plugin-enabled"; private static final String RANGER_YARN_PLUGIN_ENABLED_PROPERTY = "ranger-yarn-plugin-enabled"; private static final String RANGER_KAFKA_PLUGIN_ENABLED_PROPERTY = "ranger-kafka-plugin-enabled"; private static final String RANGER_USERSYNC_SOURCE_IMPL_CLASS_PROPERTY = "ranger.usersync.source.impl.class"; private static final String BLUEPRINT_TABLE = "blueprint"; private static final String SECURITY_TYPE_COLUMN = "security_type"; private static final String SECURITY_DESCRIPTOR_REF_COLUMN = "security_descriptor_reference"; private static final String STAGE_TABLE = "stage"; private static final String KNOX_SERVICE = "KNOX"; /** * Logger. */ private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog220.class); private static final String OOZIE_SITE_CONFIG = "oozie-site"; private static final String OOZIE_SERVICE_HADOOP_CONFIGURATIONS_PROPERTY_NAME = "oozie.service.HadoopAccessorService.hadoop.configurations"; private static final String OLD_DEFAULT_HADOOP_CONFIG_PATH = "/etc/hadoop/conf"; private static final String NEW_DEFAULT_HADOOP_CONFIG_PATH = "{{hadoop_conf_dir}}"; @Inject DaoUtils daoUtils; @Inject private RepositoryVersionDAO repositoryVersionDAO; @Inject private ClusterDAO clusterDAO; // ----- Constructors ------------------------------------------------------ /** * Don't forget to register new UpgradeCatalogs in {@link org.apache.ambari.server.upgrade.SchemaUpgradeHelper.UpgradeHelperModule#configure()} * * @param injector Guice injector to track dependencies and uses bindings to inject them. */ @Inject public UpgradeCatalog220(Injector injector) { super(injector); this.injector = injector; } // ----- UpgradeCatalog ---------------------------------------------------- /** * {@inheritDoc} */ @Override public String getTargetVersion() { return "2.2.0"; } // ----- AbstractUpgradeCatalog -------------------------------------------- /** * {@inheritDoc} */ @Override public String getSourceVersion() { return "2.1.2.1"; } /** * {@inheritDoc} */ @Override protected void executeDDLUpdates() throws AmbariException, SQLException { executeUpgradeDDLUpdates(); // Alter the host_role_command table to allow host_id to be nullable dbAccessor.alterColumn(HOST_ROLE_COMMAND_TABLE, new DBColumnInfo(HOST_ID_COL, Long.class, null, null, true)); addKerberosDescriptorTable(); executeBlueprintDDLUpdates(); executeStageDDLUpdates(); } protected void executeUpgradeDDLUpdates() throws AmbariException, SQLException { updateUpgradesDDL(); } private void addKerberosDescriptorTable() throws SQLException { List<DBAccessor.DBColumnInfo> columns = new ArrayList<DBAccessor.DBColumnInfo>(); columns.add(new DBAccessor.DBColumnInfo(KERBEROS_DESCRIPTOR_NAME_COLUMN, String.class, 255, null, false)); columns.add(new DBAccessor.DBColumnInfo(KERBEROS_DESCRIPTOR_COLUMN, char[].class, null, null, false)); LOG.debug("Creating table [ {} ] with columns [ {} ] and primary key: [ {} ]", KERBEROS_DESCRIPTOR_TABLE, columns, KERBEROS_DESCRIPTOR_NAME_COLUMN); dbAccessor.createTable(KERBEROS_DESCRIPTOR_TABLE, columns, KERBEROS_DESCRIPTOR_NAME_COLUMN); } private void executeBlueprintDDLUpdates() throws AmbariException, SQLException { dbAccessor.addColumn(BLUEPRINT_TABLE, new DBAccessor.DBColumnInfo(SECURITY_TYPE_COLUMN, String.class, 32, "NONE", false)); dbAccessor.addColumn(BLUEPRINT_TABLE, new DBAccessor.DBColumnInfo(SECURITY_DESCRIPTOR_REF_COLUMN, String.class, null, null, true)); } /** * Updates the {@code stage} table by: * <ul> * <li>Adding the {@code supports_auto_skip_failure} column</li> * </ul> * * @throws SQLException */ protected void executeStageDDLUpdates() throws SQLException { dbAccessor.addColumn(STAGE_TABLE, new DBAccessor.DBColumnInfo("supports_auto_skip_failure", Integer.class, 1, 0, false)); } /** * {@inheritDoc} */ @Override protected void executePreDMLUpdates() throws AmbariException, SQLException { // execute DDL updates executeStackUpgradeDDLUpdates(); // DDL and DML mixed code, double check here bootstrapRepoVersionForHDP21(); // execute DML updates, no DDL things after this line executeUpgradePreDMLUpdates(); } /** * Updates the following columns on the {@value #UPGRADE_TABLE} table to * default values: * <ul> * <li>{value {@link #DOWNGRADE_ALLOWED_COLUMN}}</li> * <li>{value {@link #UPGRADE_SKIP_FAILURE_COLUMN}}</li> * <li>{value {@link #UPGRADE_SKIP_SC_FAILURE_COLUMN}}</li> * </ul> * * @throws AmbariException * @throws SQLException */ protected void executeUpgradePreDMLUpdates() throws AmbariException, SQLException { UpgradeDAO upgradeDAO = injector.getInstance(UpgradeDAO.class); List<UpgradeEntity> upgrades = upgradeDAO.findAll(); for (UpgradeEntity upgrade : upgrades) { if (upgrade.isDowngradeAllowed() == null) { upgrade.setDowngradeAllowed(true); } // ensure that these are set to false for existing upgrades upgrade.setAutoSkipComponentFailures(false); upgrade.setAutoSkipServiceCheckFailures(false); // apply changes upgradeDAO.merge(upgrade); LOG.info(String.format("Updated upgrade id %s, upgrade pack %s from version %s to %s", upgrade.getId(), upgrade.getUpgradePackage(), upgrade.getFromVersion(), upgrade.getToVersion())); } // make the columns nullable now that they have defaults dbAccessor.setColumnNullable(UPGRADE_TABLE, DOWNGRADE_ALLOWED_COLUMN, false); dbAccessor.setColumnNullable(UPGRADE_TABLE, UPGRADE_SKIP_FAILURE_COLUMN, false); dbAccessor.setColumnNullable(UPGRADE_TABLE, UPGRADE_SKIP_SC_FAILURE_COLUMN, false); } @Override protected void executeDMLUpdates() throws AmbariException, SQLException { addNewConfigurationsFromXml(); updateAlertDefinitions(); updateStormConfigs(); updateAMSConfigs(); updateHDFSConfigs(); updateHbaseEnvConfig(); updateFlumeEnvConfig(); updateHadoopEnv(); updateKafkaConfigs(); updateRangerEnvConfig(); updateRangerUgsyncSiteConfig(); updateZookeeperLog4j(); updateHiveConfig(); updateAccumuloConfigs(); updateKerberosDescriptorArtifacts(); updateKnoxTopology(); } protected void updateKnoxTopology() throws AmbariException { AmbariManagementController ambariManagementController = injector .getInstance(AmbariManagementController.class); for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) { Config topology = cluster.getDesiredConfigByType(TOPOLOGY_CONFIG); if (topology != null) { String content = topology.getProperties().get(CONTENT_PROPERTY); if (content != null) { Document topologyXml = convertStringToDocument(content); if (topologyXml != null) { Element root = topologyXml.getDocumentElement(); if (root != null) { NodeList providerNodes = root.getElementsByTagName("provider"); boolean authorizationProviderExists = false; try { for (int i = 0; i < providerNodes.getLength(); i++) { Node providerNode = providerNodes.item(i); NodeList childNodes = providerNode.getChildNodes(); for (int k = 0; k < childNodes.getLength(); k++) { Node child = childNodes.item(k); child.normalize(); String childTextContent = child.getTextContent(); if (childTextContent != null && childTextContent.toLowerCase().equals("authorization")) { authorizationProviderExists = true; break; } } if (authorizationProviderExists) { break; } } } catch (Exception e) { e.printStackTrace(); LOG.error( "Error occurred during check 'authorization' provider already exists in topology." + e); return; } if (!authorizationProviderExists) { NodeList nodeList = root.getElementsByTagName("gateway"); if (nodeList != null && nodeList.getLength() > 0) { boolean rangerPluginEnabled = isConfigEnabled(cluster, AbstractUpgradeCatalog.CONFIGURATION_TYPE_RANGER_KNOX_PLUGIN_PROPERTIES, AbstractUpgradeCatalog.PROPERTY_RANGER_KNOX_PLUGIN_ENABLED); Node gatewayNode = nodeList.item(0); Element newProvider = topologyXml.createElement("provider"); Element role = topologyXml.createElement("role"); role.appendChild(topologyXml.createTextNode("authorization")); newProvider.appendChild(role); Element name = topologyXml.createElement("name"); if (rangerPluginEnabled) { name.appendChild(topologyXml.createTextNode("XASecurePDPKnox")); } else { name.appendChild(topologyXml.createTextNode("AclsAuthz")); } newProvider.appendChild(name); Element enabled = topologyXml.createElement("enabled"); enabled.appendChild(topologyXml.createTextNode("true")); newProvider.appendChild(enabled); gatewayNode.appendChild(newProvider); DOMSource topologyDomSource = new DOMSource(root); StringWriter writer = new StringWriter(); try { Transformer transformer = TransformerFactory.newInstance().newTransformer(); transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); transformer.setOutputProperty(OutputKeys.METHOD, "xml"); transformer.setOutputProperty(OutputKeys.INDENT, "yes"); transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "5"); transformer.transform(topologyDomSource, new StreamResult(writer)); } catch (TransformerConfigurationException e) { e.printStackTrace(); LOG.error( "Unable to create transformer instance, to convert Document(XML) to String. " + e); return; } catch (TransformerException e) { e.printStackTrace(); LOG.error("Unable to transform Document(XML) to StringWriter. " + e); return; } content = writer.toString(); Map<String, String> updates = Collections.singletonMap(CONTENT_PROPERTY, content); updateConfigurationPropertiesForCluster(cluster, TOPOLOGY_CONFIG, updates, true, false); } } } } } } } } /** * Move the upgrade_package column from the repo_version table to the upgrade table as follows, * add column upgrade_package to upgrade table as String 255 and nullable * populate column in the upgrade table * drop the column in the repo_version table * make the column in the upgrade table non-nullable. * This has to be called as part of DML and not DDL since the persistence service has to be started. * @throws AmbariException * @throws SQLException */ @Transactional protected void executeStackUpgradeDDLUpdates() throws SQLException, AmbariException { final Configuration.DatabaseType databaseType = configuration.getDatabaseType(); // Add columns if (!dbAccessor.tableHasColumn(UPGRADE_TABLE, UPGRADE_PACKAGE_COL)) { LOG.info("Adding upgrade_package column to upgrade table."); dbAccessor.addColumn(UPGRADE_TABLE, new DBColumnInfo(UPGRADE_PACKAGE_COL, String.class, 255, null, true)); } if (!dbAccessor.tableHasColumn(UPGRADE_TABLE, UPGRADE_TYPE_COL)) { LOG.info("Adding upgrade_type column to upgrade table."); dbAccessor.addColumn(UPGRADE_TABLE, new DBColumnInfo(UPGRADE_TYPE_COL, String.class, 32, null, true)); } // Populate values in upgrade table. boolean success = populateUpgradeTable(); if (!success) { throw new AmbariException( "Errors found while populating the upgrade table with values for columns upgrade_type and upgrade_package."); } if (dbAccessor.tableHasColumn(REPO_VERSION_TABLE, UPGRADE_PACKAGE_COL)) { LOG.info("Dropping upgrade_package column from repo_version table."); dbAccessor.dropColumn(REPO_VERSION_TABLE, UPGRADE_PACKAGE_COL); // Now, make the added column non-nullable // Make the hosts id non-null after all the values are populated LOG.info("Making upgrade_package column in the upgrade table non-nullable."); if (databaseType == Configuration.DatabaseType.DERBY) { // This is a workaround for UpgradeTest.java unit test dbAccessor.executeQuery( "ALTER TABLE " + UPGRADE_TABLE + " ALTER column " + UPGRADE_PACKAGE_COL + " NOT NULL"); } else { dbAccessor.alterColumn(UPGRADE_TABLE, new DBColumnInfo(UPGRADE_PACKAGE_COL, String.class, 255, null, false)); } } if (dbAccessor.tableHasColumn(REPO_VERSION_TABLE, UPGRADE_TYPE_COL)) { // Now, make the added column non-nullable // Make the hosts id non-null after all the values are populated LOG.info("Making upgrade_type column in the upgrade table non-nullable."); if (databaseType == Configuration.DatabaseType.DERBY) { // This is a workaround for UpgradeTest.java unit test dbAccessor.executeQuery( "ALTER TABLE " + UPGRADE_TABLE + " ALTER column " + UPGRADE_TYPE_COL + " NOT NULL"); } else { dbAccessor.alterColumn(UPGRADE_TABLE, new DBColumnInfo(UPGRADE_TYPE_COL, String.class, 32, null, false)); } } } /** * Populate the upgrade table with values for the columns upgrade_type and upgrade_package. * The upgrade_type will default to {@code org.apache.ambari.server.state.stack.upgrade.UpgradeType.ROLLING} * whereas the upgrade_package will be calculated. * @return {@code} true on success, and {@code} false otherwise. */ private boolean populateUpgradeTable() { boolean success = true; Statement statement = null; ResultSet rs = null; try { statement = dbAccessor.getConnection().createStatement(); if (statement != null) { // Need to use SQL since the schema is changing and some of the columns have not yet been added.. rs = statement.executeQuery( "SELECT upgrade_id, cluster_id, from_version, to_version, direction, upgrade_package, upgrade_type FROM upgrade"); if (rs != null) { try { while (rs.next()) { final long upgradeId = rs.getLong("upgrade_id"); final long clusterId = rs.getLong("cluster_id"); final String fromVersion = rs.getString("from_version"); final String toVersion = rs.getString("to_version"); final Direction direction = Direction.valueOf(rs.getString("direction")); // These two values are likely null. String upgradePackage = rs.getString("upgrade_package"); String upgradeType = rs.getString("upgrade_type"); LOG.info(MessageFormat.format("Populating rows for the upgrade table record with " + "upgrade_id: {0,number,#}, cluster_id: {1,number,#}, from_version: {2}, to_version: {3}, direction: {4}", upgradeId, clusterId, fromVersion, toVersion, direction)); // Set all upgrades that have been done so far to type "rolling" if (StringUtils.isEmpty(upgradeType)) { LOG.info("Updating the record's upgrade_type to " + UpgradeType.ROLLING); dbAccessor.executeQuery("UPDATE upgrade SET upgrade_type = '" + UpgradeType.ROLLING + "' WHERE upgrade_id = " + upgradeId); } if (StringUtils.isEmpty(upgradePackage)) { String version = null; StackEntity stack = null; if (direction == Direction.UPGRADE) { version = toVersion; } else if (direction == Direction.DOWNGRADE) { // TODO AMBARI-12698, this is going to be a problem. // During a downgrade, the "to_version" is overwritten to the source version, but the "from_version" // doesn't swap. E.g., // upgrade_id | from_version | to_version | direction // ------------+--------------+--------------+---------- // 1 | 2.2.6.0-2800 | 2.3.0.0-2557 | UPGRADE // 2 | 2.2.6.0-2800 | 2.2.6.0-2800 | DOWNGRADE version = fromVersion; } ClusterEntity cluster = clusterDAO.findById(clusterId); if (null != cluster) { stack = cluster.getDesiredStack(); upgradePackage = calculateUpgradePackage(stack, version); } else { LOG.error("Could not find a cluster with cluster_id " + clusterId); } if (!StringUtils.isEmpty(upgradePackage)) { LOG.info("Updating the record's upgrade_package to " + upgradePackage); dbAccessor.executeQuery("UPDATE upgrade SET upgrade_package = '" + upgradePackage + "' WHERE upgrade_id = " + upgradeId); } else { success = false; LOG.error( "Unable to populate column upgrade_package for record in table upgrade with id " + upgradeId); } } } } catch (Exception e) { success = false; e.printStackTrace(); LOG.error( "Unable to populate the upgrade_type and upgrade_package columns of the upgrade table. " + e); } } } } catch (Exception e) { success = false; e.printStackTrace(); LOG.error( "Failed to retrieve records from the upgrade table to populate the upgrade_type and upgrade_package columns. Exception: " + e); } finally { try { if (rs != null) { rs.close(); } if (statement != null) { statement.close(); } } catch (SQLException e) { ; } } return success; } /** * Find the single Repo Version for the given stack and version, and return * its upgrade_package column. Because the upgrade_package column is going to * be removed from this entity, must use raw SQL instead of the entity class. * <p/> * It's possible that there is an invalid version listed in the upgrade table. * For example: * * <pre> * upgrade * 1 2 1295 2.2.0.0-2041 2.2.4.2-2 UPGRADE * 2 2 1296 2.2.0.0-2041 2.2.0.0-2041 DOWNGRADE * 3 2 1299 2.2.0.0-2041 2.2.4.2 UPGRADE * * repo_version * 1 2.2.0.0-2041 HDP-2.2.0.0-2041 upgrade-2.2 * 2 2.2.4.2-2 HDP-2.2.4.2-2 upgrade-2.2 * </pre> * * Notice that it's possible for the {@code upgrade} table to include entries * for a repo version which does not exist; {@code 2.2.4.2}. In these cases, * this method will attempt a "best match". * * @param stack * Stack * @param version * Stack version * @return The value of the upgrade_package column, or null if not found. */ private String calculateUpgradePackage(StackEntity stack, String version) { String upgradePackage = null; // Find the corresponding repo_version, and extract its upgrade_package if (null != version && null != stack) { RepositoryVersionEntity repoVersion = repositoryVersionDAO .findByStackNameAndVersion(stack.getStackName(), version); // a null repoVersion means there's mismatch between the upgrade and repo_version table; // use a best-guess approach based on the Stack if (null == repoVersion) { List<RepositoryVersionEntity> bestMatches = repositoryVersionDAO.findByStack(new StackId(stack)); if (!bestMatches.isEmpty()) { repoVersion = bestMatches.get(0); } } // our efforts have failed; we have no idea what to use; return null as per the contract of the method if (null == repoVersion) { return null; } Statement statement = null; ResultSet rs = null; try { statement = dbAccessor.getConnection().createStatement(); if (statement != null) { // Need to use SQL since the schema is changing and the entity will no longer have the upgrade_package column. rs = statement.executeQuery("SELECT upgrade_package FROM repo_version WHERE repo_version_id = " + repoVersion.getId()); if (rs != null && rs.next()) { upgradePackage = rs.getString("upgrade_package"); } } } catch (Exception e) { LOG.error("Failed to retrieve upgrade_package for repo_version record with id " + repoVersion.getId() + ". Exception: " + e.getMessage()); } finally { try { if (rs != null) { rs.close(); } if (statement != null) { statement.close(); } } catch (SQLException e) { ; } } } return upgradePackage; } /** * {@inheritDoc} */ @Override protected void updateKerberosDescriptorArtifact(ArtifactDAO artifactDAO, ArtifactEntity artifactEntity) throws AmbariException { if (artifactEntity != null) { Map<String, Object> data = artifactEntity.getArtifactData(); if (data != null) { final KerberosDescriptor kerberosDescriptor = new KerberosDescriptorFactory().createInstance(data); if (kerberosDescriptor != null) { KerberosServiceDescriptor hdfsService = kerberosDescriptor.getService("HDFS"); if (hdfsService != null) { // before 2.2.0 hdfs indentity expected to be in HDFS service KerberosIdentityDescriptor hdfsIdentity = hdfsService.getIdentity("hdfs"); KerberosComponentDescriptor namenodeComponent = hdfsService.getComponent("NAMENODE"); hdfsIdentity.setName("hdfs"); hdfsService.removeIdentity("hdfs"); namenodeComponent.putIdentity(hdfsIdentity); } updateKerberosDescriptorIdentityReferences(kerberosDescriptor, "/HDFS/hdfs", "/HDFS/NAMENODE/hdfs"); updateKerberosDescriptorIdentityReferences(kerberosDescriptor.getServices(), "/HDFS/hdfs", "/HDFS/NAMENODE/hdfs"); artifactEntity.setArtifactData(kerberosDescriptor.toMap()); artifactDAO.merge(artifactEntity); } } } } /** * If still on HDP 2.1, then no repo versions exist, so need to bootstrap the HDP 2.1 repo version, * If still on HDP 2.1, then no repo versions exist, so need to bootstrap the HDP 2.1 repo version, * and mark it as CURRENT in the cluster_version table for the cluster, as well as the host_version table * for all hosts. */ @Transactional public void bootstrapRepoVersionForHDP21() throws AmbariException, SQLException { final String hardcodedInitialVersion = "2.1.0.0-0001"; AmbariManagementController amc = injector.getInstance(AmbariManagementController.class); AmbariMetaInfo ambariMetaInfo = amc.getAmbariMetaInfo(); StackDAO stackDAO = injector.getInstance(StackDAO.class); RepositoryVersionHelper repositoryVersionHelper = injector.getInstance(RepositoryVersionHelper.class); RepositoryVersionDAO repositoryVersionDAO = injector.getInstance(RepositoryVersionDAO.class); ClusterVersionDAO clusterVersionDAO = injector.getInstance(ClusterVersionDAO.class); HostVersionDAO hostVersionDAO = injector.getInstance(HostVersionDAO.class); Clusters clusters = amc.getClusters(); if (clusters == null) { LOG.error("Unable to get Clusters entity."); return; } for (Cluster cluster : clusters.getClusters().values()) { ClusterEntity clusterEntity = clusterDAO.findByName(cluster.getClusterName()); final StackId stackId = cluster.getCurrentStackVersion(); LOG.info(MessageFormat.format("Analyzing cluster {0}, currently at stack {1} and version {2}", cluster.getClusterName(), stackId.getStackName(), stackId.getStackVersion())); if (stackId.getStackName().equalsIgnoreCase("HDP") && stackId.getStackVersion().equalsIgnoreCase("2.1")) { final StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion()); StackEntity stackEntity = stackDAO.find(stackId.getStackName(), stackId.getStackVersion()); LOG.info("Bootstrapping the versions since using HDP-2.1"); // The actual value is not known, so use this. String displayName = stackId.getStackName() + "-" + hardcodedInitialVersion; // However, the Repo URLs should be correct. String operatingSystems = repositoryVersionHelper .serializeOperatingSystems(stackInfo.getRepositories()); // Create the Repo Version if it doesn't already exist. RepositoryVersionEntity repoVersionEntity = repositoryVersionDAO.findByDisplayName(displayName); if (null != repoVersionEntity) { LOG.info(MessageFormat.format("A Repo Version already exists with Display Name: {0}", displayName)); } else { final long repoVersionIdSeq = repositoryVersionDAO.findMaxId("id"); // Safe to attempt to add the sequence if it doesn't exist already. addSequence("repo_version_id_seq", repoVersionIdSeq, false); repoVersionEntity = repositoryVersionDAO.create(stackEntity, hardcodedInitialVersion, displayName, operatingSystems); LOG.info(MessageFormat.format( "Created Repo Version with ID: {0,number,#}\n, Display Name: {1}, Repo URLs: {2}\n", repoVersionEntity.getId(), displayName, operatingSystems)); } // Create the Cluster Version if it doesn't already exist. ClusterVersionEntity clusterVersionEntity = clusterVersionDAO.findByClusterAndStackAndVersion( cluster.getClusterName(), stackId, hardcodedInitialVersion); if (null != clusterVersionEntity) { LOG.info(MessageFormat.format( "A Cluster Version version for cluster: {0}, version: {1}, already exists; its state is {2}.", cluster.getClusterName(), clusterVersionEntity.getRepositoryVersion().getVersion(), clusterVersionEntity.getState())); // If there are not CURRENT cluster versions, make this one the CURRENT one. if (clusterVersionEntity.getState() != RepositoryVersionState.CURRENT && clusterVersionDAO .findByClusterAndState(cluster.getClusterName(), RepositoryVersionState.CURRENT) .isEmpty()) { clusterVersionEntity.setState(RepositoryVersionState.CURRENT); clusterVersionDAO.merge(clusterVersionEntity); } } else { final long clusterVersionIdSeq = clusterVersionDAO.findMaxId("id"); // Safe to attempt to add the sequence if it doesn't exist already. addSequence("cluster_version_id_seq", clusterVersionIdSeq, false); clusterVersionEntity = clusterVersionDAO.create(clusterEntity, repoVersionEntity, RepositoryVersionState.CURRENT, System.currentTimeMillis(), System.currentTimeMillis(), "admin"); LOG.info(MessageFormat.format( "Created Cluster Version with ID: {0,number,#}, cluster: {1}, version: {2}, state: {3}.", clusterVersionEntity.getId(), cluster.getClusterName(), clusterVersionEntity.getRepositoryVersion().getVersion(), clusterVersionEntity.getState())); } // Create the Host Versions if they don't already exist. Collection<HostEntity> hosts = clusterEntity.getHostEntities(); boolean addedAtLeastOneHost = false; if (null != hosts && !hosts.isEmpty()) { for (HostEntity hostEntity : hosts) { HostVersionEntity hostVersionEntity = hostVersionDAO.findByClusterStackVersionAndHost( cluster.getClusterName(), stackId, hardcodedInitialVersion, hostEntity.getHostName()); if (null != hostVersionEntity) { LOG.info(MessageFormat.format( "A Host Version version for cluster: {0}, version: {1}, host: {2}, already exists; its state is {3}.", cluster.getClusterName(), hostVersionEntity.getRepositoryVersion().getVersion(), hostEntity.getHostName(), hostVersionEntity.getState())); if (hostVersionEntity.getState() != RepositoryVersionState.CURRENT && hostVersionDAO .findByClusterHostAndState(cluster.getClusterName(), hostEntity.getHostName(), RepositoryVersionState.CURRENT) .isEmpty()) { hostVersionEntity.setState(RepositoryVersionState.CURRENT); hostVersionDAO.merge(hostVersionEntity); } } else { // This should only be done the first time. if (!addedAtLeastOneHost) { final long hostVersionIdSeq = hostVersionDAO.findMaxId("id"); // Safe to attempt to add the sequence if it doesn't exist already. addSequence("host_version_id_seq", hostVersionIdSeq, false); addedAtLeastOneHost = true; } hostVersionEntity = new HostVersionEntity(hostEntity, repoVersionEntity, RepositoryVersionState.CURRENT); hostVersionDAO.create(hostVersionEntity); LOG.info(MessageFormat.format( "Created Host Version with ID: {0,number,#}, cluster: {1}, version: {2}, host: {3}, state: {4}.", hostVersionEntity.getId(), cluster.getClusterName(), hostVersionEntity.getRepositoryVersion().getVersion(), hostEntity.getHostName(), hostVersionEntity.getState())); } } } else { LOG.info(MessageFormat.format( "Not inserting any Host Version records since cluster {0} does not have any hosts.", cluster.getClusterName())); } } } } /** * Adds the following columns to the {@value #UPGRADE_TABLE} table: * <ul> * <li>{@value #DOWNGRADE_ALLOWED_COLUMN}</li> * <li>{@value #UPGRADE_SKIP_FAILURE_COLUMN}</li> * <li>{@value #UPGRADE_SKIP_SC_FAILURE_COLUMN}</li> * </ul> * * @throws SQLException */ protected void updateUpgradesDDL() throws SQLException { dbAccessor.addColumn(UPGRADE_TABLE, new DBColumnInfo(DOWNGRADE_ALLOWED_COLUMN, Short.class, 1, null, true)); dbAccessor.addColumn(UPGRADE_TABLE, new DBColumnInfo(UPGRADE_SKIP_FAILURE_COLUMN, Short.class, 1, null, true)); dbAccessor.addColumn(UPGRADE_TABLE, new DBColumnInfo(UPGRADE_SKIP_SC_FAILURE_COLUMN, Short.class, 1, null, true)); } /** * Modifies the JSON of some of the alert definitions which have changed * between Ambari versions. */ protected void updateAlertDefinitions() { LOG.info("Updating alert definitions."); AmbariManagementController ambariManagementController = injector .getInstance(AmbariManagementController.class); AlertDefinitionDAO alertDefinitionDAO = injector.getInstance(AlertDefinitionDAO.class); Clusters clusters = ambariManagementController.getClusters(); Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters); for (final Cluster cluster : clusterMap.values()) { long clusterID = cluster.getClusterId(); final AlertDefinitionEntity journalNodeProcessAlertDefinitionEntity = alertDefinitionDAO .findByName(clusterID, "journalnode_process"); final AlertDefinitionEntity hostDiskUsageAlertDefinitionEntity = alertDefinitionDAO .findByName(clusterID, "ambari_agent_disk_usage"); if (journalNodeProcessAlertDefinitionEntity != null) { String source = journalNodeProcessAlertDefinitionEntity.getSource(); journalNodeProcessAlertDefinitionEntity.setSource(modifyJournalnodeProcessAlertSource(source)); journalNodeProcessAlertDefinitionEntity.setSourceType(SourceType.WEB); journalNodeProcessAlertDefinitionEntity.setHash(UUID.randomUUID().toString()); alertDefinitionDAO.merge(journalNodeProcessAlertDefinitionEntity); LOG.info("journalnode_process alert definition was updated."); } if (hostDiskUsageAlertDefinitionEntity != null) { hostDiskUsageAlertDefinitionEntity .setDescription("This host-level alert is triggered if the amount of disk space " + "used goes above specific thresholds. The default threshold values are 50% for WARNING and 80% for CRITICAL."); hostDiskUsageAlertDefinitionEntity.setLabel("Host Disk Usage"); alertDefinitionDAO.merge(hostDiskUsageAlertDefinitionEntity); LOG.info("ambari_agent_disk_usage alert definition was updated."); } } } /** * Modifies type of the journalnode_process alert to WEB. * Changes reporting text and uri according to the WEB type. * Removes default_port property. */ String modifyJournalnodeProcessAlertSource(String source) { JsonObject rootJson = new JsonParser().parse(source).getAsJsonObject(); rootJson.remove("type"); rootJson.addProperty("type", "WEB"); rootJson.remove("default_port"); rootJson.remove("uri"); JsonObject uriJson = new JsonObject(); uriJson.addProperty("http", "{{hdfs-site/dfs.journalnode.http-address}}"); uriJson.addProperty("https", "{{hdfs-site/dfs.journalnode.https-address}}"); uriJson.addProperty("kerberos_keytab", "{{hdfs-site/dfs.web.authentication.kerberos.keytab}}"); uriJson.addProperty("kerberos_principal", "{{hdfs-site/dfs.web.authentication.kerberos.principal}}"); uriJson.addProperty("https_property", "{{hdfs-site/dfs.http.policy}}"); uriJson.addProperty("https_property_value", "HTTPS_ONLY"); uriJson.addProperty("connection_timeout", 5.0); rootJson.add("uri", uriJson); rootJson.getAsJsonObject("reporting").getAsJsonObject("ok").remove("text"); rootJson.getAsJsonObject("reporting").getAsJsonObject("ok").addProperty("text", "HTTP {0} response in {2:.3f}s"); rootJson.getAsJsonObject("reporting").getAsJsonObject("warning").remove("text"); rootJson.getAsJsonObject("reporting").getAsJsonObject("warning").addProperty("text", "HTTP {0} response from {1} in {2:.3f}s ({3})"); rootJson.getAsJsonObject("reporting").getAsJsonObject("warning").remove("value"); rootJson.getAsJsonObject("reporting").getAsJsonObject("critical").remove("text"); rootJson.getAsJsonObject("reporting").getAsJsonObject("critical").addProperty("text", "Connection failed to {1} ({3})"); rootJson.getAsJsonObject("reporting").getAsJsonObject("critical").remove("value"); return rootJson.toString(); } protected void updateHadoopEnv() throws AmbariException { AmbariManagementController ambariManagementController = injector .getInstance(AmbariManagementController.class); for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) { Config hadoopEnvConfig = cluster.getDesiredConfigByType(HADOOP_ENV_CONFIG); if (hadoopEnvConfig != null) { String content = hadoopEnvConfig.getProperties().get(CONTENT_PROPERTY); if (content != null) { content += HADOOP_ENV_CONTENT_TO_APPEND; Map<String, String> updates = Collections.singletonMap(CONTENT_PROPERTY, content); updateConfigurationPropertiesForCluster(cluster, HADOOP_ENV_CONFIG, updates, true, false); } } } } protected void updateHDFSConfigs() throws AmbariException { AmbariManagementController ambariManagementController = injector .getInstance(AmbariManagementController.class); Map<String, Cluster> clusterMap = getCheckedClusterMap(ambariManagementController.getClusters()); for (final Cluster cluster : clusterMap.values()) { // Remove dfs.namenode.rpc-address property when NN HA is enabled if (cluster.getDesiredConfigByType(HDFS_SITE_CONFIG) != null && isNNHAEnabled(cluster)) { Set<String> removePropertiesSet = new HashSet<>(); removePropertiesSet.add("dfs.namenode.rpc-address"); removeConfigurationPropertiesFromCluster(cluster, HDFS_SITE_CONFIG, removePropertiesSet); } } } protected void updateZookeeperLog4j() throws AmbariException { AmbariManagementController ambariManagementController = injector .getInstance(AmbariManagementController.class); for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) { Config zookeeperLog4jConfig = cluster.getDesiredConfigByType(ZOOKEEPER_LOG4J_CONFIG); if (zookeeperLog4jConfig != null) { String content = zookeeperLog4jConfig.getProperties().get(CONTENT_PROPERTY); if (content != null) { content = content.replaceAll("[\n^]\\s*log4j\\.rootLogger\\s*=\\s*INFO\\s*,\\s*CONSOLE", "\nlog4j.rootLogger=INFO, ROLLINGFILE"); Map<String, String> updates = Collections.singletonMap(CONTENT_PROPERTY, content); updateConfigurationPropertiesForCluster(cluster, ZOOKEEPER_LOG4J_CONFIG, updates, true, false); } } } } protected void updateStormConfigs() throws AmbariException { AmbariManagementController ambariManagementController = injector .getInstance(AmbariManagementController.class); for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) { Config stormSiteProps = cluster.getDesiredConfigByType(STORM_SITE); if (stormSiteProps != null) { Map<String, String> updates = new HashMap<>(); String nimbusMonitorFreqSecs = stormSiteProps.getProperties().get(NIMBS_MONITOR_FREQ_SECS_PROPERTY); if (nimbusMonitorFreqSecs != null && nimbusMonitorFreqSecs.equals("10")) { updates.put(NIMBS_MONITOR_FREQ_SECS_PROPERTY, "120"); } Service amsService = null; try { amsService = cluster.getService("AMBARI_METRICS"); } catch (AmbariException ambariException) { LOG.info("AMBARI_METRICS service not found in cluster while updating storm-site properties"); } String metricsReporter = stormSiteProps.getProperties().get(STORM_METRICS_REPORTER); if (amsService != null && StringUtils.isEmpty(metricsReporter)) { updates.put(STORM_METRICS_REPORTER, "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter"); } updateConfigurationPropertiesForCluster(cluster, STORM_SITE, updates, true, false); } } } protected void updateHiveConfig() throws AmbariException { AmbariManagementController ambariManagementController = injector .getInstance(AmbariManagementController.class); for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) { Config hiveSiteConfig = cluster.getDesiredConfigByType(HIVE_SITE_CONFIG); if (hiveSiteConfig != null) { String hiveServer2OperationLogLocation = hiveSiteConfig.getProperties() .get(HIVE_SERVER2_OPERATION_LOG_LOCATION_PROPERTY); if (hiveServer2OperationLogLocation != null && hiveServer2OperationLogLocation .equals("${system:java.io.tmpdir}/${system:user.name}/operation_logs")) { Map<String, String> updates = Collections .singletonMap(HIVE_SERVER2_OPERATION_LOG_LOCATION_PROPERTY, "/tmp/hive/operation_logs"); updateConfigurationPropertiesForCluster(cluster, HIVE_SITE_CONFIG, updates, true, false); } } StackId stackId = cluster.getCurrentStackVersion(); boolean isStackNotLess23 = (stackId != null && stackId.getStackName().equals("HDP") && VersionUtils.compareVersions(stackId.getStackVersion(), "2.3") >= 0); Config hiveEnvConfig = cluster.getDesiredConfigByType(HIVE_ENV_CONFIG); if (hiveEnvConfig != null) { Map<String, String> hiveEnvProps = new HashMap<String, String>(); String content = hiveEnvConfig.getProperties().get(CONTENT_PROPERTY); // For HDP-2.3 we need to add hive heap size management to content, // for others we need to update content if (content != null) { if (isStackNotLess23) { content = updateHiveEnvContentHDP23(content); } else { content = updateHiveEnvContent(content); } hiveEnvProps.put(CONTENT_PROPERTY, content); updateConfigurationPropertiesForCluster(cluster, HIVE_ENV_CONFIG, hiveEnvProps, true, true); } } } } protected void updateHbaseEnvConfig() throws AmbariException { AmbariManagementController ambariManagementController = injector .getInstance(AmbariManagementController.class); boolean updateConfig = false; for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) { StackId stackId = cluster.getCurrentStackVersion(); Config hbaseEnvConfig = cluster.getDesiredConfigByType(HBASE_ENV_CONFIG); if (hbaseEnvConfig != null) { String content = hbaseEnvConfig.getProperties().get(CONTENT_PROPERTY); if (content != null) { if (!content.contains("-Djava.io.tmpdir")) { content += "\n\nexport HBASE_OPTS=\"${HBASE_OPTS} -Djava.io.tmpdir={{java_io_tmpdir}}\""; updateConfig = true; } if (stackId != null && stackId.getStackName().equals("HDP") && VersionUtils.compareVersions(stackId.getStackVersion(), "2.2") >= 0) { if (!content.contains("MaxDirectMemorySize={{hbase_max_direct_memory_size}}m")) { String newPartOfContent = "\n\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS {% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %}\"\n\n"; content += newPartOfContent; updateConfig = true; } if (updateConfig) { Map<String, String> updates = Collections.singletonMap(CONTENT_PROPERTY, content); updateConfigurationPropertiesForCluster(cluster, HBASE_ENV_CONFIG, updates, true, false); } } } } } } protected void updateFlumeEnvConfig() throws AmbariException { AmbariManagementController ambariManagementController = injector .getInstance(AmbariManagementController.class); for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) { Config flumeEnvConfig = cluster.getDesiredConfigByType(FLUME_ENV_CONFIG); if (flumeEnvConfig != null) { String content = flumeEnvConfig.getProperties().get(CONTENT_PROPERTY); if (content != null && !content.contains("/usr/lib/flume/lib/ambari-metrics-flume-sink.jar")) { String newPartOfContent = "\n\n" + "# Note that the Flume conf directory is always included in the classpath.\n" + "# Add flume sink to classpath\n" + "if [ -e \"/usr/lib/flume/lib/ambari-metrics-flume-sink.jar\" ]; then\n" + " export FLUME_CLASSPATH=$FLUME_CLASSPATH:/usr/lib/flume/lib/ambari-metrics-flume-sink.jar\n" + "fi\n"; content += newPartOfContent; Map<String, String> updates = Collections.singletonMap(CONTENT_PROPERTY, content); updateConfigurationPropertiesForCluster(cluster, FLUME_ENV_CONFIG, updates, true, false); } } } } protected void updateAMSConfigs() throws AmbariException { AmbariManagementController ambariManagementController = injector .getInstance(AmbariManagementController.class); Clusters clusters = ambariManagementController.getClusters(); if (clusters != null) { Map<String, Cluster> clusterMap = clusters.getClusters(); if (clusterMap != null && !clusterMap.isEmpty()) { for (final Cluster cluster : clusterMap.values()) { Config amsHbaseEnv = cluster.getDesiredConfigByType(AMS_HBASE_ENV); if (amsHbaseEnv != null) { Map<String, String> amsHbaseEnvProperties = amsHbaseEnv.getProperties(); String content = amsHbaseEnvProperties.get("content"); Map<String, String> newProperties = new HashMap<>(); newProperties.put("content", updateAmsHbaseEnvContent(content)); updateConfigurationPropertiesForCluster(cluster, AMS_HBASE_ENV, newProperties, true, true); } Config amsEnv = cluster.getDesiredConfigByType(AMS_ENV); if (amsEnv != null) { Map<String, String> amsEnvProperties = amsEnv.getProperties(); String content = amsEnvProperties.get("content"); Map<String, String> newProperties = new HashMap<>(); newProperties.put("content", updateAmsEnvContent(content)); updateConfigurationPropertiesForCluster(cluster, AMS_ENV, newProperties, true, true); } Config amsSite = cluster.getDesiredConfigByType(AMS_SITE); if (amsSite != null) { Map<String, String> currentAmsSiteProperties = amsSite.getProperties(); Map<String, String> newProperties = new HashMap<>(); //Changed AMS result set limit from 5760 to 15840. if (currentAmsSiteProperties.containsKey("timeline.metrics.service.default.result.limit") && currentAmsSiteProperties.get("timeline.metrics.service.default.result.limit") .equals(String.valueOf(5760))) { LOG.info("Updating timeline.metrics.service.default.result.limit to 15840"); newProperties.put("timeline.metrics.service.default.result.limit", String.valueOf(15840)); } //Interval newProperties.put("timeline.metrics.cluster.aggregator.second.interval", String.valueOf(120)); newProperties.put("timeline.metrics.cluster.aggregator.minute.interval", String.valueOf(300)); newProperties.put("timeline.metrics.host.aggregator.minute.interval", String.valueOf(300)); //ttl newProperties.put("timeline.metrics.cluster.aggregator.second.ttl", String.valueOf(2592000)); newProperties.put("timeline.metrics.cluster.aggregator.minute.ttl", String.valueOf(7776000)); //checkpoint newProperties.put("timeline.metrics.cluster.aggregator.second.checkpointCutOffMultiplier", String.valueOf(2)); //disabled newProperties.put("timeline.metrics.cluster.aggregator.second.disabled", String.valueOf(false)); //Add compaction policy property newProperties.put(TIMELINE_METRICS_HBASE_FIFO_COMPACTION_ENABLED, String.valueOf(true)); updateConfigurationPropertiesForCluster(cluster, AMS_SITE, newProperties, true, true); } Config amsHbaseSite = cluster.getDesiredConfigByType(AMS_HBASE_SITE); if (amsHbaseSite != null) { Map<String, String> amsHbaseSiteProperties = amsHbaseSite.getProperties(); Map<String, String> newProperties = new HashMap<>(); String zkTimeout = amsHbaseSiteProperties.get(AMS_HBASE_SITE_ZK_TIMEOUT_PROPERTY); // if old default, set new default if ("20000".equals(zkTimeout)) { newProperties.put(AMS_HBASE_SITE_ZK_TIMEOUT_PROPERTY, "120000"); } //Adding hbase.normalizer.period to upgrade if (!amsHbaseSiteProperties.containsKey(AMS_HBASE_SITE_NORMALIZER_ENABLED_PROPERTY)) { LOG.info("Enabling " + AMS_HBASE_SITE_NORMALIZER_ENABLED_PROPERTY); newProperties.put(AMS_HBASE_SITE_NORMALIZER_ENABLED_PROPERTY, String.valueOf(true)); } if (!amsHbaseSiteProperties.containsKey(AMS_HBASE_SITE_NORMALIZER_PERIOD_PROPERTY)) { LOG.info("Updating " + AMS_HBASE_SITE_NORMALIZER_PERIOD_PROPERTY); newProperties.put(AMS_HBASE_SITE_NORMALIZER_PERIOD_PROPERTY, String.valueOf(600000)); } if (!amsHbaseSiteProperties.containsKey(AMS_HBASE_SITE_NORMALIZER_CLASS_PROPERTY)) { LOG.info("Updating " + AMS_HBASE_SITE_NORMALIZER_CLASS_PROPERTY); newProperties.put(AMS_HBASE_SITE_NORMALIZER_CLASS_PROPERTY, "org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer"); } updateConfigurationPropertiesForCluster(cluster, AMS_HBASE_SITE, newProperties, true, true); } } } } } protected String updateAmsHbaseEnvContent(String content) { if (content == null) { return null; } String regSearch = "export HBASE_HEAPSIZE="; String replacement = "#export HBASE_HEAPSIZE="; content = content.replaceAll(regSearch, replacement); content += "\n" + "# The maximum amount of heap to use for hbase shell.\n" + "export HBASE_SHELL_OPTS=\"-Xmx256m\"\n"; return content; } protected String updateAmsEnvContent(String content) { if (content == null) { return null; } if (!content.contains("AMS_COLLECTOR_GC_OPTS")) { content += "\n" + "# AMS Collector GC options\n" + "export AMS_COLLECTOR_GC_OPTS=\"-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=70 " + "-XX:+UseCMSInitiatingOccupancyOnly -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps " + "-XX:+UseGCLogFileRotation -XX:GCLogFileSize=10M " + "-Xloggc:{{ams_collector_log_dir}}/collector-gc.log-`date +'%Y%m%d%H%M'`\"\n" + "export AMS_COLLECTOR_OPTS=\"$AMS_COLLECTOR_OPTS $AMS_COLLECTOR_GC_OPTS\"\n"; } if (!content.contains("HBASE_NORMALIZATION_ENABLED")) { content += "\n" + "# HBase compaction policy enabled\n" + "export HBASE_NORMALIZATION_ENABLED={{ams_hbase_normalizer_enabled}}\n"; } if (!content.contains("HBASE_FIFO_COMPACTION_POLICY_ENABLED")) { content += "\n" + "# HBase compaction policy enabled\n" + "export HBASE_FIFO_COMPACTION_POLICY_ENABLED={{ams_hbase_fifo_compaction_policy_enabled}}\n"; } return content; } protected void updateKafkaConfigs() throws AmbariException { AmbariManagementController ambariManagementController = injector .getInstance(AmbariManagementController.class); Clusters clusters = ambariManagementController.getClusters(); if (clusters != null) { Map<String, Cluster> clusterMap = clusters.getClusters(); if (clusterMap != null && !clusterMap.isEmpty()) { for (final Cluster cluster : clusterMap.values()) { Set<String> installedServices = cluster.getServices().keySet(); Config kafkaBroker = cluster.getDesiredConfigByType(KAFKA_BROKER); if (kafkaBroker != null) { Map<String, String> newProperties = new HashMap<>(); Map<String, String> kafkaBrokerProperties = kafkaBroker.getProperties(); String kafkaMetricsReporters = kafkaBrokerProperties.get("kafka.metrics.reporters"); if (kafkaMetricsReporters == null || "{{kafka_metrics_reporters}}".equals(kafkaMetricsReporters)) { if (installedServices.contains("AMBARI_METRICS")) { newProperties.put("kafka.metrics.reporters", "org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter"); } else if (installedServices.contains("GANGLIA")) { newProperties.put("kafka.metrics.reporters", "kafka.ganglia.KafkaGangliaMetricsReporter"); } else { newProperties.put("kafka.metrics.reporters", " "); } } if (!newProperties.isEmpty()) { updateConfigurationPropertiesForCluster(cluster, KAFKA_BROKER, newProperties, true, true); } } Config kafkaEnv = cluster.getDesiredConfigByType(KAFKA_ENV_CONFIG); if (kafkaEnv != null) { String kafkaEnvContent = kafkaEnv.getProperties().get(CONTENT_PROPERTY); if (kafkaEnvContent != null && !kafkaEnvContent.contains(KAFKA_ENV_CONTENT_KERBEROS_PARAMS)) { kafkaEnvContent += "\n\nexport KAFKA_KERBEROS_PARAMS=\"$KAFKA_KERBEROS_PARAMS {{kafka_kerberos_params}}\""; Map<String, String> updates = Collections.singletonMap(CONTENT_PROPERTY, kafkaEnvContent); updateConfigurationPropertiesForCluster(cluster, KAFKA_ENV_CONFIG, updates, true, false); } } } } } } protected void updateRangerEnvConfig() throws AmbariException { AmbariManagementController ambariManagementController = injector .getInstance(AmbariManagementController.class); for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) { Map<String, String> newRangerEnvProps = new HashMap<>(); Config rangerHdfsPluginProperties = cluster.getDesiredConfigByType("ranger-hdfs-plugin-properties"); if (rangerHdfsPluginProperties != null && rangerHdfsPluginProperties.getProperties() .containsKey(RANGER_HDFS_PLUGIN_ENABLED_PROPERTY)) { newRangerEnvProps.put(RANGER_HDFS_PLUGIN_ENABLED_PROPERTY, rangerHdfsPluginProperties.getProperties().get(RANGER_HDFS_PLUGIN_ENABLED_PROPERTY)); } Config hiveEnvProperties = cluster.getDesiredConfigByType("hive-env"); if (hiveEnvProperties != null && hiveEnvProperties.getProperties().containsKey("hive_security_authorization") && hiveEnvProperties.getProperties().get("hive_security_authorization").toLowerCase() .equals("ranger")) { newRangerEnvProps.put(RANGER_HIVE_PLUGIN_ENABLED_PROPERTY, "Yes"); } Config rangerHbasePluginProperties = cluster.getDesiredConfigByType("ranger-hbase-plugin-properties"); if (rangerHbasePluginProperties != null && rangerHbasePluginProperties.getProperties() .containsKey(RANGER_HBASE_PLUGIN_ENABLED_PROPERTY)) { newRangerEnvProps.put(RANGER_HBASE_PLUGIN_ENABLED_PROPERTY, rangerHbasePluginProperties.getProperties().get(RANGER_HBASE_PLUGIN_ENABLED_PROPERTY)); } Config rangerStormPluginProperties = cluster.getDesiredConfigByType("ranger-storm-plugin-properties"); if (rangerStormPluginProperties != null && rangerStormPluginProperties.getProperties() .containsKey(RANGER_STORM_PLUGIN_ENABLED_PROPERTY)) { newRangerEnvProps.put(RANGER_STORM_PLUGIN_ENABLED_PROPERTY, rangerStormPluginProperties.getProperties().get(RANGER_STORM_PLUGIN_ENABLED_PROPERTY)); } Config rangerKnoxPluginProperties = cluster.getDesiredConfigByType("ranger-knox-plugin-properties"); if (rangerKnoxPluginProperties != null && rangerKnoxPluginProperties.getProperties() .containsKey(RANGER_KNOX_PLUGIN_ENABLED_PROPERTY)) { newRangerEnvProps.put(RANGER_KNOX_PLUGIN_ENABLED_PROPERTY, rangerKnoxPluginProperties.getProperties().get(RANGER_KNOX_PLUGIN_ENABLED_PROPERTY)); } Config rangerYarnPluginProperties = cluster.getDesiredConfigByType("ranger-yarn-plugin-properties"); if (rangerYarnPluginProperties != null && rangerYarnPluginProperties.getProperties() .containsKey(RANGER_YARN_PLUGIN_ENABLED_PROPERTY)) { newRangerEnvProps.put(RANGER_YARN_PLUGIN_ENABLED_PROPERTY, rangerYarnPluginProperties.getProperties().get(RANGER_YARN_PLUGIN_ENABLED_PROPERTY)); } Config rangerKafkaPluginProperties = cluster.getDesiredConfigByType("ranger-kafka-plugin-properties"); if (rangerKafkaPluginProperties != null && rangerKafkaPluginProperties.getProperties() .containsKey(RANGER_KAFKA_PLUGIN_ENABLED_PROPERTY)) { newRangerEnvProps.put(RANGER_KAFKA_PLUGIN_ENABLED_PROPERTY, rangerKafkaPluginProperties.getProperties().get(RANGER_KAFKA_PLUGIN_ENABLED_PROPERTY)); } if (!newRangerEnvProps.isEmpty()) { updateConfigurationPropertiesForCluster(cluster, RANGER_ENV_CONFIG, newRangerEnvProps, true, true); } } } protected void updateRangerUgsyncSiteConfig() throws AmbariException { AmbariManagementController ambariManagementController = injector .getInstance(AmbariManagementController.class); for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) { Config rangerUgsyncSiteProperties = cluster.getDesiredConfigByType(RANGER_UGSYNC_SITE_CONFIG); if (rangerUgsyncSiteProperties != null && rangerUgsyncSiteProperties.getProperties() .containsKey(RANGER_USERSYNC_SOURCE_IMPL_CLASS_PROPERTY)) { String sourceClassValue = rangerUgsyncSiteProperties.getProperties() .get(RANGER_USERSYNC_SOURCE_IMPL_CLASS_PROPERTY); if (sourceClassValue != null) { if ("ldap".equals(sourceClassValue)) { Map<String, String> updates = Collections.singletonMap( RANGER_USERSYNC_SOURCE_IMPL_CLASS_PROPERTY, "org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder"); updateConfigurationPropertiesForCluster(cluster, RANGER_UGSYNC_SITE_CONFIG, updates, true, false); } else if ("unix".equals(sourceClassValue)) { Map<String, String> updates = Collections.singletonMap( RANGER_USERSYNC_SOURCE_IMPL_CLASS_PROPERTY, "org.apache.ranger.unixusersync.process.UnixUserGroupBuilder"); updateConfigurationPropertiesForCluster(cluster, RANGER_UGSYNC_SITE_CONFIG, updates, true, false); } else if ("file".equals(sourceClassValue)) { Map<String, String> updates = Collections.singletonMap( RANGER_USERSYNC_SOURCE_IMPL_CLASS_PROPERTY, "org.apache.ranger.unixusersync.process.FileSourceUserGroupBuilder"); updateConfigurationPropertiesForCluster(cluster, RANGER_UGSYNC_SITE_CONFIG, updates, true, false); } } } } } protected String updateHiveEnvContent(String hiveEnvContent) { if (hiveEnvContent == null) { return null; } // There are two cases here // We do not have "export HADOOP_CLIENT_OPTS" and we need to add it // We have "export HADOOP_CLIENT_OPTS" with wrong order String exportHadoopClientOpts = "(?s).*export\\s*HADOOP_CLIENT_OPTS.*"; if (hiveEnvContent.matches(exportHadoopClientOpts)) { String oldHeapSizeRegex = "export\\s*HADOOP_CLIENT_OPTS=\"-Xmx\\$\\{HADOOP_HEAPSIZE\\}m\\s*\\$HADOOP_CLIENT_OPTS\""; String newHeapSizeRegex = "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m\""; return hiveEnvContent.replaceAll(oldHeapSizeRegex, Matcher.quoteReplacement(newHeapSizeRegex)); } else { String oldHeapSizeRegex = "export\\s*HADOOP_HEAPSIZE\\s*=\\s*\"\\{\\{hive_heapsize\\}\\}\"\\.*\\n\\s*fi\\s*\\n"; String newHeapSizeRegex = "export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client\n" + "fi\n" + "\n" + "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m\"\n"; return hiveEnvContent.replaceAll(oldHeapSizeRegex, Matcher.quoteReplacement(newHeapSizeRegex)); } } protected String updateHiveEnvContentHDP23(String hiveEnvContent) { if (hiveEnvContent == null) { return null; } String oldHeapSizeRegex = "# The heap size of the jvm stared by hive shell script can be controlled via:\\s*\\n"; String newHeapSizeRegex = "# The heap size of the jvm stared by hive shell script can be controlled via:\n" + "\n" + "if [ \"$SERVICE\" = \"metastore\" ]; then\n" + " export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore\n" + "else\n" + " export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client\n" + "fi\n" + "\n" + "export HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m\"\n" + "\n"; return hiveEnvContent.replaceFirst(oldHeapSizeRegex, Matcher.quoteReplacement(newHeapSizeRegex)); } protected void updateAccumuloConfigs() throws AmbariException { AmbariManagementController ambariManagementController = injector .getInstance(AmbariManagementController.class); for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) { // If security type is set to Kerberos, update Kerberos-related configs if (cluster.getSecurityType() == SecurityType.KERBEROS) { Config clientProps = cluster.getDesiredConfigByType("client"); if (clientProps != null) { Map<String, String> properties = clientProps.getProperties(); if (properties == null) { properties = new HashMap<String, String>(); } // <2.2.0 did not account for a custom service principal. // Need to ensure that the client knows the server's principal (the primary) to properly authenticate. properties.put("kerberos.server.primary", "{{bare_accumulo_principal}}"); updateConfigurationPropertiesForCluster(cluster, "client", properties, true, false); } } // else -- no special client-configuration is necessary. } } }