Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.nifi.properties; import static java.util.Arrays.asList; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.util.NiFiProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Decorator class for intermediate phase when {@link NiFiPropertiesLoader} loads the * raw properties file and performs unprotection activities before returning a clean * implementation of {@link NiFiProperties}, likely {@link StandardNiFiProperties}. * This encapsulates the sensitive property access logic from external consumers * of {@code NiFiProperties}. */ class ProtectedNiFiProperties extends StandardNiFiProperties { private static final Logger logger = LoggerFactory.getLogger(ProtectedNiFiProperties.class); private NiFiProperties niFiProperties; private Map<String, SensitivePropertyProvider> localProviderCache = new HashMap<>(); // Additional "sensitive" property key public static final String ADDITIONAL_SENSITIVE_PROPERTIES_KEY = "nifi.sensitive.props.additional.keys"; // Default list of "sensitive" property keys public static final List<String> DEFAULT_SENSITIVE_PROPERTIES = new ArrayList<>( asList(SECURITY_KEY_PASSWD, SECURITY_KEYSTORE_PASSWD, SECURITY_TRUSTSTORE_PASSWD, SENSITIVE_PROPS_KEY)); public ProtectedNiFiProperties() { this(new StandardNiFiProperties()); } /** * Creates an instance containing the provided {@link NiFiProperties}. * * @param props the NiFiProperties to contain */ public ProtectedNiFiProperties(NiFiProperties props) { this.niFiProperties = props; logger.debug("Loaded {} properties (including {} protection schemes) into ProtectedNiFiProperties", getPropertyKeysIncludingProtectionSchemes().size(), getProtectedPropertyKeys().size()); } /** * Creates an instance containing the provided raw {@link Properties}. * * @param rawProps the Properties to contain */ public ProtectedNiFiProperties(Properties rawProps) { this(new StandardNiFiProperties(rawProps)); } /** * Retrieves the property value for the given property key. * * @param key the key of property value to lookup * @return value of property at given key or null if not found */ @Override public String getProperty(String key) { return getInternalNiFiProperties().getProperty(key); } /** * Retrieves all known property keys. * * @return all known property keys */ @Override public Set<String> getPropertyKeys() { Set<String> filteredKeys = getPropertyKeysIncludingProtectionSchemes(); filteredKeys.removeIf(p -> p.endsWith(".protected")); return filteredKeys; } /** * Returns the internal representation of the {@link NiFiProperties} -- protected * or not as determined by the current state. No guarantee is made to the * protection state of these properties. If the internal reference is null, a new * {@link StandardNiFiProperties} instance is created. * * @return the internal properties */ NiFiProperties getInternalNiFiProperties() { if (this.niFiProperties == null) { this.niFiProperties = new StandardNiFiProperties(); } return this.niFiProperties; } /** * Returns the number of properties, excluding protection scheme properties. * <p> * Example: * <p> * key: E(value, key) * key.protected: aes/gcm/256 * key2: value2 * <p> * would return size 2 * * @return the count of real properties */ @Override public int size() { return getPropertyKeys().size(); } /** * Returns the complete set of property keys, including any protection keys (i.e. 'x.y.z.protected'). * * @return the set of property keys */ Set<String> getPropertyKeysIncludingProtectionSchemes() { return getInternalNiFiProperties().getPropertyKeys(); } /** * Splits a single string containing multiple property keys into a List. Delimited by ',' or ';' and ignores leading and trailing whitespace around delimiter. * * @param multipleProperties a single String containing multiple properties, i.e. "nifi.property.1; nifi.property.2, nifi.property.3" * @return a List containing the split and trimmed properties */ private static List<String> splitMultipleProperties(String multipleProperties) { if (multipleProperties == null || multipleProperties.trim().isEmpty()) { return new ArrayList<>(0); } else { List<String> properties = new ArrayList<>(asList(multipleProperties.split("\\s*[,;]\\s*"))); for (int i = 0; i < properties.size(); i++) { properties.set(i, properties.get(i).trim()); } return properties; } } /** * Returns a list of the keys identifying "sensitive" properties. There is a default list, * and additional keys can be provided in the {@code nifi.sensitive.props.additional.keys} property in {@code nifi.properties}. * * @return the list of sensitive property keys */ public List<String> getSensitivePropertyKeys() { String additionalPropertiesString = getProperty(ADDITIONAL_SENSITIVE_PROPERTIES_KEY); if (additionalPropertiesString == null || additionalPropertiesString.trim().isEmpty()) { return DEFAULT_SENSITIVE_PROPERTIES; } else { List<String> additionalProperties = splitMultipleProperties(additionalPropertiesString); /* Remove this key if it was accidentally provided as a sensitive key * because we cannot protect it and read from it */ if (additionalProperties.contains(ADDITIONAL_SENSITIVE_PROPERTIES_KEY)) { logger.warn("The key '{}' contains itself. This is poor practice and should be removed", ADDITIONAL_SENSITIVE_PROPERTIES_KEY); additionalProperties.remove(ADDITIONAL_SENSITIVE_PROPERTIES_KEY); } additionalProperties.addAll(DEFAULT_SENSITIVE_PROPERTIES); return additionalProperties; } } /** * Returns true if any sensitive keys are protected. * * @return true if any key is protected; false otherwise */ public boolean hasProtectedKeys() { List<String> sensitiveKeys = getSensitivePropertyKeys(); for (String k : sensitiveKeys) { if (isPropertyProtected(k)) { return true; } } return false; } /** * Returns a Map of the keys identifying "sensitive" properties that are currently protected and the "protection" key for each. This may or may not include all properties marked as sensitive. * * @return the Map of protected property keys and the protection identifier for each */ public Map<String, String> getProtectedPropertyKeys() { List<String> sensitiveKeys = getSensitivePropertyKeys(); // This is the Java 8 way, but can likely be optimized (and not sure of correctness) // Map<String, String> protectedProperties = sensitiveKeys.stream().filter(key -> // getProperty(getProtectionKey(key)) != null).collect(Collectors.toMap(Function.identity(), key -> // getProperty(getProtectionKey(key)))); // Groovy // Map<String, String> groovyProtectedProperties = sensitiveKeys.collectEntries { key -> // [(key): getProperty(getProtectionKey(key))] }.findAll { k, v -> v } // Traditional way Map<String, String> traditionalProtectedProperties = new HashMap<>(); for (String key : sensitiveKeys) { String protection = getProperty(getProtectionKey(key)); if (!StringUtils.isBlank(protection)) { traditionalProtectedProperties.put(key, protection); } } return traditionalProtectedProperties; } /** * Returns the unique set of all protection schemes currently in use for this instance. * * @return the set of protection schemes */ public Set<String> getProtectionSchemes() { return new HashSet<>(getProtectedPropertyKeys().values()); } /** * Returns a percentage of the total number of properties marked as sensitive that are currently protected. * * @return the percent of sensitive properties marked as protected */ public int getPercentOfSensitivePropertiesProtected() { return (int) Math .round(getProtectedPropertyKeys().size() / ((double) getSensitivePropertyKeys().size()) * 100); } /** * Returns true if the property identified by this key is considered sensitive in this instance of {@code NiFiProperties}. * Some properties are sensitive by default, while others can be specified by * {@link ProtectedNiFiProperties#ADDITIONAL_SENSITIVE_PROPERTIES_KEY}. * * @param key the key * @return true if it is sensitive * @see ProtectedNiFiProperties#getSensitivePropertyKeys() */ public boolean isPropertySensitive(String key) { // If the explicit check for ADDITIONAL_SENSITIVE_PROPERTIES_KEY is not here, this will loop infinitely return key != null && !key.equals(ADDITIONAL_SENSITIVE_PROPERTIES_KEY) && getSensitivePropertyKeys().contains(key.trim()); } /** * Returns true if the property identified by this key is considered protected in this instance of {@code NiFiProperties}. * The property value is protected if the key is sensitive and the sibling key of key.protected is present. * * @param key the key * @return true if it is currently marked as protected * @see ProtectedNiFiProperties#getSensitivePropertyKeys() */ public boolean isPropertyProtected(String key) { return key != null && isPropertySensitive(key) && !StringUtils.isBlank(getProperty(getProtectionKey(key))); } /** * Returns the sibling property key which specifies the protection scheme for this key. * <p> * Example: * <p> * nifi.sensitive.key=ABCXYZ * nifi.sensitive.key.protected=aes/gcm/256 * <p> * nifi.sensitive.key -> nifi.sensitive.key.protected * * @param key the key identifying the sensitive property * @return the key identifying the protection scheme for the sensitive property */ public static String getProtectionKey(String key) { if (key == null || key.isEmpty()) { throw new IllegalArgumentException("Cannot find protection key for null key"); } return key + ".protected"; } /** * Returns the unprotected {@link NiFiProperties} instance. If none of the properties * loaded are marked as protected, it will simply pass through the internal instance. * If any are protected, it will drop the protection scheme keys and translate each * protected value (encrypted, HSM-retrieved, etc.) into the raw value and store it * under the original key. * <p> * If any property fails to unprotect, it will save that key and continue. After * attempting all properties, it will throw an exception containing all failed * properties. This is necessary because the order is not enforced, so all failed * properties should be gathered together. * * @return the NiFiProperties instance with all raw values * @throws SensitivePropertyProtectionException if there is a problem unprotecting one or more keys */ public NiFiProperties getUnprotectedProperties() throws SensitivePropertyProtectionException { if (hasProtectedKeys()) { logger.info("There are {} protected properties of {} sensitive properties ({}%)", getProtectedPropertyKeys().size(), getSensitivePropertyKeys().size(), getPercentOfSensitivePropertiesProtected()); Properties rawProperties = new Properties(); Set<String> failedKeys = new HashSet<>(); for (String key : getPropertyKeys()) { /* Three kinds of keys * 1. protection schemes -- skip * 2. protected keys -- unprotect and copy * 3. normal keys -- copy over */ if (key.endsWith(".protected")) { // Do nothing } else if (isPropertyProtected(key)) { try { rawProperties.setProperty(key, unprotectValue(key, getProperty(key))); } catch (SensitivePropertyProtectionException e) { logger.warn("Failed to unprotect '{}'", key, e); failedKeys.add(key); } } else { rawProperties.setProperty(key, getProperty(key)); } } if (!failedKeys.isEmpty()) { if (failedKeys.size() > 1) { logger.warn("Combining {} failed keys [{}] into single exception", failedKeys.size(), StringUtils.join(failedKeys, ", ")); throw new MultipleSensitivePropertyProtectionException("Failed to unprotect keys", failedKeys); } else { throw new SensitivePropertyProtectionException( "Failed to unprotect key " + failedKeys.iterator().next()); } } NiFiProperties unprotected = new StandardNiFiProperties(rawProperties); return unprotected; } else { logger.debug("No protected properties"); return getInternalNiFiProperties(); } } /** * Registers a new {@link SensitivePropertyProvider}. This method will throw a {@link UnsupportedOperationException} if a provider is already registered for the protection scheme. * * @param sensitivePropertyProvider the provider */ void addSensitivePropertyProvider(SensitivePropertyProvider sensitivePropertyProvider) { if (sensitivePropertyProvider == null) { throw new IllegalArgumentException("Cannot add null SensitivePropertyProvider"); } if (getSensitivePropertyProviders().containsKey(sensitivePropertyProvider.getIdentifierKey())) { throw new UnsupportedOperationException( "Cannot overwrite existing sensitive property provider registered for " + sensitivePropertyProvider.getIdentifierKey()); } getSensitivePropertyProviders().put(sensitivePropertyProvider.getIdentifierKey(), sensitivePropertyProvider); } private String getDefaultProtectionScheme() { if (!getSensitivePropertyProviders().isEmpty()) { List<String> schemes = new ArrayList<>(getSensitivePropertyProviders().keySet()); Collections.sort(schemes); return schemes.get(0); } else { throw new IllegalStateException("No registered protection schemes"); } } /** * Returns a new instance of {@link NiFiProperties} with all populated sensitive values protected by the default protection scheme. Plain non-sensitive values are copied directly. * * @return the protected properties in a {@link StandardNiFiProperties} object * @throws IllegalStateException if no protection schemes are registered */ NiFiProperties protectPlainProperties() { try { return protectPlainProperties(getDefaultProtectionScheme()); } catch (IllegalStateException e) { final String msg = "Cannot protect properties with default scheme if no protection schemes are registered"; logger.warn(msg); throw new IllegalStateException(msg, e); } } /** * Returns a new instance of {@link NiFiProperties} with all populated sensitive values protected by the provided protection scheme. Plain non-sensitive values are copied directly. * * @param protectionScheme the identifier key of the {@link SensitivePropertyProvider} to use * @return the protected properties in a {@link StandardNiFiProperties} object */ NiFiProperties protectPlainProperties(String protectionScheme) { SensitivePropertyProvider spp = getSensitivePropertyProvider(protectionScheme); // Make a new holder (settable) Properties protectedProperties = new Properties(); // Copy over the plain keys Set<String> plainKeys = getPropertyKeys(); plainKeys.removeAll(getSensitivePropertyKeys()); for (String key : plainKeys) { protectedProperties.setProperty(key, getInternalNiFiProperties().getProperty(key)); } // Add the protected keys and the protection schemes for (String key : getSensitivePropertyKeys()) { final String plainValue = getInternalNiFiProperties().getProperty(key); if (plainValue == null || plainValue.trim().isEmpty()) { protectedProperties.setProperty(key, plainValue); } else { final String protectedValue = spp.protect(plainValue); protectedProperties.setProperty(key, protectedValue); protectedProperties.setProperty(getProtectionKey(key), protectionScheme); } } return new StandardNiFiProperties(protectedProperties); } /** * Returns the number of properties that are marked as protected in the provided {@link NiFiProperties} instance without requiring external creation of a {@link ProtectedNiFiProperties} instance. * * @param plainProperties the instance to count protected properties * @return the number of protected properties */ public static int countProtectedProperties(NiFiProperties plainProperties) { return new ProtectedNiFiProperties(plainProperties).getProtectedPropertyKeys().size(); } /** * Returns the number of properties that are marked as sensitive in the provided {@link NiFiProperties} instance without requiring external creation of a {@link ProtectedNiFiProperties} instance. * * @param plainProperties the instance to count sensitive properties * @return the number of sensitive properties */ public static int countSensitiveProperties(NiFiProperties plainProperties) { return new ProtectedNiFiProperties(plainProperties).getSensitivePropertyKeys().size(); } @Override public String toString() { final Set<String> providers = getSensitivePropertyProviders().keySet(); return new StringBuilder("ProtectedNiFiProperties instance with ").append(size()).append(" properties (") .append(getProtectedPropertyKeys().size()).append(" protected) and ").append(providers.size()) .append(" sensitive property providers: ").append(StringUtils.join(providers, ", ")).toString(); } /** * Returns the local provider cache (null-safe) as a Map of protection schemes -> implementations. * * @return the map */ private Map<String, SensitivePropertyProvider> getSensitivePropertyProviders() { if (localProviderCache == null) { localProviderCache = new HashMap<>(); } return localProviderCache; } private SensitivePropertyProvider getSensitivePropertyProvider(String protectionScheme) { if (isProviderAvailable(protectionScheme)) { return getSensitivePropertyProviders().get(protectionScheme); } else { throw new SensitivePropertyProtectionException("No provider available for " + protectionScheme); } } private boolean isProviderAvailable(String protectionScheme) { return getSensitivePropertyProviders().containsKey(protectionScheme); } /** * If the value is protected, unprotects it and returns it. If not, returns the original value. * * @param key the retrieved property key * @param retrievedValue the retrieved property value * @return the unprotected value */ private String unprotectValue(String key, String retrievedValue) { // Checks if the key is sensitive and marked as protected if (isPropertyProtected(key)) { final String protectionScheme = getProperty(getProtectionKey(key)); // No provider registered for this scheme, so just return the value if (!isProviderAvailable(protectionScheme)) { logger.warn("No provider available for {} so passing the protected {} value back", protectionScheme, key); return retrievedValue; } try { SensitivePropertyProvider sensitivePropertyProvider = getSensitivePropertyProvider( protectionScheme); return sensitivePropertyProvider.unprotect(retrievedValue); } catch (SensitivePropertyProtectionException e) { throw new SensitivePropertyProtectionException("Error unprotecting value for " + key, e.getCause()); } } return retrievedValue; } }