Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hdfs; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYPASSWORD_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY; import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.IOException; import java.io.PrintStream; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.net.UnknownHostException; import java.security.SecureRandom; import java.util.Arrays; import java.util.Collection; import java.util.Comparator; import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.cli.PosixParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.ToolRunner; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.protobuf.BlockingService; @InterfaceAudience.Private public class DFSUtil { public static final Logger LOG = LoggerFactory.getLogger(DFSUtil.class.getName()); private DFSUtil() { /* Hidden constructor */ } private static final ThreadLocal<SecureRandom> SECURE_RANDOM = new ThreadLocal<SecureRandom>() { @Override protected SecureRandom initialValue() { return new SecureRandom(); } }; /** @return a pseudo secure random number generator. */ public static SecureRandom getSecureRandom() { return SECURE_RANDOM.get(); } /** Shuffle the elements in the given array. */ public static <T> T[] shuffle(final T[] array) { if (array != null && array.length > 0) { for (int n = array.length; n > 1;) { final int randomIndex = ThreadLocalRandom.current().nextInt(n); n--; if (n != randomIndex) { final T tmp = array[randomIndex]; array[randomIndex] = array[n]; array[n] = tmp; } } } return array; } /** * Comparator for sorting DataNodeInfo[] based on * decommissioned and entering_maintenance states. */ public static class ServiceComparator implements Comparator<DatanodeInfo> { @Override public int compare(DatanodeInfo a, DatanodeInfo b) { // Decommissioned nodes will still be moved to the end of the list if (a.isDecommissioned()) { return b.isDecommissioned() ? 0 : 1; } else if (b.isDecommissioned()) { return -1; } // ENTERING_MAINTENANCE nodes should be after live nodes. if (a.isEnteringMaintenance()) { return b.isEnteringMaintenance() ? 0 : 1; } else if (b.isEnteringMaintenance()) { return -1; } else { return 0; } } } /** * Comparator for sorting DataNodeInfo[] based on * stale, decommissioned and entering_maintenance states. * Order: live -> stale -> entering_maintenance -> decommissioned */ @InterfaceAudience.Private public static class ServiceAndStaleComparator extends ServiceComparator { private final long staleInterval; /** * Constructor of ServiceAndStaleComparator * * @param interval * The time interval for marking datanodes as stale is passed from * outside, since the interval may be changed dynamically */ public ServiceAndStaleComparator(long interval) { this.staleInterval = interval; } @Override public int compare(DatanodeInfo a, DatanodeInfo b) { int ret = super.compare(a, b); if (ret != 0) { return ret; } // Stale nodes will be moved behind the normal nodes boolean aStale = a.isStale(staleInterval); boolean bStale = b.isStale(staleInterval); return aStale == bStale ? 0 : (aStale ? 1 : -1); } } /** * Address matcher for matching an address to local address */ static final AddressMatcher LOCAL_ADDRESS_MATCHER = new AddressMatcher() { @Override public boolean match(InetSocketAddress s) { return NetUtils.isLocalAddress(s.getAddress()); }; }; /** * Whether the pathname is valid. Currently prohibits relative paths, * names which contain a ":" or "//", or other non-canonical paths. */ public static boolean isValidName(String src) { return DFSUtilClient.isValidName(src); } /** * Checks if a string is a valid path component. For instance, components * cannot contain a ":" or "/", and cannot be equal to a reserved component * like ".snapshot". * <p> * The primary use of this method is for validating paths when loading the * FSImage. During normal NN operation, paths are sometimes allowed to * contain reserved components. * * @return If component is valid */ public static boolean isValidNameForComponent(String component) { if (component.equals(".") || component.equals("..") || component.indexOf(":") >= 0 || component.indexOf("/") >= 0) { return false; } return !isReservedPathComponent(component); } /** * Returns if the component is reserved. * * <p> * Note that some components are only reserved under certain directories, e.g. * "/.reserved" is reserved, while "/hadoop/.reserved" is not. * @return true, if the component is reserved */ public static boolean isReservedPathComponent(String component) { for (String reserved : HdfsServerConstants.RESERVED_PATH_COMPONENTS) { if (component.equals(reserved)) { return true; } } return false; } /** * Converts a byte array to a string using UTF8 encoding. */ public static String bytes2String(byte[] bytes) { return bytes2String(bytes, 0, bytes.length); } /** * Decode a specific range of bytes of the given byte array to a string * using UTF8. * * @param bytes The bytes to be decoded into characters * @param offset The index of the first byte to decode * @param length The number of bytes to decode * @return The decoded string */ public static String bytes2String(byte[] bytes, int offset, int length) { return DFSUtilClient.bytes2String(bytes, 0, bytes.length); } /** * Converts a string to a byte array using UTF8 encoding. */ public static byte[] string2Bytes(String str) { return DFSUtilClient.string2Bytes(str); } /** * Given a list of path components returns a path as a UTF8 String */ public static String byteArray2PathString(final byte[][] components, final int offset, final int length) { // specifically not using StringBuilder to more efficiently build // string w/o excessive byte[] copies and charset conversions. final int range = offset + length; Preconditions.checkPositionIndexes(offset, range, components.length); if (length == 0) { return ""; } // absolute paths start with either null or empty byte[] byte[] firstComponent = components[offset]; boolean isAbsolute = (offset == 0 && (firstComponent == null || firstComponent.length == 0)); if (offset == 0 && length == 1) { return isAbsolute ? Path.SEPARATOR : bytes2String(firstComponent); } // compute length of full byte[], seed with 1st component and delimiters int pos = isAbsolute ? 0 : firstComponent.length; int size = pos + length - 1; for (int i = offset + 1; i < range; i++) { size += components[i].length; } final byte[] result = new byte[size]; if (!isAbsolute) { System.arraycopy(firstComponent, 0, result, 0, firstComponent.length); } // append remaining components as "/component". for (int i = offset + 1; i < range; i++) { result[pos++] = (byte) Path.SEPARATOR_CHAR; int len = components[i].length; System.arraycopy(components[i], 0, result, pos, len); pos += len; } return bytes2String(result); } public static String byteArray2PathString(byte[][] pathComponents) { return byteArray2PathString(pathComponents, 0, pathComponents.length); } /** * Converts a list of path components into a path using Path.SEPARATOR. * * @param components Path components * @return Combined path as a UTF-8 string */ public static String strings2PathString(String[] components) { if (components.length == 0) { return ""; } if (components.length == 1) { if (components[0] == null || components[0].isEmpty()) { return Path.SEPARATOR; } } return Joiner.on(Path.SEPARATOR).join(components); } /** Convert an object representing a path to a string. */ public static String path2String(final Object path) { return path == null ? null : path instanceof String ? (String) path : path instanceof byte[][] ? byteArray2PathString((byte[][]) path) : path.toString(); } /** * Convert a UTF8 string to an array of byte arrays. */ public static byte[][] getPathComponents(String path) { // avoid intermediate split to String[] final byte[] bytes = string2Bytes(path); return DFSUtilClient.bytes2byteArray(bytes, bytes.length, (byte) Path.SEPARATOR_CHAR); } /** * Splits the array of bytes into array of arrays of bytes * on byte separator * @param bytes the array of bytes to split * @param separator the delimiting byte */ public static byte[][] bytes2byteArray(byte[] bytes, byte separator) { return bytes2byteArray(bytes, bytes.length, separator); } /** * Splits first len bytes in bytes to array of arrays of bytes * on byte separator * @param bytes the byte array to split * @param len the number of bytes to split * @param separator the delimiting byte */ public static byte[][] bytes2byteArray(byte[] bytes, int len, byte separator) { return DFSUtilClient.bytes2byteArray(bytes, len, separator); } /** * Return configuration key of format key.suffix1.suffix2...suffixN */ public static String addKeySuffixes(String key, String... suffixes) { String keySuffix = DFSUtilClient.concatSuffixes(suffixes); return DFSUtilClient.addSuffix(key, keySuffix); } /** * Get all of the RPC addresses of the individual NNs in a given nameservice. * * @param conf Configuration * @param nsId the nameservice whose NNs addresses we want. * @param defaultValue default address to return in case key is not found. * @return A map from nnId -> RPC address of each NN in the nameservice. */ public static Map<String, InetSocketAddress> getRpcAddressesForNameserviceId(Configuration conf, String nsId, String defaultValue) { return DFSUtilClient.getAddressesForNameserviceId(conf, nsId, defaultValue, DFS_NAMENODE_RPC_ADDRESS_KEY); } /** * @return a collection of all configured NN Kerberos principals. */ public static Set<String> getAllNnPrincipals(Configuration conf) throws IOException { Set<String> principals = new HashSet<String>(); for (String nsId : DFSUtilClient.getNameServiceIds(conf)) { if (HAUtil.isHAEnabled(conf, nsId)) { for (String nnId : DFSUtilClient.getNameNodeIds(conf, nsId)) { Configuration confForNn = new Configuration(conf); NameNode.initializeGenericKeys(confForNn, nsId, nnId); String principal = SecurityUtil.getServerPrincipal( confForNn.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY), DFSUtilClient.getNNAddress(confForNn).getHostName()); principals.add(principal); } } else { Configuration confForNn = new Configuration(conf); NameNode.initializeGenericKeys(confForNn, nsId, null); String principal = SecurityUtil.getServerPrincipal( confForNn.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY), DFSUtilClient.getNNAddress(confForNn).getHostName()); principals.add(principal); } } return principals; } /** * Returns list of Journalnode addresses from the configuration. * * @param conf configuration * @return list of journalnode host names * @throws URISyntaxException * @throws IOException */ public static Set<String> getJournalNodeAddresses(Configuration conf) throws URISyntaxException, IOException { Set<String> journalNodeList = new HashSet<>(); String journalsUri = ""; try { journalsUri = conf.get(DFS_NAMENODE_SHARED_EDITS_DIR_KEY); if (journalsUri == null) { Collection<String> nameserviceIds = DFSUtilClient.getNameServiceIds(conf); for (String nsId : nameserviceIds) { journalsUri = DFSUtilClient.getConfValue(null, nsId, conf, DFS_NAMENODE_SHARED_EDITS_DIR_KEY); if (journalsUri == null) { Collection<String> nnIds = DFSUtilClient.getNameNodeIds(conf, nsId); for (String nnId : nnIds) { String suffix = DFSUtilClient.concatSuffixes(nsId, nnId); journalsUri = DFSUtilClient.getConfValue(null, suffix, conf, DFS_NAMENODE_SHARED_EDITS_DIR_KEY); if (journalsUri == null || !journalsUri.startsWith("qjournal://")) { return journalNodeList; } else { LOG.warn(DFS_NAMENODE_SHARED_EDITS_DIR_KEY + " is to be " + "configured as nameservice" + " specific key(append it with nameserviceId), no need" + " to append it with namenodeId"); URI uri = new URI(journalsUri); List<InetSocketAddress> socketAddresses = Util.getAddressesList(uri); for (InetSocketAddress is : socketAddresses) { journalNodeList.add(is.getHostName()); } } } } else if (!journalsUri.startsWith("qjournal://")) { return journalNodeList; } else { URI uri = new URI(journalsUri); List<InetSocketAddress> socketAddresses = Util.getAddressesList(uri); for (InetSocketAddress is : socketAddresses) { journalNodeList.add(is.getHostName()); } } } } else { if (!journalsUri.startsWith("qjournal://")) { return journalNodeList; } else { URI uri = new URI(journalsUri); List<InetSocketAddress> socketAddresses = Util.getAddressesList(uri); for (InetSocketAddress is : socketAddresses) { journalNodeList.add(is.getHostName()); } } } } catch (UnknownHostException e) { LOG.error("The conf property " + DFS_NAMENODE_SHARED_EDITS_DIR_KEY + " is not properly set with correct journal node hostnames"); throw new UnknownHostException(journalsUri); } catch (URISyntaxException e) { LOG.error("The conf property " + DFS_NAMENODE_SHARED_EDITS_DIR_KEY + "is not set properly with correct journal node uri"); throw new URISyntaxException(journalsUri, "The conf property " + DFS_NAMENODE_SHARED_EDITS_DIR_KEY + "is not" + " properly set with correct journal node uri"); } return journalNodeList; } /** * Returns list of InetSocketAddress corresponding to backup node rpc * addresses from the configuration. * * @param conf configuration * @return list of InetSocketAddresses * @throws IOException on error */ public static Map<String, Map<String, InetSocketAddress>> getBackupNodeAddresses(Configuration conf) throws IOException { Map<String, Map<String, InetSocketAddress>> addressList = DFSUtilClient.getAddresses(conf, null, DFS_NAMENODE_BACKUP_ADDRESS_KEY); if (addressList.isEmpty()) { throw new IOException("Incorrect configuration: backup node address " + DFS_NAMENODE_BACKUP_ADDRESS_KEY + " is not configured."); } return addressList; } /** * Returns list of InetSocketAddresses of corresponding to secondary namenode * http addresses from the configuration. * * @param conf configuration * @return list of InetSocketAddresses * @throws IOException on error */ public static Map<String, Map<String, InetSocketAddress>> getSecondaryNameNodeAddresses(Configuration conf) throws IOException { Map<String, Map<String, InetSocketAddress>> addressList = DFSUtilClient.getAddresses(conf, null, DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY); if (addressList.isEmpty()) { throw new IOException("Incorrect configuration: secondary namenode address " + DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY + " is not configured."); } return addressList; } /** * Returns list of InetSocketAddresses corresponding to namenodes from the * configuration. * * Returns namenode address specifically configured for datanodes (using * service ports), if found. If not, regular RPC address configured for other * clients is returned. * * @param conf configuration * @return list of InetSocketAddress * @throws IOException on error */ public static Map<String, Map<String, InetSocketAddress>> getNNServiceRpcAddresses(Configuration conf) throws IOException { // Use default address as fall back String defaultAddress; try { defaultAddress = NetUtils.getHostPortString(DFSUtilClient.getNNAddress(conf)); } catch (IllegalArgumentException e) { defaultAddress = null; } Map<String, Map<String, InetSocketAddress>> addressList = DFSUtilClient.getAddresses(conf, defaultAddress, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY); if (addressList.isEmpty()) { throw new IOException( "Incorrect configuration: namenode address " + DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY + " or " + DFS_NAMENODE_RPC_ADDRESS_KEY + " is not configured."); } return addressList; } /** * Returns list of InetSocketAddresses corresponding to the namenode * that manages this cluster. Note this is to be used by datanodes to get * the list of namenode addresses to talk to. * * Returns namenode address specifically configured for datanodes (using * service ports), if found. If not, regular RPC address configured for other * clients is returned. * * @param conf configuration * @return list of InetSocketAddress * @throws IOException on error */ public static Map<String, Map<String, InetSocketAddress>> getNNServiceRpcAddressesForCluster(Configuration conf) throws IOException { // Use default address as fall back String defaultAddress; try { defaultAddress = NetUtils.getHostPortString(DFSUtilClient.getNNAddress(conf)); } catch (IllegalArgumentException e) { defaultAddress = null; } Collection<String> parentNameServices = conf .getTrimmedStringCollection(DFSConfigKeys.DFS_INTERNAL_NAMESERVICES_KEY); if (parentNameServices.isEmpty()) { parentNameServices = conf.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES); } else { // Ensure that the internal service is ineed in the list of all available // nameservices. Set<String> availableNameServices = Sets .newHashSet(conf.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES)); for (String nsId : parentNameServices) { if (!availableNameServices.contains(nsId)) { throw new IOException("Unknown nameservice: " + nsId); } } } Map<String, Map<String, InetSocketAddress>> addressList = DFSUtilClient.getAddressesForNsIds(conf, parentNameServices, defaultAddress, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY); if (addressList.isEmpty()) { throw new IOException( "Incorrect configuration: namenode address " + DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY + " or " + DFS_NAMENODE_RPC_ADDRESS_KEY + " is not configured."); } return addressList; } /** * Returns list of InetSocketAddresses corresponding to lifeline RPC servers * at namenodes from the configuration. * * @param conf configuration * @return list of InetSocketAddress * @throws IOException on error */ public static Map<String, Map<String, InetSocketAddress>> getNNLifelineRpcAddressesForCluster( Configuration conf) throws IOException { Collection<String> parentNameServices = conf .getTrimmedStringCollection(DFSConfigKeys.DFS_INTERNAL_NAMESERVICES_KEY); if (parentNameServices.isEmpty()) { parentNameServices = conf.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES); } else { // Ensure that the internal service is indeed in the list of all available // nameservices. Set<String> availableNameServices = Sets .newHashSet(conf.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES)); for (String nsId : parentNameServices) { if (!availableNameServices.contains(nsId)) { throw new IOException("Unknown nameservice: " + nsId); } } } return DFSUtilClient.getAddressesForNsIds(conf, parentNameServices, null, DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY); } /** * Map a logical namenode ID to its lifeline address. Use the given * nameservice if specified, or the configured one if none is given. * * @param conf Configuration * @param nsId which nameservice nnId is a part of, optional * @param nnId the namenode ID to get the service addr for * @return the lifeline addr, null if it could not be determined */ public static String getNamenodeLifelineAddr(final Configuration conf, String nsId, String nnId) { if (nsId == null) { nsId = getOnlyNameServiceIdOrNull(conf); } String lifelineAddrKey = DFSUtilClient.concatSuffixes(DFSConfigKeys.DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY, nsId, nnId); return conf.get(lifelineAddrKey); } /** * Flatten the given map, as returned by other functions in this class, * into a flat list of {@link ConfiguredNNAddress} instances. */ public static List<ConfiguredNNAddress> flattenAddressMap(Map<String, Map<String, InetSocketAddress>> map) { List<ConfiguredNNAddress> ret = Lists.newArrayList(); for (Map.Entry<String, Map<String, InetSocketAddress>> entry : map.entrySet()) { String nsId = entry.getKey(); Map<String, InetSocketAddress> nnMap = entry.getValue(); for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) { String nnId = e2.getKey(); InetSocketAddress addr = e2.getValue(); ret.add(new ConfiguredNNAddress(nsId, nnId, addr)); } } return ret; } /** * Format the given map, as returned by other functions in this class, * into a string suitable for debugging display. The format of this string * should not be considered an interface, and is liable to change. */ public static String addressMapToString(Map<String, Map<String, InetSocketAddress>> map) { StringBuilder b = new StringBuilder(); for (Map.Entry<String, Map<String, InetSocketAddress>> entry : map.entrySet()) { String nsId = entry.getKey(); Map<String, InetSocketAddress> nnMap = entry.getValue(); b.append("Nameservice <").append(nsId).append(">:").append("\n"); for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) { b.append(" NN ID ").append(e2.getKey()).append(" => ").append(e2.getValue()).append("\n"); } } return b.toString(); } public static String nnAddressesAsString(Configuration conf) { Map<String, Map<String, InetSocketAddress>> addresses = DFSUtilClient.getHaNnRpcAddresses(conf); return addressMapToString(addresses); } /** * Represent one of the NameNodes configured in the cluster. */ public static class ConfiguredNNAddress { private final String nameserviceId; private final String namenodeId; private final InetSocketAddress addr; private ConfiguredNNAddress(String nameserviceId, String namenodeId, InetSocketAddress addr) { this.nameserviceId = nameserviceId; this.namenodeId = namenodeId; this.addr = addr; } public String getNameserviceId() { return nameserviceId; } public String getNamenodeId() { return namenodeId; } public InetSocketAddress getAddress() { return addr; } @Override public String toString() { return "ConfiguredNNAddress[nsId=" + nameserviceId + ";" + "nnId=" + namenodeId + ";addr=" + addr + "]"; } } /** @return Internal name services specified in the conf. */ static Collection<String> getInternalNameServices(Configuration conf) { final Collection<String> ids = conf.getTrimmedStringCollection(DFSConfigKeys.DFS_INTERNAL_NAMESERVICES_KEY); return !ids.isEmpty() ? ids : DFSUtilClient.getNameServiceIds(conf); } /** * Get a URI for each internal nameservice. If a nameservice is * HA-enabled, and the configured failover proxy provider supports logical * URIs, then the logical URI of the nameservice is returned. * Otherwise, a URI corresponding to an RPC address of the single NN for that * nameservice is returned, preferring the service RPC address over the * client RPC address. * * @param conf configuration * @return a collection of all configured NN URIs, preferring service * addresses */ public static Collection<URI> getInternalNsRpcUris(Configuration conf) { return getNameServiceUris(conf, getInternalNameServices(conf), DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY); } /** * Get a URI for each configured nameservice. If a nameservice is * HA-enabled, and the configured failover proxy provider supports logical * URIs, then the logical URI of the nameservice is returned. * Otherwise, a URI corresponding to the address of the single NN for that * nameservice is returned. * * @param conf configuration * @param keys configuration keys to try in order to get the URI for non-HA * nameservices * @return a collection of all configured NN URIs */ static Collection<URI> getNameServiceUris(Configuration conf, Collection<String> nameServices, String... keys) { Set<URI> ret = new HashSet<URI>(); // We're passed multiple possible configuration keys for any given NN or HA // nameservice, and search the config in order of these keys. In order to // make sure that a later config lookup (e.g. fs.defaultFS) doesn't add a // URI for a config key for which we've already found a preferred entry, we // keep track of non-preferred keys here. Set<URI> nonPreferredUris = new HashSet<URI>(); for (String nsId : nameServices) { URI nsUri = createUri(HdfsConstants.HDFS_URI_SCHEME, nsId, -1); /** * Determine whether the logical URI of the name service can be resolved * by the configured failover proxy provider. If not, we should try to * resolve the URI here */ boolean useLogicalUri = false; try { useLogicalUri = HAUtil.useLogicalUri(conf, nsUri); } catch (IOException e) { LOG.warn("Getting exception while trying to determine if nameservice " + nsId + " can use logical URI: " + e); } if (HAUtil.isHAEnabled(conf, nsId) && useLogicalUri) { // Add the logical URI of the nameservice. ret.add(nsUri); } else { // Add the URI corresponding to the address of the NN. boolean uriFound = false; for (String key : keys) { String addr = conf.get(DFSUtilClient.concatSuffixes(key, nsId)); if (addr != null) { URI uri = createUri(HdfsConstants.HDFS_URI_SCHEME, NetUtils.createSocketAddr(addr)); if (!uriFound) { uriFound = true; ret.add(uri); } else { nonPreferredUris.add(uri); } } } } } // Add the generic configuration keys. boolean uriFound = false; for (String key : keys) { String addr = conf.get(key); if (addr != null) { URI uri = createUri(HdfsConstants.HDFS_URI_SCHEME, NetUtils.createSocketAddr(addr)); if (!uriFound) { uriFound = true; ret.add(uri); } else { nonPreferredUris.add(uri); } } } // Add the default URI if it is an HDFS URI and we haven't come up with a // valid non-nameservice NN address yet. Consider the servicerpc-address // and rpc-address to be the "unnamed" nameservice. defaultFS is our // fallback when rpc-address isn't given. We therefore only want to add // the defaultFS when neither the servicerpc-address (which is preferred) // nor the rpc-address (which overrides defaultFS) is given. if (!uriFound) { URI defaultUri = FileSystem.getDefaultUri(conf); if (defaultUri != null) { // checks if defaultUri is ip:port format // and convert it to hostname:port format if (defaultUri.getPort() != -1) { defaultUri = createUri(defaultUri.getScheme(), NetUtils.createSocketAddr(defaultUri.getHost(), defaultUri.getPort())); } defaultUri = trimUri(defaultUri); if (HdfsConstants.HDFS_URI_SCHEME.equals(defaultUri.getScheme()) && !nonPreferredUris.contains(defaultUri)) { ret.add(defaultUri); } } } return ret; } /** * Given the InetSocketAddress this method returns the nameservice Id * corresponding to the key with matching address, by doing a reverse * lookup on the list of nameservices until it finds a match. * * Since the process of resolving URIs to Addresses is slightly expensive, * this utility method should not be used in performance-critical routines. * * @param conf - configuration * @param address - InetSocketAddress for configured communication with NN. * Configured addresses are typically given as URIs, but we may have to * compare against a URI typed in by a human, or the server name may be * aliased, so we compare unambiguous InetSocketAddresses instead of just * comparing URI substrings. * @param keys - list of configured communication parameters that should * be checked for matches. For example, to compare against RPC addresses, * provide the list DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, * DFS_NAMENODE_RPC_ADDRESS_KEY. Use the generic parameter keys, * not the NameServiceId-suffixed keys. * @return nameserviceId, or null if no match found */ public static String getNameServiceIdFromAddress(final Configuration conf, final InetSocketAddress address, String... keys) { // Configuration with a single namenode and no nameserviceId String[] ids = getSuffixIDs(conf, address, keys); return (ids != null) ? ids[0] : null; } /** * return server http or https address from the configuration for a * given namenode rpc address. * @param namenodeAddr - namenode RPC address * @param conf configuration * @param scheme - the scheme (http / https) * @return server http or https address * @throws IOException */ public static URI getInfoServer(InetSocketAddress namenodeAddr, Configuration conf, String scheme) throws IOException { String[] suffixes = null; if (namenodeAddr != null) { // if non-default namenode, try reverse look up // the nameServiceID if it is available suffixes = getSuffixIDs(conf, namenodeAddr, DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY); } String authority; if ("http".equals(scheme)) { authority = getSuffixedConf(conf, DFS_NAMENODE_HTTP_ADDRESS_KEY, DFS_NAMENODE_HTTP_ADDRESS_DEFAULT, suffixes); } else if ("https".equals(scheme)) { authority = getSuffixedConf(conf, DFS_NAMENODE_HTTPS_ADDRESS_KEY, DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT, suffixes); } else { throw new IllegalArgumentException("Invalid scheme:" + scheme); } if (namenodeAddr != null) { authority = substituteForWildcardAddress(authority, namenodeAddr.getHostName()); } return URI.create(scheme + "://" + authority); } /** * Lookup the HTTP / HTTPS address of the namenode, and replace its hostname * with defaultHost when it found out that the address is a wildcard / local * address. * * @param defaultHost * The default host name of the namenode. * @param conf * The configuration * @param scheme * HTTP or HTTPS * @throws IOException */ public static URI getInfoServerWithDefaultHost(String defaultHost, Configuration conf, final String scheme) throws IOException { URI configuredAddr = getInfoServer(null, conf, scheme); String authority = substituteForWildcardAddress(configuredAddr.getAuthority(), defaultHost); return URI.create(scheme + "://" + authority); } /** * Determine whether HTTP or HTTPS should be used to connect to the remote * server. Currently the client only connects to the server via HTTPS if the * policy is set to HTTPS_ONLY. * * @return the scheme (HTTP / HTTPS) */ public static String getHttpClientScheme(Configuration conf) { HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf); return policy == HttpConfig.Policy.HTTPS_ONLY ? "https" : "http"; } /** * Substitute a default host in the case that an address has been configured * with a wildcard. This is used, for example, when determining the HTTP * address of the NN -- if it's configured to bind to 0.0.0.0, we want to * substitute the hostname from the filesystem URI rather than trying to * connect to 0.0.0.0. * @param configuredAddress the address found in the configuration * @param defaultHost the host to substitute with, if configuredAddress * is a local/wildcard address. * @return the substituted address * @throws IOException if it is a wildcard address and security is enabled */ @VisibleForTesting static String substituteForWildcardAddress(String configuredAddress, String defaultHost) { InetSocketAddress sockAddr = NetUtils.createSocketAddr(configuredAddress); final InetAddress addr = sockAddr.getAddress(); if (addr != null && addr.isAnyLocalAddress()) { return defaultHost + ":" + sockAddr.getPort(); } else { return configuredAddress; } } private static String getSuffixedConf(Configuration conf, String key, String defaultVal, String[] suffixes) { String ret = conf.get(DFSUtil.addKeySuffixes(key, suffixes)); if (ret != null) { return ret; } return conf.get(key, defaultVal); } /** * Sets the node specific setting into generic configuration key. Looks up * value of "key.nameserviceId.namenodeId" and if found sets that value into * generic key in the conf. If this is not found, falls back to * "key.nameserviceId" and then the unmodified key. * * Note that this only modifies the runtime conf. * * @param conf * Configuration object to lookup specific key and to set the value * to the key passed. Note the conf object is modified. * @param nameserviceId * nameservice Id to construct the node specific key. Pass null if * federation is not configuration. * @param nnId * namenode Id to construct the node specific key. Pass null if * HA is not configured. * @param keys * The key for which node specific value is looked up */ public static void setGenericConf(Configuration conf, String nameserviceId, String nnId, String... keys) { for (String key : keys) { String value = conf.get(addKeySuffixes(key, nameserviceId, nnId)); if (value != null) { conf.set(key, value); continue; } value = conf.get(addKeySuffixes(key, nameserviceId)); if (value != null) { conf.set(key, value); } } } /** * Round bytes to GiB (gibibyte) * @param bytes number of bytes * @return number of GiB */ public static int roundBytesToGB(long bytes) { return Math.round((float) bytes / 1024 / 1024 / 1024); } /** * Get nameservice Id for the {@link NameNode} based on namenode RPC address * matching the local node address. */ public static String getNamenodeNameServiceId(Configuration conf) { return getNameServiceId(conf, DFS_NAMENODE_RPC_ADDRESS_KEY); } /** * Get nameservice Id for the BackupNode based on backup node RPC address * matching the local node address. */ public static String getBackupNameServiceId(Configuration conf) { return getNameServiceId(conf, DFS_NAMENODE_BACKUP_ADDRESS_KEY); } /** * Get nameservice Id for the secondary node based on secondary http address * matching the local node address. */ public static String getSecondaryNameServiceId(Configuration conf) { return getNameServiceId(conf, DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY); } /** * Get the nameservice Id by matching the {@code addressKey} with the * the address of the local node. * * If {@link DFSConfigKeys#DFS_NAMESERVICE_ID} is not specifically * configured, and more than one nameservice Id is configured, this method * determines the nameservice Id by matching the local node's address with the * configured addresses. When a match is found, it returns the nameservice Id * from the corresponding configuration key. * * @param conf Configuration * @param addressKey configuration key to get the address. * @return nameservice Id on success, null if federation is not configured. * @throws HadoopIllegalArgumentException on error */ private static String getNameServiceId(Configuration conf, String addressKey) { String nameserviceId = conf.get(DFS_NAMESERVICE_ID); if (nameserviceId != null) { return nameserviceId; } Collection<String> nsIds = DFSUtilClient.getNameServiceIds(conf); if (1 == nsIds.size()) { return nsIds.toArray(new String[1])[0]; } String nnId = conf.get(DFS_HA_NAMENODE_ID_KEY); return getSuffixIDs(conf, addressKey, null, nnId, LOCAL_ADDRESS_MATCHER)[0]; } /** * Determine the {@link InetSocketAddress} to bind to, for any service. * In case of HA or federation, the address is assumed to specified as * {@code confKey}.NAMESPACEID.NAMENODEID as appropriate. * * @param conf configuration. * @param confKey configuration key (prefix if HA/federation) used to * specify the address for the service. * @param defaultValue default value for the address. * @param bindHostKey configuration key (prefix if HA/federation) * specifying host to bind to. * @return the address to bind to. */ public static InetSocketAddress getBindAddress(Configuration conf, String confKey, String defaultValue, String bindHostKey) { InetSocketAddress address; String nsId = DFSUtil.getNamenodeNameServiceId(conf); String bindHostActualKey; if (nsId != null) { String namenodeId = HAUtil.getNameNodeId(conf, nsId); address = DFSUtilClient.getAddressesForNameserviceId(conf, nsId, null, confKey).get(namenodeId); bindHostActualKey = DFSUtil.addKeySuffixes(bindHostKey, nsId, namenodeId); } else { address = NetUtils.createSocketAddr(conf.get(confKey, defaultValue)); bindHostActualKey = bindHostKey; } String bindHost = conf.get(bindHostActualKey); if (bindHost == null || bindHost.isEmpty()) { bindHost = address.getHostName(); } return new InetSocketAddress(bindHost, address.getPort()); } /** * Returns nameservice Id and namenode Id when the local host matches the * configuration parameter {@code addressKey}.<nameservice Id>.<namenode Id> * * @param conf Configuration * @param addressKey configuration key corresponding to the address. * @param knownNsId only look at configs for the given nameservice, if not-null * @param knownNNId only look at configs for the given namenode, if not null * @param matcher matching criteria for matching the address * @return Array with nameservice Id and namenode Id on success. First element * in the array is nameservice Id and second element is namenode Id. * Null value indicates that the configuration does not have the the * Id. * @throws HadoopIllegalArgumentException on error */ static String[] getSuffixIDs(final Configuration conf, final String addressKey, String knownNsId, String knownNNId, final AddressMatcher matcher) { String nameserviceId = null; String namenodeId = null; int found = 0; Collection<String> nsIds = DFSUtilClient.getNameServiceIds(conf); for (String nsId : DFSUtilClient.emptyAsSingletonNull(nsIds)) { if (knownNsId != null && !knownNsId.equals(nsId)) { continue; } Collection<String> nnIds = DFSUtilClient.getNameNodeIds(conf, nsId); for (String nnId : DFSUtilClient.emptyAsSingletonNull(nnIds)) { if (LOG.isTraceEnabled()) { LOG.trace(String.format("addressKey: %s nsId: %s nnId: %s", addressKey, nsId, nnId)); } if (knownNNId != null && !knownNNId.equals(nnId)) { continue; } String key = addKeySuffixes(addressKey, nsId, nnId); String addr = conf.get(key); if (addr == null) { continue; } InetSocketAddress s = null; try { s = NetUtils.createSocketAddr(addr); } catch (Exception e) { LOG.warn("Exception in creating socket address " + addr, e); continue; } if (!s.isUnresolved() && matcher.match(s)) { nameserviceId = nsId; namenodeId = nnId; found++; } } } if (found > 1) { // Only one address must match the local address String msg = "Configuration has multiple addresses that match " + "local node's address. Please configure the system with " + DFS_NAMESERVICE_ID + " and " + DFS_HA_NAMENODE_ID_KEY; throw new HadoopIllegalArgumentException(msg); } return new String[] { nameserviceId, namenodeId }; } /** * For given set of {@code keys} adds nameservice Id and or namenode Id * and returns {nameserviceId, namenodeId} when address match is found. * @see #getSuffixIDs(Configuration, String, String, String, AddressMatcher) */ static String[] getSuffixIDs(final Configuration conf, final InetSocketAddress address, final String... keys) { AddressMatcher matcher = new AddressMatcher() { @Override public boolean match(InetSocketAddress s) { return address.equals(s); } }; for (String key : keys) { String[] ids = getSuffixIDs(conf, key, null, null, matcher); if (ids != null && (ids[0] != null || ids[1] != null)) { return ids; } } return null; } private interface AddressMatcher { public boolean match(InetSocketAddress s); } /** Create an URI from scheme and address. */ public static URI createUri(String scheme, InetSocketAddress address) { return createUri(scheme, address.getHostName(), address.getPort()); } /** Create an URI from scheme, host, and port. */ public static URI createUri(String scheme, String host, int port) { try { return new URI(scheme, null, host, port, null, null, null); } catch (URISyntaxException x) { throw new IllegalArgumentException(x.getMessage(), x); } } /** Remove unnecessary path from HDFS URI. */ static URI trimUri(URI uri) { String path = uri.getPath(); if (HdfsConstants.HDFS_URI_SCHEME.equals(uri.getScheme()) && path != null && !path.isEmpty()) { uri = createUri(uri.getScheme(), uri.getHost(), uri.getPort()); } return uri; } /** * Add protobuf based protocol to the {@link org.apache.hadoop.ipc.RPC.Server} * @param conf configuration * @param protocol Protocol interface * @param service service that implements the protocol * @param server RPC server to which the protocol & implementation is added to * @throws IOException */ public static void addPBProtocol(Configuration conf, Class<?> protocol, BlockingService service, RPC.Server server) throws IOException { RPC.setProtocolEngine(conf, protocol, ProtobufRpcEngine.class); server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, protocol, service); } /** * Map a logical namenode ID to its service address. Use the given * nameservice if specified, or the configured one if none is given. * * @param conf Configuration * @param nsId which nameservice nnId is a part of, optional * @param nnId the namenode ID to get the service addr for * @return the service addr, null if it could not be determined */ public static String getNamenodeServiceAddr(final Configuration conf, String nsId, String nnId) { if (nsId == null) { nsId = getOnlyNameServiceIdOrNull(conf); } String serviceAddrKey = DFSUtilClient.concatSuffixes(DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, nsId, nnId); String addrKey = DFSUtilClient.concatSuffixes(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, nsId, nnId); String serviceRpcAddr = conf.get(serviceAddrKey); if (serviceRpcAddr == null) { serviceRpcAddr = conf.get(addrKey); } return serviceRpcAddr; } /** * Map a logical namenode ID to its web address. Use the given nameservice if * specified, or the configured one if none is given. * * @param conf Configuration * @param nsId which nameservice nnId is a part of, optional * @param nnId the namenode ID to get the service addr for * @return the service addr, null if it could not be determined */ public static String getNamenodeWebAddr(final Configuration conf, String nsId, String nnId) { if (nsId == null) { nsId = getOnlyNameServiceIdOrNull(conf); } String webAddrKey = DFSUtilClient.concatSuffixes(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, nsId, nnId); String webAddr = conf.get(webAddrKey, DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT); return webAddr; } /** * Get all of the Web addresses of the individual NNs in a given nameservice. * * @param conf Configuration * @param nsId the nameservice whose NNs addresses we want. * @param defaultValue default address to return in case key is not found. * @return A map from nnId -> Web address of each NN in the nameservice. */ public static Map<String, InetSocketAddress> getWebAddressesForNameserviceId(Configuration conf, String nsId, String defaultValue) { return DFSUtilClient.getAddressesForNameserviceId(conf, nsId, defaultValue, DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY); } /** * If the configuration refers to only a single nameservice, return the * name of that nameservice. If it refers to 0 or more than 1, return null. */ public static String getOnlyNameServiceIdOrNull(Configuration conf) { Collection<String> nsIds = DFSUtilClient.getNameServiceIds(conf); if (1 == nsIds.size()) { return nsIds.toArray(new String[1])[0]; } else { // No nameservice ID was given and more than one is configured return null; } } public static final Options helpOptions = new Options(); public static final Option helpOpt = new Option("h", "help", false, "get help information"); static { helpOptions.addOption(helpOpt); } /** * Parse the arguments for commands * * @param args the argument to be parsed * @param helpDescription help information to be printed out * @param out Printer * @param printGenericCommandUsage whether to print the * generic command usage defined in ToolRunner * @return true when the argument matches help option, false if not */ public static boolean parseHelpArgument(String[] args, String helpDescription, PrintStream out, boolean printGenericCommandUsage) { if (args.length == 1) { try { CommandLineParser parser = new PosixParser(); CommandLine cmdLine = parser.parse(helpOptions, args); if (cmdLine.hasOption(helpOpt.getOpt()) || cmdLine.hasOption(helpOpt.getLongOpt())) { // should print out the help information out.println(helpDescription + "\n"); if (printGenericCommandUsage) { ToolRunner.printGenericCommandUsage(out); } return true; } } catch (ParseException pe) { return false; } } return false; } /** * Get DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION from configuration. * * @param conf Configuration * @return Value of DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION */ public static float getInvalidateWorkPctPerIteration(Configuration conf) { float blocksInvalidateWorkPct = conf.getFloat(DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT); Preconditions.checkArgument((blocksInvalidateWorkPct > 0 && blocksInvalidateWorkPct <= 1.0f), DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION + " = '" + blocksInvalidateWorkPct + "' is invalid. " + "It should be a positive, non-zero float value, not greater than 1.0f, " + "to indicate a percentage."); return blocksInvalidateWorkPct; } /** * Get DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION from * configuration. * * @param conf Configuration * @return Value of DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION */ public static int getReplWorkMultiplier(Configuration conf) { int blocksReplWorkMultiplier = conf.getInt( DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION, DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT); Preconditions.checkArgument((blocksReplWorkMultiplier > 0), DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION + " = '" + blocksReplWorkMultiplier + "' is invalid. " + "It should be a positive, non-zero integer value."); return blocksReplWorkMultiplier; } /** * Get SPNEGO keytab Key from configuration * * @param conf Configuration * @param defaultKey default key to be used for config lookup * @return DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY if the key is not empty * else return defaultKey */ public static String getSpnegoKeytabKey(Configuration conf, String defaultKey) { String value = conf.get(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY); return (value == null || value.isEmpty()) ? defaultKey : DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY; } /** * Get http policy. */ public static HttpConfig.Policy getHttpPolicy(Configuration conf) { String policyStr = conf.get(DFSConfigKeys.DFS_HTTP_POLICY_KEY, DFSConfigKeys.DFS_HTTP_POLICY_DEFAULT); HttpConfig.Policy policy = HttpConfig.Policy.fromString(policyStr); if (policy == null) { throw new HadoopIllegalArgumentException( "Unregonized value '" + policyStr + "' for " + DFSConfigKeys.DFS_HTTP_POLICY_KEY); } conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, policy.name()); return policy; } public static HttpServer2.Builder loadSslConfToHttpServerBuilder(HttpServer2.Builder builder, Configuration sslConf) { return builder .needsClientAuth( sslConf.getBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY, DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT)) .keyPassword(getPassword(sslConf, DFS_SERVER_HTTPS_KEYPASSWORD_KEY)) .keyStore(sslConf.get("ssl.server.keystore.location"), getPassword(sslConf, DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY), sslConf.get("ssl.server.keystore.type", "jks")) .trustStore(sslConf.get("ssl.server.truststore.location"), getPassword(sslConf, DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY), sslConf.get("ssl.server.truststore.type", "jks")) .excludeCiphers(sslConf.get("ssl.server.exclude.cipher.list")); } /** * Leverages the Configuration.getPassword method to attempt to get * passwords from the CredentialProvider API before falling back to * clear text in config - if falling back is allowed. * @param conf Configuration instance * @param alias name of the credential to retreive * @return String credential value or null */ static String getPassword(Configuration conf, String alias) { String password = null; try { char[] passchars = conf.getPassword(alias); if (passchars != null) { password = new String(passchars); } } catch (IOException ioe) { LOG.warn("Setting password to null since IOException is caught" + " when getting password", ioe); password = null; } return password; } /** * Converts a Date into an ISO-8601 formatted datetime string. */ public static String dateToIso8601String(Date date) { return DFSUtilClient.dateToIso8601String(date); } /** * Converts a time duration in milliseconds into DDD:HH:MM:SS format. */ public static String durationToString(long durationMs) { return DFSUtilClient.durationToString(durationMs); } /** * Converts a relative time string into a duration in milliseconds. */ public static long parseRelativeTime(String relTime) throws IOException { if (relTime.length() < 2) { throw new IOException("Unable to parse relative time value of " + relTime + ": too short"); } String ttlString = relTime.substring(0, relTime.length() - 1); long ttl; try { ttl = Long.parseLong(ttlString); } catch (NumberFormatException e) { throw new IOException( "Unable to parse relative time value of " + relTime + ": " + ttlString + " is not a number"); } if (relTime.endsWith("s")) { // pass } else if (relTime.endsWith("m")) { ttl *= 60; } else if (relTime.endsWith("h")) { ttl *= 60 * 60; } else if (relTime.endsWith("d")) { ttl *= 60 * 60 * 24; } else { throw new IOException("Unable to parse relative time value of " + relTime + ": unknown time unit " + relTime.charAt(relTime.length() - 1)); } return ttl * 1000; } /** * Load HTTPS-related configuration. */ public static Configuration loadSslConfiguration(Configuration conf) { Configuration sslConf = new Configuration(false); sslConf.addResource(conf.get(DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_DEFAULT)); final String[] reqSslProps = { DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_LOCATION_KEY, DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_LOCATION_KEY, DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY, DFSConfigKeys.DFS_SERVER_HTTPS_KEYPASSWORD_KEY }; // Check if the required properties are included for (String sslProp : reqSslProps) { if (sslConf.get(sslProp) == null) { LOG.warn("SSL config " + sslProp + " is missing. If " + DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY + " is specified, make sure it is a relative path"); } } boolean requireClientAuth = conf.getBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY, DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT); sslConf.setBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY, requireClientAuth); return sslConf; } /** * Return a HttpServer.Builder that the journalnode / namenode / secondary * namenode can use to initialize their HTTP / HTTPS server. * */ public static HttpServer2.Builder httpServerTemplateForNNAndJN(Configuration conf, final InetSocketAddress httpAddr, final InetSocketAddress httpsAddr, String name, String spnegoUserNameKey, String spnegoKeytabFileKey) throws IOException { HttpConfig.Policy policy = getHttpPolicy(conf); HttpServer2.Builder builder = new HttpServer2.Builder().setName(name).setConf(conf) .setACL(new AccessControlList(conf.get(DFS_ADMIN, " "))) .setSecurityEnabled(UserGroupInformation.isSecurityEnabled()).setUsernameConfKey(spnegoUserNameKey) .setKeytabConfKey(getSpnegoKeytabKey(conf, spnegoKeytabFileKey)); // initialize the webserver for uploading/downloading files. if (UserGroupInformation.isSecurityEnabled()) { LOG.info("Starting web server as: " + SecurityUtil.getServerPrincipal(conf.get(spnegoUserNameKey), httpAddr.getHostName())); } if (policy.isHttpEnabled()) { if (httpAddr.getPort() == 0) { builder.setFindPort(true); } URI uri = URI.create("http://" + NetUtils.getHostPortString(httpAddr)); builder.addEndpoint(uri); LOG.info("Starting Web-server for " + name + " at: " + uri); } if (policy.isHttpsEnabled() && httpsAddr != null) { Configuration sslConf = loadSslConfiguration(conf); loadSslConfToHttpServerBuilder(builder, sslConf); if (httpsAddr.getPort() == 0) { builder.setFindPort(true); } URI uri = URI.create("https://" + NetUtils.getHostPortString(httpsAddr)); builder.addEndpoint(uri); LOG.info("Starting Web-server for " + name + " at: " + uri); } return builder; } /** * Assert that all objects in the collection are equal. Returns silently if * so, throws an AssertionError if any object is not equal. All null values * are considered equal. * * @param objects the collection of objects to check for equality. */ public static void assertAllResultsEqual(Collection<?> objects) throws AssertionError { if (objects.size() == 0 || objects.size() == 1) return; Object[] resultsArray = objects.toArray(); for (int i = 1; i < resultsArray.length; i++) { Object currElement = resultsArray[i]; Object lastElement = resultsArray[i - 1]; if ((currElement == null && currElement != lastElement) || (currElement != null && !currElement.equals(lastElement))) { throw new AssertionError("Not all elements match in results: " + Arrays.toString(resultsArray)); } } } /** * Creates a new KeyProviderCryptoExtension by wrapping the * KeyProvider specified in the given Configuration. * * @param conf Configuration * @return new KeyProviderCryptoExtension, or null if no provider was found. * @throws IOException if the KeyProvider is improperly specified in * the Configuration */ public static KeyProviderCryptoExtension createKeyProviderCryptoExtension(final Configuration conf) throws IOException { KeyProvider keyProvider = HdfsKMSUtil.createKeyProvider(conf); if (keyProvider == null) { return null; } KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension .createKeyProviderCryptoExtension(keyProvider); return cryptoProvider; } /** * Decodes an HDFS delegation token to its identifier. * * @param token the token * @return the decoded identifier. * @throws IOException */ public static DelegationTokenIdentifier decodeDelegationToken(final Token<DelegationTokenIdentifier> token) throws IOException { final DelegationTokenIdentifier id = new DelegationTokenIdentifier(); final ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); try (DataInputStream in = new DataInputStream(buf)) { id.readFields(in); } return id; } }