Java tutorial
/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hive.llap; import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto.Builder; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class LlapUtil { private static final Logger LOG = LoggerFactory.getLogger(LlapUtil.class); public static String getDaemonLocalDirString(Configuration conf, String workDirsEnvString) { String localDirList = HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_WORK_DIRS); if (localDirList != null && !localDirList.isEmpty()) { LOG.info("Local dirs from Configuration: {}", localDirList); if (!localDirList.equalsIgnoreCase("useYarnEnvDirs") && !StringUtils.isBlank(localDirList)) { LOG.info("Using local dirs from Configuration"); return localDirList; } } // Fallback to picking up the value from environment. if (StringUtils.isNotBlank(workDirsEnvString)) { LOG.info("Using local dirs from environment: {}", workDirsEnvString); return workDirsEnvString; } else { throw new RuntimeException( "Cannot determined local dirs from specified configuration and env. ValueFromConf=" + localDirList + ", ValueFromEnv=" + workDirsEnvString); } } /** * Login using kerberos. But does not change the current logged in user. * * @param principal - kerberos principal * @param keytabFile - keytab file * @return UGI * @throws IOException - if keytab file cannot be found */ public static UserGroupInformation loginWithKerberos(String principal, String keytabFile) throws IOException { if (!UserGroupInformation.isSecurityEnabled()) { return null; } if (principal == null || principal.isEmpty() || keytabFile == null || keytabFile.isEmpty()) { throw new RuntimeException("Kerberos principal and/or keytab are null or empty"); } final String serverPrincipal = SecurityUtil.getServerPrincipal(principal, "0.0.0.0"); LOG.info("Logging in as " + serverPrincipal + " via " + keytabFile); return UserGroupInformation.loginUserFromKeytabAndReturnUGI(serverPrincipal, keytabFile); } /** * Login using kerberos and also updates the current logged in user * * @param principal - kerberos principal * @param keytabFile - keytab file * @throws IOException - if keytab file cannot be found */ public static void loginWithKerberosAndUpdateCurrentUser(String principal, String keytabFile) throws IOException { if (!UserGroupInformation.isSecurityEnabled()) { return; } if (principal == null || principal.isEmpty() || keytabFile == null || keytabFile.isEmpty()) { throw new RuntimeException("Kerberos principal and/or keytab is null or empty"); } final String serverPrincipal = SecurityUtil.getServerPrincipal(principal, "0.0.0.0"); LOG.info( "Logging in as " + serverPrincipal + " via " + keytabFile + " and updating current logged in user"); UserGroupInformation.loginUserFromKeytab(serverPrincipal, keytabFile); } private final static Pattern hostsRe = Pattern.compile("[^A-Za-z0-9_-]"); public static String generateClusterName(Configuration conf) { String hosts = HiveConf.getTrimmedVar(conf, ConfVars.LLAP_DAEMON_SERVICE_HOSTS); return hostsRe.matcher(hosts.startsWith("@") ? hosts.substring(1) : hosts).replaceAll("_"); } public static String getUserNameFromPrincipal(String principal) { // Based on SecurityUtil. if (principal == null) return null; String[] components = principal.split("[/@]"); return (components == null || components.length != 3) ? principal : components[0]; } public static List<StatisticsData> getStatisticsForScheme(final String scheme, final List<StatisticsData> stats) { List<StatisticsData> result = new ArrayList<>(); if (stats != null && scheme != null) { for (StatisticsData s : stats) { if (s.getScheme().equalsIgnoreCase(scheme)) { result.add(s); } } } return result; } public static Map<String, FileSystem.Statistics> getCombinedFileSystemStatistics() { final List<FileSystem.Statistics> allStats = FileSystem.getAllStatistics(); final Map<String, FileSystem.Statistics> result = new HashMap<>(); for (FileSystem.Statistics statistics : allStats) { final String scheme = statistics.getScheme(); if (result.containsKey(scheme)) { FileSystem.Statistics existing = result.get(scheme); FileSystem.Statistics combined = combineFileSystemStatistics(existing, statistics); result.put(scheme, combined); } else { result.put(scheme, statistics); } } return result; } private static FileSystem.Statistics combineFileSystemStatistics(final FileSystem.Statistics s1, final FileSystem.Statistics s2) { FileSystem.Statistics result = new FileSystem.Statistics(s1); result.incrementReadOps(s2.getReadOps()); result.incrementLargeReadOps(s2.getLargeReadOps()); result.incrementWriteOps(s2.getWriteOps()); result.incrementBytesRead(s2.getBytesRead()); result.incrementBytesWritten(s2.getBytesWritten()); return result; } public static List<StatisticsData> cloneThreadLocalFileSystemStatistics() { List<StatisticsData> result = new ArrayList<>(); // thread local filesystem stats is private and cannot be cloned. So make a copy to new class for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { result.add(new StatisticsData(statistics.getScheme(), statistics.getThreadStatistics())); } return result; } public static class StatisticsData { long bytesRead; long bytesWritten; int readOps; int largeReadOps; int writeOps; String scheme; public StatisticsData(String scheme, FileSystem.Statistics.StatisticsData fsStats) { this.scheme = scheme; this.bytesRead = fsStats.getBytesRead(); this.bytesWritten = fsStats.getBytesWritten(); this.readOps = fsStats.getReadOps(); this.largeReadOps = fsStats.getLargeReadOps(); this.writeOps = fsStats.getWriteOps(); } public long getBytesRead() { return bytesRead; } public long getBytesWritten() { return bytesWritten; } public int getReadOps() { return readOps; } public int getLargeReadOps() { return largeReadOps; } public int getWriteOps() { return writeOps; } public String getScheme() { return scheme; } @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append(" scheme: ").append(scheme); sb.append(" bytesRead: ").append(bytesRead); sb.append(" bytesWritten: ").append(bytesWritten); sb.append(" readOps: ").append(readOps); sb.append(" largeReadOps: ").append(largeReadOps); sb.append(" writeOps: ").append(writeOps); return sb.toString(); } } public static String getAmHostNameFromAddress(InetSocketAddress address, Configuration conf) { if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_DAEMON_AM_USE_FQDN)) { return address.getHostName(); } InetAddress ia = address.getAddress(); // getCanonicalHostName would either return FQDN, or an IP. return (ia == null) ? address.getHostName() : ia.getCanonicalHostName(); } public static String humanReadableByteCount(long bytes) { int unit = 1024; if (bytes < unit) { return bytes + "B"; } int exp = (int) (Math.log(bytes) / Math.log(unit)); String suffix = "KMGTPE".charAt(exp - 1) + ""; return String.format("%.2f%sB", bytes / Math.pow(unit, exp), suffix); } }