Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hive.jdbc.miniHS2; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.LlapItUtils; import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper; import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim; import org.apache.hadoop.hive.shims.HadoopShims.MiniMrShim; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hive.service.Service; import org.apache.hive.service.cli.CLIServiceClient; import org.apache.hive.service.cli.SessionHandle; import org.apache.hive.service.cli.thrift.ThriftBinaryCLIService; import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient; import org.apache.hive.service.cli.thrift.ThriftHttpCLIService; import org.apache.hive.service.server.HiveServer2; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class MiniHS2 extends AbstractHiveService { private static final Logger LOG = LoggerFactory.getLogger(MiniHS2.class); public static final String HS2_BINARY_MODE = "binary"; public static final String HS2_HTTP_MODE = "http"; private static final String driverName = "org.apache.hive.jdbc.HiveDriver"; private static final FsPermission FULL_PERM = new FsPermission((short) 00777); private static final FsPermission WRITE_ALL_PERM = new FsPermission((short) 00733); private static final String tmpDir = System.getProperty("test.tmp.dir"); private HiveServer2 hiveServer2 = null; private final File baseDir; private final Path baseFsDir; private MiniMrShim mr; private MiniDFSShim dfs; private MiniLlapCluster llapCluster = null; private final FileSystem localFS; private boolean useMiniKdc = false; private final String serverPrincipal; private final boolean isMetastoreRemote; private final boolean cleanupLocalDirOnStartup; private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY; public enum MiniClusterType { MR, TEZ, LLAP, LOCALFS_ONLY; } public static class Builder { private HiveConf hiveConf = new HiveConf(); private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY; private boolean useMiniKdc = false; private String serverPrincipal; private String serverKeytab; private boolean isHTTPTransMode = false; private boolean isMetastoreRemote; private boolean usePortsFromConf = false; private String authType = "KERBEROS"; private boolean isHA = false; private boolean cleanupLocalDirOnStartup = true; public Builder() { } public Builder withMiniMR() { this.miniClusterType = MiniClusterType.MR; return this; } public Builder withMiniKdc(String serverPrincipal, String serverKeytab) { this.useMiniKdc = true; this.serverPrincipal = serverPrincipal; this.serverKeytab = serverKeytab; return this; } public Builder withAuthenticationType(String authType) { this.authType = authType; return this; } public Builder withRemoteMetastore() { this.isMetastoreRemote = true; return this; } public Builder withConf(HiveConf hiveConf) { this.hiveConf = hiveConf; return this; } public Builder withHA() { this.isHA = true; return this; } /** * Start HS2 with HTTP transport mode, default is binary mode * @return this Builder */ public Builder withHTTPTransport() { this.isHTTPTransMode = true; return this; } public Builder cleanupLocalDirOnStartup(boolean val) { this.cleanupLocalDirOnStartup = val; return this; } public MiniHS2 build() throws Exception { if (miniClusterType == MiniClusterType.MR && useMiniKdc) { throw new IOException("Can't create secure miniMr ... yet"); } if (isHTTPTransMode) { hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_HTTP_MODE); } else { hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_BINARY_MODE); } return new MiniHS2(hiveConf, miniClusterType, useMiniKdc, serverPrincipal, serverKeytab, isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup); } } public MiniMrShim getMr() { return mr; } public void setMr(MiniMrShim mr) { this.mr = mr; } public MiniDFSShim getDfs() { return dfs; } public void setDfs(MiniDFSShim dfs) { this.dfs = dfs; } public FileSystem getLocalFS() { return localFS; } public MiniClusterType getMiniClusterType() { return miniClusterType; } public void setMiniClusterType(MiniClusterType miniClusterType) { this.miniClusterType = miniClusterType; } public boolean isUseMiniKdc() { return useMiniKdc; } private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useMiniKdc, String serverPrincipal, String serverKeytab, boolean isMetastoreRemote, boolean usePortsFromConf, String authType, boolean isHA, boolean cleanupLocalDirOnStartup) throws Exception { // Always use localhost for hostname as some tests like SSL CN validation ones // are tied to localhost being present in the certificate name super(hiveConf, "localhost", (usePortsFromConf ? hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT) : MetaStoreUtils.findFreePort()), (usePortsFromConf ? hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT) : MetaStoreUtils.findFreePort())); hiveConf.setLongVar(ConfVars.HIVE_SERVER2_MAX_START_ATTEMPTS, 3l); hiveConf.setTimeVar(ConfVars.HIVE_SERVER2_SLEEP_INTERVAL_BETWEEN_START_ATTEMPTS, 10, TimeUnit.SECONDS); this.miniClusterType = miniClusterType; this.useMiniKdc = useMiniKdc; this.serverPrincipal = serverPrincipal; this.isMetastoreRemote = isMetastoreRemote; this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup; baseDir = getBaseDir(); localFS = FileSystem.getLocal(hiveConf); FileSystem fs; if (miniClusterType != MiniClusterType.LOCALFS_ONLY) { // Initialize dfs dfs = ShimLoader.getHadoopShims().getMiniDfs(hiveConf, 4, true, null, isHA); fs = dfs.getFileSystem(); String uriString = fs.getUri().toString(); // Initialize the execution engine based on cluster type switch (miniClusterType) { case TEZ: // Change the engine to tez hiveConf.setVar(ConfVars.HIVE_EXECUTION_ENGINE, "tez"); // TODO: This should be making use of confDir to load configs setup for Tez, etc. mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString, false); break; case LLAP: if (usePortsFromConf) { hiveConf.setBoolean("minillap.usePortsFromConf", true); } llapCluster = LlapItUtils.startAndGetMiniLlapCluster(hiveConf, null, null); mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString, true); break; case MR: mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 2, uriString, 1); break; default: throw new IllegalArgumentException("Unsupported cluster type " + mr); } // store the config in system properties mr.setupConfiguration(getHiveConf()); baseFsDir = new Path(new Path(fs.getUri()), "/base"); } else { // This is FS only mode, just initialize the dfs root directory. fs = FileSystem.getLocal(hiveConf); baseFsDir = new Path("file://" + baseDir.toURI().getPath()); if (cleanupLocalDirOnStartup) { // Cleanup baseFsDir since it can be shared across tests. LOG.info("Attempting to cleanup baseFsDir: {} while setting up MiniHS2", baseDir); Preconditions.checkState(baseFsDir.depth() >= 3); // Avoid "/tmp", directories closer to "/" fs.delete(baseFsDir, true); } } if (useMiniKdc) { hiveConf.setVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL, serverPrincipal); hiveConf.setVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB, serverKeytab); hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, authType); } String metaStoreURL = "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + File.separator + "test_metastore;create=true"; fs.mkdirs(baseFsDir); Path wareHouseDir = new Path(baseFsDir, "warehouse"); // Create warehouse with 777, so that user impersonation has no issues. FileSystem.mkdirs(fs, wareHouseDir, FULL_PERM); fs.mkdirs(wareHouseDir); setWareHouseDir(wareHouseDir.toString()); System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, metaStoreURL); hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, metaStoreURL); if (!usePortsFromConf) { // reassign a new port, just in case if one of the MR services grabbed the last one setBinaryPort(MetaStoreUtils.findFreePort()); } hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, getHost()); hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, getBinaryPort()); hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, getHttpPort()); Path scratchDir = new Path(baseFsDir, "scratch"); // Create root scratchdir with write all, so that user impersonation has no issues. Utilities.createDirsWithPermission(hiveConf, scratchDir, WRITE_ALL_PERM, true); System.setProperty(HiveConf.ConfVars.SCRATCHDIR.varname, scratchDir.toString()); hiveConf.setVar(ConfVars.SCRATCHDIR, scratchDir.toString()); String localScratchDir = baseDir.getPath() + File.separator + "scratch"; System.setProperty(HiveConf.ConfVars.LOCALSCRATCHDIR.varname, localScratchDir); hiveConf.setVar(ConfVars.LOCALSCRATCHDIR, localScratchDir); } public MiniHS2(HiveConf hiveConf) throws Exception { this(hiveConf, MiniClusterType.LOCALFS_ONLY); } public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType) throws Exception { this(hiveConf, clusterType, false); } public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, boolean usePortsFromConf) throws Exception { this(hiveConf, clusterType, false, null, null, false, usePortsFromConf, "KERBEROS", false, true); } public void start(Map<String, String> confOverlay) throws Exception { if (isMetastoreRemote) { int metaStorePort = MetaStoreUtils.findFreePort(); getHiveConf().setVar(ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort); MetaStoreUtils.startMetaStore(metaStorePort, ShimLoader.getHadoopThriftAuthBridge(), getHiveConf()); } hiveServer2 = new HiveServer2(); // Set confOverlay parameters for (Map.Entry<String, String> entry : confOverlay.entrySet()) { setConfProperty(entry.getKey(), entry.getValue()); } hiveServer2.init(getHiveConf()); hiveServer2.start(); waitForStartup(); setStarted(true); } public void stop() { verifyStarted(); // Currently there is no way to stop the MetaStore service. It will be stopped when the // test JVM exits. This is how other tests are also using MetaStore server. hiveServer2.stop(); setStarted(false); try { if (llapCluster != null) { llapCluster.stop(); } if (mr != null) { mr.shutdown(); mr = null; } if (dfs != null) { dfs.shutdown(); dfs = null; } } catch (IOException e) { // Ignore errors cleaning up miniMR } } public void cleanup() { FileUtils.deleteQuietly(baseDir); } public CLIServiceClient getServiceClient() { verifyStarted(); return getServiceClientInternal(); } public HiveConf getServerConf() { if (hiveServer2 != null) { return hiveServer2.getHiveConf(); } return null; } public CLIServiceClient getServiceClientInternal() { for (Service service : hiveServer2.getServices()) { if (service instanceof ThriftBinaryCLIService) { return new ThriftCLIServiceClient((ThriftBinaryCLIService) service); } if (service instanceof ThriftHttpCLIService) { return new ThriftCLIServiceClient((ThriftHttpCLIService) service); } } throw new IllegalStateException("HiveServer2 not running Thrift service"); } /** * return connection URL for this server instance * @return * @throws Exception */ public String getJdbcURL() throws Exception { return getJdbcURL("default"); } /** * return connection URL for this server instance * @param dbName - DB name to be included in the URL * @return * @throws Exception */ public String getJdbcURL(String dbName) throws Exception { return getJdbcURL(dbName, ""); } /** * return connection URL for this server instance * @param dbName - DB name to be included in the URL * @param sessionConfExt - Addional string to be appended to sessionConf part of url * @return * @throws Exception */ public String getJdbcURL(String dbName, String sessionConfExt) throws Exception { return getJdbcURL(dbName, sessionConfExt, ""); } /** * return connection URL for this server instance * @param dbName - DB name to be included in the URL * @param sessionConfExt - Addional string to be appended to sessionConf part of url * @param hiveConfExt - Additional string to be appended to HiveConf part of url (excluding the ?) * @return * @throws Exception */ public String getJdbcURL(String dbName, String sessionConfExt, String hiveConfExt) throws Exception { sessionConfExt = (sessionConfExt == null ? "" : sessionConfExt); hiveConfExt = (hiveConfExt == null ? "" : hiveConfExt); // Strip the leading ";" if provided // (this is the assumption with which we're going to start configuring sessionConfExt) if (sessionConfExt.startsWith(";")) { sessionConfExt = sessionConfExt.substring(1); } if (isUseMiniKdc()) { sessionConfExt = "principal=" + serverPrincipal + ";" + sessionConfExt; } if (isHttpTransportMode()) { sessionConfExt = "transportMode=http;httpPath=cliservice" + ";" + sessionConfExt; } String baseJdbcURL; if (isDynamicServiceDiscovery()) { sessionConfExt = "serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=" + getServerConf().getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE) + ";" + sessionConfExt; baseJdbcURL = getZKBaseJdbcURL(); } else { baseJdbcURL = getBaseJdbcURL(); } baseJdbcURL = baseJdbcURL + dbName; if (!sessionConfExt.isEmpty()) { baseJdbcURL = baseJdbcURL + ";" + sessionConfExt; } if ((hiveConfExt != null) && (!hiveConfExt.trim().isEmpty())) { baseJdbcURL = baseJdbcURL + "?" + hiveConfExt; } return baseJdbcURL; } /** * Build base JDBC URL * @return */ public String getBaseJdbcURL() { if (isHttpTransportMode()) { return "jdbc:hive2://" + getHost() + ":" + getHttpPort() + "/"; } else { return "jdbc:hive2://" + getHost() + ":" + getBinaryPort() + "/"; } } /** * Build zk base JDBC URL * @return */ private String getZKBaseJdbcURL() throws Exception { HiveConf hiveConf = getServerConf(); if (hiveConf != null) { String zkEnsemble = ZooKeeperHiveHelper.getQuorumServers(hiveConf); return "jdbc:hive2://" + zkEnsemble + "/"; } throw new Exception("Server's HiveConf is null. Unable to read ZooKeeper configs."); } private boolean isHttpTransportMode() { String transportMode = getConfProperty(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname); return transportMode != null && (transportMode.equalsIgnoreCase(HS2_HTTP_MODE)); } private boolean isDynamicServiceDiscovery() throws Exception { HiveConf hiveConf = getServerConf(); if (hiveConf == null) { throw new Exception("Server's HiveConf is null. Unable to read ZooKeeper configs."); } if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) { return true; } return false; } public static String getJdbcDriverName() { return driverName; } public MiniMrShim getMR() { return mr; } public MiniDFSShim getDFS() { return dfs; } private void waitForStartup() throws Exception { int waitTime = 0; long startupTimeout = 1000L * 1000L; CLIServiceClient hs2Client = getServiceClientInternal(); SessionHandle sessionHandle = null; do { Thread.sleep(500L); waitTime += 500L; if (waitTime > startupTimeout) { throw new TimeoutException("Couldn't access new HiveServer2: " + getJdbcURL()); } try { Map<String, String> sessionConf = new HashMap<String, String>(); /** if (isUseMiniKdc()) { getMiniKdc().loginUser(getMiniKdc().getDefaultUserPrincipal()); sessionConf.put("principal", serverPrincipal); } */ sessionHandle = hs2Client.openSession("foo", "bar", sessionConf); } catch (Exception e) { // service not started yet continue; } hs2Client.closeSession(sessionHandle); break; } while (true); } public Service.STATE getState() { return hiveServer2.getServiceState(); } static File getBaseDir() { File baseDir = new File(tmpDir + "/local_base"); return baseDir; } public static void cleanupLocalDir() throws IOException { File baseDir = getBaseDir(); try { org.apache.hadoop.hive.common.FileUtils.deleteDirectory(baseDir); } catch (FileNotFoundException e) { // Ignore. Safe if it does not exist. } } }