Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hive.ql; import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.PrintStream; import java.io.Serializable; import java.io.StringWriter; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.StandardOpenOption; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Comparator; import java.util.Deque; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hive.cli.CliDriver; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.cli.control.AbstractCliConfig; import org.apache.hadoop.hive.common.io.CachingPrintStream; import org.apache.hadoop.hive.common.io.DigestPrintStream; import org.apache.hadoop.hive.common.io.SortAndDigestPrintStream; import org.apache.hadoop.hive.common.io.SortPrintStream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.LlapItUtils; import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; import org.apache.hadoop.hive.llap.io.api.LlapProxy; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.hbase.HBaseStore; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession; import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl; import org.apache.hadoop.hive.ql.exec.tez.TezSessionState; import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton; import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.InvalidTableException; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.ParseDriver; import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.processors.CommandProcessor; import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.processors.HiveCommand; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hive.common.util.StreamPrinter; import org.apache.logging.log4j.util.Strings; import org.apache.tools.ant.BuildException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooKeeper; import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableList; import junit.framework.TestSuite; /** * QTestUtil. * */ public class QTestUtil { public static final String UTF_8 = "UTF-8"; // security property names private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri"; private static final String CRLF = System.getProperty("line.separator"); public static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES"; private static final Logger LOG = LoggerFactory.getLogger("QTestUtil"); private final static String defaultInitScript = "q_test_init.sql"; private final static String defaultCleanupScript = "q_test_cleanup.sql"; private final String[] testOnlyCommands = new String[] { "crypto" }; private static final String TEST_TMP_DIR_PROPERTY = "test.tmp.dir"; // typically target/tmp private static final String BUILD_DIR_PROPERTY = "build.dir"; // typically target private String testWarehouse; private final String testFiles; protected final String outDir; protected final String logDir; private final TreeMap<String, String> qMap; private final Set<String> qSkipSet; private final Set<String> qSortSet; private final Set<String> qSortQuerySet; private final Set<String> qHashQuerySet; private final Set<String> qSortNHashQuerySet; private final Set<String> qNoSessionReuseQuerySet; private final Set<String> qJavaVersionSpecificOutput; private static final String SORT_SUFFIX = ".sorted"; private final Set<String> srcTables; private final Set<String> srcUDFs; private final MiniClusterType clusterType; private final FsType fsType; private ParseDriver pd; protected Hive db; protected QueryState queryState; protected HiveConf conf; private Driver drv; private BaseSemanticAnalyzer sem; protected final boolean overWrite; private CliDriver cliDriver; private HadoopShims.MiniMrShim mr = null; private HadoopShims.MiniDFSShim dfs = null; private FileSystem fs; private HadoopShims.HdfsEncryptionShim hes = null; private MiniLlapCluster llapCluster = null; private String hadoopVer = null; private QTestSetup setup = null; private TezSessionState tezSessionState = null; private SparkSession sparkSession = null; private boolean isSessionStateStarted = false; private static final String javaVersion = getJavaVersion(); private final String initScript; private final String cleanupScript; private boolean useHBaseMetastore = false; public interface SuiteAddTestFunctor { public void addTestToSuite(TestSuite suite, Object setup, String tName); } private HBaseTestingUtility utility; public static Set<String> getSrcTables() { HashSet<String> srcTables = new HashSet<String>(); // FIXME: moved default value to here...for now // i think this features is never really used from the command line String defaultTestSrcTables = "src,src1,srcbucket,srcbucket2,src_json,src_thrift,src_sequencefile,srcpart,alltypesorc,src_hbase,cbo_t1,cbo_t2,cbo_t3,src_cbo,part,lineitem"; for (String srcTable : System.getProperty("test.src.tables", defaultTestSrcTables).trim().split(",")) { srcTable = srcTable.trim(); if (!srcTable.isEmpty()) { srcTables.add(srcTable); } } if (srcTables.isEmpty()) { throw new RuntimeException("Source tables cannot be empty"); } return srcTables; } /** * Returns the default UDF names which should not be removed when resetting the test database * @return The list of the UDF names not to remove */ private Set<String> getSrcUDFs() { HashSet<String> srcUDFs = new HashSet<String>(); // FIXME: moved default value to here...for now // i think this features is never really used from the command line String defaultTestSrcUDFs = "qtest_get_java_boolean"; for (String srcUDF : System.getProperty("test.src.udfs", defaultTestSrcUDFs).trim().split(",")) { srcUDF = srcUDF.trim(); if (!srcUDF.isEmpty()) { srcUDFs.add(srcUDF); } } if (srcUDFs.isEmpty()) { throw new RuntimeException("Source UDFs cannot be empty"); } return srcUDFs; } public HiveConf getConf() { return conf; } public boolean deleteDirectory(File path) { if (path.exists()) { File[] files = path.listFiles(); for (File file : files) { if (file.isDirectory()) { deleteDirectory(file); } else { file.delete(); } } } return (path.delete()); } public void copyDirectoryToLocal(Path src, Path dest) throws Exception { FileSystem srcFs = src.getFileSystem(conf); FileSystem destFs = dest.getFileSystem(conf); if (srcFs.exists(src)) { FileStatus[] files = srcFs.listStatus(src); for (FileStatus file : files) { String name = file.getPath().getName(); Path dfs_path = file.getPath(); Path local_path = new Path(dest, name); // If this is a source table we do not copy it out if (srcTables.contains(name)) { continue; } if (file.isDirectory()) { if (!destFs.exists(local_path)) { destFs.mkdirs(local_path); } copyDirectoryToLocal(dfs_path, local_path); } else { srcFs.copyToLocalFile(dfs_path, local_path); } } } } static Pattern mapTok = Pattern.compile("(\\.?)(.*)_map_(.*)"); static Pattern reduceTok = Pattern.compile("(.*)(reduce_[^\\.]*)((\\..*)?)"); public void normalizeNames(File path) throws Exception { if (path.isDirectory()) { File[] files = path.listFiles(); for (File file : files) { normalizeNames(file); } } else { Matcher m = reduceTok.matcher(path.getName()); if (m.matches()) { String name = m.group(1) + "reduce" + m.group(3); path.renameTo(new File(path.getParent(), name)); } else { m = mapTok.matcher(path.getName()); if (m.matches()) { String name = m.group(1) + "map_" + m.group(3); path.renameTo(new File(path.getParent(), name)); } } } } public String getOutputDirectory() { return outDir; } public String getLogDirectory() { return logDir; } private String getHadoopMainVersion(String input) { if (input == null) { return null; } Pattern p = Pattern.compile("^(\\d+\\.\\d+).*"); Matcher m = p.matcher(input); if (m.matches()) { return m.group(1); } return null; } public void initConf() throws Exception { String vectorizationEnabled = System.getProperty("test.vectorization.enabled"); if (vectorizationEnabled != null && vectorizationEnabled.equalsIgnoreCase("true")) { conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, true); } if (!useHBaseMetastore) { // Plug verifying metastore in for testing DirectSQL. conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, "org.apache.hadoop.hive.metastore.VerifyingObjectStore"); } else { conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL, HBaseStore.class.getName()); conf.setBoolVar(ConfVars.METASTORE_FASTPATH, true); } if (mr != null) { mr.setupConfiguration(conf); // TODO Ideally this should be done independent of whether mr is setup or not. setFsRelatedProperties(conf, fs.getScheme().equals("file"), fs); } if (llapCluster != null) { Configuration clusterSpecificConf = llapCluster.getClusterSpecificConfiguration(); for (Map.Entry<String, String> confEntry : clusterSpecificConf) { // Conf.get takes care of parameter replacement, iterator.value does not. conf.set(confEntry.getKey(), clusterSpecificConf.get(confEntry.getKey())); } } } private void setFsRelatedProperties(HiveConf conf, boolean isLocalFs, FileSystem fs) { String fsUriString = fs.getUri().toString(); // Different paths if running locally vs a remote fileSystem. Ideally this difference should not exist. Path warehousePath; Path jarPath; Path userInstallPath; if (isLocalFs) { String buildDir = System.getProperty(BUILD_DIR_PROPERTY); Preconditions.checkState(Strings.isNotBlank(buildDir)); Path path = new Path(fsUriString, buildDir); // Create a fake fs root for local fs Path localFsRoot = new Path(path, "localfs"); warehousePath = new Path(localFsRoot, "warehouse"); jarPath = new Path(localFsRoot, "jar"); userInstallPath = new Path(localFsRoot, "user_install"); } else { // TODO Why is this changed from the default in hive-conf? warehousePath = new Path(fsUriString, "/build/ql/test/data/warehouse/"); jarPath = new Path(new Path(fsUriString, "/user"), "hive"); userInstallPath = new Path(fsUriString, "/user"); } warehousePath = fs.makeQualified(warehousePath); jarPath = fs.makeQualified(jarPath); userInstallPath = fs.makeQualified(userInstallPath); conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsUriString); // Remote dirs conf.setVar(ConfVars.METASTOREWAREHOUSE, warehousePath.toString()); conf.setVar(ConfVars.HIVE_JAR_DIRECTORY, jarPath.toString()); conf.setVar(ConfVars.HIVE_USER_INSTALL_DIR, userInstallPath.toString()); // ConfVars.SCRATCHDIR - {test.tmp.dir}/scratchdir // Local dirs // ConfVars.LOCALSCRATCHDIR - {test.tmp.dir}/localscratchdir // TODO Make sure to cleanup created dirs. } private void createRemoteDirs() { assert fs != null; Path warehousePath = fs.makeQualified(new Path(conf.getVar(ConfVars.METASTOREWAREHOUSE))); assert warehousePath != null; Path hiveJarPath = fs.makeQualified(new Path(conf.getVar(ConfVars.HIVE_JAR_DIRECTORY))); assert hiveJarPath != null; Path userInstallPath = fs.makeQualified(new Path(conf.getVar(ConfVars.HIVE_USER_INSTALL_DIR))); assert userInstallPath != null; try { fs.mkdirs(warehousePath); } catch (IOException e) { LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath, e.getMessage()); } try { fs.mkdirs(hiveJarPath); } catch (IOException e) { LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath, e.getMessage()); } try { fs.mkdirs(userInstallPath); } catch (IOException e) { LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath, e.getMessage()); } } private enum CoreClusterType { MR, TEZ, SPARK } public enum FsType { local, hdfs, encrypted_hdfs, } public enum MiniClusterType { mr(CoreClusterType.MR, FsType.hdfs), tez(CoreClusterType.TEZ, FsType.hdfs), tez_local(CoreClusterType.TEZ, FsType.local), spark(CoreClusterType.SPARK, FsType.local), miniSparkOnYarn(CoreClusterType.SPARK, FsType.hdfs), llap(CoreClusterType.TEZ, FsType.hdfs), llap_local(CoreClusterType.TEZ, FsType.local), none(CoreClusterType.MR, FsType.local); private final CoreClusterType coreClusterType; private final FsType defaultFsType; MiniClusterType(CoreClusterType coreClusterType, FsType defaultFsType) { this.coreClusterType = coreClusterType; this.defaultFsType = defaultFsType; } public CoreClusterType getCoreClusterType() { return coreClusterType; } public FsType getDefaultFsType() { return defaultFsType; } public static MiniClusterType valueForString(String type) { // Replace this with valueOf. if (type.equals("miniMR")) { return mr; } else if (type.equals("tez")) { return tez; } else if (type.equals("tez_local")) { return tez_local; } else if (type.equals("spark")) { return spark; } else if (type.equals("miniSparkOnYarn")) { return miniSparkOnYarn; } else if (type.equals("llap")) { return llap; } else if (type.equals("llap_local")) { return llap_local; } else { return none; } } } private String getKeyProviderURI() { // Use the target directory if it is not specified String HIVE_ROOT = AbstractCliConfig.HIVE_ROOT; String keyDir = HIVE_ROOT + "ql/target/"; // put the jks file in the current test path only for test purpose return "jceks://file" + new Path(keyDir, "test.jks").toUri(); } private void startMiniHBaseCluster() throws Exception { Configuration hbaseConf = HBaseConfiguration.create(); hbaseConf.setInt("hbase.master.info.port", -1); utility = new HBaseTestingUtility(hbaseConf); utility.startMiniCluster(); conf = new HiveConf(utility.getConfiguration(), Driver.class); HBaseAdmin admin = utility.getHBaseAdmin(); // Need to use reflection here to make compilation pass since HBaseIntegrationTests // is not compiled in hadoop-1. All HBaseMetastore tests run under hadoop-2, so this // guarantee HBaseIntegrationTests exist when we hitting this code path java.lang.reflect.Method initHBaseMetastoreMethod = Class .forName("org.apache.hadoop.hive.metastore.hbase.HBaseStoreTestUtil") .getMethod("initHBaseMetastore", HBaseAdmin.class, HiveConf.class); initHBaseMetastoreMethod.invoke(null, admin, conf); conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL, HBaseStore.class.getName()); conf.setBoolVar(ConfVars.METASTORE_FASTPATH, true); } public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, String cleanupScript, boolean useHBaseMetastore, boolean withLlapIo) throws Exception { this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, withLlapIo, null); } public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadzoopVer, String initScript, String cleanupScript, boolean useHBaseMetastore, boolean withLlapIo, FsType fsType) throws Exception { LOG.info( "Setting up QTestUtil with outDir={}, logDir={}, clusterType={}, confDir={}," + " hadoopVer={}, initScript={}, cleanupScript={}, useHbaseMetaStore={}, withLlapIo={}," + " fsType={}", outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, withLlapIo, fsType); Preconditions.checkNotNull(clusterType, "ClusterType cannot be null"); if (fsType != null) { this.fsType = fsType; } else { this.fsType = clusterType.getDefaultFsType(); } this.outDir = outDir; this.logDir = logDir; this.useHBaseMetastore = useHBaseMetastore; this.srcTables = getSrcTables(); this.srcUDFs = getSrcUDFs(); // HIVE-14443 move this fall-back logic to CliConfigs if (confDir != null && !confDir.isEmpty()) { HiveConf.setHiveSiteLocation( new URL("file://" + new File(confDir).toURI().getPath() + "/hive-site.xml")); System.out.println("Setting hive-site: " + HiveConf.getHiveSiteLocation()); } queryState = new QueryState.Builder().withHiveConf(new HiveConf(Driver.class)).build(); if (useHBaseMetastore) { startMiniHBaseCluster(); } else { conf = queryState.getConf(); } this.hadoopVer = getHadoopMainVersion(hadoopVer); qMap = new TreeMap<String, String>(); qSkipSet = new HashSet<String>(); qSortSet = new HashSet<String>(); qSortQuerySet = new HashSet<String>(); qHashQuerySet = new HashSet<String>(); qSortNHashQuerySet = new HashSet<String>(); qNoSessionReuseQuerySet = new HashSet<String>(); qJavaVersionSpecificOutput = new HashSet<String>(); this.clusterType = clusterType; HadoopShims shims = ShimLoader.getHadoopShims(); setupFileSystem(shims); setup = new QTestSetup(); setup.preTest(conf); setupMiniCluster(shims, confDir); initConf(); if (withLlapIo && (clusterType == MiniClusterType.none)) { LOG.info("initializing llap IO"); LlapProxy.initializeLlapIo(conf); } // Use the current directory if it is not specified String dataDir = conf.get("test.data.files"); if (dataDir == null) { dataDir = new File(".").getAbsolutePath() + "/data/files"; } testFiles = dataDir; // Use the current directory if it is not specified String scriptsDir = conf.get("test.data.scripts"); if (scriptsDir == null) { scriptsDir = new File(".").getAbsolutePath() + "/data/scripts"; } this.initScript = scriptsDir + File.separator + initScript; this.cleanupScript = scriptsDir + File.separator + cleanupScript; overWrite = "true".equalsIgnoreCase(System.getProperty("test.output.overwrite")); init(); } private void setupFileSystem(HadoopShims shims) throws IOException { if (fsType == FsType.local) { fs = FileSystem.getLocal(conf); } else if (fsType == FsType.hdfs || fsType == FsType.encrypted_hdfs) { int numDataNodes = 4; if (fsType == FsType.encrypted_hdfs) { // Set the security key provider so that the MiniDFS cluster is initialized // with encryption conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI()); conf.setInt("fs.trash.interval", 50); dfs = shims.getMiniDfs(conf, numDataNodes, true, null); fs = dfs.getFileSystem(); // set up the java key provider for encrypted hdfs cluster hes = shims.createHdfsEncryptionShim(fs, conf); LOG.info("key provider is initialized"); } else { dfs = shims.getMiniDfs(conf, numDataNodes, true, null); fs = dfs.getFileSystem(); } } else { throw new IllegalArgumentException("Unknown or unhandled fsType [" + fsType + "]"); } } private void setupMiniCluster(HadoopShims shims, String confDir) throws IOException { String uriString = fs.getUri().toString(); if (clusterType.getCoreClusterType() == CoreClusterType.TEZ) { if (confDir != null && !confDir.isEmpty()) { conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + "/tez-site.xml")); } int numTrackers = 2; if (EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType)) { llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir); } else { } if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local).contains(clusterType)) { mr = shims.getLocalMiniTezCluster(conf, clusterType == MiniClusterType.llap_local); } else { mr = shims.getMiniTezCluster(conf, numTrackers, uriString, EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType)); } } else if (clusterType == MiniClusterType.miniSparkOnYarn) { mr = shims.getMiniSparkCluster(conf, 2, uriString, 1); } else if (clusterType == MiniClusterType.mr) { mr = shims.getMiniMrCluster(conf, 2, uriString, 1); } } public void shutdown() throws Exception { if (System.getenv(QTEST_LEAVE_FILES) == null) { cleanUp(); } if (clusterType.getCoreClusterType() == CoreClusterType.TEZ) { SessionState.get().getTezSession().close(false); } setup.tearDown(); if (sparkSession != null) { try { SparkSessionManagerImpl.getInstance().closeSession(sparkSession); } catch (Exception ex) { LOG.error("Error closing spark session.", ex); } finally { sparkSession = null; } } if (useHBaseMetastore) { utility.shutdownMiniCluster(); } if (mr != null) { mr.shutdown(); mr = null; } FileSystem.closeAll(); if (dfs != null) { dfs.shutdown(); dfs = null; } Hive.closeCurrent(); } public String readEntireFileIntoString(File queryFile) throws IOException { InputStreamReader isr = new InputStreamReader(new BufferedInputStream(new FileInputStream(queryFile)), QTestUtil.UTF_8); StringWriter sw = new StringWriter(); try { IOUtils.copy(isr, sw); } finally { if (isr != null) { isr.close(); } } return sw.toString(); } public void addFile(String queryFile) throws IOException { addFile(queryFile, false); } public void addFile(String queryFile, boolean partial) throws IOException { addFile(new File(queryFile)); } public void addFile(File qf) throws IOException { addFile(qf, false); } public void addFile(File qf, boolean partial) throws IOException { String query = readEntireFileIntoString(qf); qMap.put(qf.getName(), query); if (partial) return; if (checkHadoopVersionExclude(qf.getName(), query)) { qSkipSet.add(qf.getName()); } if (checkNeedJavaSpecificOutput(qf.getName(), query)) { qJavaVersionSpecificOutput.add(qf.getName()); } if (matches(SORT_BEFORE_DIFF, query)) { qSortSet.add(qf.getName()); } else if (matches(SORT_QUERY_RESULTS, query)) { qSortQuerySet.add(qf.getName()); } else if (matches(HASH_QUERY_RESULTS, query)) { qHashQuerySet.add(qf.getName()); } else if (matches(SORT_AND_HASH_QUERY_RESULTS, query)) { qSortNHashQuerySet.add(qf.getName()); } if (matches(NO_SESSION_REUSE, query)) { qNoSessionReuseQuerySet.add(qf.getName()); } } private static final Pattern SORT_BEFORE_DIFF = Pattern.compile("-- SORT_BEFORE_DIFF"); private static final Pattern SORT_QUERY_RESULTS = Pattern.compile("-- SORT_QUERY_RESULTS"); private static final Pattern HASH_QUERY_RESULTS = Pattern.compile("-- HASH_QUERY_RESULTS"); private static final Pattern SORT_AND_HASH_QUERY_RESULTS = Pattern.compile("-- SORT_AND_HASH_QUERY_RESULTS"); private static final Pattern NO_SESSION_REUSE = Pattern.compile("-- NO_SESSION_REUSE"); private boolean matches(Pattern pattern, String query) { Matcher matcher = pattern.matcher(query); if (matcher.find()) { return true; } return false; } private boolean checkHadoopVersionExclude(String fileName, String query) { // Look for a hint to not run a test on some Hadoop versions Pattern pattern = Pattern.compile("-- (EX|IN)CLUDE_HADOOP_MAJOR_VERSIONS\\((.*)\\)"); boolean excludeQuery = false; boolean includeQuery = false; Set<String> versionSet = new HashSet<String>(); String hadoopVer = ShimLoader.getMajorVersion(); Matcher matcher = pattern.matcher(query); // Each qfile may include at most one INCLUDE or EXCLUDE directive. // // If a qfile contains an INCLUDE directive, and hadoopVer does // not appear in the list of versions to include, then the qfile // is skipped. // // If a qfile contains an EXCLUDE directive, and hadoopVer is // listed in the list of versions to EXCLUDE, then the qfile is // skipped. // // Otherwise, the qfile is included. if (matcher.find()) { String prefix = matcher.group(1); if ("EX".equals(prefix)) { excludeQuery = true; } else { includeQuery = true; } String versions = matcher.group(2); for (String s : versions.split("\\,")) { s = s.trim(); versionSet.add(s); } } if (matcher.find()) { //2nd match is not supposed to be there String message = "QTestUtil: qfile " + fileName + " contains more than one reference to (EX|IN)CLUDE_HADOOP_MAJOR_VERSIONS"; throw new UnsupportedOperationException(message); } if (excludeQuery && versionSet.contains(hadoopVer)) { System.out.println("QTestUtil: " + fileName + " EXCLUDE list contains Hadoop Version " + hadoopVer + ". Skipping..."); return true; } else if (includeQuery && !versionSet.contains(hadoopVer)) { System.out.println("QTestUtil: " + fileName + " INCLUDE list does not contain Hadoop Version " + hadoopVer + ". Skipping..."); return true; } return false; } private boolean checkNeedJavaSpecificOutput(String fileName, String query) { Pattern pattern = Pattern.compile("-- JAVA_VERSION_SPECIFIC_OUTPUT"); Matcher matcher = pattern.matcher(query); if (matcher.find()) { System.out.println("Test is flagged to generate Java version specific " + "output. Since we are using Java version " + javaVersion + ", we will generated Java " + javaVersion + " specific " + "output file for query file " + fileName); return true; } return false; } /** * Get formatted Java version to include minor version, but * exclude patch level. * * @return Java version formatted as major_version.minor_version */ private static String getJavaVersion() { String version = System.getProperty("java.version"); if (version == null) { throw new NullPointerException("No java version could be determined " + "from system properties"); } // "java version" system property is formatted // major_version.minor_version.patch_level. // Find second dot, instead of last dot, to be safe int pos = version.indexOf('.'); pos = version.indexOf('.', pos + 1); return version.substring(0, pos); } /** * Clear out any side effects of running tests */ public void clearPostTestEffects() throws Exception { setup.postTest(conf); } public void clearKeysCreatedInTests() { if (hes == null) { return; } try { for (String keyAlias : hes.getKeys()) { hes.deleteKey(keyAlias); } } catch (IOException e) { LOG.error("Fail to clean the keys created in test due to the error", e); } } public void clearUDFsCreatedDuringTests() throws Exception { if (System.getenv(QTEST_LEAVE_FILES) != null) { return; } // Delete functions created by the tests // It is enough to remove functions from the default database, other databases are dropped for (String udfName : db.getFunctions(DEFAULT_DATABASE_NAME, ".*")) { if (!srcUDFs.contains(udfName)) { db.dropFunction(DEFAULT_DATABASE_NAME, udfName); } } } /** * Clear out any side effects of running tests */ public void clearTablesCreatedDuringTests() throws Exception { if (System.getenv(QTEST_LEAVE_FILES) != null) { return; } conf.set("hive.metastore.filter.hook", "org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl"); db = Hive.get(conf); // Delete any tables other than the source tables // and any databases other than the default database. for (String dbName : db.getAllDatabases()) { SessionState.get().setCurrentDatabase(dbName); for (String tblName : db.getAllTables()) { if (!DEFAULT_DATABASE_NAME.equals(dbName) || !srcTables.contains(tblName)) { Table tblObj = null; try { tblObj = db.getTable(tblName); } catch (InvalidTableException e) { LOG.warn("Trying to drop table " + e.getTableName() + ". But it does not exist."); continue; } // dropping index table can not be dropped directly. Dropping the base // table will automatically drop all its index table if (tblObj.isIndexTable()) { continue; } db.dropTable(dbName, tblName, true, true, fsType == FsType.encrypted_hdfs); } else { // this table is defined in srcTables, drop all indexes on it List<Index> indexes = db.getIndexes(dbName, tblName, (short) -1); if (indexes != null && indexes.size() > 0) { for (Index index : indexes) { db.dropIndex(dbName, tblName, index.getIndexName(), true, true); } } } } if (!DEFAULT_DATABASE_NAME.equals(dbName)) { // Drop cascade, functions dropped by cascade db.dropDatabase(dbName, true, true, true); } } // delete remaining directories for external tables (can affect stats for following tests) try { Path p = new Path(testWarehouse); FileSystem fileSystem = p.getFileSystem(conf); if (fileSystem.exists(p)) { for (FileStatus status : fileSystem.listStatus(p)) { if (status.isDirectory() && !srcTables.contains(status.getPath().getName())) { fileSystem.delete(status.getPath(), true); } } } } catch (IllegalArgumentException e) { // ignore.. provides invalid url sometimes intentionally } SessionState.get().setCurrentDatabase(DEFAULT_DATABASE_NAME); List<String> roleNames = db.getAllRoleNames(); for (String roleName : roleNames) { if (!"PUBLIC".equalsIgnoreCase(roleName) && !"ADMIN".equalsIgnoreCase(roleName)) { db.dropRole(roleName); } } } /** * Clear out any side effects of running tests */ public void clearTestSideEffects() throws Exception { if (System.getenv(QTEST_LEAVE_FILES) != null) { return; } // allocate and initialize a new conf since a test can // modify conf by using 'set' commands conf = new HiveConf(Driver.class); initConf(); initConfFromSetup(); // renew the metastore since the cluster type is unencrypted db = Hive.get(conf); // propagate new conf to meta store clearTablesCreatedDuringTests(); clearUDFsCreatedDuringTests(); clearKeysCreatedInTests(); } protected void initConfFromSetup() throws Exception { setup.preTest(conf); } public void cleanUp() throws Exception { cleanUp(null); } public void cleanUp(String tname) throws Exception { boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname); if (!isSessionStateStarted) { startSessionState(canReuseSession); } if (System.getenv(QTEST_LEAVE_FILES) != null) { return; } clearTablesCreatedDuringTests(); clearUDFsCreatedDuringTests(); clearKeysCreatedInTests(); File cleanupFile = new File(cleanupScript); if (cleanupFile.isFile()) { String cleanupCommands = readEntireFileIntoString(cleanupFile); LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands); if (cliDriver == null) { cliDriver = new CliDriver(); } SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true); int result = cliDriver.processLine(cleanupCommands); if (result != 0) { LOG.error("Failed during cleanup processLine with code={}. Ignoring", result); // TODO Convert this to an Assert.fail once HIVE-14682 is fixed } SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", false); } else { LOG.info("No cleanup script detected. Skipping."); } // delete any contents in the warehouse dir Path p = new Path(testWarehouse); FileSystem fs = p.getFileSystem(conf); try { FileStatus[] ls = fs.listStatus(p); for (int i = 0; (ls != null) && (i < ls.length); i++) { fs.delete(ls[i].getPath(), true); } } catch (FileNotFoundException e) { // Best effort } // TODO: Clean up all the other paths that are created. FunctionRegistry.unregisterTemporaryUDF("test_udaf"); FunctionRegistry.unregisterTemporaryUDF("test_error"); } protected void runCreateTableCmd(String createTableCmd) throws Exception { int ecode = 0; ecode = drv.run(createTableCmd).getResponseCode(); if (ecode != 0) { throw new Exception("create table command: " + createTableCmd + " failed with exit code= " + ecode); } return; } protected void runCmd(String cmd) throws Exception { int ecode = 0; ecode = drv.run(cmd).getResponseCode(); drv.close(); if (ecode != 0) { throw new Exception("command: " + cmd + " failed with exit code= " + ecode); } return; } public void createSources() throws Exception { createSources(null); } public void createSources(String tname) throws Exception { boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname); if (!isSessionStateStarted) { startSessionState(canReuseSession); } if (cliDriver == null) { cliDriver = new CliDriver(); } cliDriver.processLine("set test.data.dir=" + testFiles + ";"); File scriptFile = new File(this.initScript); if (!scriptFile.isFile()) { LOG.info("No init script detected. Skipping"); return; } conf.setBoolean("hive.test.init.phase", true); String initCommands = readEntireFileIntoString(scriptFile); LOG.info("Initial setup (" + initScript + "):\n" + initCommands); int result = cliDriver.processLine(initCommands); LOG.info("Result from cliDrriver.processLine in createSources=" + result); if (result != 0) { Assert.fail("Failed during createSources processLine with code=" + result); } conf.setBoolean("hive.test.init.phase", false); } public void init() throws Exception { // Create remote dirs once. if (mr != null) { createRemoteDirs(); } testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); String execEngine = conf.get("hive.execution.engine"); conf.set("hive.execution.engine", "mr"); SessionState.start(conf); conf.set("hive.execution.engine", execEngine); db = Hive.get(conf); drv = new Driver(conf); drv.init(); pd = new ParseDriver(); sem = new SemanticAnalyzer(queryState); } public void init(String tname) throws Exception { cleanUp(tname); createSources(tname); cliDriver.processCmd("set hive.cli.print.header=true;"); } public void cliInit(String tname) throws Exception { cliInit(tname, true); } public String cliInit(String tname, boolean recreate) throws Exception { if (recreate) { cleanUp(tname); createSources(tname); } HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, "org.apache.hadoop.hive.ql.security.DummyAuthenticator"); Utilities.clearWorkMap(conf); CliSessionState ss = createSessionState(); assert ss != null; ss.in = System.in; String outFileExtension = getOutFileExtension(tname); String stdoutName = null; if (outDir != null) { // TODO: why is this needed? File qf = new File(outDir, tname); stdoutName = qf.getName().concat(outFileExtension); } else { stdoutName = tname + outFileExtension; } File outf = new File(logDir, stdoutName); OutputStream fo = new BufferedOutputStream(new FileOutputStream(outf)); if (qSortQuerySet.contains(tname)) { ss.out = new SortPrintStream(fo, "UTF-8"); } else if (qHashQuerySet.contains(tname)) { ss.out = new DigestPrintStream(fo, "UTF-8"); } else if (qSortNHashQuerySet.contains(tname)) { ss.out = new SortAndDigestPrintStream(fo, "UTF-8"); } else { ss.out = new PrintStream(fo, true, "UTF-8"); } ss.err = new CachingPrintStream(fo, true, "UTF-8"); ss.setIsSilent(true); SessionState oldSs = SessionState.get(); boolean canReuseSession = !qNoSessionReuseQuerySet.contains(tname); if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusterType.TEZ) { // Copy the tezSessionState from the old CliSessionState. tezSessionState = oldSs.getTezSession(); oldSs.setTezSession(null); ss.setTezSession(tezSessionState); oldSs.close(); } if (oldSs != null && clusterType.getCoreClusterType() == CoreClusterType.SPARK) { sparkSession = oldSs.getSparkSession(); ss.setSparkSession(sparkSession); oldSs.setSparkSession(null); oldSs.close(); } if (oldSs != null && oldSs.out != null && oldSs.out != System.out) { oldSs.out.close(); } SessionState.start(ss); cliDriver = new CliDriver(); if (tname.equals("init_file.q")) { ss.initFiles.add(AbstractCliConfig.HIVE_ROOT + "/data/scripts/test_init_file.sql"); } cliDriver.processInitFiles(ss); return outf.getAbsolutePath(); } private CliSessionState createSessionState() { return new CliSessionState(conf) { @Override public void setSparkSession(SparkSession sparkSession) { super.setSparkSession(sparkSession); if (sparkSession != null) { try { // Wait a little for cluster to init, at most 4 minutes long endTime = System.currentTimeMillis() + 240000; while (sparkSession.getMemoryAndCores().getSecond() <= 1) { if (System.currentTimeMillis() >= endTime) { String msg = "Timed out waiting for Spark cluster to init"; throw new IllegalStateException(msg); } Thread.sleep(100); } } catch (Exception e) { String msg = "Error trying to obtain executor info: " + e; LOG.error(msg, e); throw new IllegalStateException(msg, e); } } } }; } private CliSessionState startSessionState(boolean canReuseSession) throws IOException { HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, "org.apache.hadoop.hive.ql.security.DummyAuthenticator"); String execEngine = conf.get("hive.execution.engine"); conf.set("hive.execution.engine", "mr"); CliSessionState ss = createSessionState(); assert ss != null; ss.in = System.in; ss.out = System.out; ss.err = System.out; SessionState oldSs = SessionState.get(); if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusterType.TEZ) { // Copy the tezSessionState from the old CliSessionState. tezSessionState = oldSs.getTezSession(); ss.setTezSession(tezSessionState); oldSs.setTezSession(null); oldSs.close(); } if (oldSs != null && clusterType.getCoreClusterType() == CoreClusterType.SPARK) { sparkSession = oldSs.getSparkSession(); ss.setSparkSession(sparkSession); oldSs.setSparkSession(null); oldSs.close(); } if (oldSs != null && oldSs.out != null && oldSs.out != System.out) { oldSs.out.close(); } SessionState.start(ss); isSessionStateStarted = true; conf.set("hive.execution.engine", execEngine); return ss; } public int executeAdhocCommand(String q) { if (!q.contains(";")) { return -1; } String q1 = q.split(";")[0] + ";"; LOG.debug("Executing " + q1); return cliDriver.processLine(q1); } public int executeOne(String tname) { String q = qMap.get(tname); if (q.indexOf(";") == -1) { return -1; } String q1 = q.substring(0, q.indexOf(";") + 1); String qrest = q.substring(q.indexOf(";") + 1); qMap.put(tname, qrest); System.out.println("Executing " + q1); return cliDriver.processLine(q1); } public int execute(String tname) { try { return drv.run(qMap.get(tname)).getResponseCode(); } catch (CommandNeedRetryException e) { LOG.error("driver failed to run the command: " + tname + " due to the exception: ", e); e.printStackTrace(); return -1; } } public int executeClient(String tname1, String tname2) { String commands = getCommand(tname1) + CRLF + getCommand(tname2); return executeClientInternal(commands); } public int executeClient(String tname) { return executeClientInternal(getCommand(tname)); } private int executeClientInternal(String commands) { List<String> cmds = CliDriver.splitSemiColon(commands); int rc = 0; String command = ""; for (String oneCmd : cmds) { if (StringUtils.endsWith(oneCmd, "\\")) { command += StringUtils.chop(oneCmd) + "\\;"; continue; } else { if (isHiveCommand(oneCmd)) { command = oneCmd; } else { command += oneCmd; } } if (StringUtils.isBlank(command)) { continue; } if (isCommandUsedForTesting(command)) { rc = executeTestCommand(command); } else { rc = cliDriver.processLine(command); } if (rc != 0 && !ignoreErrors()) { break; } command = ""; } if (rc == 0 && SessionState.get() != null) { SessionState.get().setLastCommand(null); // reset } return rc; } /** * This allows a .q file to continue executing after a statement runs into an error which is convenient * if you want to use another hive cmd after the failure to sanity check the state of the system. */ private boolean ignoreErrors() { return conf.getBoolVar(HiveConf.ConfVars.CLIIGNOREERRORS); } private boolean isHiveCommand(String command) { String[] cmd = command.trim().split("\\s+"); if (HiveCommand.find(cmd) != null) { return true; } else if (HiveCommand.find(cmd, HiveCommand.ONLY_FOR_TESTING) != null) { return true; } else { return false; } } private int executeTestCommand(final String command) { String commandName = command.trim().split("\\s+")[0]; String commandArgs = command.trim().substring(commandName.length()); if (commandArgs.endsWith(";")) { commandArgs = StringUtils.chop(commandArgs); } //replace ${hiveconf:hive.metastore.warehouse.dir} with actual dir if existed. //we only want the absolute path, so remove the header, such as hdfs://localhost:57145 String wareHouseDir = SessionState.get().getConf().getVar(ConfVars.METASTOREWAREHOUSE) .replaceAll("^[a-zA-Z]+://.*?:\\d+", ""); commandArgs = commandArgs.replaceAll("\\$\\{hiveconf:hive\\.metastore\\.warehouse\\.dir\\}", wareHouseDir); if (SessionState.get() != null) { SessionState.get().setLastCommand(commandName + " " + commandArgs.trim()); } enableTestOnlyCmd(SessionState.get().getConf()); try { CommandProcessor proc = getTestCommand(commandName); if (proc != null) { CommandProcessorResponse response = proc.run(commandArgs.trim()); int rc = response.getResponseCode(); if (rc != 0) { SessionState.getConsole().printError(response.toString(), response.getException() != null ? Throwables.getStackTraceAsString(response.getException()) : ""); } return rc; } else { throw new RuntimeException("Could not get CommandProcessor for command: " + commandName); } } catch (Exception e) { throw new RuntimeException("Could not execute test command", e); } } private CommandProcessor getTestCommand(final String commandName) throws SQLException { HiveCommand testCommand = HiveCommand.find(new String[] { commandName }, HiveCommand.ONLY_FOR_TESTING); if (testCommand == null) { return null; } return CommandProcessorFactory.getForHiveCommandInternal(new String[] { commandName }, SessionState.get().getConf(), testCommand.isOnlyForTesting()); } private void enableTestOnlyCmd(HiveConf conf) { StringBuilder securityCMDs = new StringBuilder( conf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST)); for (String c : testOnlyCommands) { securityCMDs.append(","); securityCMDs.append(c); } conf.set(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST.toString(), securityCMDs.toString()); } private boolean isCommandUsedForTesting(final String command) { String commandName = command.trim().split("\\s+")[0]; HiveCommand testCommand = HiveCommand.find(new String[] { commandName }, HiveCommand.ONLY_FOR_TESTING); return testCommand != null; } private String getCommand(String tname) { String commands = qMap.get(tname); StringBuilder newCommands = new StringBuilder(commands.length()); int lastMatchEnd = 0; Matcher commentMatcher = Pattern.compile("^--.*$", Pattern.MULTILINE).matcher(commands); // remove the comments while (commentMatcher.find()) { newCommands.append(commands.substring(lastMatchEnd, commentMatcher.start())); lastMatchEnd = commentMatcher.end(); } newCommands.append(commands.substring(lastMatchEnd, commands.length())); commands = newCommands.toString(); return commands; } public boolean shouldBeSkipped(String tname) { return qSkipSet.contains(tname); } private String getOutFileExtension(String fname) { String outFileExtension = ".out"; if (qJavaVersionSpecificOutput.contains(fname)) { outFileExtension = ".java" + javaVersion + ".out"; } return outFileExtension; } public void convertSequenceFileToTextFile() throws Exception { // Create an instance of hive in order to create the tables testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); db = Hive.get(conf); // Move all data from dest4_sequencefile to dest4 drv.run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*"); // Drop dest4_sequencefile db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "dest4_sequencefile", true, true); } public QTestProcessExecResult checkNegativeResults(String tname, Exception e) throws Exception { String outFileExtension = getOutFileExtension(tname); File qf = new File(outDir, tname); String expf = outPath(outDir.toString(), tname.concat(outFileExtension)); File outf = null; outf = new File(logDir); outf = new File(outf, qf.getName().concat(outFileExtension)); FileWriter outfd = new FileWriter(outf); if (e instanceof ParseException) { outfd.write("Parse Error: "); } else if (e instanceof SemanticException) { outfd.write("Semantic Exception: \n"); } else { throw e; } outfd.write(e.getMessage()); outfd.close(); QTestProcessExecResult result = executeDiffCommand(outf.getPath(), expf, false, qSortSet.contains(qf.getName())); if (overWrite) { overwriteResults(outf.getPath(), expf); return QTestProcessExecResult.createWithoutOutput(0); } return result; } public QTestProcessExecResult checkParseResults(String tname, ASTNode tree) throws Exception { if (tree != null) { String outFileExtension = getOutFileExtension(tname); File parseDir = new File(outDir, "parse"); String expf = outPath(parseDir.toString(), tname.concat(outFileExtension)); File outf = null; outf = new File(logDir); outf = new File(outf, tname.concat(outFileExtension)); FileWriter outfd = new FileWriter(outf); outfd.write(tree.toStringTree()); outfd.close(); QTestProcessExecResult exitVal = executeDiffCommand(outf.getPath(), expf, false, false); if (overWrite) { overwriteResults(outf.getPath(), expf); return QTestProcessExecResult.createWithoutOutput(0); } return exitVal; } else { throw new Exception("Parse tree is null"); } } /** * Given the current configurations (e.g., hadoop version and execution mode), return * the correct file name to compare with the current test run output. * @param outDir The directory where the reference log files are stored. * @param testName The test file name (terminated by ".out"). * @return The file name appended with the configuration values if it exists. */ public String outPath(String outDir, String testName) { String ret = (new File(outDir, testName)).getPath(); // List of configurations. Currently the list consists of hadoop version and execution mode only List<String> configs = new ArrayList<String>(); configs.add(this.hadoopVer); Deque<String> stack = new LinkedList<String>(); StringBuilder sb = new StringBuilder(); sb.append(testName); stack.push(sb.toString()); // example file names are input1.q.out_0.20.0_minimr or input2.q.out_0.17 for (String s : configs) { sb.append('_'); sb.append(s); stack.push(sb.toString()); } while (stack.size() > 0) { String fileName = stack.pop(); File f = new File(outDir, fileName); if (f.exists()) { ret = f.getPath(); break; } } return ret; } private Pattern[] toPattern(String[] patternStrs) { Pattern[] patterns = new Pattern[patternStrs.length]; for (int i = 0; i < patternStrs.length; i++) { patterns[i] = Pattern.compile(patternStrs[i]); } return patterns; } private void maskPatterns(Pattern[] patterns, String fname) throws Exception { String maskPattern = "#### A masked pattern was here ####"; String partialMaskPattern = "#### A PARTIAL masked pattern was here ####"; String line; BufferedReader in; BufferedWriter out; File file = new File(fname); File fileOrig = new File(fname + ".orig"); FileUtils.copyFile(file, fileOrig); in = new BufferedReader(new InputStreamReader(new FileInputStream(fileOrig), "UTF-8")); out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "UTF-8")); boolean lastWasMasked = false; boolean partialMaskWasMatched = false; Matcher matcher; while (null != (line = in.readLine())) { if (fsType == FsType.encrypted_hdfs) { for (Pattern pattern : partialReservedPlanMask) { matcher = pattern.matcher(line); if (matcher.find()) { line = partialMaskPattern + " " + matcher.group(0); partialMaskWasMatched = true; break; } } } if (!partialMaskWasMatched) { for (Pair<Pattern, String> pair : patternsWithMaskComments) { Pattern pattern = pair.getLeft(); String maskComment = pair.getRight(); matcher = pattern.matcher(line); if (matcher.find()) { line = matcher.replaceAll(maskComment); partialMaskWasMatched = true; break; } } for (Pattern pattern : patterns) { line = pattern.matcher(line).replaceAll(maskPattern); } } if (line.equals(maskPattern)) { // We're folding multiple masked lines into one. if (!lastWasMasked) { out.write(line); out.write("\n"); lastWasMasked = true; partialMaskWasMatched = false; } } else { out.write(line); out.write("\n"); lastWasMasked = false; partialMaskWasMatched = false; } } in.close(); out.close(); } private final Pattern[] planMask = toPattern(new String[] { ".*file:.*", ".*pfile:.*", ".*hdfs:.*", ".*/tmp/.*", ".*invalidscheme:.*", ".*lastUpdateTime.*", ".*lastAccessTime.*", ".*lastModifiedTime.*", ".*[Oo]wner.*", ".*CreateTime.*", ".*LastAccessTime.*", ".*Location.*", ".*LOCATION '.*", ".*transient_lastDdlTime.*", ".*last_modified_.*", ".*at org.*", ".*at sun.*", ".*at java.*", ".*at junit.*", ".*Caused by:.*", ".*LOCK_QUERYID:.*", ".*LOCK_TIME:.*", ".*grantTime.*", ".*[.][.][.] [0-9]* more.*", ".*job_[0-9_]*.*", ".*job_local[0-9_]*.*", ".*USING 'java -cp.*", "^Deleted.*", ".*DagName:.*", ".*DagId:.*", ".*Input:.*/data/files/.*", ".*Output:.*/data/files/.*", ".*total number of created files now is.*", ".*.hive-staging.*", "pk_-?[0-9]*_[0-9]*_[0-9]*", "fk_-?[0-9]*_[0-9]*_[0-9]*", "uk_-?[0-9]*_[0-9]*_[0-9]*", "nn_-?[0-9]*_[0-9]*_[0-9]*", ".*at com\\.sun\\.proxy.*", ".*at com\\.jolbox.*", ".*at com\\.zaxxer.*", "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*", "^Repair: Added partition to metastore.*" }); private final Pattern[] partialReservedPlanMask = toPattern( new String[] { "data/warehouse/(.*?/)+\\.hive-staging" // the directory might be db/table/partition //TODO: add more expected test result here }); /* This list may be modified by specific cli drivers to mask strings that change on every test */ private final List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<Pair<Pattern, String>>() { { add(toPatternPair("(pblob|s3.?|swift|wasb.?).*hive-staging.*", "### BLOBSTORE_STAGING_PATH ###")); } }; private Pair<Pattern, String> toPatternPair(String patternStr, String maskComment) { return ImmutablePair.of(Pattern.compile(patternStr), maskComment); } public void addPatternWithMaskComment(String patternStr, String maskComment) { patternsWithMaskComments.add(toPatternPair(patternStr, maskComment)); } public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception { assert (qMap.containsKey(tname)); String outFileExtension = getOutFileExtension(tname); String outFileName = outPath(outDir, tname + outFileExtension); File f = new File(logDir, tname + outFileExtension); maskPatterns(planMask, f.getPath()); QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(), outFileName, false, qSortSet.contains(tname)); if (overWrite) { overwriteResults(f.getPath(), outFileName); return QTestProcessExecResult.createWithoutOutput(0); } return exitVal; } public QTestProcessExecResult checkCompareCliDriverResults(String tname, List<String> outputs) throws Exception { assert outputs.size() > 1; maskPatterns(planMask, outputs.get(0)); for (int i = 1; i < outputs.size(); ++i) { maskPatterns(planMask, outputs.get(i)); QTestProcessExecResult result = executeDiffCommand(outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname)); if (result.getReturnCode() != 0) { System.out.println("Files don't match: " + outputs.get(i - 1) + " and " + outputs.get(i)); return result; } } return QTestProcessExecResult.createWithoutOutput(0); } private static void overwriteResults(String inFileName, String outFileName) throws Exception { // This method can be replaced with Files.copy(source, target, REPLACE_EXISTING) // once Hive uses JAVA 7. System.out.println("Overwriting results " + inFileName + " to " + outFileName); int result = executeCmd(new String[] { "cp", getQuotedString(inFileName), getQuotedString(outFileName) }) .getReturnCode(); if (result != 0) throw new IllegalStateException( "Unexpected error while overwriting " + inFileName + " with " + outFileName); } private static QTestProcessExecResult executeDiffCommand(String inFileName, String outFileName, boolean ignoreWhiteSpace, boolean sortResults) throws Exception { QTestProcessExecResult result; if (sortResults) { // sort will try to open the output file in write mode on windows. We need to // close it first. SessionState ss = SessionState.get(); if (ss != null && ss.out != null && ss.out != System.out) { ss.out.close(); } String inSorted = inFileName + SORT_SUFFIX; String outSorted = outFileName + SORT_SUFFIX; sortFiles(inFileName, inSorted); sortFiles(outFileName, outSorted); inFileName = inSorted; outFileName = outSorted; } ArrayList<String> diffCommandArgs = new ArrayList<String>(); diffCommandArgs.add("diff"); // Text file comparison diffCommandArgs.add("-a"); // Ignore changes in the amount of white space if (ignoreWhiteSpace) { diffCommandArgs.add("-b"); } // Add files to compare to the arguments list diffCommandArgs.add(getQuotedString(inFileName)); diffCommandArgs.add(getQuotedString(outFileName)); result = executeCmd(diffCommandArgs); if (sortResults) { new File(inFileName).delete(); new File(outFileName).delete(); } return result; } private static void sortFiles(String in, String out) throws Exception { int result = executeCmd(new String[] { "sort", getQuotedString(in), }, out, null).getReturnCode(); if (result != 0) throw new IllegalStateException("Unexpected error while sorting " + in); } private static QTestProcessExecResult executeCmd(Collection<String> args) throws Exception { return executeCmd(args, null, null); } private static QTestProcessExecResult executeCmd(String[] args) throws Exception { return executeCmd(args, null, null); } private static QTestProcessExecResult executeCmd(Collection<String> args, String outFile, String errFile) throws Exception { String[] cmdArray = args.toArray(new String[args.size()]); return executeCmd(cmdArray, outFile, errFile); } private static QTestProcessExecResult executeCmd(String[] args, String outFile, String errFile) throws Exception { System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' ')); PrintStream out = outFile == null ? SessionState.getConsole().getChildOutStream() : new PrintStream(new FileOutputStream(outFile), true, "UTF-8"); PrintStream err = errFile == null ? SessionState.getConsole().getChildErrStream() : new PrintStream(new FileOutputStream(errFile), true, "UTF-8"); Process executor = Runtime.getRuntime().exec(args); ByteArrayOutputStream bos = new ByteArrayOutputStream(); PrintStream str = new PrintStream(bos, true, "UTF-8"); StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err); StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out, str); outPrinter.start(); errPrinter.start(); int result = executor.waitFor(); outPrinter.join(); errPrinter.join(); if (outFile != null) { out.close(); } if (errFile != null) { err.close(); } return QTestProcessExecResult.create(result, new String(bos.toByteArray(), StandardCharsets.UTF_8)); } private static String getQuotedString(String str) { return str; } public ASTNode parseQuery(String tname) throws Exception { return pd.parse(qMap.get(tname)); } public void resetParser() throws SemanticException { drv.init(); pd = new ParseDriver(); queryState = new QueryState.Builder().withHiveConf(conf).build(); sem = new SemanticAnalyzer(queryState); } public List<Task<? extends Serializable>> analyzeAST(ASTNode ast) throws Exception { // Do semantic analysis and plan generation Context ctx = new Context(conf); while ((ast.getToken() == null) && (ast.getChildCount() > 0)) { ast = (ASTNode) ast.getChild(0); } sem.getOutputs().clear(); sem.getInputs().clear(); sem.analyze(ast, ctx); ctx.clear(); return sem.getRootTasks(); } public TreeMap<String, String> getQMap() { return qMap; } /** * QTestSetup defines test fixtures which are reused across testcases, * and are needed before any test can be run */ public static class QTestSetup { private MiniZooKeeperCluster zooKeeperCluster = null; private int zkPort; private ZooKeeper zooKeeper; public QTestSetup() { } public void preTest(HiveConf conf) throws Exception { if (zooKeeperCluster == null) { //create temp dir String tmpBaseDir = System.getProperty(TEST_TMP_DIR_PROPERTY); File tmpDir = Utilities.createTempDir(tmpBaseDir); zooKeeperCluster = new MiniZooKeeperCluster(); zkPort = zooKeeperCluster.startup(tmpDir); } if (zooKeeper != null) { zooKeeper.close(); } int sessionTimeout = (int) conf.getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT, TimeUnit.MILLISECONDS); zooKeeper = new ZooKeeper("localhost:" + zkPort, sessionTimeout, new Watcher() { @Override public void process(WatchedEvent arg0) { } }); String zkServer = "localhost"; conf.set("hive.zookeeper.quorum", zkServer); conf.set("hive.zookeeper.client.port", "" + zkPort); } public void postTest(HiveConf conf) throws Exception { if (zooKeeperCluster == null) { return; } if (zooKeeper != null) { zooKeeper.close(); } ZooKeeperHiveLockManager.releaseAllLocks(conf); } public void tearDown() throws Exception { CuratorFrameworkSingleton.closeAndReleaseInstance(); if (zooKeeperCluster != null) { zooKeeperCluster.shutdown(); zooKeeperCluster = null; } } } /** * QTRunner: Runnable class for running a a single query file. * **/ public static class QTRunner implements Runnable { private final QTestUtil qt; private final String fname; public QTRunner(QTestUtil qt, String fname) { this.qt = qt; this.fname = fname; } @Override public void run() { try { // assumption is that environment has already been cleaned once globally // hence each thread does not call cleanUp() and createSources() again qt.cliInit(fname, false); qt.executeClient(fname); } catch (Throwable e) { System.err.println("Query file " + fname + " failed with exception " + e.getMessage()); e.printStackTrace(); outputTestFailureHelpMessage(); } } } /** * Setup to execute a set of query files. Uses QTestUtil to do so. * * @param qfiles * array of input query files containing arbitrary number of hive * queries * @param resDir * output directory * @param logDir * log directory * @return one QTestUtil for each query file */ public static QTestUtil[] queryListRunnerSetup(File[] qfiles, String resDir, String logDir, String initScript, String cleanupScript) throws Exception { QTestUtil[] qt = new QTestUtil[qfiles.length]; for (int i = 0; i < qfiles.length; i++) { qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20", initScript == null ? defaultInitScript : initScript, cleanupScript == null ? defaultCleanupScript : cleanupScript, false, false); qt[i].addFile(qfiles[i]); qt[i].clearTestSideEffects(); } return qt; } /** * Executes a set of query files in sequence. * * @param qfiles * array of input query files containing arbitrary number of hive * queries * @param qt * array of QTestUtils, one per qfile * @return true if all queries passed, false otw */ public static boolean queryListRunnerSingleThreaded(File[] qfiles, QTestUtil[] qt) throws Exception { boolean failed = false; qt[0].cleanUp(); qt[0].createSources(); for (int i = 0; i < qfiles.length && !failed; i++) { qt[i].clearTestSideEffects(); qt[i].cliInit(qfiles[i].getName(), false); qt[i].executeClient(qfiles[i].getName()); QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName()); if (result.getReturnCode() != 0) { failed = true; StringBuilder builder = new StringBuilder(); builder.append("Test ").append(qfiles[i].getName()).append(" results check failed with error code ") .append(result.getReturnCode()); if (Strings.isNotEmpty(result.getCapturedOutput())) { builder.append(" and diff value ").append(result.getCapturedOutput()); } System.err.println(builder.toString()); outputTestFailureHelpMessage(); } qt[i].clearPostTestEffects(); } return (!failed); } /** * Executes a set of query files parallel. * * Each query file is run in a separate thread. The caller has to arrange * that different query files do not collide (in terms of destination tables) * * @param qfiles * array of input query files containing arbitrary number of hive * queries * @param qt * array of QTestUtils, one per qfile * @return true if all queries passed, false otw * */ public static boolean queryListRunnerMultiThreaded(File[] qfiles, QTestUtil[] qt) throws Exception { boolean failed = false; // in multithreaded mode - do cleanup/initialization just once qt[0].cleanUp(); qt[0].createSources(); qt[0].clearTestSideEffects(); QTRunner[] qtRunners = new QTRunner[qfiles.length]; Thread[] qtThread = new Thread[qfiles.length]; for (int i = 0; i < qfiles.length; i++) { qtRunners[i] = new QTRunner(qt[i], qfiles[i].getName()); qtThread[i] = new Thread(qtRunners[i]); } for (int i = 0; i < qfiles.length; i++) { qtThread[i].start(); } for (int i = 0; i < qfiles.length; i++) { qtThread[i].join(); QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName()); if (result.getReturnCode() != 0) { failed = true; StringBuilder builder = new StringBuilder(); builder.append("Test ").append(qfiles[i].getName()).append(" results check failed with error code ") .append(result.getReturnCode()); if (Strings.isNotEmpty(result.getCapturedOutput())) { builder.append(" and diff value ").append(result.getCapturedOutput()); } System.err.println(builder.toString()); outputTestFailureHelpMessage(); } } return (!failed); } public static void outputTestFailureHelpMessage() { System.err.println("See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, or check " + "./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific " + "test cases logs."); System.err.flush(); } public static String ensurePathEndsInSlash(String path) { if (path == null) { throw new NullPointerException("Path cannot be null"); } if (path.endsWith(File.separator)) { return path; } else { return path + File.separator; } } private static String[] cachedQvFileList = null; private static ImmutableList<String> cachedDefaultQvFileList = null; private static Pattern qvSuffix = Pattern.compile("_[0-9]+.qv$", Pattern.CASE_INSENSITIVE); public static List<String> getVersionFiles(String queryDir, String tname) { ensureQvFileList(queryDir); List<String> result = getVersionFilesInternal(tname); if (result == null) { result = cachedDefaultQvFileList; } return result; } private static void ensureQvFileList(String queryDir) { if (cachedQvFileList != null) return; // Not thread-safe. System.out.println("Getting versions from " + queryDir); cachedQvFileList = (new File(queryDir)).list(new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.toLowerCase().endsWith(".qv"); } }); if (cachedQvFileList == null) return; // no files at all Arrays.sort(cachedQvFileList, String.CASE_INSENSITIVE_ORDER); List<String> defaults = getVersionFilesInternal("default"); cachedDefaultQvFileList = (defaults != null) ? ImmutableList.copyOf(defaults) : ImmutableList.<String>of(); } private static List<String> getVersionFilesInternal(String tname) { if (cachedQvFileList == null) { return new ArrayList<String>(); } int pos = Arrays.binarySearch(cachedQvFileList, tname, String.CASE_INSENSITIVE_ORDER); if (pos >= 0) { throw new BuildException("Unexpected file list element: " + cachedQvFileList[pos]); } List<String> result = null; for (pos = (-pos - 1); pos < cachedQvFileList.length; ++pos) { String candidate = cachedQvFileList[pos]; if (candidate.length() <= tname.length() || !tname.equalsIgnoreCase(candidate.substring(0, tname.length())) || !qvSuffix.matcher(candidate.substring(tname.length())).matches()) { break; } if (result == null) { result = new ArrayList<String>(); } result.add(candidate); } return result; } public void failed(int ecode, String fname, String debugHint) { String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null; String message = "Client execution failed with error code = " + ecode + (command != null ? " running \"" + command : "") + "\" fname=" + fname + " " + (debugHint != null ? debugHint : ""); LOG.error(message); Assert.fail(message); } // for negative tests, which is succeeded.. no need to print the query string public void failed(String fname, String debugHint) { Assert.fail("Client Execution was expected to fail, but succeeded with error code 0 for fname=" + fname + (debugHint != null ? (" " + debugHint) : "")); } public void failedDiff(int ecode, String fname, String debugHint) { String message = "Client Execution succeeded but contained differences " + "(error code = " + ecode + ") after executing " + fname + (debugHint != null ? (" " + debugHint) : ""); LOG.error(message); Assert.fail(message); } public void failed(Exception e, String fname, String debugHint) { String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null; System.err.println("Failed query: " + fname); System.err.flush(); Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e) + "\n" + (command != null ? " running " + command : "") + (debugHint != null ? debugHint : "")); } public static void addTestsToSuiteFromQfileNames(String qFileNamesFile, Set<String> qFilesToExecute, TestSuite suite, Object setup, SuiteAddTestFunctor suiteAddTestCallback) { try { File qFileNames = new File(qFileNamesFile); FileReader fr = new FileReader(qFileNames.getCanonicalFile()); BufferedReader br = new BufferedReader(fr); String fName = null; while ((fName = br.readLine()) != null) { if (fName.isEmpty() || fName.trim().equals("")) { continue; } int eIdx = fName.indexOf('.'); if (eIdx == -1) { continue; } String tName = fName.substring(0, eIdx); if (qFilesToExecute.isEmpty() || qFilesToExecute.contains(fName)) { suiteAddTestCallback.addTestToSuite(suite, setup, tName); } } br.close(); } catch (Exception e) { Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e)); } } public static void setupMetaStoreTableColumnStatsFor30TBTPCDSWorkload(HiveConf conf) { Connection conn = null; ArrayList<Statement> statements = new ArrayList<Statement>(); // list of Statements, PreparedStatements try { Properties props = new Properties(); // connection properties props.put("user", conf.get("javax.jdo.option.ConnectionUserName")); props.put("password", conf.get("javax.jdo.option.ConnectionPassword")); conn = DriverManager.getConnection(conf.get("javax.jdo.option.ConnectionURL"), props); ResultSet rs = null; Statement s = conn.createStatement(); if (LOG.isDebugEnabled()) { LOG.debug("Connected to metastore database "); } String mdbPath = AbstractCliConfig.HIVE_ROOT + "/data/files/tpcds-perf/metastore_export/"; // Setup the table column stats BufferedReader br = new BufferedReader(new FileReader(new File( AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql"))); String command; s.execute("DROP TABLE APP.TABLE_PARAMS"); s.execute("DROP TABLE APP.TAB_COL_STATS"); // Create the column stats table while ((command = br.readLine()) != null) { if (!command.endsWith(";")) { continue; } if (LOG.isDebugEnabled()) { LOG.debug("Going to run command : " + command); } try { PreparedStatement psCommand = conn.prepareStatement(command.substring(0, command.length() - 1)); statements.add(psCommand); psCommand.execute(); if (LOG.isDebugEnabled()) { LOG.debug("successfully completed " + command); } } catch (SQLException e) { LOG.info("Got SQL Exception " + e.getMessage()); } } br.close(); java.nio.file.Path tabColStatsCsv = FileSystems.getDefault().getPath(mdbPath, "csv", "TAB_COL_STATS.txt.bz2"); java.nio.file.Path tabParamsCsv = FileSystems.getDefault().getPath(mdbPath, "csv", "TABLE_PARAMS.txt.bz2"); // Set up the foreign key constraints properly in the TAB_COL_STATS data String tmpBaseDir = System.getProperty(TEST_TMP_DIR_PROPERTY); java.nio.file.Path tmpFileLoc1 = FileSystems.getDefault().getPath(tmpBaseDir, "TAB_COL_STATS.txt"); java.nio.file.Path tmpFileLoc2 = FileSystems.getDefault().getPath(tmpBaseDir, "TABLE_PARAMS.txt"); class MyComp implements Comparator<String> { @Override public int compare(String str1, String str2) { if (str2.length() != str1.length()) { return str2.length() - str1.length(); } return str1.compareTo(str2); } } final SortedMap<String, Integer> tableNameToID = new TreeMap<String, Integer>(new MyComp()); rs = s.executeQuery("SELECT * FROM APP.TBLS"); while (rs.next()) { String tblName = rs.getString("TBL_NAME"); Integer tblId = rs.getInt("TBL_ID"); tableNameToID.put(tblName, tblId); if (LOG.isDebugEnabled()) { LOG.debug("Resultset : " + tblName + " | " + tblId); } } final Map<String, Map<String, String>> data = new HashMap<>(); rs = s.executeQuery("select TBLS.TBL_NAME, a.COLUMN_NAME, a.TYPE_NAME from " + "(select COLUMN_NAME, TYPE_NAME, SDS.SD_ID from APP.COLUMNS_V2 join APP.SDS on SDS.CD_ID = COLUMNS_V2.CD_ID) a" + " join APP.TBLS on TBLS.SD_ID = a.SD_ID"); while (rs.next()) { String tblName = rs.getString(1); String colName = rs.getString(2); String typeName = rs.getString(3); Map<String, String> cols = data.get(tblName); if (null == cols) { cols = new HashMap<>(); } cols.put(colName, typeName); data.put(tblName, cols); } BufferedReader reader = new BufferedReader(new InputStreamReader( new BZip2CompressorInputStream(Files.newInputStream(tabColStatsCsv, StandardOpenOption.READ)))); Stream<String> replaced = reader.lines().parallel().map(str -> { String[] splits = str.split(","); String tblName = splits[0]; String colName = splits[1]; Integer tblID = tableNameToID.get(tblName); StringBuilder sb = new StringBuilder( "default@" + tblName + "@" + colName + "@" + data.get(tblName).get(colName) + "@"); for (int i = 2; i < splits.length; i++) { sb.append(splits[i] + "@"); } // Add tbl_id and empty bitvector return sb.append(tblID).append("@").toString(); }); Files.write(tmpFileLoc1, (Iterable<String>) replaced::iterator); replaced.close(); reader.close(); BufferedReader reader2 = new BufferedReader(new InputStreamReader( new BZip2CompressorInputStream(Files.newInputStream(tabParamsCsv, StandardOpenOption.READ)))); final Map<String, String> colStats = new ConcurrentHashMap<>(); Stream<String> replacedStream = reader2.lines().parallel().map(str -> { String[] splits = str.split("_@"); String tblName = splits[0]; Integer tblId = tableNameToID.get(tblName); Map<String, String> cols = data.get(tblName); StringBuilder sb = new StringBuilder(); sb.append("{\"COLUMN_STATS\":{"); for (String colName : cols.keySet()) { sb.append("\"" + colName + "\":\"true\","); } sb.append("},\"BASIC_STATS\":\"true\"}"); colStats.put(tblId.toString(), sb.toString()); return tblId.toString() + "@" + splits[1]; }); Files.write(tmpFileLoc2, (Iterable<String>) replacedStream::iterator); Files.write(tmpFileLoc2, (Iterable<String>) colStats.entrySet().stream() .map(map -> map.getKey() + "@COLUMN_STATS_ACCURATE@" + map.getValue())::iterator, StandardOpenOption.APPEND); replacedStream.close(); reader2.close(); // Load the column stats and table params with 30 TB scale String importStatement1 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TAB_COL_STATS" + "', '" + tmpFileLoc1.toAbsolutePath().toString() + "', '@', null, 'UTF-8', 1)"; String importStatement2 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TABLE_PARAMS" + "', '" + tmpFileLoc2.toAbsolutePath().toString() + "', '@', null, 'UTF-8', 1)"; try { PreparedStatement psImport1 = conn.prepareStatement(importStatement1); if (LOG.isDebugEnabled()) { LOG.debug("Going to execute : " + importStatement1); } statements.add(psImport1); psImport1.execute(); if (LOG.isDebugEnabled()) { LOG.debug("successfully completed " + importStatement1); } PreparedStatement psImport2 = conn.prepareStatement(importStatement2); if (LOG.isDebugEnabled()) { LOG.debug("Going to execute : " + importStatement2); } statements.add(psImport2); psImport2.execute(); if (LOG.isDebugEnabled()) { LOG.debug("successfully completed " + importStatement2); } } catch (SQLException e) { LOG.info("Got SQL Exception " + e.getMessage()); } } catch (FileNotFoundException e1) { LOG.info("Got File not found Exception " + e1.getMessage()); } catch (IOException e1) { LOG.info("Got IOException " + e1.getMessage()); } catch (SQLException e1) { LOG.info("Got SQLException " + e1.getMessage()); } finally { // Statements and PreparedStatements int i = 0; while (!statements.isEmpty()) { // PreparedStatement extend Statement Statement st = statements.remove(i); try { if (st != null) { st.close(); st = null; } } catch (SQLException sqle) { } } //Connection try { if (conn != null) { conn.close(); conn = null; } } catch (SQLException sqle) { } } } }