Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hive.sql; import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME; import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.PrintStream; import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.Arrays; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hive.cli.CliDriver; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.common.io.CachingPrintStream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.Utilities.StreamPrinter; import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.ParseDriver; import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer; import org.apache.hadoop.hive.serde2.thrift.test.Complex; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.zookeeper.ZooKeeper; /** * QTestUtil. * */ public class QTestUtil { private static final Log LOG = LogFactory.getLog("QTestUtil"); private String testWarehouse; private final String testFiles; protected final String outDir; protected final String logDir; private final TreeMap<String, String> qMap; private final Set<String> qSkipSet; public static final HashSet<String> srcTables = new HashSet<String>(Arrays.asList(new String[] {})); private ParseDriver pd; private Hive db; protected HiveConf conf; private Driver drv; private SemanticAnalyzer sem; private FileSystem fs; protected final boolean overWrite; private CliDriver cliDriver; private MiniMRCluster mr = null; private HadoopShims.MiniDFSShim dfs = null; private boolean miniMr = false; private String hadoopVer = null; private QTestSetup setup = null; public boolean deleteDirectory(File path) { if (path.exists()) { File[] files = path.listFiles(); for (File file : files) { if (file.isDirectory()) { deleteDirectory(file); } else { file.delete(); } } } return (path.delete()); } public void copyDirectoryToLocal(Path src, Path dest) throws Exception { FileSystem srcFs = src.getFileSystem(conf); FileSystem destFs = dest.getFileSystem(conf); if (srcFs.exists(src)) { FileStatus[] files = srcFs.listStatus(src); for (FileStatus file : files) { String name = file.getPath().getName(); Path dfs_path = file.getPath(); Path local_path = new Path(dest, name); // If this is a source table we do not copy it out if (srcTables.contains(name)) { continue; } if (file.isDir()) { if (!destFs.exists(local_path)) { destFs.mkdirs(local_path); } copyDirectoryToLocal(dfs_path, local_path); } else { srcFs.copyToLocalFile(dfs_path, local_path); } } } } static Pattern mapTok = Pattern.compile("(\\.?)(.*)_map_(.*)"); static Pattern reduceTok = Pattern.compile("(.*)(reduce_[^\\.]*)((\\..*)?)"); public void normalizeNames(File path) throws Exception { if (path.isDirectory()) { File[] files = path.listFiles(); for (File file : files) { normalizeNames(file); } } else { // System.out.println("Trying to match: " + path.getPath()); Matcher m = reduceTok.matcher(path.getName()); if (m.matches()) { String name = m.group(1) + "reduce" + m.group(3); // System.out.println("Matched new name: " + name); path.renameTo(new File(path.getParent(), name)); } else { m = mapTok.matcher(path.getName()); if (m.matches()) { String name = m.group(1) + "map_" + m.group(3); // System.out.println("Matched new name: " + name); path.renameTo(new File(path.getParent(), name)); } } } } public QTestUtil(String outDir, String logDir) throws Exception { this(outDir, logDir, false, "0.20"); } private String getHadoopMainVersion(String input) { if (input == null) { return null; } Pattern p = Pattern.compile("^(\\d+\\.\\d+).*"); Matcher m = p.matcher(input); if (m.matches()) { return m.group(1); } return null; } public void initConf() throws Exception { if (miniMr) { assert dfs != null; assert mr != null; // set fs.default.name to the uri of mini-dfs conf.setVar(HiveConf.ConfVars.HADOOPFS, dfs.getFileSystem().getUri().toString()); // hive.metastore.warehouse.dir needs to be set relative to the mini-dfs conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, (new Path(dfs.getFileSystem().getUri().toString(), "/build/ql/test/data/warehouse/")) .toString()); conf.setVar(HiveConf.ConfVars.HADOOPJT, "localhost:" + mr.getJobTrackerPort()); } } public QTestUtil(String outDir, String logDir, boolean miniMr, String hadoopVer) throws Exception { this.outDir = outDir; this.logDir = logDir; conf = new HiveConf(Driver.class); this.miniMr = miniMr; this.hadoopVer = getHadoopMainVersion(hadoopVer); qMap = new TreeMap<String, String>(); qSkipSet = new HashSet<String>(); if (miniMr) { dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null); FileSystem fs = dfs.getFileSystem(); mr = new MiniMRCluster(4, fs.getUri().toString(), 1); } initConf(); // Use the current directory if it is not specified String dataDir = conf.get("test.data.files"); if (dataDir == null) { dataDir = new File(".").getAbsolutePath() + "/data/files"; } testFiles = dataDir.replace('\\', '/').replace("c:", ""); String ow = System.getProperty("test.output.overwrite"); if ((ow != null) && ow.equalsIgnoreCase("true")) { overWrite = true; } else { overWrite = false; } setup = new QTestSetup(); setup.preTest(conf); init(); } public void shutdown() throws Exception { cleanUp(); setup.tearDown(); if (dfs != null) { dfs.shutdown(); dfs = null; } if (mr != null) { mr.shutdown(); mr = null; } } public void addFile(String qFile) throws Exception { File qf = new File(qFile); addFile(qf); } public void addFile(File qf) throws Exception { FileInputStream fis = new FileInputStream(qf); BufferedInputStream bis = new BufferedInputStream(fis); DataInputStream dis = new DataInputStream(bis); StringBuilder qsb = new StringBuilder(); // Look for a hint to not run a test on some Hadoop versions Pattern pattern = Pattern.compile("-- EXCLUDE_HADOOP_MAJOR_VERSIONS(.*)"); // Read the entire query boolean excludeQuery = false; String hadoopVer = ShimLoader.getMajorVersion(); while (dis.available() != 0) { String line = dis.readLine(); // While we are reading the lines, detect whether this query wants to be // excluded from running because the Hadoop version is incorrect Matcher matcher = pattern.matcher(line); if (matcher.find()) { String group = matcher.group(); int start = group.indexOf('('); int end = group.indexOf(')'); assert end > start; // versions might be something like '0.17, 0.19' String versions = group.substring(start + 1, end); Set<String> excludedVersionSet = new HashSet<String>(); for (String s : versions.split("\\,")) { s = s.trim(); excludedVersionSet.add(s); } if (excludedVersionSet.contains(hadoopVer)) { excludeQuery = true; } } qsb.append(line + "\n"); } qMap.put(qf.getName(), qsb.toString()); if (excludeQuery) { System.out.println("Due to the Hadoop Version (" + hadoopVer + "), " + "adding query " + qf.getName() + " to the set of tests to skip"); qSkipSet.add(qf.getName()); } dis.close(); } /** * Clear out any side effects of running tests */ public void clearPostTestEffects() throws Exception { setup.postTest(conf); } /** * Clear out any side effects of running tests */ public void clearTestSideEffects() throws Exception { // Delete any tables other than the source tables // and any databases other than the default database. for (String dbName : db.getAllDatabases()) { db.setCurrentDatabase(dbName); for (String tblName : db.getAllTables()) { if (!DEFAULT_DATABASE_NAME.equals(dbName) || !srcTables.contains(tblName)) { Table tblObj = db.getTable(tblName); // dropping index table can not be dropped directly. Dropping the base // table will automatically drop all its index table if (tblObj.isIndexTable()) { continue; } db.dropTable(dbName, tblName); } else { // this table is defined in srcTables, drop all indexes on it List<Index> indexes = db.getIndexes(dbName, tblName, (short) -1); if (indexes != null && indexes.size() > 0) { for (Index index : indexes) { db.dropIndex(dbName, tblName, index.getIndexName(), true); } } } } if (!DEFAULT_DATABASE_NAME.equals(dbName)) { db.dropDatabase(dbName); } } Hive.get().setCurrentDatabase(DEFAULT_DATABASE_NAME); List<String> roleNames = db.getAllRoleNames(); for (String roleName : roleNames) { db.dropRole(roleName); } // allocate and initialize a new conf since a test can // modify conf by using 'set' commands conf = new HiveConf(Driver.class); initConf(); setup.preTest(conf); } public void cleanUp() throws Exception { // Drop any tables that remain due to unsuccessful runs for (String s : new String[] {}) { db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, s); } // delete any contents in the warehouse dir Path p = new Path(testWarehouse); FileSystem fs = p.getFileSystem(conf); try { FileStatus[] ls = fs.listStatus(p); for (int i = 0; (ls != null) && (i < ls.length); i++) { fs.delete(ls[i].getPath(), true); } } catch (FileNotFoundException e) { // Best effort } FunctionRegistry.unregisterTemporaryUDF("test_udaf"); FunctionRegistry.unregisterTemporaryUDF("test_error"); preProcedureFile(); } private void runLoadCmd(String loadCmd) throws Exception { int ecode = 0; ecode = drv.run(loadCmd).getResponseCode(); drv.close(); if (ecode != 0) { throw new Exception("load command: " + loadCmd + " failed with exit code= " + ecode); } return; } private void runCreateTableCmd(String createTableCmd) throws Exception { int ecode = 0; ecode = drv.run(createTableCmd).getResponseCode(); if (ecode != 0) { throw new Exception("create table command: " + createTableCmd + " failed with exit code= " + ecode); } return; } public void createSources() throws Exception { startSessionState(); conf.setBoolean("hive.test.init.phase", true); // Create a bunch of tables with columns key and value LinkedList<String> cols = new LinkedList<String>(); cols.add("key"); cols.add("value"); LinkedList<String> part_cols = new LinkedList<String>(); part_cols.add("ds"); part_cols.add("hr"); db.createTable("srcpart", cols, part_cols, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); Path fpath; HashMap<String, String> part_spec = new HashMap<String, String>(); for (String ds : new String[] { "2008-04-08", "2008-04-09" }) { for (String hr : new String[] { "11", "12" }) { part_spec.clear(); part_spec.put("ds", ds); part_spec.put("hr", hr); // System.out.println("Loading partition with spec: " + part_spec); // db.createPartition(srcpart, part_spec); fpath = new Path(testFiles, "kv1.txt"); // db.loadPartition(fpath, srcpart.getName(), part_spec, true); runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' OVERWRITE INTO TABLE srcpart PARTITION (ds='" + ds + "',hr='" + hr + "')"); } } ArrayList<String> bucketCols = new ArrayList<String>(); bucketCols.add("key"); runCreateTableCmd( "CREATE TABLE srcbucket(key int, value string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE"); // db.createTable("srcbucket", cols, null, TextInputFormat.class, // IgnoreKeyTextOutputFormat.class, 2, bucketCols); for (String fname : new String[] { "srcbucket0.txt", "srcbucket1.txt" }) { fpath = new Path(testFiles, fname); runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE srcbucket"); } runCreateTableCmd("CREATE TABLE srcbucket2(key int, value string) " + "CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE"); // db.createTable("srcbucket", cols, null, TextInputFormat.class, // IgnoreKeyTextOutputFormat.class, 2, bucketCols); for (String fname : new String[] { "srcbucket20.txt", "srcbucket21.txt", "srcbucket22.txt", "srcbucket23.txt" }) { fpath = new Path(testFiles, fname); runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE srcbucket2"); } for (String tname : new String[] { "src", "src1" }) { db.createTable(tname, cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); } db.createTable("src_sequencefile", cols, null, SequenceFileInputFormat.class, SequenceFileOutputFormat.class); Table srcThrift = new Table(db.getCurrentDatabase(), "src_thrift"); srcThrift.setInputFormatClass(SequenceFileInputFormat.class.getName()); srcThrift.setOutputFormatClass(SequenceFileOutputFormat.class.getName()); srcThrift.setSerializationLib(ThriftDeserializer.class.getName()); srcThrift.setSerdeParam(Constants.SERIALIZATION_CLASS, Complex.class.getName()); srcThrift.setSerdeParam(Constants.SERIALIZATION_FORMAT, TBinaryProtocol.class.getName()); db.createTable(srcThrift); LinkedList<String> json_cols = new LinkedList<String>(); json_cols.add("json"); db.createTable("src_json", json_cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); // create table for sql testcases runCreateTableCmd( "CREATE TABLE src_sql_t1(a string, b string, c int, d int, e int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE"); runCreateTableCmd( "CREATE TABLE src_sql_t2(x string, y string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE"); runCreateTableCmd( "CREATE TABLE src_sql_t3(x string, y string, z int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE"); // load the input data into the src table fpath = new Path(testFiles, "kv1.txt"); runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src"); // load the input data into the src table fpath = new Path(testFiles, "kv3.txt"); runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src1"); // load the input data into the src_sequencefile table fpath = new Path(testFiles, "kv1.seq"); runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src_sequencefile"); // load the input data into the src_thrift table fpath = new Path(testFiles, "complex.seq"); runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src_thrift"); // load the json data into the src_json table fpath = new Path(testFiles, "json.txt"); runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src_json"); // load data for sql test cases fpath = new Path(testFiles, "sql_t1.txt"); runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src_sql_t1"); fpath = new Path(testFiles, "sql_t2.txt"); runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src_sql_t2"); fpath = new Path(testFiles, "sql_t3.txt"); runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src_sql_t3"); createOthers();// add by frank conf.setBoolean("hive.test.init.phase", false); } public void init() throws Exception { // System.out.println(conf.toString()); testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); // conf.logVars(System.out); // System.out.flush(); SessionState.start(conf); db = Hive.get(conf); fs = FileSystem.get(conf); drv = new Driver(conf); drv.init(); pd = new ParseDriver(); sem = new SemanticAnalyzer(conf); } public void init(String tname) throws Exception { cleanUp(); createSources(); LinkedList<String> cols = new LinkedList<String>(); cols.add("key"); cols.add("value"); LinkedList<String> part_cols = new LinkedList<String>(); part_cols.add("ds"); part_cols.add("hr"); db.createTable("dest1", cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); db.createTable("dest2", cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); db.createTable("dest3", cols, part_cols, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); Table dest3 = db.getTable("dest3"); HashMap<String, String> part_spec = new HashMap<String, String>(); part_spec.put("ds", "2008-04-08"); part_spec.put("hr", "12"); db.createPartition(dest3, part_spec); db.createTable("dest4", cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); db.createTable("dest4_sequencefile", cols, null, SequenceFileInputFormat.class, SequenceFileOutputFormat.class); } public void cliInit(String tname) throws Exception { cliInit(tname, true); } public void cliInit(String tname, boolean recreate) throws Exception { if (recreate) { cleanUp(); createSources(); } HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, "org.apache.hadoop.hive.ql.security.DummyAuthenticator"); CliSessionState ss = new CliSessionState(conf); assert ss != null; ss.in = System.in; File qf = new File(outDir, tname); File outf = null; outf = new File(logDir); outf = new File(outf, qf.getName().concat(".out")); FileOutputStream fo = new FileOutputStream(outf); ss.out = new PrintStream(fo, true, "UTF-8"); ss.err = new CachingPrintStream(fo, true, "UTF-8"); ss.setIsSilent(true); SessionState oldSs = SessionState.get(); if (oldSs != null && oldSs.out != null && oldSs.out != System.out) { oldSs.out.close(); } SessionState.start(ss); cliDriver = new CliDriver(); if (tname.equals("init_file.q")) { ss.initFiles.add("../data/scripts/test_init_file.sql"); } cliDriver.processInitFiles(ss); } private CliSessionState startSessionState() throws FileNotFoundException, UnsupportedEncodingException { HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, "org.apache.hadoop.hive.ql.security.DummyAuthenticator"); CliSessionState ss = new CliSessionState(conf); assert ss != null; SessionState.start(ss); return ss; } public int executeOne(String tname) { String q = qMap.get(tname); if (q.indexOf(";") == -1) { return -1; } String q1 = q.substring(0, q.indexOf(";") + 1); String qrest = q.substring(q.indexOf(";") + 1); qMap.put(tname, qrest); System.out.println("Executing " + q1); return cliDriver.processLine(q1); } public int execute(String tname) { try { return drv.run(qMap.get(tname)).getResponseCode(); } catch (CommandNeedRetryException e) { // TODO Auto-generated catch block e.printStackTrace(); return -1; } } public int executeClient(String tname) { return cliDriver.processLine(qMap.get(tname)); } public boolean shouldBeSkipped(String tname) { return qSkipSet.contains(tname); } public void convertSequenceFileToTextFile() throws Exception { // Create an instance of hive in order to create the tables testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); db = Hive.get(conf); // Create dest4 to replace dest4_sequencefile LinkedList<String> cols = new LinkedList<String>(); cols.add("key"); cols.add("value"); // Move all data from dest4_sequencefile to dest4 drv.run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*"); // Drop dest4_sequencefile db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "dest4_sequencefile", true, true); } public int checkNegativeResults(String tname, Exception e) throws Exception { File qf = new File(outDir, tname); String expf = outPath(outDir.toString(), tname.concat(".out")); File outf = null; outf = new File(logDir); outf = new File(outf, qf.getName().concat(".out")); FileWriter outfd = new FileWriter(outf); if (e instanceof ParseException) { outfd.write("Parse Error: "); } else if (e instanceof SemanticException) { outfd.write("Semantic Exception: \n"); } else { throw e; } outfd.write(e.getMessage()); outfd.close(); String cmdLine = "diff " + outf.getPath() + " " + expf; System.out.println(cmdLine); Process executor = Runtime.getRuntime().exec(cmdLine); StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, SessionState.getConsole().getChildOutStream()); StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, SessionState.getConsole().getChildErrStream()); outPrinter.start(); errPrinter.start(); int exitVal = executor.waitFor(); if (exitVal != 0 && overWrite) { System.out.println("Overwriting results"); cmdLine = "cp " + outf.getPath() + " " + expf; executor = Runtime.getRuntime().exec(cmdLine); exitVal = executor.waitFor(); } return exitVal; } public int checkParseResults(String tname, ASTNode tree) throws Exception { if (tree != null) { File parseDir = new File(outDir, "parse"); String expf = outPath(parseDir.toString(), tname.concat(".out")); File outf = null; outf = new File(logDir); outf = new File(outf, tname.concat(".out")); FileWriter outfd = new FileWriter(outf); outfd.write(tree.toStringTree()); outfd.close(); String cmdLine = "diff " + outf.getPath() + " " + expf; System.out.println(cmdLine); Process executor = Runtime.getRuntime().exec(cmdLine); StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, SessionState.getConsole().getChildOutStream()); StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, SessionState.getConsole().getChildErrStream()); outPrinter.start(); errPrinter.start(); int exitVal = executor.waitFor(); if (exitVal != 0 && overWrite) { System.out.println("Overwriting results"); cmdLine = "cp " + outf.getPath() + " " + expf; executor = Runtime.getRuntime().exec(cmdLine); exitVal = executor.waitFor(); } return exitVal; } else { throw new Exception("Parse tree is null"); } } public int checkPlan(String tname, List<Task<? extends Serializable>> tasks) throws Exception { if (tasks != null) { File planDir = new File(outDir, "plan"); String planFile = outPath(planDir.toString(), tname + ".xml"); File outf = null; outf = new File(logDir); outf = new File(outf, tname.concat(".xml")); FileOutputStream ofs = new FileOutputStream(outf); for (Task<? extends Serializable> plan : tasks) { Utilities.serializeTasks(plan, ofs); } String[] patterns = new String[] { "<java version=\".*\" class=\"java.beans.XMLDecoder\">", "<string>.*/tmp/.*</string>", "<string>file:.*</string>", "<string>pfile:.*</string>", "<string>[0-9]{10}</string>", "<string>/.*/warehouse/.*</string>" }; maskPatterns(patterns, outf.getPath()); String[] cmdArray = new String[] { "diff", "-b", outf.getPath(), planFile }; System.out.println(org.apache.commons.lang.StringUtils.join(cmdArray, ' ')); Process executor = Runtime.getRuntime().exec(cmdArray); StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, SessionState.getConsole().getChildOutStream()); StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, SessionState.getConsole().getChildErrStream()); outPrinter.start(); errPrinter.start(); int exitVal = executor.waitFor(); if (exitVal != 0 && overWrite) { System.out.println("Overwriting results"); String cmdLine = "cp " + outf.getPath() + " " + planFile; executor = Runtime.getRuntime().exec(cmdLine); exitVal = executor.waitFor(); } return exitVal; } else { throw new Exception("Plan is null"); } } /** * Given the current configurations (e.g., hadoop version and execution mode), return * the correct file name to compare with the current test run output. * * @param outDir * The directory where the reference log files are stored. * @param testName * The test file name (terminated by ".out"). * @return The file name appended with the configuration values if it exists. */ public String outPath(String outDir, String testName) { String ret = (new File(outDir, testName)).getPath(); // List of configurations. Currently the list consists of hadoop version and execution mode only List<String> configs = new ArrayList<String>(); configs.add(this.hadoopVer); Deque<String> stack = new LinkedList<String>(); StringBuilder sb = new StringBuilder(); sb.append(testName); stack.push(sb.toString()); // example file names are input1.q.out_0.20.0_minimr or input2.q.out_0.17 for (String s : configs) { sb.append('_'); sb.append(s); stack.push(sb.toString()); } while (stack.size() > 0) { String fileName = stack.pop(); File f = new File(outDir, fileName); if (f.exists()) { ret = f.getPath(); break; } } return ret; } private void maskPatterns(String[] patterns, String fname) throws Exception { String maskPattern = "#### A masked pattern was here ####"; String line; BufferedReader in; BufferedWriter out; in = new BufferedReader(new FileReader(fname)); out = new BufferedWriter(new FileWriter(fname + ".orig")); while (null != (line = in.readLine())) { out.write(line); out.write('\n'); } in.close(); out.close(); in = new BufferedReader(new FileReader(fname + ".orig")); out = new BufferedWriter(new FileWriter(fname)); boolean lastWasMasked = false; while (null != (line = in.readLine())) { for (String pattern : patterns) { line = line.replaceAll(pattern, maskPattern); } if (line.equals(maskPattern)) { // We're folding multiple masked lines into one. if (!lastWasMasked) { out.write(line); out.write("\n"); lastWasMasked = true; } } else { out.write(line); out.write("\n"); lastWasMasked = false; } } in.close(); out.close(); } public int checkCliDriverResults(String tname) throws Exception { String[] cmdArray; String[] patterns; assert (qMap.containsKey(tname)); String outFileName = outPath(outDir, tname + ".out"); patterns = new String[] { ".*file:.*", ".*pfile:.*", ".*hdfs:.*", ".*/tmp/.*", ".*invalidscheme:.*", ".*lastUpdateTime.*", ".*lastAccessTime.*", ".*lastModifiedTime.*", ".*[Oo]wner.*", ".*CreateTime.*", ".*LastAccessTime.*", ".*Location.*", ".*LOCATION '.*", ".*transient_lastDdlTime.*", ".*last_modified_.*", ".*java.lang.RuntimeException.*", ".*at org.*", ".*at sun.*", ".*at java.*", ".*at junit.*", ".*Caused by:.*", ".*LOCK_QUERYID:.*", ".*LOCK_TIME:.*", ".*grantTime.*", ".*[.][.][.] [0-9]* more.*", ".*job_[0-9]*_[0-9]*.*", ".*USING 'java -cp.*", "^Deleted.*", }; maskPatterns(patterns, (new File(logDir, tname + ".out")).getPath()); cmdArray = new String[] { "diff", "-a", (new File(logDir, tname + ".out")).getPath(), outFileName }; System.out.println(org.apache.commons.lang.StringUtils.join(cmdArray, ' ')); Process executor = Runtime.getRuntime().exec(cmdArray); StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, SessionState.getConsole().getChildOutStream()); StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, SessionState.getConsole().getChildErrStream()); outPrinter.start(); errPrinter.start(); int exitVal = executor.waitFor(); if (exitVal != 0 && overWrite) { System.out.println("Overwriting results"); cmdArray = new String[3]; cmdArray[0] = "cp"; cmdArray[1] = (new File(logDir, tname + ".out")).getPath(); cmdArray[2] = outFileName; executor = Runtime.getRuntime().exec(cmdArray); exitVal = executor.waitFor(); } return exitVal; } public ASTNode parseQuery(String tname) throws Exception { return pd.parse(qMap.get(tname)); } public void resetParser() throws SemanticException { drv.init(); pd = new ParseDriver(); sem = new SemanticAnalyzer(conf); } public List<Task<? extends Serializable>> analyzeAST(ASTNode ast) throws Exception { // Do semantic analysis and plan generation Context ctx = new Context(conf); while ((ast.getToken() == null) && (ast.getChildCount() > 0)) { ast = (ASTNode) ast.getChild(0); } sem.getOutputs().clear(); sem.getInputs().clear(); sem.analyze(ast, ctx); ctx.clear(); return sem.getRootTasks(); } public TreeMap<String, String> getQMap() { return qMap; } /** * QTestSetup defines test fixtures which are reused across testcases, * and are needed before any test can be run */ public static class QTestSetup { private MiniZooKeeperCluster zooKeeperCluster = null; private int zkPort; private ZooKeeper zooKeeper; public QTestSetup() { } public void preTest(HiveConf conf) throws Exception { if (zooKeeperCluster == null) { String tmpdir = System.getProperty("user.dir") + "/../build/ql/tmp"; zooKeeperCluster = new MiniZooKeeperCluster(); zkPort = zooKeeperCluster.startup(new File(tmpdir, "zookeeper")); } if (zooKeeper != null) { zooKeeper.close(); } int sessionTimeout = conf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT); zooKeeper = new ZooKeeper("localhost:" + zkPort, sessionTimeout, null); String zkServer = "localhost"; conf.set("hive.zookeeper.quorum", zkServer); conf.set("hive.zookeeper.client.port", "" + zkPort); } public void postTest(HiveConf conf) throws Exception { if (zooKeeperCluster == null) { return; } if (zooKeeper != null) { zooKeeper.close(); } ZooKeeperHiveLockManager.releaseAllLocks(conf); } public void tearDown() throws Exception { if (zooKeeperCluster != null) { zooKeeperCluster.shutdown(); zooKeeperCluster = null; } } } /** * QTRunner: Runnable class for running a a single query file. * **/ public static class QTRunner implements Runnable { private final QTestUtil qt; private final String fname; public QTRunner(QTestUtil qt, String fname) { this.qt = qt; this.fname = fname; } public void run() { try { // assumption is that environment has already been cleaned once globally // hence each thread does not call cleanUp() and createSources() again qt.cliInit(fname, false); qt.executeClient(fname); } catch (Throwable e) { System.err.println("Query file " + fname + " failed with exception " + e.getMessage()); e.printStackTrace(); outputTestFailureHelpMessage(); } } } /** * Setup to execute a set of query files. Uses QTestUtil to do so. * * @param qfiles * array of input query files containing arbitrary number of hive * queries * @param resDir * output directory * @param logDir * log directory * @return one QTestUtil for each query file */ public static QTestUtil[] queryListRunnerSetup(File[] qfiles, String resDir, String logDir) throws Exception { QTestUtil[] qt = new QTestUtil[qfiles.length]; for (int i = 0; i < qfiles.length; i++) { qt[i] = new QTestUtil(resDir, logDir, false, "0.20"); qt[i].addFile(qfiles[i]); qt[i].clearTestSideEffects(); } return qt; } /** * Executes a set of query files in sequence. * * @param qfiles * array of input query files containing arbitrary number of hive * queries * @param qt * array of QTestUtils, one per qfile * @return true if all queries passed, false otw */ public static boolean queryListRunnerSingleThreaded(File[] qfiles, QTestUtil[] qt) throws Exception { boolean failed = false; qt[0].cleanUp(); qt[0].createSources(); for (int i = 0; i < qfiles.length && !failed; i++) { qt[i].clearTestSideEffects(); qt[i].cliInit(qfiles[i].getName(), false); qt[i].executeClient(qfiles[i].getName()); int ecode = qt[i].checkCliDriverResults(qfiles[i].getName()); if (ecode != 0) { failed = true; System.err .println("Test " + qfiles[i].getName() + " results check failed with error code " + ecode); outputTestFailureHelpMessage(); } qt[i].clearPostTestEffects(); } return (!failed); } /** * Executes a set of query files parallel. * * Each query file is run in a separate thread. The caller has to arrange * that different query files do not collide (in terms of destination tables) * * @param qfiles * array of input query files containing arbitrary number of hive * queries * @param qt * array of QTestUtils, one per qfile * @return true if all queries passed, false otw * */ public static boolean queryListRunnerMultiThreaded(File[] qfiles, QTestUtil[] qt) throws Exception { boolean failed = false; // in multithreaded mode - do cleanup/initialization just once qt[0].cleanUp(); qt[0].createSources(); qt[0].clearTestSideEffects(); QTRunner[] qtRunners = new QTestUtil.QTRunner[qfiles.length]; Thread[] qtThread = new Thread[qfiles.length]; for (int i = 0; i < qfiles.length; i++) { qtRunners[i] = new QTestUtil.QTRunner(qt[i], qfiles[i].getName()); qtThread[i] = new Thread(qtRunners[i]); } for (int i = 0; i < qfiles.length; i++) { qtThread[i].start(); } for (int i = 0; i < qfiles.length; i++) { qtThread[i].join(); int ecode = qt[i].checkCliDriverResults(qfiles[i].getName()); if (ecode != 0) { failed = true; System.err .println("Test " + qfiles[i].getName() + " results check failed with error code " + ecode); outputTestFailureHelpMessage(); } } return (!failed); } public static void outputTestFailureHelpMessage() { System.err.println( "See build/ql/tmp/hive.log, " + "or try \"ant test ... -Dtest.silent=false\" to get more logs."); System.err.flush(); } /////////////////////////////////////////////next is added by Frank///////////////////////////// public Hive getDb() { return db; } public String getTestFiles() { return testFiles; } public void runLoad(String loadCmd) throws Exception { runLoadCmd(loadCmd); } public void createOthers() throws Exception { // TODO abstract method } public void runCreateTable(String createTableCmd) throws Exception { runCreateTableCmd(createTableCmd); } public void preProcedureFile() { } }