org.apache.hadoop.hive.ql.QTestUtil2.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hive.ql.QTestUtil2.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hive.ql;

import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;

import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.PrintStream;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import junit.framework.Test;

import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.hive.cli.CliDriver;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.Utilities.StreamPrinter;
import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.ParseDriver;
import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer;
import org.apache.hadoop.hive.serde2.thrift.test.Complex;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.zookeeper.ZooKeeper;

/**
 * QTestUtil.
 * 
 */
public class QTestUtil2 {

    private String testWarehouse;
    private final String tmpdir = System.getProperty("test.tmp.dir");
    // private final Path tmppath = new Path(tmpdir);
    private Path tmppath = new Path("/home/seoeun/tmp/hive-test");

    private final String testFiles;
    private final String outDir;
    private final String logDir;
    private final TreeMap<String, String> qMap;
    private final Set<String> qSkipSet;
    public static final HashSet<String> srcTables = new HashSet<String>(Arrays.asList(new String[] { "src", "src1",
            "srcbucket", "srcbucket2", "src_json", "src_thrift", "src_sequencefile", "srcpart" }));

    private ParseDriver pd;
    private Hive db;
    protected HiveConf conf;
    private Driver drv;
    private SemanticAnalyzer sem;
    private FileSystem fs;
    private boolean overWrite;
    private CliDriver cliDriver;
    private MiniMRCluster mr = null;
    private HadoopShims.MiniDFSShim dfs = null;
    private boolean miniMr = false;
    private String hadoopVer = null;
    private QTestSetup setup = null;

    public boolean deleteDirectory(File path) {
        if (path.exists()) {
            File[] files = path.listFiles();
            for (File file : files) {
                if (file.isDirectory()) {
                    deleteDirectory(file);
                } else {
                    file.delete();
                }
            }
        }
        return (path.delete());
    }

    public void copyDirectoryToLocal(Path src, Path dest) throws Exception {

        FileSystem srcFs = src.getFileSystem(conf);
        FileSystem destFs = dest.getFileSystem(conf);
        if (srcFs.exists(src)) {
            FileStatus[] files = srcFs.listStatus(src);
            for (FileStatus file : files) {
                String name = file.getPath().getName();
                Path dfs_path = file.getPath();
                Path local_path = new Path(dest, name);

                // If this is a source table we do not copy it out
                if (srcTables.contains(name)) {
                    continue;
                }

                if (file.isDir()) {
                    if (!destFs.exists(local_path)) {
                        destFs.mkdirs(local_path);
                    }
                    copyDirectoryToLocal(dfs_path, local_path);
                } else {
                    srcFs.copyToLocalFile(dfs_path, local_path);
                }
            }
        }
    }

    static Pattern mapTok = Pattern.compile("(\\.?)(.*)_map_(.*)");
    static Pattern reduceTok = Pattern.compile("(.*)(reduce_[^\\.]*)((\\..*)?)");

    public void normalizeNames(File path) throws Exception {
        if (path.isDirectory()) {
            File[] files = path.listFiles();
            for (File file : files) {
                normalizeNames(file);
            }
        } else {
            // System.out.println("Trying to match: " + path.getPath());
            Matcher m = reduceTok.matcher(path.getName());
            if (m.matches()) {
                String name = m.group(1) + "reduce" + m.group(3);
                // System.out.println("Matched new name: " + name);
                path.renameTo(new File(path.getParent(), name));
            } else {
                m = mapTok.matcher(path.getName());
                if (m.matches()) {
                    String name = m.group(1) + "map_" + m.group(3);
                    // System.out.println("Matched new name: " + name);
                    path.renameTo(new File(path.getParent(), name));
                }
            }
        }
    }

    public QTestUtil2(String outDir, String logDir) throws Exception {
        this(outDir, logDir, false, "0.20");
    }

    private String getHadoopMainVersion(String input) {
        if (input == null) {
            return null;
        }
        Pattern p = Pattern.compile("^(\\d+\\.\\d+).*");
        Matcher m = p.matcher(input);
        if (m.matches()) {
            return m.group(1);
        }
        return null;
    }

    public void initConf() throws Exception {
        if (miniMr) {
            assert dfs != null;
            assert mr != null;
            // set fs.default.name to the uri of mini-dfs
            conf.setVar(HiveConf.ConfVars.HADOOPFS, dfs.getFileSystem().getUri().toString());
            // hive.metastore.warehouse.dir needs to be set relative to the
            // mini-dfs
            conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE,
                    (new Path(dfs.getFileSystem().getUri().toString(), "/build/ql/test/data/warehouse/"))
                            .toString());
            conf.setVar(HiveConf.ConfVars.HADOOPJT, "localhost:" + mr.getJobTrackerPort());
        }
    }

    public QTestUtil2(String outDir, String logDir, boolean miniMr, String hadoopVer) throws Exception {
        this.outDir = outDir;
        this.logDir = logDir;
        conf = new HiveConf(Driver.class);
        this.miniMr = miniMr;
        this.hadoopVer = getHadoopMainVersion(hadoopVer);
        qMap = new TreeMap<String, String>();
        qSkipSet = new HashSet<String>();

        if (miniMr) {
            dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null);
            FileSystem fs = dfs.getFileSystem();
            mr = new MiniMRCluster(4, fs.getUri().toString(), 1);
        }

        initConf();

        tmppath = new Path(conf.get("test.tmp.dir"));

        testFiles = conf.get("test.data.files").replace('\\', '/').replace("c:", "");

        String ow = System.getProperty("test.output.overwrite");
        overWrite = false;
        if ((ow != null) && ow.equalsIgnoreCase("true")) {
            overWrite = true;
        }

        setup = new QTestSetup();
        //TODO @seoeun  ?. : Zookeeper connection 
        //setup.preTest(conf);
        init();
    }

    public void shutdown() throws Exception {
        cleanUp();

        if (dfs != null) {
            dfs.shutdown();
            dfs = null;
        }

        if (mr != null) {
            mr.shutdown();
            mr = null;
        }
    }

    public void addFile(String qFile) throws Exception {

        File qf = new File(qFile);
        addFile(qf);
    }

    public void addFile(File qf) throws Exception {

        FileInputStream fis = new FileInputStream(qf);
        BufferedInputStream bis = new BufferedInputStream(fis);
        DataInputStream dis = new DataInputStream(bis);
        StringBuilder qsb = new StringBuilder();

        // Look for a hint to not run a test on some Hadoop versions
        Pattern pattern = Pattern.compile("-- EXCLUDE_HADOOP_MAJOR_VERSIONS(.*)");

        // Read the entire query
        boolean excludeQuery = false;
        String hadoopVer = ShimLoader.getMajorVersion();
        while (dis.available() != 0) {
            String line = dis.readLine();

            // While we are reading the lines, detect whether this query wants
            // to be
            // excluded from running because the Hadoop version is incorrect
            Matcher matcher = pattern.matcher(line);
            if (matcher.find()) {
                String group = matcher.group();
                int start = group.indexOf('(');
                int end = group.indexOf(')');
                assert end > start;
                // versions might be something like '0.17, 0.19'
                String versions = group.substring(start + 1, end);

                Set<String> excludedVersionSet = new HashSet<String>();
                for (String s : versions.split("\\,")) {
                    s = s.trim();
                    excludedVersionSet.add(s);
                }
                if (excludedVersionSet.contains(hadoopVer)) {
                    excludeQuery = true;
                }
            }
            qsb.append(line + "\n");
        }
        qMap.put(qf.getName(), qsb.toString());
        if (excludeQuery) {
            System.out.println("Due to the Hadoop Version (" + hadoopVer + "), " + "adding query " + qf.getName()
                    + " to the set of tests to skip");
            qSkipSet.add(qf.getName());
        }
        dis.close();
    }

    /**
     * Clear out any side effects of running tests
     */
    public void clearPostTestEffects() throws Exception {
        setup.postTest(conf);
    }

    /**
     * Clear out any side effects of running tests
     */
    public void clearTestSideEffects() throws Exception {
        // Delete any tables other than the source tables
        // and any databases other than the default database.
        for (String dbName : db.getAllDatabases()) {
            db.setCurrentDatabase(dbName);
            for (String tblName : db.getAllTables()) {
                if (!DEFAULT_DATABASE_NAME.equals(dbName) || !srcTables.contains(tblName)) {
                    db.dropTable(dbName, tblName);
                }
            }
            if (!DEFAULT_DATABASE_NAME.equals(dbName)) {
                db.dropDatabase(dbName);
            }
        }
        db.setCurrentDatabase(DEFAULT_DATABASE_NAME);

        List<String> roleNames = db.getAllRoleNames();
        for (String roleName : roleNames) {
            db.dropRole(roleName);
        }
        // allocate and initialize a new conf since a test can
        // modify conf by using 'set' commands
        conf = new HiveConf(Driver.class);
        initConf();
        setup.preTest(conf);
    }

    public void cleanUp() throws Exception {
        Path p = new Path(testWarehouse);
        FileSystem fs = p.getFileSystem(conf);

        // Drop any tables that remain due to unsuccessful runs
        String[] tables = new String[] { "src", "src1", "src_json", "src_thrift", "src_sequencefile", "srcpart",
                "srcbucket", "srcbucket2", "dest1", "dest2", "dest3", "dest4", "dest4_sequencefile", "dest_j1",
                "dest_j2", "dest_g1", "dest_g2", "fetchtask_ioexception" };
        for (String s : tables) {
            db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, s);
            fs.delete(new Path(p, s), true);
        }

        // delete any contents in the warehouse dir
        FileStatus[] ls = fs.listStatus(p);
        //      for (int i = 0; (ls != null) && (i < ls.length); i++) {
        //         fs.delete(ls[i].getPath(), true);
        //      }

        FunctionRegistry.unregisterTemporaryUDF("test_udaf");
        FunctionRegistry.unregisterTemporaryUDF("test_error");
        setup.tearDown();
    }

    private void runLoadCmd(String loadCmd) throws Exception {
        int ecode = 0;
        ecode = drv.run(loadCmd).getResponseCode();
        drv.close();
        if (ecode != 0) {
            throw new Exception("load command: " + loadCmd + " failed with exit code= " + ecode);
        }
        return;
    }

    private void runCreateTableCmd(String createTableCmd) throws Exception {
        int ecode = 0;
        ecode = drv.run(createTableCmd).getResponseCode();
        // TODO @seoeun
        // if (ecode != 0) {
        // throw new Exception("create table command: " + createTableCmd
        // + " failed with exit code= " + ecode);
        // }

        return;
    }

    public void runCreateTableCmd2(String createTableCmd) throws Exception {
        // TODO : @seoeun create test code.
        db.setCurrentDatabase(DEFAULT_DATABASE_NAME);

        int ecode = 0;
        ecode = drv.run(createTableCmd).getResponseCode();
        // TODO @seoeun
        // if (ecode != 0) {
        // throw new Exception("create table command: " + createTableCmd
        // + " failed with exit code= " + ecode);
        // }

        return;
    }

    public void createSources() throws Exception {

        startSessionState();

        // Create a bunch of tables with columns key and value
        LinkedList<String> cols = new LinkedList<String>();
        cols.add("key");
        cols.add("value");

        LinkedList<String> part_cols = new LinkedList<String>();
        part_cols.add("ds");
        part_cols.add("hr");
        db.createTable("srcpart", cols, part_cols, TextInputFormat.class, IgnoreKeyTextOutputFormat.class);

        Path fpath;
        HashMap<String, String> part_spec = new HashMap<String, String>();
        for (String ds : new String[] { "2008-04-08", "2008-04-09" }) {
            for (String hr : new String[] { "11", "12" }) {
                part_spec.clear();
                part_spec.put("ds", ds);
                part_spec.put("hr", hr);
                // System.out.println("Loading partition with spec: " +
                // part_spec);
                // db.createPartition(srcpart, part_spec);
                fpath = new Path(testFiles, "kv1.txt");
                // db.loadPartition(fpath, srcpart.getName(), part_spec, true);
                runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString()
                        + "' OVERWRITE INTO TABLE srcpart PARTITION (ds='" + ds + "',hr='" + hr + "')");
            }
        }
        ArrayList<String> bucketCols = new ArrayList<String>();
        bucketCols.add("key");
        runCreateTableCmd(
                "CREATE TABLE srcbucket(key int, value string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE");
        // db.createTable("srcbucket", cols, null, TextInputFormat.class,
        // IgnoreKeyTextOutputFormat.class, 2, bucketCols);
        for (String fname : new String[] { "srcbucket0.txt", "srcbucket1.txt" }) {
            fpath = new Path(testFiles, fname);
            runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE srcbucket");
        }

        runCreateTableCmd("CREATE TABLE srcbucket2(key int, value string) "
                + "CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE");
        // db.createTable("srcbucket", cols, null, TextInputFormat.class,
        // IgnoreKeyTextOutputFormat.class, 2, bucketCols);
        for (String fname : new String[] { "srcbucket20.txt", "srcbucket21.txt", "srcbucket22.txt",
                "srcbucket23.txt" }) {
            fpath = new Path(testFiles, fname);
            runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE srcbucket2");
        }

        for (String tname : new String[] { "src", "src1" }) {
            db.createTable(tname, cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class);
        }
        db.createTable("src_sequencefile", cols, null, SequenceFileInputFormat.class,
                SequenceFileOutputFormat.class);

        Table srcThrift = new Table(db.getCurrentDatabase(), "src_thrift");
        srcThrift.setInputFormatClass(SequenceFileInputFormat.class.getName());
        srcThrift.setOutputFormatClass(SequenceFileOutputFormat.class.getName());
        srcThrift.setSerializationLib(ThriftDeserializer.class.getName());
        srcThrift.setSerdeParam(Constants.SERIALIZATION_CLASS, Complex.class.getName());
        srcThrift.setSerdeParam(Constants.SERIALIZATION_FORMAT, TBinaryProtocol.class.getName());
        db.createTable(srcThrift);

        LinkedList<String> json_cols = new LinkedList<String>();
        json_cols.add("json");
        db.createTable("src_json", json_cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class);

        // load the input data into the src table
        fpath = new Path(testFiles, "kv1.txt");
        runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src");

        // load the input data into the src table
        fpath = new Path(testFiles, "kv3.txt");
        runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src1");

        // load the input data into the src_sequencefile table
        fpath = new Path(testFiles, "kv1.seq");
        runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src_sequencefile");

        // load the input data into the src_thrift table
        fpath = new Path(testFiles, "complex.seq");
        runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src_thrift");

        // load the json data into the src_json table
        fpath = new Path(testFiles, "json.txt");
        runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src_json");

    }

    /**
     * Create table used for test.
     * @param tableName
     *            "records2"
     * @param tableScheme
     *            "(year STRING, temperature INT, quality STRING)"
     * @param sourceFile
     *            "records2.txt" It must be in 'resources/data/files'
     *            
     */
    public void createTestData(String tableName, String tableScheme, String sourceFile) throws Exception {
        // TODO : @seoeun create test code.
        db.setCurrentDatabase(DEFAULT_DATABASE_NAME);

        System.out.println("Drop table : " + drv.run("drop table " + tableName).getResponseCode());

        String tableCmd = "create table " + tableName + " " + tableScheme
                + " row format delimited fields terminated by '\\t'";

        runCreateTableCmd(tableCmd);

        Path fpath = new Path(testFiles, sourceFile);
        Path newfpath = new Path(tmppath, sourceFile);
        fs.copyFromLocalFile(false, true, fpath, newfpath);

        String loadCmd = "LOAD DATA INPATH '" + newfpath.toString() + "' OVERWRITE INTO TABLE " + tableName;

        runLoadCmd(loadCmd);
    }

    public void createRecords2() throws Exception {
        // TODO : @seoeun create test code.
        db.setCurrentDatabase(DEFAULT_DATABASE_NAME);

        String tableCmd = "create table separator (separator STRING) row format delimited fields terminated by '\\t'";

        runCreateTableCmd(tableCmd);

        Path fpath = new Path(testFiles, "separator.txt");
        Path newfpath = new Path(tmppath, "separator.txt");
        fs.copyFromLocalFile(false, true, fpath, newfpath);

        runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + "' OVERWRITE INTO TABLE separator");
    }

    public void createSQDTable() throws Exception {
        // TODO : @seoeun create test code.
        db.setCurrentDatabase(DEFAULT_DATABASE_NAME);

        String tableCmd = "create external table z_sqd_ipas_cust(" + "i_yyyymmdd string,   "
                + "i_service_grp string," + "i_service string," + "i_ncn string," + "i_bonbu string,"
                + "i_switch string," + "i_bsc string," + "i_cell string," + "i_net_cls string" + ")"
                + "row format delimited fields terminated by '\\t'";

        runCreateTableCmd(tableCmd);

        Path fpath = new Path(testFiles, "z_sqd_ipas_cust.txt");
        Path newfpath = new Path(tmppath, "z_sqd_ipas_cust.txt");
        fs.copyFromLocalFile(false, true, fpath, newfpath);

        runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + "' OVERWRITE INTO TABLE z_sqd_ipas_cust");
    }

    public void init() throws Exception {
        // System.out.println(conf.toString());
        testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
        // conf.logVars(System.out);
        // System.out.flush();

        SessionState.start(conf);
        db = Hive.get(conf);
        fs = FileSystem.get(conf);
        drv = new Driver(conf);
        drv.init();
        pd = new ParseDriver();
        sem = new SemanticAnalyzer(conf);
    }

    public void init(String tname) throws Exception {
        cleanUp();
        createSources();

        LinkedList<String> cols = new LinkedList<String>();
        cols.add("key");
        cols.add("value");

        LinkedList<String> part_cols = new LinkedList<String>();
        part_cols.add("ds");
        part_cols.add("hr");

        db.createTable("dest1", cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class);
        db.createTable("dest2", cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class);

        db.createTable("dest3", cols, part_cols, TextInputFormat.class, IgnoreKeyTextOutputFormat.class);
        Table dest3 = db.getTable("dest3");

        HashMap<String, String> part_spec = new HashMap<String, String>();
        part_spec.put("ds", "2008-04-08");
        part_spec.put("hr", "12");
        db.createPartition(dest3, part_spec);

        db.createTable("dest4", cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class);
        db.createTable("dest4_sequencefile", cols, null, SequenceFileInputFormat.class,
                SequenceFileOutputFormat.class);
    }

    public void cliInit(String tname) throws Exception {
        cliInit(tname, true);
    }

    public void cliInit(String tname, boolean recreate) throws Exception {
        if (recreate) {
            cleanUp();
            //TODO @seoeun  table src   ?.
            //createSources();
            //createRecords2();
        }

        HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
                "org.apache.hadoop.hive.ql.security.DummyAuthenticator");
        CliSessionState ss = new CliSessionState(conf);
        assert ss != null;
        ss.in = System.in;

        File qf = new File(outDir, tname);
        File outf = null;
        outf = new File(logDir);
        outf = new File(outf, qf.getName().concat(".out"));
        FileOutputStream fo = new FileOutputStream(outf);
        ss.out = new PrintStream(fo, true, "UTF-8");
        ss.err = ss.out;
        ss.setIsSilent(true);
        SessionState oldSs = SessionState.get();
        if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
            oldSs.out.close();
        }
        SessionState.start(ss);

        cliDriver = new CliDriver();
        if (tname.equals("init_file.q")) {
            ss.initFiles.add("../data/scripts/test_init_file.sql");
        }
        cliDriver.processInitFiles(ss);
    }

    private CliSessionState startSessionState() throws FileNotFoundException, UnsupportedEncodingException {

        HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
                "org.apache.hadoop.hive.ql.security.DummyAuthenticator");

        CliSessionState ss = new CliSessionState(conf);
        assert ss != null;

        SessionState.start(ss);
        return ss;
    }

    public int executeOne(String tname) {
        String q = qMap.get(tname);

        if (q.indexOf(";") == -1) {
            return -1;
        }

        String q1 = q.substring(0, q.indexOf(";") + 1);
        String qrest = q.substring(q.indexOf(";") + 1);
        qMap.put(tname, qrest);

        System.out.println("Executing " + q1);
        return cliDriver.processLine(q1);
    }

    public int execute(String tname) {
        return drv.run(qMap.get(tname)).getResponseCode();
    }

    public int executeClient(String tname) {
        return cliDriver.processLine(qMap.get(tname));
    }

    public boolean shouldBeSkipped(String tname) {
        return qSkipSet.contains(tname);
    }

    public void convertSequenceFileToTextFile() throws Exception {
        // Create an instance of hive in order to create the tables
        testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
        db = Hive.get(conf);
        // Create dest4 to replace dest4_sequencefile
        LinkedList<String> cols = new LinkedList<String>();
        cols.add("key");
        cols.add("value");

        // Move all data from dest4_sequencefile to dest4
        drv.run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*");

        // Drop dest4_sequencefile
        db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "dest4_sequencefile", true, true);
    }

    public int checkNegativeResults(String tname, Exception e) throws Exception {

        File qf = new File(outDir, tname);
        String expf = outPath(outDir.toString(), tname.concat(".out"));

        File outf = null;
        outf = new File(logDir);
        outf = new File(outf, qf.getName().concat(".out"));

        FileWriter outfd = new FileWriter(outf);
        if (e instanceof ParseException) {
            outfd.write("Parse Error: ");
        } else if (e instanceof SemanticException) {
            outfd.write("Semantic Exception: \n");
        } else {
            throw e;
        }

        outfd.write(e.getMessage());
        outfd.close();

        String cmdLine = "diff " + outf.getPath() + " " + expf;
        System.out.println(cmdLine);

        Process executor = Runtime.getRuntime().exec(cmdLine);

        StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null,
                SessionState.getConsole().getChildOutStream());
        StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null,
                SessionState.getConsole().getChildErrStream());

        outPrinter.start();
        errPrinter.start();

        int exitVal = executor.waitFor();

        if (exitVal != 0 && overWrite) {
            System.out.println("Overwriting results");
            cmdLine = "cp " + outf.getPath() + " " + expf;
            executor = Runtime.getRuntime().exec(cmdLine);
            exitVal = executor.waitFor();
        }

        return exitVal;
    }

    public int checkParseResults(String tname, ASTNode tree) throws Exception {

        if (tree != null) {
            File parseDir = new File(outDir, "parse");
            String expf = outPath(parseDir.toString(), tname.concat(".out"));

            File outf = null;
            outf = new File(logDir);
            outf = new File(outf, tname.concat(".out"));

            FileWriter outfd = new FileWriter(outf);
            outfd.write(tree.toStringTree());
            outfd.close();

            String cmdLine = "diff " + outf.getPath() + " " + expf;
            System.out.println(cmdLine);

            Process executor = Runtime.getRuntime().exec(cmdLine);

            StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null,
                    SessionState.getConsole().getChildOutStream());
            StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null,
                    SessionState.getConsole().getChildErrStream());

            outPrinter.start();
            errPrinter.start();

            int exitVal = executor.waitFor();

            if (exitVal != 0 && overWrite) {
                System.out.println("Overwriting results");
                cmdLine = "cp " + outf.getPath() + " " + expf;
                executor = Runtime.getRuntime().exec(cmdLine);
                exitVal = executor.waitFor();
            }

            return exitVal;
        } else {
            throw new Exception("Parse tree is null");
        }
    }

    public int checkPlan(String tname, List<Task<? extends Serializable>> tasks) throws Exception {

        if (tasks != null) {
            File planDir = new File(outDir, "plan");
            String planFile = outPath(planDir.toString(), tname + ".xml");

            File outf = null;
            outf = new File(logDir);
            outf = new File(outf, tname.concat(".xml"));

            FileOutputStream ofs = new FileOutputStream(outf);
            for (Task<? extends Serializable> plan : tasks) {
                Utilities.serializeTasks(plan, ofs);
            }

            String[] cmdArray = new String[6];
            cmdArray[0] = "diff";
            cmdArray[1] = "-b";
            cmdArray[2] = "-I";
            cmdArray[3] = "\\(\\(<java version=\".*\" class=\"java.beans.XMLDecoder\">\\)"
                    + "\\|\\(<string>.*/tmp/.*</string>\\)" + "\\|\\(<string>file:.*</string>\\)"
                    + "\\|\\(<string>pfile:.*</string>\\)" + "\\|\\(<string>[0-9]\\{10\\}</string>\\)"
                    + "\\|\\(<string>/.*/warehouse/.*</string>\\)\\)";
            cmdArray[4] = outf.getPath();
            cmdArray[5] = planFile;
            System.out.println(cmdArray[0] + " " + cmdArray[1] + " " + cmdArray[2] + "\'" + cmdArray[3] + "\'" + " "
                    + cmdArray[4] + " " + cmdArray[5]);

            Process executor = Runtime.getRuntime().exec(cmdArray);

            StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null,
                    SessionState.getConsole().getChildOutStream());
            StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null,
                    SessionState.getConsole().getChildErrStream());

            outPrinter.start();
            errPrinter.start();

            int exitVal = executor.waitFor();

            if (exitVal != 0 && overWrite) {
                System.out.println("Overwriting results");
                String cmdLine = "cp " + outf.getPath() + " " + planFile;
                executor = Runtime.getRuntime().exec(cmdLine);
                exitVal = executor.waitFor();
            }

            return exitVal;
        } else {
            throw new Exception("Plan is null");
        }

    }

    /*
     * This seems unused. Comment out first in case it is used somewhere. public
     * int checkResults(String tname) throws Exception { Path warehousePath =
     * new Path(FileSystem.get(conf).getUri().getPath()); warehousePath = new
     * Path(warehousePath, (new URI(testWarehouse)).getPath());
     * 
     * Path localPath = new Path(FileSystem.getLocal(conf).getUri().getPath());
     * localPath = new Path(localPath, logDir); localPath = new Path(localPath,
     * "warehouse_local_copy"); System.out.println("warehousePath = " +
     * warehousePath.toString() + " localPath = " + localPath.toString());
     * 
     * if (FileSystem.getLocal(conf).exists(localPath)) {
     * FileSystem.getLocal(conf).delete(localPath, true); }
     * 
     * copyDirectoryToLocal(warehousePath, localPath); normalizeNames(new
     * File(localPath.toUri().getPath()));
     * 
     * String[] cmdArray; if (overWrite == false) { cmdArray = new String[6];
     * cmdArray[0] = "diff"; cmdArray[1] = "-r"; cmdArray[2] = "--exclude=tmp";
     * cmdArray[3] = "--exclude=.svn"; cmdArray[4] =
     * localPath.toUri().getPath(); cmdArray[5] = (new File(outDir,
     * tname)).getPath() + "/warehouse"; System.out.println(cmdArray[0] + " " +
     * cmdArray[1] + " " + cmdArray[2] + " " + cmdArray[3] + " " + cmdArray[4] +
     * " " + cmdArray[5]); } else { System.out.println("overwritting"); //
     * Remove any existing output String[] cmdArray1 = new String[5];
     * cmdArray1[0] = "rm"; cmdArray1[1] = "-rf"; cmdArray1[2] = (new
     * File(outDir, tname)).getPath(); System.out .println(cmdArray1[0] + " " +
     * cmdArray1[1] + " " + cmdArray1[2]);
     * 
     * Process executor = Runtime.getRuntime().exec(cmdArray1);
     * 
     * StreamPrinter outPrinter = new StreamPrinter( executor.getInputStream(),
     * null, SessionState.getConsole().getChildOutStream()); StreamPrinter
     * errPrinter = new StreamPrinter( executor.getErrorStream(), null,
     * SessionState.getConsole().getChildErrStream());
     * 
     * outPrinter.start(); errPrinter.start(); int exitVal = executor.waitFor();
     * if (exitVal != 0) { return exitVal; }
     * 
     * // Capture code cmdArray = new String[5]; cmdArray[0] = "cp"; cmdArray[1]
     * = "-r"; cmdArray[2] = localPath.toUri().getPath(); cmdArray[3] = (new
     * File(outDir, tname)).getPath(); System.out.println(cmdArray[0] + " " +
     * cmdArray[1] + " " + cmdArray[2] + " " + cmdArray[3]); }
     * 
     * Process executor = Runtime.getRuntime().exec(cmdArray);
     * 
     * StreamPrinter outPrinter = new StreamPrinter( executor.getInputStream(),
     * null, SessionState.getConsole().getChildOutStream()); StreamPrinter
     * errPrinter = new StreamPrinter( executor.getErrorStream(), null,
     * SessionState.getConsole().getChildErrStream());
     * 
     * outPrinter.start(); errPrinter.start();
     * 
     * int exitVal = executor.waitFor();
     * 
     * return exitVal; }
     */

    /**
     * Given the current configurations (e.g., hadoop version and execution
     * mode), return the correct file name to compare with the current test run
     * output.
     * 
     * @param outDir
     *            The directory where the reference log files are stored.
     * @param testName
     *            The test file name (terminated by ".out").
     * @return The file name appended with the configuration values if it
     *         exists.
     */
    public String outPath(String outDir, String testName) {
        String ret = (new File(outDir, testName)).getPath();
        // List of configurations. Currently the list consists of hadoop version
        // and execution mode only
        List<String> configs = new ArrayList<String>();
        configs.add(this.hadoopVer);

        Deque<String> stack = new LinkedList<String>();
        StringBuilder sb = new StringBuilder();
        sb.append(testName);
        stack.push(sb.toString());

        // example file names are input1.q.out_0.20.0_minimr or
        // input2.q.out_0.17
        for (String s : configs) {
            sb.append('_');
            sb.append(s);
            stack.push(sb.toString());
        }
        while (stack.size() > 0) {
            String fileName = stack.pop();
            File f = new File(outDir, fileName);
            if (f.exists()) {
                ret = f.getPath();
                break;
            }
        }
        return ret;
    }

    public int checkCliDriverResults(String tname) throws Exception {
        String[] cmdArray;
        assert (qMap.containsKey(tname));

        String outFileName = outPath(outDir, tname + ".out");

        cmdArray = new String[] { "diff", "-a", "-I", "file:", "-I", "pfile:", "-I", "hdfs:", "-I", "/tmp/", "-I",
                "invalidscheme:", "-I", "lastUpdateTime", "-I", "lastAccessTime", "-I", "[Oo]wner", "-I",
                "CreateTime", "-I", "LastAccessTime", "-I", "Location", "-I", "transient_lastDdlTime", "-I",
                "last_modified_", "-I", "java.lang.RuntimeException", "-I", "at org", "-I", "at sun", "-I",
                "at java", "-I", "at junit", "-I", "Caused by:", "-I", "LOCK_QUERYID:", "-I", "grantTime", "-I",
                "[.][.][.] [0-9]* more", "-I", "USING 'java -cp", (new File(logDir, tname + ".out")).getPath(),
                outFileName };

        System.out.println(org.apache.commons.lang.StringUtils.join(cmdArray, ' '));

        Process executor = Runtime.getRuntime().exec(cmdArray);

        StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null,
                SessionState.getConsole().getChildOutStream());
        StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null,
                SessionState.getConsole().getChildErrStream());

        outPrinter.start();
        errPrinter.start();

        int exitVal = executor.waitFor();

        if (exitVal != 0 && overWrite) {
            System.out.println("Overwriting results");
            cmdArray = new String[3];
            cmdArray[0] = "cp";
            cmdArray[1] = (new File(logDir, tname + ".out")).getPath();
            cmdArray[2] = outFileName;
            executor = Runtime.getRuntime().exec(cmdArray);
            exitVal = executor.waitFor();
        }

        return exitVal;
    }

    public ASTNode parseQuery(String tname) throws Exception {
        return pd.parse(qMap.get(tname));
    }

    public void resetParser() throws SemanticException {
        drv.init();
        pd = new ParseDriver();
        sem = new SemanticAnalyzer(conf);
    }

    public List<Task<? extends Serializable>> analyzeAST(ASTNode ast) throws Exception {

        // Do semantic analysis and plan generation
        Context ctx = new Context(conf);
        while ((ast.getToken() == null) && (ast.getChildCount() > 0)) {
            ast = (ASTNode) ast.getChild(0);
        }

        sem.analyze(ast, ctx);
        ctx.clear();
        return sem.getRootTasks();
    }

    public TreeMap<String, String> getQMap() {
        return qMap;
    }

    /**
     * QTestSetup defines test fixtures which are reused across testcases, and
     * are needed before any test can be run
     */
    public static class QTestSetup {
        private MiniZooKeeperCluster zooKeeperCluster = null;
        private int zkPort;
        private ZooKeeper zooKeeper;

        public QTestSetup() {
        }

        public void preTest(HiveConf conf) throws Exception {

            if (zooKeeperCluster == null) {
                String tmpdir = System.getProperty("user.dir") + "/../build/ql/tmp";
                zooKeeperCluster = new MiniZooKeeperCluster();
                zkPort = zooKeeperCluster.startup(new File(tmpdir, "zookeeper"));
            }

            if (zooKeeper != null) {
                zooKeeper.close();
            }

            int sessionTimeout = conf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT);
            zooKeeper = new ZooKeeper("localhost:" + zkPort, sessionTimeout, null);

            String zkServer = "localhost";
            conf.set("hive.zookeeper.quorum", zkServer);
            conf.set("hive.zookeeper.client.port", "" + zkPort);
        }

        public void postTest(HiveConf conf) throws Exception {
            if (zooKeeperCluster == null) {
                return;
            }

            if (zooKeeper != null) {
                zooKeeper.close();
            }

            ZooKeeperHiveLockManager.releaseAllLocks(conf);
        }

        public void tearDown() throws Exception {
            if (zooKeeperCluster != null) {
                zooKeeperCluster.shutdown();
                zooKeeperCluster = null;
            }
        }
    }

    /**
     * QTRunner: Runnable class for running a a single query file.
     * 
     **/
    public static class QTRunner implements Runnable {
        private final QTestUtil2 qt;
        private final String fname;

        public QTRunner(QTestUtil2 qt, String fname) {
            this.qt = qt;
            this.fname = fname;
        }

        public void run() {
            try {
                // assumption is that environment has already been cleaned once
                // globally
                // hence each thread does not call cleanUp() and createSources()
                // again
                qt.cliInit(fname, false);
                qt.executeClient(fname);
            } catch (Throwable e) {
                System.err.println("Query file " + fname + " failed with exception " + e.getMessage());
                e.printStackTrace();
                outputTestFailureHelpMessage();
            }
        }
    }

    /**
     * executes a set of query files either in sequence or in parallel. Uses
     * QTestUtil to do so
     * 
     * @param qfiles
     *            array of input query files containing arbitrary number of hive
     *            queries
     * @param resDirs
     *            array of output directories one corresponding to each input
     *            query file
     * @param mt
     *            whether to run in multithreaded mode or not
     * @return true if all the query files were executed successfully, else
     *         false
     * 
     *         In multithreaded mode each query file is run in a separate
     *         thread. the caller has to arrange that different query files do
     *         not collide (in terms of destination tables)
     */
    public static boolean queryListRunner(File[] qfiles, String[] resDirs, String[] logDirs, boolean mt,
            Test test) {

        assert (qfiles.length == resDirs.length);
        assert (qfiles.length == logDirs.length);
        boolean failed = false;
        try {
            QTestUtil2[] qt = new QTestUtil2[qfiles.length];
            QTestSetup[] qsetup = new QTestSetup[qfiles.length];
            for (int i = 0; i < qfiles.length; i++) {
                qt[i] = new QTestUtil2(resDirs[i], logDirs[i], false, "0.20");
                qt[i].addFile(qfiles[i]);
                qt[i].clearTestSideEffects();
            }

            if (mt) {
                // in multithreaded mode - do cleanup/initialization just once

                qt[0].cleanUp();
                qt[0].createSources();
                qt[0].clearTestSideEffects();

                QTRunner[] qtRunners = new QTestUtil2.QTRunner[qfiles.length];
                Thread[] qtThread = new Thread[qfiles.length];

                for (int i = 0; i < qfiles.length; i++) {
                    qtRunners[i] = new QTestUtil2.QTRunner(qt[i], qfiles[i].getName());
                    qtThread[i] = new Thread(qtRunners[i]);
                }

                for (int i = 0; i < qfiles.length; i++) {
                    qtThread[i].start();
                }

                for (int i = 0; i < qfiles.length; i++) {
                    qtThread[i].join();
                    int ecode = qt[i].checkCliDriverResults(qfiles[i].getName());
                    if (ecode != 0) {
                        failed = true;
                        System.err.println(
                                "Test " + qfiles[i].getName() + " results check failed with error code " + ecode);
                        outputTestFailureHelpMessage();
                    }
                }

            } else {

                for (int i = 0; i < qfiles.length && !failed; i++) {
                    qt[i].cliInit(qfiles[i].getName());
                    qt[i].executeClient(qfiles[i].getName());
                    int ecode = qt[i].checkCliDriverResults(qfiles[i].getName());
                    if (ecode != 0) {
                        failed = true;
                        System.err.println(
                                "Test " + qfiles[i].getName() + " results check failed with error code " + ecode);
                        outputTestFailureHelpMessage();
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }
        return (!failed);
    }

    public static void outputTestFailureHelpMessage() {
        System.err.println(
                "See build/ql/tmp/hive.log, " + "or try \"ant test ... -Dtest.silent=false\" to get more logs.");
        System.err.flush();
    }
}