org.apache.tajo.QueryTestCaseBase.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.tajo.QueryTestCaseBase.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.tajo;

import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.protobuf.ServiceException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.algebra.*;
import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.cli.tsql.InvalidStatementException;
import org.apache.tajo.cli.tsql.ParsedResult;
import org.apache.tajo.cli.tsql.SimpleParser;
import org.apache.tajo.client.TajoClient;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.exception.InsufficientPrivilegeException;
import org.apache.tajo.exception.TajoException;
import org.apache.tajo.exception.UndefinedTableException;
import org.apache.tajo.jdbc.FetchResultSet;
import org.apache.tajo.jdbc.TajoMemoryResultSet;
import org.apache.tajo.jdbc.TajoResultSetBase;
import org.apache.tajo.master.GlobalEngine;
import org.apache.tajo.parser.sql.SQLAnalyzer;
import org.apache.tajo.plan.LogicalOptimizer;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.LogicalPlanner;
import org.apache.tajo.plan.verifier.LogicalPlanVerifier;
import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier;
import org.apache.tajo.plan.verifier.VerificationState;
import org.apache.tajo.schema.IdentifierUtil;
import org.apache.tajo.storage.BufferPool;
import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.util.FileUtil;
import org.junit.*;
import org.junit.rules.TestName;
import org.junit.rules.TestRule;
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.lang.annotation.*;
import java.lang.management.BufferPoolMXBean;
import java.lang.reflect.Method;
import java.net.URL;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.*;

import static org.junit.Assert.*;

/**
 * (Note that this class is not thread safe. Do not execute maven test in any parallel mode.)
 * <br />
 * <code>QueryTestCaseBase</code> provides useful methods to easily execute queries and verify their results.
 *
 * This class basically uses four resource directories:
 * <ul>
 *   <li>src/test/resources/dataset - contains a set of data files. It contains sub directories, each of which
 *   corresponds each test class. All data files in each sub directory can be used in the corresponding test class.</li>
 *
 *   <li>src/test/resources/queries - This is the query directory. It contains sub directories, each of which
 *   corresponds each test class. All query files in each sub directory can be used in the corresponding test
 *   class. This directory also can include <code>positive</code> and <code>negative</code> directories, each
 *   of which include multiple files containing multiple queries. They are used by <code>runPositiveTests()</code> and
 *   <code>runNegativeTests()</code>, which test just successfully completed and failed respectively
 *   without compararing query results.</li>
 *
 *   <li>src/test/resources/results - This is the result directory. It contains sub directories, each of which
 *   corresponds each test class. All result files in each sub directory can be used in the corresponding test class.
 *   </li>
 * </ul>
 *
 * For example, if you create a test class named <code>TestJoinQuery</code>, you should create a pair of query and
 * result set directories as follows:
 *
 * <pre>
 *   src-|
 *       |- resources
 *             |- dataset
 *             |     |- TestJoinQuery
 *             |              |- table1.tbl
 *             |              |- table2.tbl
 *             |
 *             |- queries
 *             |     |- TestJoinQuery
 *             |              |- positive
 *             |                    |- valid_join_conditions.sql
 *             |              |- negative
 *             |                    |- invalid_join_conditions.sql
 *             |              |- TestInnerJoin.sql
 *             |              |- table1_ddl.sql
 *             |              |- table2_ddl.sql
 *             |
 *             |- results
 *                   |- TestJoinQuery
 *                            |- TestInnerJoin.result
 * </pre>
 *
 * <code>QueryTestCaseBase</code> basically provides the following methods:
 * <ul>
 *  <li><code>{@link #executeQuery()}</code> - executes a corresponding query and returns an ResultSet instance</li>
 *  <li><code>{@link #executeFile(String)}</code> - executes a given query file included in the corresponding query
 *  file in the current class's query directory</li>
 *  <li><code>assertResultSet()</code> - check if the query result is equivalent to the expected result included
 *  in the corresponding result file in the current class's result directory.</li>
 *  <li><code>cleanQuery()</code> - clean up all resources</li>
 *  <li><code>executeDDL()</code> - execute a DDL query like create or drop table.</li>
 * </ul>
 *
 * In order to make use of the above methods, query files and results file must be as follows:
 * <ul>
 *  <li>Each query file must be located on the subdirectory whose structure must be src/resources/queries/${ClassName},
 *  where ${ClassName} indicates an actual test class's simple name.</li>
 *  <li>Each result file must be located on the subdirectory whose structure must be src/resources/results/${ClassName},
 *  where ${ClassName} indicates an actual test class's simple name.</li>
 * </ul>
 *
 * Especially, {@link #executeQuery() and {@link #assertResultSet(java.sql.ResultSet)} methods automatically finds
 * a query file to be executed and a result to be compared, which are corresponding to the running class and method.
 * For them, query and result files additionally must be follows as:
 * <ul>
 *  <li>Each result file must have the file extension '.result'</li>
 *  <li>Each query file must have the file extension '.sql'.</li>
 * </ul>
 */
public class QueryTestCaseBase {
    private static final Log LOG = LogFactory.getLog(QueryTestCaseBase.class);
    protected static final TpchTestBase testBase;
    protected static final TajoTestingCluster testingCluster;
    protected static TajoConf conf;
    protected static TajoClient client;
    protected static final CatalogService catalog;
    protected static final SQLAnalyzer sqlParser;
    protected static PreLogicalPlanVerifier verifier;
    protected static LogicalPlanner planner;
    protected static LogicalOptimizer optimizer;
    protected static LogicalPlanVerifier postVerifier;

    /** the base path of dataset directories */
    protected static Path datasetBasePath;
    /** the base path of query directories */
    protected static Path queryBasePath;
    /** the base path of result directories */
    protected static Path resultBasePath;

    static {
        testBase = TpchTestBase.getInstance();
        testingCluster = testBase.getTestingCluster();
        conf = testBase.getTestingCluster().getConfiguration();
        catalog = testBase.getTestingCluster().getMaster().getCatalog();

        GlobalEngine engine = testingCluster.getMaster().getContext().getGlobalEngine();
        sqlParser = engine.getAnalyzer();
        verifier = engine.getPreLogicalPlanVerifier();
        planner = engine.getLogicalPlanner();
        optimizer = engine.getLogicalOptimizer();
        postVerifier = engine.getLogicalPlanVerifier();
    }

    /** It transiently contains created tables for the running test class. */
    private static String currentDatabase;
    private static Set<String> createdTableGlobalSet = new HashSet<>();
    // queries and results directory corresponding to subclass class.
    protected Path currentQueryPath;
    protected Path namedQueryPath;
    protected Path currentResultPath;
    protected Path currentDatasetPath;
    protected Path namedDatasetPath;

    protected FileSystem currentResultFS;

    protected final String testParameter;

    // for getting a method name
    @Rule
    public TestName name = new TestName();

    @BeforeClass
    public static void setUpClass() throws Exception {
        client = testBase.getTestingCluster().newTajoClient();

        URL datasetBaseURL = ClassLoader.getSystemResource("dataset");
        Preconditions.checkNotNull(datasetBaseURL, "dataset directory is absent.");
        datasetBasePath = new Path(datasetBaseURL.toString());
        URL queryBaseURL = ClassLoader.getSystemResource("queries");
        Preconditions.checkNotNull(queryBaseURL, "queries directory is absent.");
        queryBasePath = new Path(queryBaseURL.toString());
        URL resultBaseURL = ClassLoader.getSystemResource("results");
        Preconditions.checkNotNull(resultBaseURL, "results directory is absent.");
        resultBasePath = new Path(resultBaseURL.toString());
    }

    @AfterClass
    public static void tearDownClass() throws Exception {
        for (String tableName : createdTableGlobalSet) {
            client.updateQuery("DROP TABLE IF EXISTS " + IdentifierUtil.denormalizeIdentifier(tableName));
        }
        createdTableGlobalSet.clear();

        // if the current database is "default", shouldn't drop it.
        if (!currentDatabase.equals(TajoConstants.DEFAULT_DATABASE_NAME)) {
            for (String tableName : catalog.getAllTableNames(currentDatabase)) {
                try {
                    client.updateQuery("DROP TABLE IF EXISTS " + tableName);
                } catch (InsufficientPrivilegeException i) {
                    LOG.warn("relation '" + tableName + "' is read only.");
                }
            }

            client.selectDatabase(TajoConstants.DEFAULT_DATABASE_NAME);
            try {
                client.dropDatabase(currentDatabase);
            } catch (InsufficientPrivilegeException e) {
                LOG.warn("database '" + currentDatabase + "' is read only.");
            }
        }
        client.close();
    }

    @Before
    public void printTestName() {
        /* protect a travis stalled build */
        BufferPoolMXBean direct = BufferPool.getDirectBufferPool();
        BufferPoolMXBean mapped = BufferPool.getMappedBufferPool();
        System.out.println(
                String.format("Used heap: %s/%s, direct:%s/%s, mapped:%s/%s, Active Threads: %d, Run: %s.%s",
                        FileUtil.humanReadableByteCount(
                                Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(), false),
                        FileUtil.humanReadableByteCount(Runtime.getRuntime().maxMemory(), false),
                        FileUtil.humanReadableByteCount(direct.getMemoryUsed(), false),
                        FileUtil.humanReadableByteCount(direct.getTotalCapacity(), false),
                        FileUtil.humanReadableByteCount(mapped.getMemoryUsed(), false),
                        FileUtil.humanReadableByteCount(mapped.getTotalCapacity(), false), Thread.activeCount(),
                        getClass().getSimpleName(), name.getMethodName()));
    }

    @After
    public void clear() {
        getClient().unsetSessionVariables(Lists.newArrayList(SessionVars.TIMEZONE.name()));
    }

    public QueryTestCaseBase() {
        // hive 0.12 does not support quoted identifier.
        // So, we use lower case database names when Tajo uses HiveCatalogStore.
        if (testingCluster.isHiveCatalogStoreRunning()) {
            this.currentDatabase = getClass().getSimpleName().toLowerCase();
        } else {
            this.currentDatabase = getClass().getSimpleName();
        }
        testParameter = null;
        init();
    }

    public QueryTestCaseBase(String currentDatabase) {
        this(currentDatabase, null);
    }

    public QueryTestCaseBase(String currentDatabase, String testParameter) {
        this.currentDatabase = currentDatabase;
        this.testParameter = testParameter;
        init();
    }

    private void init() {
        String className = getClass().getSimpleName();
        currentQueryPath = new Path(queryBasePath, className);
        currentResultPath = new Path(resultBasePath, className);
        currentDatasetPath = new Path(datasetBasePath, className);
        NamedTest namedTest = getClass().getAnnotation(NamedTest.class);
        if (namedTest != null) {
            namedQueryPath = new Path(queryBasePath, namedTest.value());
            namedDatasetPath = new Path(datasetBasePath, namedTest.value());
        }

        try {
            // if the current database is "default", we don't need create it because it is already prepated at startup time.
            if (!currentDatabase.equals(TajoConstants.DEFAULT_DATABASE_NAME)) {
                client.updateQuery(
                        "CREATE DATABASE IF NOT EXISTS " + IdentifierUtil.denormalizeIdentifier(currentDatabase));
            }
            client.selectDatabase(currentDatabase);
            currentResultFS = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());

        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false");
    }

    protected TajoClient getClient() {
        return client;
    }

    public String getCurrentDatabase() {
        return currentDatabase;
    }

    private static VerificationState verify(String query) throws TajoException {

        VerificationState state = new VerificationState();
        QueryContext context = LocalTajoTestingUtility.createDummyContext(conf);

        Expr expr = sqlParser.parse(query);

        verifier.verify(context, state, expr);

        if (state.getErrors().size() > 0) {
            return state;
        }
        LogicalPlan plan = planner.createPlan(context, expr);
        optimizer.optimize(plan);
        postVerifier.verify(state, plan);

        return state;
    }

    public void assertValidSQL(String query) throws IOException {
        VerificationState state = null;
        try {
            state = verify(query);
            if (state.getErrors().size() > 0) {
                fail(state.getErrors().get(0).getMessage());
            }
        } catch (TajoException e) {
            throw new RuntimeException(e);
        }
    }

    public void assertValidSQLFromFile(String fileName) throws IOException {
        Path queryFilePath = getQueryFilePath(fileName);
        String query = FileUtil.readTextFile(new File(queryFilePath.toUri()));
        assertValidSQL(query);
    }

    public void assertInvalidSQL(String query) throws IOException {
        VerificationState state = null;
        try {
            state = verify(query);

            if (state.getErrors().size() == 0) {
                fail(PreLogicalPlanVerifier.class.getSimpleName() + " cannot catch any verification error: "
                        + query);
            }

        } catch (TajoException e) {
            throw new RuntimeException(e);
        }
    }

    public void assertInvalidSQLFromFile(String fileName) throws IOException {
        Path queryFilePath = getQueryFilePath(fileName);
        String query = FileUtil.readTextFile(new File(queryFilePath.toUri()));
        assertInvalidSQL(query);
    }

    public void assertPlanError(String fileName) throws IOException {
        Path queryFilePath = getQueryFilePath(fileName);
        String query = FileUtil.readTextFile(new File(queryFilePath.toUri()));
        try {
            verify(query);
        } catch (TajoException e) {
            return;
        }
        fail("Cannot catch any planning error from: " + query);
    }

    protected ResultSet executeString(String sql) throws TajoException {
        return client.executeQueryAndGetResult(sql);
    }

    /**
     * It executes the query file and compare the result against the the result file.
     *
     * @throws Exception
     */
    public void assertQuery() throws Exception {
        ResultSet res = null;
        try {
            res = executeQuery();
            assertResultSet(res);
        } finally {
            if (res != null) {
                res.close();
            }
        }
    }

    /**
     * It executes a given query statement and verifies the result against the the result file.
     *
     * @param query A query statement
     * @throws Exception
     */
    public void assertQueryStr(String query) throws Exception {
        ResultSet res = null;
        try {
            res = executeString(query);
            assertResultSet(res);
        } finally {
            if (res != null) {
                res.close();
            }
        }
    }

    /**
     * Execute a query contained in the file located in src/test/resources/results/<i>ClassName</i>/<i>MethodName</i>.
     * <i>ClassName</i> and <i>MethodName</i> will be replaced by actual executed class and methods.
     *
     * @return ResultSet of query execution.
     */
    public ResultSet executeQuery() throws Exception {
        return executeFile(getMethodName() + ".sql");
    }

    private volatile Description current;

    @Rule
    public TestRule watcher = new TestWatcher() {
        @Override
        protected void starting(Description description) {
            QueryTestCaseBase.this.current = description;
        }
    };

    @Target({ ElementType.METHOD })
    @Retention(RetentionPolicy.RUNTIME)
    protected @interface SimpleTest {
        String[] prepare() default {};

        QuerySpec[] queries() default {};

        String[] cleanup() default {};
    }

    @Target({ ElementType.METHOD })
    @Retention(RetentionPolicy.RUNTIME)
    protected @interface QuerySpec {
        String value();

        boolean override() default false;

        Option option() default @Option;
    }

    @Target({ ElementType.METHOD })
    @Retention(RetentionPolicy.RUNTIME)
    protected @interface Option {
        boolean withExplain() default false;

        boolean withExplainGlobal() default false;

        boolean parameterized() default false;

        boolean sort() default false;

        boolean resultClose() default true;
    }

    private static class DummyQuerySpec implements QuerySpec {
        private final String value;
        private final Option option;

        public DummyQuerySpec(String value, Option option) {
            this.value = value;
            this.option = option;
        }

        public Class<? extends Annotation> annotationType() {
            return QuerySpec.class;
        }

        public String value() {
            return value;
        }

        public boolean override() {
            return option != null;
        }

        public Option option() {
            return option;
        }
    }

    private static class DummyOption implements Option {
        private final boolean explain;
        private final boolean withExplainGlobal;
        private final boolean parameterized;
        private final boolean sort;
        private final boolean resultClose;

        public DummyOption(boolean explain, boolean withExplainGlobal, boolean parameterized, boolean sort,
                boolean resultClose) {
            this.explain = explain;
            this.withExplainGlobal = withExplainGlobal;
            this.parameterized = parameterized;
            this.sort = sort;
            this.resultClose = resultClose;
        }

        public Class<? extends Annotation> annotationType() {
            return Option.class;
        }

        public boolean withExplain() {
            return explain;
        }

        public boolean withExplainGlobal() {
            return withExplainGlobal;
        }

        public boolean parameterized() {
            return parameterized;
        }

        public boolean sort() {
            return sort;
        }

        @Override
        public boolean resultClose() {
            return resultClose;
        }
    }

    protected Collection<String> getBatchQueries(Collection<Path> paths)
            throws IOException, InvalidStatementException {
        List<String> queries = Lists.newArrayList();

        for (Path p : paths) {
            for (ParsedResult statement : SimpleParser.parseScript(FileUtil.readTextFile(new File(p.toUri())))) {
                queries.add(statement.getStatement());
            }
        }

        return queries;
    }

    /**
     * Run all positive tests
     *
     * @throws Exception
     */
    protected void runPositiveTests() throws Exception {
        Collection<String> queries = getBatchQueries(getPositiveQueryFiles());

        ResultSet result = null;
        for (String query : queries) {
            try {
                result = client.executeQueryAndGetResult(query);
            } catch (TajoException e) {
                fail("Positive Test Failed: " + e.getMessage());
            } finally {
                if (result != null) {
                    result.close();
                }
            }
        }
    }

    /**
     * Run all negative tests
     *
     * @throws Exception
     */
    protected void runNegativeTests() throws Exception {
        Collection<String> queries = getBatchQueries(getNegativeQueryFiles());

        ResultSet result = null;
        for (String query : queries) {
            try {
                result = client.executeQueryAndGetResult(query);
                fail("Negative Test Failed: " + query);
            } catch (TajoException e) {
            } finally {
                if (result != null) {
                    result.close();
                }
            }
        }
    }

    protected Optional<TajoResultSetBase[]> runSimpleTests() throws Exception {
        String methodName = getMethodName();
        Method method = current.getTestClass().getMethod(methodName);
        SimpleTest annotation = method.getAnnotation(SimpleTest.class);
        if (annotation == null) {
            throw new IllegalStateException("Cannot find test annotation");
        }

        List<String> prepares = new ArrayList<>(Arrays.asList(annotation.prepare()));
        QuerySpec[] queries = annotation.queries();
        Option defaultOption = method.getAnnotation(Option.class);
        if (defaultOption == null) {
            defaultOption = new DummyOption(false, false, false, false, true);
        }

        boolean fromFile = false;
        if (queries.length == 0) {
            Path queryFilePath = getQueryFilePath(getMethodName() + ".sql");
            List<ParsedResult> parsedResults = SimpleParser
                    .parseScript(FileUtil.readTextFile(new File(queryFilePath.toUri())));
            int i = 0;
            for (; i < parsedResults.size() - 1; i++) {
                prepares.add(parsedResults.get(i).getStatement());
            }
            queries = new QuerySpec[] { new DummyQuerySpec(parsedResults.get(i).getHistoryStatement(), null) };
            fromFile = true; // do not append query index to result file
        }

        try {
            for (String prepare : prepares) {
                client.executeQueryAndGetResult(prepare).close();
            }
            List<TajoResultSetBase> resultSetBases = new ArrayList<>();
            for (int i = 0; i < queries.length; i++) {
                QuerySpec spec = queries[i];
                Option option = spec.override() ? spec.option() : defaultOption;
                String prefix = "";
                testingCluster.getConfiguration().set(TajoConf.ConfVars.$TEST_PLAN_SHAPE_FIX_ENABLED.varname,
                        "true");
                if (option.withExplain()) {// Enable this option to fix the shape of the generated plans.
                    prefix += resultSetToString(executeString("explain " + spec.value()));
                }
                if (option.withExplainGlobal()) {
                    // Enable this option to fix the shape of the generated plans.
                    prefix += resultSetToString(executeString("explain global " + spec.value()));
                }

                // plan test
                if (prefix.length() > 0) {
                    String planResultName = methodName + (fromFile ? "" : "." + (i + 1))
                            + ((option.parameterized() && testParameter != null) ? "." + testParameter : "")
                            + ".plan";
                    Path resultPath = StorageUtil.concatPath(currentResultPath, planResultName);
                    if (currentResultFS.exists(resultPath)) {
                        assertEquals("Plan Verification for: " + (i + 1) + " th test",
                                FileUtil.readTextFromStream(currentResultFS.open(resultPath)), prefix);
                    } else if (prefix.length() > 0) {
                        // If there is no result file expected, create gold files for new tests.
                        FileUtil.writeTextToStream(prefix, currentResultFS.create(resultPath));
                        LOG.info(
                                "New test output for " + current.getDisplayName() + " is written to " + resultPath);
                        // should be copied to src directory
                    }
                }

                testingCluster.getConfiguration().set(TajoConf.ConfVars.$TEST_PLAN_SHAPE_FIX_ENABLED.varname,
                        "false");
                ResultSet result = client.executeQueryAndGetResult(spec.value());
                resultSetBases.add((TajoResultSetBase) result);

                // result test
                String fileName = methodName + (fromFile ? "" : "." + (i + 1)) + ".result";
                Path resultPath = StorageUtil.concatPath(currentResultPath, fileName);
                if (currentResultFS.exists(resultPath)) {
                    assertEquals("Result Verification for: " + (i + 1) + " th test",
                            FileUtil.readTextFromStream(currentResultFS.open(resultPath)),
                            resultSetToString(result, option.sort()));
                } else if (!isNull(result)) {
                    // If there is no result file expected, create gold files for new tests.
                    FileUtil.writeTextToStream(resultSetToString(result, option.sort()),
                            currentResultFS.create(resultPath));
                    LOG.info("New test output for " + current.getDisplayName() + " is written to " + resultPath);
                    // should be copied to src directory
                }
                if (option.resultClose()) {
                    result.close();
                }
            }

            if (resultSetBases.size() > 0) {
                return Optional.of(resultSetBases.toArray(new TajoResultSetBase[resultSetBases.size()]));
            } else {
                return Optional.empty();
            }
        } finally {
            for (String cleanup : annotation.cleanup()) {
                try {
                    client.executeQueryAndGetResult(cleanup).close();
                } catch (SQLException e) {
                    // ignore
                }
            }
        }
    }

    protected void closeResultSets(ResultSet... resultSets) throws SQLException {
        for (ResultSet rs : resultSets) {
            rs.close();
        }
    }

    private boolean isNull(ResultSet result) throws SQLException {
        return result.getMetaData().getColumnCount() == 0;
    }

    protected String getMethodName() {
        String methodName = name.getMethodName();
        // In the case of parameter execution name's pattern is methodName[0]
        if (methodName.endsWith("]")) {
            int index = methodName.indexOf('[');
            methodName = methodName.substring(0, index);
        }
        return methodName;
    }

    public ResultSet executeJsonQuery() throws Exception {
        return executeJsonFile(getMethodName() + ".json");
    }

    /**
     * Execute a query contained in the given named file. This methods tries to find the given file within the directory
     * src/test/resources/results/<i>ClassName</i>.
     *
     * @param queryFileName The file name to be used to execute a query.
     * @return ResultSet of query execution.
     */
    public ResultSet executeFile(String queryFileName) throws Exception {
        Path queryFilePath = getQueryFilePath(queryFileName);

        List<ParsedResult> parsedResults = SimpleParser
                .parseScript(FileUtil.readTextFile(new File(queryFilePath.toUri())));
        if (parsedResults.size() > 1) {
            assertNotNull("This script \"" + queryFileName + "\" includes two or more queries");
        }

        int idx = 0;
        for (; idx < parsedResults.size() - 1; idx++) {
            client.executeQueryAndGetResult(parsedResults.get(idx).getHistoryStatement()).close();
        }

        ResultSet result = client.executeQueryAndGetResult(parsedResults.get(idx).getHistoryStatement());
        assertNotNull("Query succeeded test", result);
        return result;
    }

    public ResultSet executeJsonFile(String jsonFileName) throws Exception {
        Path queryFilePath = getQueryFilePath(jsonFileName);

        ResultSet result = client
                .executeJsonQueryAndGetResult(FileUtil.readTextFile(new File(queryFilePath.toUri())));
        assertNotNull("Query succeeded test", result);
        return result;
    }

    /**
     * Assert the equivalence between the expected result and an actual query result.
     * If it isn't it throws an AssertionError.
     *
     * @param result Query result to be compared.
     */
    public final void assertResultSet(ResultSet result) throws IOException {
        assertResultSet("Result Verification", result, getMethodName() + ".result");
    }

    /**
     * Assert the equivalence between the expected result and an actual query result.
     * If it isn't it throws an AssertionError.
     *
     * @param result Query result to be compared.
     * @param resultFileName The file name containing the result to be compared
     */
    public final void assertResultSet(ResultSet result, String resultFileName) throws IOException {
        assertResultSet("Result Verification", result, resultFileName);
    }

    /**
     * Assert the equivalence between the expected result and an actual query result.
     * If it isn't it throws an AssertionError with the given message.
     *
     * @param message message The message to printed if the assertion is failed.
     * @param result Query result to be compared.
     */
    public final void assertResultSet(String message, ResultSet result, String resultFileName) throws IOException {
        Path resultFile = getResultFile(resultFileName);
        try {
            verifyResultText(message, result, resultFile);
        } catch (SQLException e) {
            throw new IOException(e);
        }
    }

    public final void assertStrings(String actual) throws IOException {
        assertStrings(actual, getMethodName() + ".result");
    }

    public final void assertStrings(String actual, String resultFileName) throws IOException {
        assertStrings("Result Verification", actual, resultFileName);
    }

    public final void assertStrings(String message, String actual, String resultFileName) throws IOException {
        Path resultFile = getResultFile(resultFileName);
        String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri()));
        assertEquals(message, expectedResult, actual);
    }

    /**
     * Release all resources
     *
     * @param resultSet ResultSet
     */
    public final void cleanupQuery(ResultSet resultSet) throws IOException {
        if (resultSet == null) {
            return;
        }
        try {
            resultSet.close();
        } catch (SQLException e) {
            throw new IOException(e);
        }
    }

    /**
     * Assert that the database exists.
     * @param databaseName The database name to be checked. This name is case sensitive.
     */
    public void assertDatabaseExists(String databaseName) throws SQLException {
        assertTrue(client.existDatabase(databaseName));
    }

    /**
     * Assert that the database does not exists.
     * @param databaseName The database name to be checked. This name is case sensitive.
     */
    public void assertDatabaseNotExists(String databaseName) {
        assertTrue(!client.existDatabase(databaseName));
    }

    /**
     * Assert that the table exists.
     *
     * @param tableName The table name to be checked. This name is case sensitive.
     * @throws ServiceException
     */
    public void assertTableExists(String tableName) {
        assertTrue(client.existTable(tableName));
    }

    /**
     * Assert that the table does not exist.
     *
     * @param tableName The table name to be checked. This name is case sensitive.
     */
    public void assertTableNotExists(String tableName) {
        assertTrue(!client.existTable(tableName));
    }

    public void assertColumnExists(String tableName, String columnName) throws UndefinedTableException {
        TableDesc tableDesc = getTableDesc(tableName);
        assertTrue(tableDesc.getSchema().containsByName(columnName));
    }

    private TableDesc getTableDesc(String tableName) throws UndefinedTableException {
        return client.getTableDesc(tableName);
    }

    public void assertTablePropertyEquals(String tableName, String key, String expectedValue)
            throws UndefinedTableException {

        TableDesc tableDesc = getTableDesc(tableName);
        assertEquals(expectedValue, tableDesc.getMeta().getProperty(key));
    }

    public String resultSetToString(ResultSet resultSet) throws SQLException {
        return resultSetToString(resultSet, false);
    }

    /**
     * It transforms a ResultSet instance to rows represented as strings.
     *
     * @param resultSet ResultSet that contains a query result
     * @return String
     * @throws SQLException
     */
    public String resultSetToString(ResultSet resultSet, boolean sort) throws SQLException {
        StringBuilder sb = new StringBuilder();
        ResultSetMetaData rsmd = resultSet.getMetaData();
        int numOfColumns = rsmd.getColumnCount();

        for (int i = 1; i <= numOfColumns; i++) {
            if (i > 1)
                sb.append(",");
            String columnName = rsmd.getColumnName(i);
            sb.append(columnName);
        }
        sb.append("\n-------------------------------\n");

        List<String> results = new ArrayList<>();
        while (resultSet.next()) {
            StringBuilder line = new StringBuilder();
            for (int i = 1; i <= numOfColumns; i++) {
                if (i > 1)
                    line.append(",");
                String columnValue = resultSet.getString(i);
                if (resultSet.wasNull()) {
                    columnValue = "null";
                }
                line.append(columnValue);
            }
            results.add(line.toString());
        }
        if (sort) {
            Collections.sort(results);
        }
        for (String line : results) {
            sb.append(line).append('\n');
        }
        return sb.toString();
    }

    private void verifyResultText(String message, ResultSet res, Path resultFile) throws SQLException, IOException {
        String actualResult = resultSetToString(res);
        String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri()));
        assertEquals(message, expectedResult.trim(), actualResult.trim());
    }

    private Collection<Path> getPositiveQueryFiles() throws IOException {
        Path positiveQueryDir = StorageUtil.concatPath(currentQueryPath, "positive");
        FileSystem fs = currentQueryPath.getFileSystem(testBase.getTestingCluster().getConfiguration());

        if (!fs.exists(positiveQueryDir)) {
            throw new IOException("Cannot find " + positiveQueryDir);
        }

        return Collections2.transform(Lists.newArrayList(fs.listStatus(positiveQueryDir)),
                new Function<FileStatus, Path>() {
                    @Override
                    public Path apply(@Nullable FileStatus fileStatus) {
                        return fileStatus.getPath();
                    }
                });
    }

    private Collection<Path> getNegativeQueryFiles() throws IOException {
        Path positiveQueryDir = StorageUtil.concatPath(currentQueryPath, "negative");
        FileSystem fs = currentQueryPath.getFileSystem(testBase.getTestingCluster().getConfiguration());

        if (!fs.exists(positiveQueryDir)) {
            throw new IOException("Cannot find " + positiveQueryDir);
        }

        return Collections2.transform(Lists.newArrayList(fs.listStatus(positiveQueryDir)),
                new Function<FileStatus, Path>() {
                    @Override
                    public Path apply(@Nullable FileStatus fileStatus) {
                        return fileStatus.getPath();
                    }
                });
    }

    private Path getQueryFilePath(String fileName) throws IOException {
        Path queryFilePath = StorageUtil.concatPath(currentQueryPath, fileName);
        FileSystem fs = currentQueryPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
        if (!fs.exists(queryFilePath)) {
            if (namedQueryPath != null) {
                queryFilePath = StorageUtil.concatPath(namedQueryPath, fileName);
                fs = namedQueryPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
                if (!fs.exists(queryFilePath)) {
                    throw new IOException(
                            "Cannot find " + fileName + " at " + currentQueryPath + " and " + namedQueryPath);
                }
            } else {
                throw new IOException("Cannot find " + fileName + " at " + currentQueryPath);
            }
        }
        return queryFilePath;
    }

    protected String getResultContents(String fileName) throws IOException {
        Path resultFile = getResultFile(getMethodName() + ".result");
        return FileUtil.readTextFile(new File(resultFile.toUri()));
    }

    protected Path getResultFile(String fileName) throws IOException {
        Path resultPath = StorageUtil.concatPath(currentResultPath, fileName);
        FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
        assertTrue(resultPath.toString() + " existence check", fs.exists(resultPath));
        return resultPath;
    }

    protected Path getDataSetFile(String fileName) throws IOException {
        Path dataFilePath = StorageUtil.concatPath(currentDatasetPath, fileName);
        FileSystem fs = currentDatasetPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
        if (!fs.exists(dataFilePath)) {
            if (namedDatasetPath != null) {
                dataFilePath = StorageUtil.concatPath(namedDatasetPath, fileName);
                fs = namedDatasetPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
                if (!fs.exists(dataFilePath)) {
                    throw new IOException("Cannot find " + fileName + " at " + currentDatasetPath);
                }
            } else {
                throw new IOException("Cannot find " + fileName + " at " + currentDatasetPath);
            }
        }
        return dataFilePath;
    }

    public List<String> executeDDL(String ddlFileName, @Nullable String[] args) throws Exception {
        return executeDDL(ddlFileName, null, true, args);
    }

    /**
     *
     * Execute a data definition language (DDL) template. A general SQL DDL statement can be included in this file. But,
     * for user-specified table name or exact external table path, you must use some format string to indicate them.
     * The format string will be replaced by the corresponding arguments.
     *
     * The below is predefined format strings:
     * <ul>
     *   <li>${table.path} - It is replaced by the absolute file path that <code>dataFileName</code> points. </li>
     *   <li>${i} - It is replaced by the corresponding element of <code>args</code>. For example, ${0} and ${1} are
     *   replaced by the first and second elements of <code>args</code> respectively</li>. It uses zero-based index.
     * </ul>
     *
     * Example ddl
     * <pre>
     *   CREATE EXTERNAL TABLE ${0} (
     *     t_timestamp  TIMESTAMP,
     *     t_date    DATE
     *   ) USING TEXT LOCATION ${table.path}
     * </pre>
     *
     * @param ddlFileName A file name, containing a data definition statement.
     * @param dataFileName A file name, containing data rows, which columns have to be separated by vertical bar '|'.
     *                     This file name is used for replacing some format string indicating an external table location.
     * @param args A list of arguments, each of which is used to replace corresponding variable which has a form of ${i}.
     * @return The table names created
     */
    public List<String> executeDDL(String ddlFileName, @Nullable String dataFileName, @Nullable String... args)
            throws Exception {

        return executeDDL(ddlFileName, dataFileName, true, args);
    }

    private List<String> executeDDL(String ddlFileName, @Nullable String dataFileName, boolean isLocalTable,
            @Nullable String[] args) throws Exception {

        Path ddlFilePath = getQueryFilePath(ddlFileName);

        String template = FileUtil.readTextFile(new File(ddlFilePath.toUri()));
        String dataFilePath = null;
        if (dataFileName != null) {
            dataFilePath = getDataSetFile(dataFileName).toString();
        }
        String compiled = compileTemplate(template, dataFilePath, args);

        List<ParsedResult> parsedResults = SimpleParser.parseScript(compiled);
        List<String> createdTableNames = new ArrayList<>();

        for (ParsedResult parsedResult : parsedResults) {
            // parse a statement
            Expr expr = sqlParser.parse(parsedResult.getHistoryStatement());
            assertNotNull(ddlFilePath + " cannot be parsed", expr);

            if (expr.getType() == OpType.CreateTable) {
                CreateTable createTable = (CreateTable) expr;
                String tableName = createTable.getTableName();
                assertTrue("Table [" + tableName + "] creation is failed.",
                        client.updateQuery(parsedResult.getHistoryStatement()));

                TableDesc createdTable = client.getTableDesc(tableName);
                String createdTableName = createdTable.getName();

                assertTrue("table '" + createdTableName + "' creation check", client.existTable(createdTableName));
                if (isLocalTable) {
                    createdTableGlobalSet.add(createdTableName);
                    createdTableNames.add(tableName);
                }
            } else if (expr.getType() == OpType.DropTable) {
                DropTable dropTable = (DropTable) expr;
                String tableName = dropTable.getTableName();
                assertTrue("table '" + tableName + "' existence check",
                        client.existTable(IdentifierUtil.buildFQName(currentDatabase, tableName)));
                assertTrue("table drop is failed.", client.updateQuery(parsedResult.getHistoryStatement()));
                assertFalse("table '" + tableName + "' dropped check",
                        client.existTable(IdentifierUtil.buildFQName(currentDatabase, tableName)));
                if (isLocalTable) {
                    createdTableGlobalSet.remove(tableName);
                }
            } else if (expr.getType() == OpType.AlterTable) {
                AlterTable alterTable = (AlterTable) expr;
                String tableName = alterTable.getTableName();
                assertTrue("table '" + tableName + "' existence check", client.existTable(tableName));
                client.updateQuery(compiled);
                if (isLocalTable) {
                    createdTableGlobalSet.remove(tableName);
                }
            } else if (expr.getType() == OpType.CreateIndex) {
                // TODO: index existence check
                client.executeQuery(compiled);
            } else {
                assertTrue(ddlFilePath + " is not a Create or Drop Table statement", false);
            }
        }

        return createdTableNames;
    }

    /**
     * Replace format strings by a given parameters.
     *
     * @param template
     * @param dataFileName The data file name to replace <code>${table.path}</code>
     * @param args The list argument to replace each corresponding format string ${i}. ${i} uses zero-based index.
     * @return A string compiled
     */
    private String compileTemplate(String template, @Nullable String dataFileName, @Nullable String... args) {
        String result;
        if (dataFileName != null) {
            result = template.replace("${table.path}", "\'" + dataFileName + "'");
        } else {
            result = template;
        }

        if (args != null) {
            for (int i = 0; i < args.length; i++) {
                result = result.replace("${" + i + "}", args[i]);
            }
        }
        return result;
    }

    /**
     * Reads data file from Test Cluster's HDFS
     * @param path data parent path
     * @return data file's contents
     * @throws Exception
     */
    public String getTableFileContents(Path path) throws Exception {
        FileSystem fs = path.getFileSystem(conf);

        FileStatus[] files = fs.listStatus(path);

        if (files == null || files.length == 0) {
            return "";
        }

        StringBuilder sb = new StringBuilder();
        byte[] buf = new byte[1024];

        for (FileStatus file : files) {
            if (file.isDirectory()) {
                sb.append(getTableFileContents(file.getPath()));
                continue;
            }

            try (InputStream in = fs.open(file.getPath())) {
                while (true) {
                    int readBytes = in.read(buf);
                    if (readBytes <= 0) {
                        break;
                    }

                    sb.append(new String(buf, 0, readBytes));
                }
            }
        }

        return sb.toString();
    }

    /**
     * Reads data file from Test Cluster's HDFS
     * @param tableName
     * @return data file's contents
     * @throws Exception
     */
    public String getTableFileContents(String tableName) throws Exception {
        TableDesc tableDesc = testingCluster.getMaster().getCatalog().getTableDesc(getCurrentDatabase(), tableName);
        if (tableDesc == null) {
            return null;
        }

        Path path = new Path(tableDesc.getUri());
        return getTableFileContents(path);
    }

    public List<Path> listTableFiles(String tableName) throws Exception {
        TableDesc tableDesc = testingCluster.getMaster().getCatalog().getTableDesc(getCurrentDatabase(), tableName);
        if (tableDesc == null) {
            return null;
        }

        Path path = new Path(tableDesc.getUri());
        FileSystem fs = path.getFileSystem(conf);

        return listFiles(fs, path);
    }

    private List<Path> listFiles(FileSystem fs, Path path) throws Exception {
        List<Path> result = new ArrayList<>();
        FileStatus[] files = fs.listStatus(path);
        if (files == null || files.length == 0) {
            return result;
        }

        for (FileStatus eachFile : files) {
            if (eachFile.isDirectory()) {
                result.addAll(listFiles(fs, eachFile.getPath()));
            } else {
                result.add(eachFile.getPath());
            }
        }
        return result;
    }

    public static QueryId getQueryId(ResultSet resultSet) {
        if (resultSet instanceof TajoMemoryResultSet) {
            return ((TajoMemoryResultSet) resultSet).getQueryId();
        } else if (resultSet instanceof FetchResultSet) {
            return ((FetchResultSet) resultSet).getQueryId();
        } else {
            throw new IllegalArgumentException(resultSet.toString());
        }
    }
}