Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hive.beeline; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.InputStream; import java.io.PrintStream; import java.io.StringBufferInputStream; import java.io.UnsupportedEncodingException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.google.common.base.Function; import com.google.common.collect.Lists; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hive.jdbc.Utils; import org.apache.hive.jdbc.miniHS2.MiniHS2; import org.apache.hive.jdbc.miniHS2.MiniHS2.MiniClusterType; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; /** * TestBeeLineWithArgs - executes tests of the command-line arguments to BeeLine * */ public class TestBeeLineWithArgs { private enum OutStream { ERR, OUT } // Default location of HiveServer2 private static final String tableName = "TestBeelineTable1"; private static final String tableComment = "Test table comment"; private static MiniHS2 miniHS2; private static final String userName = System.getProperty("user.name"); private List<String> getBaseArgs(String jdbcUrl) { List<String> argList = new ArrayList<>(8); argList.add("-d"); argList.add(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER); argList.add("-u"); argList.add(jdbcUrl); argList.add("-n"); argList.add(userName); return argList; } /** * Start up a local Hive Server 2 for these tests */ @BeforeClass public static void preTests() throws Exception { HiveConf hiveConf = new HiveConf(); hiveConf.setVar(HiveConf.ConfVars.HIVE_LOCK_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); hiveConf.setBoolVar(HiveConf.ConfVars.HIVEOPTIMIZEMETADATAQUERIES, false); hiveConf.set(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "verbose"); miniHS2 = new MiniHS2(hiveConf, MiniClusterType.TEZ); Map<String, String> confOverlay = new HashMap<String, String>(); miniHS2.start(confOverlay); createTable(); } /** * Create table for use by tests * @throws ClassNotFoundException * @throws SQLException */ private static void createTable() throws ClassNotFoundException, SQLException { Class.forName(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER); Connection con = DriverManager.getConnection(miniHS2.getBaseJdbcURL(), userName, ""); assertNotNull("Connection is null", con); assertFalse("Connection should not be closed", con.isClosed()); Statement stmt = con.createStatement(); assertNotNull("Statement is null", stmt); stmt.execute("set hive.support.concurrency = false"); HiveConf conf = new HiveConf(); String dataFileDir = conf.get("test.data.files").replace('\\', '/').replace("c:", ""); Path dataFilePath = new Path(dataFileDir, "kv1.txt"); // drop table. ignore error. try { stmt.execute("drop table " + tableName); } catch (Exception ex) { fail(ex.toString() + " " + ExceptionUtils.getStackTrace(ex)); } // create table stmt.execute("create table " + tableName + " (under_col int comment 'the under column', value string) comment '" + tableComment + "'"); // load data stmt.execute("load data local inpath '" + dataFilePath.toString() + "' into table " + tableName); } /** * Shut down a local Hive Server 2 for these tests */ @AfterClass public static void postTests() { if (miniHS2.isStarted()) { miniHS2.stop(); } } /** * Execute a script with "beeline -f or -i" * @param argList List of arguments for beeline * @param inputStream input stream if any * @param streamType if output from STDERR or STDOUT needs to be returned * @return The stderr and stdout from running the script * @throws Throwable */ private static String testCommandLineScript(List<String> argList, InputStream inputStream, OutStream streamType) throws Throwable { BeeLine beeLine = new BeeLine(); ByteArrayOutputStream os = new ByteArrayOutputStream(); PrintStream beelineOutputStream = new PrintStream(os); switch (streamType) { case OUT: beeLine.setOutputStream(beelineOutputStream); break; case ERR: beeLine.setErrorStream(beelineOutputStream); break; default: throw new RuntimeException("Unexpected outstream type " + streamType); } String[] args = argList.toArray(new String[argList.size()]); beeLine.begin(args, inputStream); String output = os.toString("UTF8"); beeLine.close(); return output; } /** * Attempt to execute a simple script file with the -f and -i option to * BeeLine to test for presence of an expected pattern in the output (stdout * or stderr), fail if not found. Print PASSED or FAILED * * @param expectedRegex * Text to look for in command output (stdout) * @param shouldMatch * true if the pattern should be found, false if it should not * @throws Exception * on command execution error */ private void testScriptFile(String scriptText, List<String> argList, String expectedRegex, boolean shouldMatch) throws Throwable { testScriptFile(scriptText, argList, OutStream.OUT, Collections.singletonList(new Tuple<>(expectedRegex, shouldMatch))); } /** * Attempt to execute a simple script file with the -f and -i option * to BeeLine to test for presence of an expected pattern * in the output (stdout or stderr), fail if not found. * Print PASSED or FAILED * @param argList arguments * @param outType output stream type * @param expectedRegex Text to look for in command output (stdout) * @param shouldMatch true if the pattern should be found, false if it should not * @throws Throwable */ private void testScriptFile(String scriptText, List<String> argList, OutStream outType, String expectedRegex, boolean shouldMatch) throws Throwable { testScriptFile(scriptText, argList, outType, Collections.singletonList(new Tuple<>(expectedRegex, shouldMatch))); } private void testScriptFile(String scriptText, List<String> argList, OutStream streamType, List<Tuple<String>> expectedMatches) throws Throwable { testScriptFile(scriptText, argList, streamType, expectedMatches, Arrays.asList(Modes.values())); } /** * Attempt to execute a simple script file with the -f or -i option * to BeeLine (or both) to test for presence of an expected pattern * in the output (stdout or stderr), fail if not found. * Print PASSED or FAILED * @param scriptText script to test the output for * @param argList arguments to be passed to the script file to execute and produce output * @param streamType Whether match should be done against STDERR or STDOUT * @param expectedMatches List of Tuple's defining the pattern to match and result of matching * @param modes testing modes we have to run the script as * @throws Exception on command execution error */ private void testScriptFile(String scriptText, List<String> argList, OutStream streamType, List<Tuple<String>> expectedMatches, List<Modes> modes) throws Throwable { // Put the script content in a temp file File scriptFile = File.createTempFile(this.getClass().getSimpleName(), "temp"); System.out.println("script file is " + scriptFile.getAbsolutePath()); scriptFile.deleteOnExit(); PrintStream os = new PrintStream(new FileOutputStream(scriptFile)); os.print(scriptText); os.close(); List<Tuple<Pattern>> patternsToBeMatched = Lists.transform(expectedMatches, new Function<Tuple<String>, Tuple<Pattern>>() { @Override public Tuple<Pattern> apply(Tuple<String> tuple) { return new Tuple<>(Pattern.compile(".*" + tuple.pattern + ".*", Pattern.DOTALL), tuple.shouldMatch); } }); for (Modes mode : modes) { String output = mode.output(scriptFile, argList, streamType); for (Tuple<Pattern> patternToMatch : patternsToBeMatched) { Matcher m = patternToMatch.pattern.matcher(output); boolean matches = m.matches(); if (patternToMatch.shouldMatch != matches) { //failed fail("Output" + output + " should" + (patternToMatch.shouldMatch ? "" : " not") + " contain " + patternToMatch.pattern.pattern()); } } } scriptFile.delete(); } /* We are testing for both type of modes always so not passing that as a parameter for now */ enum Modes { INIT { @Override String output(File scriptFile, List<String> argList, OutStream streamType) throws Throwable { List<String> copy = new ArrayList<>(argList); copy.add("-i"); copy.add(scriptFile.getAbsolutePath()); return testCommandLineScript(copy, new StringBufferInputStream("!quit\n"), streamType); } }, SCRIPT { @Override String output(File scriptFile, List<String> argList, OutStream streamType) throws Throwable { List<String> copy = new ArrayList<>(argList); copy.add("-f"); copy.add(scriptFile.getAbsolutePath()); return testCommandLineScript(copy, null, streamType); } }; abstract String output(File scriptFile, List<String> argList, OutStream streamType) throws Throwable; } /** * Attempt to execute the enclosed query with the -e option to BeeLine * Test for presence of an expected pattern * in the output (stdout or stderr), fail if not found * Print PASSED or FAILED * @param expectedPattern Text to look for in command output/error * @param shouldMatch true if the pattern should be found, false if it should not * @throws Exception on command execution error */ private void testCommandEnclosedQuery(String enclosedQuery, String expectedPattern, boolean shouldMatch, List<String> argList, OutStream out) throws Throwable { List<String> copy = new ArrayList<String>(argList); copy.add("-e"); copy.add(enclosedQuery); String output = testCommandLineScript(copy, null, out); boolean matches = output.contains(expectedPattern); if (shouldMatch != matches) { //failed fail("Output" + output + " should" + (shouldMatch ? "" : " not") + " contain " + expectedPattern); } } /** * Test that BeeLine will read comment lines that start with whitespace * @throws Throwable */ @Test public void testWhitespaceBeforeCommentScriptFile() throws Throwable { final String SCRIPT_TEXT = " -- comment has spaces and tabs before it\n # comment has spaces and tabs before it\n"; final String EXPECTED_PATTERN = "cannot recognize input near '<EOF>'"; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, false); } /** * Attempt to execute a simple script file with the -f option to BeeLine * Test for presence of an expected pattern * in the output (stdout or stderr), fail if not found * Print PASSED or FAILED */ @Test public void testPositiveScriptFile() throws Throwable { final String SCRIPT_TEXT = "show databases;\n"; final String EXPECTED_PATTERN = " default "; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** * Fix to HIVE-10541: Beeline requires a newline at the end of each query in a file. * Otherwise, the last line of cmd in the script will be ignored. */ @Test public void testLastLineCmdInScriptFile() throws Throwable { final String SCRIPT_TEXT = "show databases;\nshow tables;"; final String EXPECTED_PATTERN = " testbeelinetable1 "; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** * Test Beeline -hivevar option. User can specify --hivevar name=value on Beeline command line. * In the script, user should be able to use it in the form of ${name}, which will be substituted with * the value. * @throws Throwable */ @Test public void testBeelineHiveVariable() throws Throwable { List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--hivevar"); argList.add("DUMMY_TBL=dummy"); final String SCRIPT_TEXT = "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};"; final String EXPECTED_PATTERN = "dummy"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test public void testBeelineHiveConfVariable() throws Throwable { List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--hiveconf"); argList.add("test.hive.table.name=dummy"); final String SCRIPT_TEXT = "create table ${hiveconf:test.hive.table.name} (d int);\nshow tables;\n" + " drop table ${hiveconf:test.hive.table.name};\n"; final String EXPECTED_PATTERN = "dummy"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** * Test Beeline -hivevar option. User can specify --hivevar name=value on Beeline command line. * This test defines multiple variables using repeated --hivevar or --hiveconf flags. * @throws Throwable */ @Test public void testBeelineMultiHiveVariable() throws Throwable { List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--hivevar"); argList.add("TABLE_NAME=dummy2"); argList.add("--hiveconf"); argList.add("COLUMN_NAME=d"); argList.add("--hivevar"); argList.add("COMMAND=create"); argList.add("--hivevar"); argList.add("OBJECT=table"); argList.add("--hiveconf"); argList.add("COLUMN_TYPE=int"); final String SCRIPT_TEXT = "${COMMAND} ${OBJECT} ${TABLE_NAME} " + "(${hiveconf:COLUMN_NAME} ${hiveconf:COLUMN_TYPE});" + "\nshow tables;\n drop ${OBJECT} ${TABLE_NAME};\n"; final String EXPECTED_PATTERN = "dummy2"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** * Attempt to execute a simple script file with the -f option to BeeLine * The first command should fail and the second command should not execute * Print PASSED or FAILED */ @Test public void testBreakOnErrorScriptFile() throws Throwable { List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "select * from abcdefg01;\nshow databases;\n"; final String EXPECTED_PATTERN = " default "; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, false); } @Test public void testTabInScriptFile() throws Throwable { List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "CREATE\tTABLE IF NOT EXISTS testTabInScriptFile\n(id\tint);\nSHOW TABLES;" + "\ndrop table testTabInScriptFile"; final String EXPECTED_PATTERN = "testTabInScriptFile"; testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, EXPECTED_PATTERN, false); } @Test public void testBeelineShellCommand() throws Throwable { List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "!sh echo \"hello world.\" > hw.txt\n!sh cat hw.txt\n!rm hw.txt"; final String EXPECTED_PATTERN = "hello world"; testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), Collections.singletonList(Modes.SCRIPT)); } /** * Select null from table , check how null is printed * Print PASSED or FAILED */ @Test public void testNullDefault() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "select null from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "NULL"; testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true); } /** * Select null from table , check if default null is printed differently * Print PASSED or FAILED */ @Test public void testNullNonEmpty() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "!set nullemptystring false\n select null from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "NULL"; testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true); } @Test public void testGetVariableValue() throws Throwable { final String SCRIPT_TEXT = "set env:TERM;"; final String EXPECTED_PATTERN = "env:TERM"; testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, EXPECTED_PATTERN, true); } /** * Select null from table , check if setting null to empty string works. * Original beeline/sqlline used to print nulls as empty strings. * Also test csv2 output format * Print PASSED or FAILED */ @Test public void testNullEmpty() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "abc,,def"; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=csv2"); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** * Test writing output using DSV format, with custom delimiter ";" */ @Test public void testDSVOutput() throws Throwable { String SCRIPT_TEXT = getFormatTestQuery(); List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=dsv"); argList.add("--delimiterForDSV=;"); final String EXPECTED_PATTERN = "1;NULL;defg;ab\"c;1.0"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** * Test writing output using TSV (new) format */ @Test public void testTSV2Output() throws Throwable { String SCRIPT_TEXT = getFormatTestQuery(); List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=tsv2"); final String EXPECTED_PATTERN = "1\tNULL\tdefg\tab\"c\t1.0"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** * Test writing output using TSV deprecated format */ @Test public void testTSVOutput() throws Throwable { String SCRIPT_TEXT = getFormatTestQuery(); List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=tsv"); final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c\'\t'1.0'"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** * Test writing output using new TSV format */ @Test public void testTSV2OutputWithDoubleQuotes() throws Throwable { String SCRIPT_TEXT = getFormatTestQueryForEableQuotes(); List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=tsv2"); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "1\tNULL\tdefg\t\"ab\"\"c\"\t\"\"\"aa\"\"\"\t1.0"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } /** * Test writing output using TSV deprecated format */ @Test public void testTSVOutputWithDoubleQuotes() throws Throwable { String SCRIPT_TEXT = getFormatTestQueryForEableQuotes(); List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=tsv"); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c'\t'\"aa\"'\t'1.0'"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } /** * Test writing output using new CSV format */ @Test public void testCSV2OutputWithDoubleQuotes() throws Throwable { String SCRIPT_TEXT = getFormatTestQueryForEableQuotes(); List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=csv2"); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "1,NULL,defg,\"ab\"\"c\",\"\"\"aa\"\"\",1.0"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } /** * Test writing output using CSV deprecated format */ @Test public void testCSVOutputWithDoubleQuotes() throws Throwable { String SCRIPT_TEXT = getFormatTestQueryForEableQuotes(); List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=csv"); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c','\"aa\"','1.0'"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } /** * Test writing output using DSV format, with custom delimiter ";" */ @Test public void testDSVOutputWithDoubleQuotes() throws Throwable { String SCRIPT_TEXT = getFormatTestQueryForEableQuotes(); List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=dsv"); argList.add("--delimiterForDSV=;"); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "1;NULL;defg;\"ab\"\"c\";\"\"\"aa\"\"\";1.0"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } /** * Test writing output using TSV deprecated format * Check for deprecation message */ @Test public void testTSVOutputDeprecation() throws Throwable { String SCRIPT_TEXT = getFormatTestQuery(); List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=tsv"); final String EXPECTED_PATTERN = "Format tsv is deprecated, please use tsv2"; testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); } /** * Test writing output using CSV deprecated format * Check for deprecation message */ @Test public void testCSVOutputDeprecation() throws Throwable { String SCRIPT_TEXT = getFormatTestQuery(); List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=csv"); final String EXPECTED_PATTERN = "Format csv is deprecated, please use csv2"; testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true))); } /** * Test writing output using CSV deprecated format */ @Test public void testCSVOutput() throws Throwable { String SCRIPT_TEXT = getFormatTestQuery(); List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=csv"); final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c\','1.0'"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } private String getFormatTestQuery() { return "set hive.support.concurrency = false;\n" + "select 1, null, 'defg', 'ab\"c', 1.0D from " + tableName + " limit 1 ;\n"; } private String getFormatTestQueryForEableQuotes() { return "set hive.support.concurrency = false;\n" + "select 1, null, 'defg', 'ab\"c', '\"aa\"', 1.0D from " + tableName + " limit 1 ;\n"; } /** * Select null from table , check if setting null to empty string works - Using beeling cmd line * argument. * Original beeline/sqlline used to print nulls as empty strings * Print PASSED or FAILED */ @Test public void testNullEmptyCmdArg() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "'abc','','def'"; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--nullemptystring=true"); argList.add("--outputformat=csv"); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** * Attempt to execute a missing script file with the -f option to BeeLine */ @Test public void testNegativeScriptFile() throws Throwable { final String EXPECTED_PATTERN = " default "; // Create and delete a temp file File scriptFile = File.createTempFile("beelinenegative", "temp"); scriptFile.delete(); List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("-f"); argList.add(scriptFile.getAbsolutePath()); try { String output = testCommandLineScript(argList, null, OutStream.OUT); if (output.contains(EXPECTED_PATTERN)) { fail("Output: " + output + " Negative pattern: " + EXPECTED_PATTERN); } } catch (Throwable e) { e.printStackTrace(); throw e; } } /** * HIVE-4566 * @throws UnsupportedEncodingException */ @Test public void testNPE() throws UnsupportedEncodingException { BeeLine beeLine = new BeeLine(); ByteArrayOutputStream os = new ByteArrayOutputStream(); PrintStream beelineOutputStream = new PrintStream(os); beeLine.setOutputStream(beelineOutputStream); beeLine.setErrorStream(beelineOutputStream); beeLine.runCommands(new String[] { "!typeinfo" }); String output = os.toString("UTF8"); Assert.assertFalse(output.contains("java.lang.NullPointerException")); assertTrue(output.contains("No current connection")); beeLine.runCommands(new String[] { "!nativesql" }); output = os.toString("UTF8"); Assert.assertFalse(output.contains("java.lang.NullPointerException")); assertTrue(output.contains("No current connection")); System.out.println(">>> PASSED " + "testNPE"); } @Test public void testHiveVarSubstitution() throws Throwable { List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL() + "#D_TBL=dummy_t"); final String SCRIPT_TEXT = "create table ${D_TBL} (d int);\nshow tables;\ndrop table ${D_TBL};\n"; final String EXPECTED_PATTERN = "dummy_t"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test public void testEmbeddedBeelineConnection() throws Throwable { String embeddedJdbcURL = Utils.URL_PREFIX + "/Default"; List<String> argList = getBaseArgs(embeddedJdbcURL); argList.add("--hivevar"); argList.add("DUMMY_TBL=embedded_table"); // Set to non-zk lock manager to avoid trying to connect to zookeeper final String SCRIPT_TEXT = "set hive.lock.manager=org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager;\n" + "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};\n"; final String EXPECTED_PATTERN = "embedded_table"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** * Test Beeline could show the query progress for time-consuming query. * @throws Throwable */ @Test public void testQueryProgress() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "set hive.server2.logging.operation.level=execution;\n" + "select count(*) from " + tableName + ";\n"; // Check for part of log message as well as part of progress information final String EXPECTED_PATTERN = "ELAPSED TIME"; final String UNEXPECTED_PATTERN = "(?=Reducer 2\\:).*(?=Map 1\\:)"; testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, Arrays.asList(new Tuple<>(EXPECTED_PATTERN, true), new Tuple<>(UNEXPECTED_PATTERN, false))); } /** * Test Beeline could show the query progress for time-consuming query when hive.exec.parallel * is true * * We have changed the pattern to not look of the progress bar as the test runs fine individually * and also as part of the whole class, on CI however they are batched and that might have caused * some issue, it needs more investigation for the same * * @throws Throwable */ @Test public void testQueryProgressParallel() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "set hive.exec.parallel = true;\n" + "select count(*) from " + tableName + ";\n"; // Check for part of log message as well as part of progress information final String EXPECTED_PATTERN = "Number of reducers determined to be."; testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, EXPECTED_PATTERN, true); } /** * Test Beeline will hide the query progress when silent option is set. * @throws Throwable */ @Test public void testQueryProgressHidden() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "!set silent true\n" + "select count(*) from " + tableName + ";\n"; final String EXPECTED_PATTERN = "Executing command"; testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, EXPECTED_PATTERN, false); } @Test public void testQueryProgressWithHiveServer2ProgressBarDisabled() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\nset hive.server2.in.place.progress=false;\n" + "select count(*) from " + tableName + ";\n"; // Check for part of log message as well as part of progress information final String EXPECTED_PATTERN = "(?=Reducer 2\\:).*(?=Map 1\\:)"; testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, Arrays.asList(new Tuple<>(EXPECTED_PATTERN, true), new Tuple<>("ELAPSED TIME", false))); } @Test public void testMultiCommandsInOneline() throws Throwable { final String SCRIPT_TEXT = "drop table if exists multiCmdTbl;create table multiCmdTbl " + "(key int);show tables; --multicommands in one line"; final String EXPECTED_PATTERN = " multicmdtbl "; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); final String SCRIPT_TEXT_DROP = "drop table multiCmdTbl;show tables;"; testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false); } @Test public void testMultiCommandsInOneEnclosedQuery() throws Throwable { final String QUERY_TEXT = "drop table if exists multiCmdTbl;create table multiCmdTbl " + "(key int);show tables; --multicommands in one line"; final String EXPECTED_PATTERN = " multicmdtbl "; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); testCommandEnclosedQuery(QUERY_TEXT, EXPECTED_PATTERN, true, argList, OutStream.OUT); final String QUERY_TEXT_DROP = "drop table multiCmdTbl;show tables;"; testCommandEnclosedQuery(QUERY_TEXT_DROP, EXPECTED_PATTERN, false, argList, OutStream.OUT); } @Test public void testOneCommandInMultiLines() throws Throwable { final String SCRIPT_TEXT = "drop table if exists multiCmdTbl;create table \nmultiCmdTbl " + "(key int);show tables; --one command in multiple lines"; final String EXPECTED_PATTERN = " multicmdtbl "; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); final String SCRIPT_TEXT_DROP = "drop table\nmultiCmdTbl;show tables;"; testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false); } @Test public void testEscapeSemiColonInQueries() throws Throwable { final String SCRIPT_TEXT = "drop table if exists multiCmdTbl;create table multiCmdTbl " + "(key int, value string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\;' LINES " + " TERMINATED BY '\\n';show tables; --one command in multiple lines"; final String EXPECTED_PATTERN = " multicmdtbl "; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); final String SCRIPT_TEXT_DROP = "drop table\nmultiCmdTbl;show tables;"; testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false); } @Test public void testEscapeSemiColonInEnclosedQuery() throws Throwable { final String QUERY_TEXT = "drop table if exists multiCmdTbl;create table multiCmdTbl " + "(key int, value string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\;' LINES " + " TERMINATED BY '\\n';show tables;"; final String EXPECTED_PATTERN = " multicmdtbl "; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); testCommandEnclosedQuery(QUERY_TEXT, EXPECTED_PATTERN, true, argList, OutStream.OUT); final String QUERY_TEXT_DROP = "drop table multiCmdTbl;show tables;"; testCommandEnclosedQuery(QUERY_TEXT_DROP, EXPECTED_PATTERN, false, argList, OutStream.OUT); } @Test public void testEmbeddedBeelineOutputs() throws Throwable { String embeddedJdbcURL = Utils.URL_PREFIX + "/Default"; List<String> argList = getBaseArgs(embeddedJdbcURL); // Set to non-zk lock manager to avoid trying to connect to zookeeper final String SCRIPT_TEXT = "set hive.lock.manager=org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager;\n" + "set hive.compute.query.using.stats=false;\n" + "create table if not exists embeddedBeelineOutputs(d int);\n" + "set a=1;\nselect count(*) from embeddedBeelineOutputs;\n" + "drop table embeddedBeelineOutputs;\n"; final String EXPECTED_PATTERN = "Stage-1 map ="; testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); } @Test public void testConnectionUrlWithSemiColon() throws Throwable { List<String> argList = getBaseArgs(miniHS2.getJdbcURL("default", "sess_var_list?var1=value1")); final String SCRIPT_TEXT = "set var1"; final String EXPECTED_PATTERN = "var1=value1"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** * Test Beeline !connect with beeline saved vars * @throws Throwable */ @Test public void testBeelineConnectEnvVar() throws Throwable { final String jdbcUrl = miniHS2.getBaseJdbcURL(); List<String> argList = new ArrayList<>(); argList.add("-u"); argList.add("blue"); argList.add("-d"); argList.add(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER); final String SCRIPT_TEXT = "create table blueconnecttest (d int);\nshow tables;\ndrop table blueconnecttest;\n"; final String EXPECTED_PATTERN = "blueconnecttest"; // We go through these hijinxes because java considers System.getEnv // to be read-only, and offers no way to set an env var from within // a process, only for processes that we sub-spawn. final BeeLineOpts.Env baseEnv = BeeLineOpts.getEnv(); BeeLineOpts.Env newEnv = new BeeLineOpts.Env() { @Override public String get(String envVar) { if (envVar.equalsIgnoreCase("BEELINE_URL_BLUE")) { return jdbcUrl; } else { return baseEnv.get(envVar); } } }; BeeLineOpts.setEnv(newEnv); testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), Collections.singletonList(Modes.SCRIPT)); } /** * Test that if we !close, we can still !reconnect * @throws Throwable */ @Test public void testBeelineReconnect() throws Throwable { List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "!close\n" + "!reconnect\n\n\n" + "create table reconnecttest (d int);\nshow tables;\ndrop table reconnecttest;\n"; final String EXPECTED_PATTERN = "reconnecttest"; testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), Collections.singletonList(Modes.SCRIPT)); } /** * Attempt to execute a simple script file with the usage of user & password variables in URL. * Test for presence of an expected pattern * in the output (stdout or stderr), fail if not found * Print PASSED or FAILED */ @Test public void testConnectionWithURLParams() throws Throwable { final String EXPECTED_PATTERN = " hivetest "; List<String> argList = new ArrayList<>(); argList.add("-d"); argList.add(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER); argList.add("-u"); argList.add(miniHS2.getBaseJdbcURL() + ";user=hivetest;password=hive"); String SCRIPT_TEXT = "select current_user();"; testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** * Test that Beeline queries don't treat semicolons inside quotations as query-ending characters. */ @Test public void testQueryNonEscapedSemiColon() throws Throwable { String SCRIPT_TEXT = "drop table if exists nonEscapedSemiColon;create table nonEscapedSemiColon " + "(key int, value int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';';show tables;"; String EXPECTED_PATTERN = "nonescapedsemicolon"; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); //look for the " nonEscapedSemiColon " in the query text not the table name which comes //in the result EXPECTED_PATTERN = " nonEscapedSemiColon "; testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, EXPECTED_PATTERN, false); } @Test public void testSelectQueryWithNonEscapedSemiColon() throws Throwable { String SCRIPT_TEXT = "select ';', \"';'\", '\";\"', '\\';', ';\\'', '\\\";', ';\\\"' from " + tableName + ";"; final String EXPECTED_PATTERN = ";\t';'\t\";\"\t';\t;'\t\";\t;\""; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=tsv2"); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** * Attempt to execute a simple script file with the usage of user & password variables in URL. * Test for presence of an expected pattern * in the output (stdout or stderr), fail if not found * Print PASSED or FAILED */ @Test public void testShowDbInPrompt() throws Throwable { final String EXPECTED_PATTERN = " \\(default\\)>"; List<String> argList = new ArrayList<>(); argList.add("--showDbInPrompt"); argList.add("-u"); argList.add(miniHS2.getBaseJdbcURL() + ";user=hivetest;password=hive"); String SCRIPT_TEXT = "select current_user();"; testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); } @Test public void testBeelineShellCommandWithoutConn() throws Throwable { List<String> argList = new ArrayList<>(); final String SCRIPT_TEXT = "!sh echo hello world"; final String EXPECTED_PATTERN = "hello world"; testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), Collections.singletonList(Modes.SCRIPT)); } /** * Attempt to execute Beeline with force option to continue running script even after errors. * Test for presence of an expected pattern to match the output of a valid command at the end. */ @Test public void testBeelineWithForce() throws Throwable { final String SCRIPT_TEXT = "drop table does_not_exist;\ncreate table incomplete_syntax(a, string, );\n " + "drop table if exists new_table;\n create table new_table(foo int, bar string);\n " + "desc new_table;\n"; final String EXPECTED_PATTERN = "2 rows selected"; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--force"); testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); } private static class Tuple<K> { final K pattern; final boolean shouldMatch; Tuple(K pattern, boolean shouldMatch) { this.pattern = pattern; this.shouldMatch = shouldMatch; } } /** * Test that Beeline can handle \\ characters within a string literal. Either at the beginning, middle, or end of the * literal. */ @Test public void testBackslashInLiteral() throws Throwable { String SCRIPT_TEXT = "select 'hello\\\\', '\\\\hello', 'hel\\\\lo', '\\\\' as literal;"; final String EXPECTED_PATTERN = "hello\\\\\t\\\\hello\thel\\\\lo\t\\\\"; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=tsv2"); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test public void testCustomDelimiter() throws Throwable { String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$"; final String EXPECTED_PATTERN = "hello\thello\thello"; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--delimiter=$"); argList.add("--outputformat=tsv2"); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test public void testCustomMultiCharDelimiter() throws Throwable { String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$$"; final String EXPECTED_PATTERN = "hello\thello\thello"; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--delimiter=$$"); argList.add("--outputformat=tsv2"); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test public void testCustomDelimiterWithMultiQuery() throws Throwable { String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$select 'world', 'world', 'world'$"; final String EXPECTED_PATTERN1 = "hello\thello\thello"; final String EXPECTED_PATTERN2 = "world\tworld\tworld"; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--delimiter=$"); argList.add("--outputformat=tsv2"); List<Tuple<String>> expectedMatches = Arrays.asList(new Tuple<>(EXPECTED_PATTERN1, true), new Tuple<>(EXPECTED_PATTERN2, true)); testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, expectedMatches); } @Test public void testCustomDelimiterBeelineCmd() throws Throwable { String SCRIPT_TEXT = "!delimiter $\n select 'hello', 'hello', 'hello'$"; final String EXPECTED_PATTERN = "hello\thello\thello"; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=tsv2"); testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } }