Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.pig.test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintWriter; import java.util.Properties; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.util.Shell; import org.apache.pig.ExecType; import org.apache.pig.impl.PigContext; import org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor; import org.apache.pig.tools.parameters.ParseException; import org.junit.Test; public class TestParamSubPreproc { private final Log log = LogFactory.getLog(getClass()); private BufferedReader pigIStream; private FileWriter pigOStream; private FileInputStream pigExResultStream; private String basedir = "test/org/apache/pig/test/data"; private void compareResults(InputStream expected, InputStream result) throws IOException { BufferedReader inExpected = new BufferedReader(new InputStreamReader(expected)); BufferedReader inResult = new BufferedReader(new InputStreamReader(result)); String exLine; String resLine; int lineNum = 0; while (true) { lineNum++; exLine = inExpected.readLine(); resLine = inResult.readLine(); if (exLine == null || resLine == null) break; assertEquals("Command line parameter substitution failed. " + "Expected : " + exLine + " , but got : " + resLine + " in line num : " + lineNum, exLine.trim(), resLine.trim()); } if (!(exLine == null && resLine == null)) { fail("Command line parameter substitution failed. " + "Expected : " + exLine + " , but got : " + resLine + " in line num : " + lineNum); } inExpected.close(); inResult.close(); } /* Test case 1 * Use a parameter within a pig script and provide value on the command line. */ @Test public void testCmdlineParam() throws Exception { log.info("Starting test testCmdlineParam() ..."); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date=20080228" }; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } /* Test case 2 * Use a parameter within a pig script and provide value through a file. */ @Test public void testFileParam() throws Exception { log.info("Starting test testFileParam()"); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; String[] argFiles = { basedir + "/ConfFile1.txt" }; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } /* Test case 3 * Use a parameter within a pig script and provide value through running a binary or script. */ @Test public void testShellCommand() throws Exception { log.info("Starting test testShellCommand()"); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = WithConditionalReplacement(basedir + "/input4.pig", "sh test/org/apache/pig/test/data/generate_date.sh", "test/org/apache/pig/test/data/generate_date.bat", Shell.WINDOWS); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; //{"date=`sh generate_date.sh`"}; //`date \\T`"}; String[] argFiles = null; // {basedir+"/ConfFile1.txt"}; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResultDefault.pig"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } /* Test case 4 * Use a Pig-supported parameter like "$0" and ensure that the Pig-supported parameter is not resolved */ @Test public void testPigParamNotResolved() throws Exception { log.info("Starting test testPigParamNotResolved()"); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input3.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; String[] argFiles = null; try { ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); } catch (RuntimeException e) { if (e.getMessage().equals("Undefined parameter : 4")) { fail("Pig supported parameter $4 should not have been resolved."); } } log.info("Done"); } /* Test case 5 * Use the %declare statement after the command has been used. */ @Test public void testUndefinedParam() throws Exception { log.info("Starting test testUndefinedParam()"); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input2.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; String[] argFiles = null; try { ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); fail("Should have thrown an Undefined parameter exception"); } catch (ParseException e) { assertEquals(e.getMessage(), "Undefined parameter : param"); } log.info("Done"); } /* Test case 7 * Use a parameter in %declare that is defined in terms of other parameters */ @Test public void testSubstitutionWithinValue() throws Exception { log.info("Starting test testSubstitutionWithinValue()"); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputSubstitutionWithinValue.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; String[] argFiles = { basedir + "/ConfFile1.txt" }; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } /* Test case 8 * Use a parameter within a pig script, provide value for it through running a binary or script. * The script itself takes an argument that is a parameter and should be resolved */ @Test public void testSubstitutionWithinShellCommand() throws Exception { log.info("Starting test testSubstitutionWithinShellCommand()"); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = WithConditionalReplacement(basedir + "/inputSubstitutionWithinShellCommand.pig", "sh test/org/apache/pig/test/data/generate_date.sh", "test/org/apache/pig/test/data/generate_date.bat", Shell.WINDOWS); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } /* Test case 9 * Use parameters passed on the command line/file that are not resolved prior to the declare statement. */ @Test public void testCmdlineParamPriortoDeclare() throws Exception { log.info("Starting test testCmdlineParamPriortoDeclare()"); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input2.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "param='20080228'" }; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResultCmdLnPriorDeclare.pig"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } /* Test case 10 * Use a command name in a %declare statement as a parameter */ @Test public void testCmdnameAsParamDeclare() throws Exception { log.info("Starting test testCmdnameAsParamDeclare()"); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = WithConditionalReplacement(basedir + "/inputCmdnameAsParamDeclare.pig", "sh \\$cmd.sh \\$date", "\\$cmd.bat \\$date", Shell.WINDOWS); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } /* Test case 11 * Use the same parameter multiple times on the command line. * Result : last value used and warning should be thrown */ @Test public void testMultipleCmdlineParam() throws Exception { log.info("Starting test testCmdnameAsParamDeclare()"); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date='092487'", "date='20080228'" }; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } /* Test case 12 * Read parameters from multiple files. */ @Test public void testFileParamsFromMultipleFiles() throws Exception { log.info("Starting test testFileParamsFromMultipleFiles()"); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputMultipleParams.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; String[] argFiles = { basedir + "/ConfFile1.txt", basedir + "/ConfFile2.txt" }; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } /* Test case 13 * Use the same parameter in multiple files. * Result: last value used and warning should be thrown */ @Test public void testSameParamInMultipleFiles() throws Exception { log.info("Starting test testSameParamInMultipleFiles()"); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputMultipleParams.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; String[] argFiles = { basedir + "/ConfFile3.txt", basedir + "/ConfFile2.txt", basedir + "/ConfFile1.txt" }; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } /* Test case 14 * Use the same parameter multiple times in a single file. * Result: last value used and warning should be thrown. */ @Test public void testMultipleParamsFromSingleFile() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; String[] argFiles = { basedir + "/ConfFileSameParamMultipleTimes.txt" }; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); } /* Test case 15,16 * Use an empty lines and Comment lines in the parameter file. * Result: Allowed */ @Test public void testEmptyCommentLineinConfigfile() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; String[] argFiles = { basedir + "/ConfFileWithEmptyComments.txt" }; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); } /* Test case 17 * Use a line in the file that is not empty or a comment but does not conform to param_name=param_value. */ @Test public void testInvalidLineinConfigfile() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; String[] argFiles = { basedir + "/ConfFileWithInvalidLines.txt" }; try { ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); fail("Should have thrown an exception"); } catch (ParseException e) { assertTrue(e.getMessage().startsWith("Encountered \" <IDENTIFIER> \"is \"\" at line 2, column 6.")); } } /* Test case 18,19 * Check a parameter line of form param_name=param_value is allowed. * Check a parameter line of form param_name<white space>=<white space>param_value is allowed. */ @Test public void testValidLinesinConfigfile() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputMultipleParams.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; String[] argFiles = { basedir + "/ConfFileWithValidLines.txt" }; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); } /* Test case 20 * Use a combination of command line and file parameters. */ @Test public void testCmdlineFileCombo() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputMultipleParams.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date='20080228'" }; String[] argFiles = { basedir + "/ConfFile2.txt" }; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); } /* Test case 21 * Use a combination of command line and file parameters where there are duplicate parameters. * Result: Command line parameters take precedence over files. */ @Test public void testCmdlineFileComboDuplicate() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputMultipleParams.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date='20080228'" }; String[] argFiles = { basedir + "/ConfFileDuplicates.txt" }; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); } /* Test case 22 * Use a combination of command line, file and declare. */ @Test public void testCmdlineFileDeclareCombo() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputThreeParams.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date='20080228'" }; String[] argFiles = { basedir + "/ConfFile2.txt" }; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); } /* Test case 23 * Use a combination of command line, file and declare where there are duplicate parameters. * Result: Declare has highest priority. */ @Test public void testCmdlineFileDeclareComboDuplicates() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputThreeParams.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date='20080228'", "tableName=\"skip this\"" }; String[] argFiles = { basedir + "/ConfFile2.txt" }; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); } /* Test case 24 * Use multiple declare statement that specify the same parameter. * Result: Scope of a parameter declared using declare is until the next declare statement that defines the same parameter. */ @Test public void testMultipleDeclareScope() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputMultipleDeclares.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date='20080228'" }; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResultMulDecs.pig"); compareResults(pigExResultStream, pigResultStream); } /* Test case 25 * Use %default to define param values */ @Test public void testDefaultParam() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputDefault.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); } /* Test case 26 * Use a combination of file, command line, declare and default. Default has the lowest precedence. */ @Test public void testCmdlineFileDeclareDefaultComboDuplicates() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputThreeParams.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date='20080228'", "tableName=\"skip this\"" }; String[] argFiles = { basedir + "/ConfFile2.txt" }; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); } /* Test case 28 * 28. More than 1 parameter is present and needs to be substituted within a line e.g. A = load '/data/$name/$date'; */ @Test public void testMultipleParamsinSingleLine() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputThreeParams.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date='20080228'" }; String[] argFiles = { basedir + "/ConfFile2.txt" }; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); } /* Test case 29 * Parameter is substituted within a literal e.g. store A into 'output/$name'; */ @Test public void testSubstituteWithinLiteral() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date='20080228'" }; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); } /* Test case 30 * Make sure that escaped values are not substituted e.g. A = load '/data/\$name' */ @Test public void testEscaping() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputEscape.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date='20080228'" }; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult2.pig"); compareResults(pigExResultStream, pigResultStream); } /* Test case 31 * Use of inline command */ @Test public void testCmdlineParamWithInlineCmd() throws Exception { log.info("Starting test testCmdlineParamWithInlineCmd() ..."); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date=`perl -e \"print qq@20080228\\n20070101@\" | head -n 1`" }; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } /* Test case 32 * No substitution */ @Test public void testNoVars() throws Exception { log.info("Starting test testNoVars() ..."); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputNoVars.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = null; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/inputNoVars.pig"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } /* Test case 32 * Test special values in characters */ @Test public void testComplexVals() throws Exception { log.info("Starting test testComplexVals() ..."); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input5.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "loadfile = /user/pig/tests/data/singlefile/textdoc.txt" }; String[] argFiles = { basedir + "/ConfFileComplexVal.txt" }; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult3.txt"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } /* Test case 25 * Test that params in comments are untouched */ @Test public void testCommentWithParam() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputComment.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date='20080228'" }; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResultComment.pig"); compareResults(pigExResultStream, pigResultStream); } /* Test case * Use a parameter within a pig function argument (containing newline characters). * provide value for it. */ @Test public void testSubstitutionInFuncArgs() throws Exception { log.info("Starting test testSubstitutionInFuncArgs()"); final String queryString = " avro = LOAD '/data/part-m-00000.avro' USING PigStorage ();\n" + " avro2 = FOREACH avro GENERATE browser_id, component_version, " + "member_id, page_key, session_id, tracking_time, type;\n" + " fs -rmr testOut/out1;\n" + " STORE avro2 INTO 'testOut/out2'\n" + " USING PigStorage (\n" + " ' {\n" + " \"debug\": $debug,\n" + " \"schema\":\n" + " { \"type\":\"record\",\"name\":\"$name\", \n" + " \"fields\": [ {\"name\":\"browser_id\", \"type\":[\"null\",\"string\"]}, \n" + " {\"name\":\"component_version\",\"type\":\"int\"},\n" + " {\"name\":\"member_id\",\"type\":\"int\"},\n" + " {\"name\":\"page_key\",\"type\":[\"null\",\"string\"]},\n" + " {\"name\":\"session_id\",\"type\":\"long\"},\n" + " {\"name\":\"tracking_time\",\"type\":\"long\"},\n" + " {\"name\":\"type\",\"type\":[\"null\",\"string\"]}\n" + " ]\n" + " }\n" + " }\n" + " ');"; final String expectedString = " avro = LOAD '/data/part-m-00000.avro' USING PigStorage ();\n" + " avro2 = FOREACH avro GENERATE browser_id, component_version, " + "member_id, page_key, session_id, tracking_time, type;\n" + " fs -rmr testOut/out1;\n" + " STORE avro2 INTO 'testOut/out2'\n" + " USING PigStorage (\n" + " ' {\n" + " \"debug\": 5,\n" + " \"schema\":\n" + " { \"type\":\"record\",\"name\":\"TestRecord\", \n" + " \"fields\": [ {\"name\":\"browser_id\", \"type\":[\"null\",\"string\"]}, \n" + " {\"name\":\"component_version\",\"type\":\"int\"},\n" + " {\"name\":\"member_id\",\"type\":\"int\"},\n" + " {\"name\":\"page_key\",\"type\":[\"null\",\"string\"]},\n" + " {\"name\":\"session_id\",\"type\":\"long\"},\n" + " {\"name\":\"tracking_time\",\"type\":\"long\"},\n" + " {\"name\":\"type\",\"type\":[\"null\",\"string\"]}\n" + " ]\n" + " }\n" + " }\n" + " ');"; ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader( new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8")))); pigOStream = new FileWriter(basedir + "/output26.pig"); String[] arg = { "debug = '5'", "name = 'TestRecord'" }; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output26.pig"); InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8")); compareResults(expected, pigResultStream); log.info("Done"); } /* Test case * Test that $ signs in substitution value are treated as literal replacement strings. */ @Test public void testSubstitutionWithDollarSign() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/inputDollarSign.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "filter=\"($0 == 'x') and ($1 == 'y')\"" }; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResultDollarSign.pig"); compareResults(pigExResultStream, pigResultStream); } @Test public void testMacroDef() throws Exception { log.info("Starting test testMacroDef() ..."); ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input6.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date=20080228" }; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult6.pig"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } @Test public void testCmdlineParamCurlySyntax() throws Exception { ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader(new FileReader(basedir + "/input7.pig")); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "date=20080228" }; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); pigExResultStream = new FileInputStream(basedir + "/ExpectedResult7.pig"); compareResults(pigExResultStream, pigResultStream); log.info("Done"); } @Test public void testGruntWithParamSub() throws Exception { log.info("Starting test testGruntWithParamSub()"); File inputFile = Util.createFile(new String[] { "daniel\t10", "jenny\t20" }); File outputFile = File.createTempFile("tmp", ""); outputFile.delete(); PigContext pc = new PigContext(ExecType.LOCAL, new Properties()); String command = "%default agelimit `echo 15`\n" + "rmf $outputFile;\n" + "a = load '" + Util.generateURI(inputFile.toString(), pc) + "' as ($param1:chararray, $param2:int);\n" + "b = filter a by age > $agelimit;" + "store b into '$outputFile';\n" + "quit\n"; System.setProperty("jline.WindowsTerminal.directConsole", "false"); System.setIn(new ByteArrayInputStream(command.getBytes())); org.apache.pig.PigRunner.run(new String[] { "-x", "local", "-p", "param1=name", "-p", "param2=age", "-p", "outputFile=" + Util.generateURI(outputFile.toString(), pc) }, null); File[] partFiles = outputFile.listFiles(new FilenameFilter() { public boolean accept(File dir, String name) { return name.startsWith("part"); } }); String resultContent = Util.readFile(partFiles[0]); assertEquals(resultContent, "jenny\t20\n"); } @Test public void testGruntMultilineDefine() throws Exception { log.info("Starting test testGruntMultilineDefine()"); File inputFile = Util.createFile(new String[] { "daniel\t10", "jenny\t20" }); File outputFile = File.createTempFile("tmp", ""); outputFile.delete(); PigContext pc = new PigContext(ExecType.LOCAL, new Properties()); String command = "DEFINE process(input_file) returns data {\n" + "$data = load '$input_file' using PigStorage(',');};\n" + "b = process('" + Util.generateURI(inputFile.toString(), pc) + "');\n" + "store b into '" + Util.generateURI(outputFile.toString(), pc) + "';" + "quit\n"; System.setProperty("jline.WindowsTerminal.directConsole", "false"); System.setIn(new ByteArrayInputStream(command.getBytes())); org.apache.pig.PigRunner.run(new String[] { "-x", "local" }, null); File[] partFiles = outputFile.listFiles(new FilenameFilter() { public boolean accept(File dir, String name) { return name.startsWith("part"); } }); String resultContent = Util.readFile(partFiles[0]); assertEquals(resultContent, "daniel\t10\njenny\t20\n"); } /* * Test parameter substition when register contains /* globbing */ @Test public void testSubstitutionWithRegisterGlobbing() throws Exception { log.info("Starting test testSubstitutionWithRegisterGlobbing()"); final String queryString = "register /abc/$regdir/*.jar;\n" + "A = LOAD '$input' USING PigStorage ();\n" + "STORE A INTO '$output';\n" + " /* comment that would make register globbing to be part of the multi-line comment */\n"; ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader( new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8")))); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "input = 'input.txt'", "output = 'output.txt'", "regdir = 'def'" }; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); String expectedString = queryString.replaceAll("\\$input", "input.txt") .replaceAll("\\$output", "output.txt").replaceAll("\\$regdir", "def"); InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8")); compareResults(expected, pigResultStream); log.info("Done"); } /* * Test parameter substition when load contains /* globbing */ @Test public void testSubstitutionWithLoadGlobbing() throws Exception { log.info("Starting test testSubstitutionWithLoadGlobbing()"); final String queryString = "A = LOAD '/zzz/*' USING PigStorage ();\n" + "STORE A INTO '$output';\n" + " /* comment that would make register globbing to be part of the multi-line comment */\n"; ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50); pigIStream = new BufferedReader( new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8")))); pigOStream = new FileWriter(basedir + "/output1.pig"); String[] arg = { "output = 'output.txt'" }; String[] argFiles = null; ps.genSubstitutedFile(pigIStream, pigOStream, arg, argFiles); FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig"); String expectedString = queryString.replaceAll("\\$output", "output.txt"); InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8")); compareResults(expected, pigResultStream); log.info("Done"); } @SuppressWarnings("resource") private BufferedReader WithConditionalReplacement(String filename, String orig, String dest, boolean replace) throws IOException { BufferedReader pigOrigIStream = new BufferedReader(new FileReader(filename)); BufferedReader result; if (replace) { File tmpInputFile = File.createTempFile("tmp", ""); PrintWriter tmppw = new PrintWriter(tmpInputFile); String line; while ((line = pigOrigIStream.readLine()) != null) { line = line.replaceAll(orig, dest); tmppw.println(line); } pigOrigIStream.close(); tmppw.close(); result = new BufferedReader(new FileReader(tmpInputFile)); } else { result = pigOrigIStream; } return result; } }