Java tutorial
/** * (C) Copyright IBM Corp. 2010, 2015 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package com.ibm.bi.dml.test.integration; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import com.ibm.bi.dml.lops.Lop; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.wink.json4j.JSONObject; import org.junit.After; import org.junit.Assert; import org.junit.Before; import com.ibm.bi.dml.api.DMLScript; import com.ibm.bi.dml.api.DMLScript.RUNTIME_PLATFORM; import com.ibm.bi.dml.api.MLContext; import com.ibm.bi.dml.conf.DMLConfig; import com.ibm.bi.dml.hops.OptimizerUtils; import com.ibm.bi.dml.parser.DMLTranslator; import com.ibm.bi.dml.parser.DataExpression; import com.ibm.bi.dml.parser.Expression.ValueType; import com.ibm.bi.dml.runtime.DMLRuntimeException; import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContext; import com.ibm.bi.dml.runtime.controlprogram.context.ExecutionContextFactory; import com.ibm.bi.dml.runtime.controlprogram.context.SparkExecutionContext; import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics; import com.ibm.bi.dml.runtime.matrix.data.MatrixValue.CellIndex; import com.ibm.bi.dml.runtime.matrix.data.OutputInfo; import com.ibm.bi.dml.runtime.util.MapReduceTool; import com.ibm.bi.dml.test.utils.TestUtils; import com.ibm.bi.dml.utils.ParameterBuilder; import com.ibm.bi.dml.utils.Statistics; /** * <p> * Extend this class to easily * </p> * <ul> * <li>set up an environment for DML script execution</li> * <li>use multiple test cases</li> * <li>generate test data></li> * <li>check results</li> * <li>clean up after test run</li> * </ul> * */ public abstract class AutomatedTestBase { public enum ScriptType { DML, PYDML; public String lowerCase() { return super.toString().toLowerCase(); } }; public static final boolean EXCEPTION_EXPECTED = true; public static final boolean EXCEPTION_NOT_EXPECTED = false; protected ScriptType scriptType; // *** HACK ALERT *** HACK ALERT *** HACK ALERT *** // Hadoop 2.4.1 doesn't work on Windows unless winutils.exe is available // under $HADOOP_HOME/bin and hadoop.dll is available in the Java library // path. The following static initializer sets up JVM variables so that // Hadoop can find these native binaries, assuming that any Hadoop code // loads after this class and that the JVM's current working directory // is the root of this project. static { String osname = System.getProperty("os.name").toLowerCase(); if (osname.contains("win")) { System.err.printf("AutomatedTestBase has detected a Windows OS and is overriding\n" + "hadoop.home.dir and java.library.path.\n"); String cwd = System.getProperty("user.dir"); System.setProperty("hadoop.home.dir", cwd + File.separator + "\\src\\test\\config\\hadoop_bin_windows"); System.setProperty("java.library.path", cwd + File.separator + "\\src\\test\\config\\hadoop_bin_windows\\bin"); // Need to muck around with the classloader to get it to use the new // value of java.library.path. try { final Field sysPathsField = ClassLoader.class.getDeclaredField("sys_paths"); sysPathsField.setAccessible(true); sysPathsField.set(null, null); } catch (Exception e) { // IBM Java throws an exception here, so don't print the stack trace. //e.printStackTrace(); //System.err.printf("Caught exception while attempting to override library path. Attempting to continue."); } } } // *** END HACK *** /** * Script source directory for .dml and .r files only * (TEST_DATA_DIR for generated test data artifacts). */ protected static final String SCRIPT_DIR = "./src/test/scripts/"; protected static final String INPUT_DIR = "in/"; protected static final String OUTPUT_DIR = "out/"; protected static final String EXPECTED_DIR = "expected/"; /** Location where this class writes files for inspection if DEBUG is set to true. */ private static final String DEBUG_TEMP_DIR = "./tmp/"; /** Directory under which config files shared across tests are located. */ private static final String CONFIG_DIR = "./src/test/config/"; /** * Location of the SystemML config file that we use as a template when * generating the configs for each test case. */ private static final File CONFIG_TEMPLATE_FILE = new File(CONFIG_DIR, "SystemML-config.xml"); /** Location under which we create local temporary directories for test cases. */ private static final String LOCAL_TEMP_ROOT_DIR = "target/testTemp"; private static final File LOCAL_TEMP_ROOT = new File(LOCAL_TEMP_ROOT_DIR); /** Base directory for generated IN, OUT, EXPECTED test data artifacts instead of SCRIPT_DIR. */ protected static final String TEST_DATA_DIR = LOCAL_TEMP_ROOT_DIR + "/"; protected static final boolean TEST_CACHE_ENABLED = true; /** Optional sub-directory under EXPECTED_DIR for reusing R script test results */ private String cacheDir = ""; /** * Runtime backend to use for all integration tests. Some individual tests * override this value, but the rest will use whatever is the default here. * <p> * Also set DMLScript.USE_LOCAL_SPARK_CONFIG to true for running the test * suite in spark mode */ protected static RUNTIME_PLATFORM rtplatform = RUNTIME_PLATFORM.HYBRID; protected static final boolean DEBUG = false; protected static final boolean VISUALIZE = false; protected static final boolean RUNNETEZZA = false; protected String fullDMLScriptName; // utilize for both DML and PyDML, should probably be renamed. // protected String fullPYDMLScriptName; protected String fullRScriptName; protected static String baseDirectory; protected static String sourceDirectory; protected HashMap<String, TestConfiguration> availableTestConfigurations; /* For testing in the old way */ protected HashMap<String, String> testVariables; /* variables and their values */ /* For testing in the new way */ //protected String[] dmlArgs; /* program-independent arguments to SystemML (e.g., debug, execution mode) */ protected String[] programArgs; /* program-specific arguments, which are passed to SystemML via -args option */ protected String rCmd; /* Rscript foo.R arg1, arg2 ... */ protected String selectedTest; protected String[] outputDirectories; protected String[] comparisonFiles; protected ArrayList<String> inputDirectories; protected ArrayList<String> inputRFiles; protected ArrayList<String> expectedFiles; private File curLocalTempDir = null; private boolean isOutAndExpectedDeletionDisabled = false; private long lTimeBeforeTest = 0; private String expectedStdOut; private int iExpectedStdOutState = 0; private PrintStream originalPrintStreamStd = null; private String expectedStdErr; private int iExpectedStdErrState = 0; private PrintStream originalErrStreamStd = null; @Before public abstract void setUp(); /** * <p> * Adds a test configuration to the list of available test configurations. * </p> * * @param testName * test name * @param config * test configuration */ protected void addTestConfiguration(String testName, TestConfiguration config) { availableTestConfigurations.put(testName, config); } /** * <p> * Adds a test configuration to the list of available test configurations based * on the test directory and the test name. * </p> * * @param testDirectory * test directory * @param testName * test name */ protected void addTestConfiguration(String testDirectory, String testName) { TestConfiguration config = new TestConfiguration(testDirectory, testName); availableTestConfigurations.put(testName, config); } @Before public final void setUpBase() { availableTestConfigurations = new HashMap<String, TestConfiguration>(); testVariables = new HashMap<String, String>(); inputDirectories = new ArrayList<String>(); inputRFiles = new ArrayList<String>(); expectedFiles = new ArrayList<String>(); outputDirectories = new String[0]; setOutAndExpectedDeletionDisabled(false); lTimeBeforeTest = System.currentTimeMillis(); TestUtils.clearAssertionInformation(); } /** * <p> * Returns a test configuration from the list of available configurations. * If no configuration is added for the specified name, the test will fail. * </p> * * @param testName * test name * @return test configuration */ protected TestConfiguration getTestConfiguration(String testName) { if (!availableTestConfigurations.containsKey(testName)) fail("unable to load test configuration"); return availableTestConfigurations.get(testName); } /** * <p> * Gets a test configuration from the list of available configurations * and loads it if it's available. It is then returned. * If no configuration exists for the specified name, the test will fail. * * </p> * * @param testName * test name * @return test configuration */ protected TestConfiguration getAndLoadTestConfiguration(String testName) { TestConfiguration testConfiguration = getTestConfiguration(testName); loadTestConfiguration(testConfiguration); return testConfiguration; } /** * Subclasses must call {@link #loadTestConfiguration(TestConfiguration)} * before calling this method. * * @return the directory where the current test case should write temp * files. This directory also contains the current test's customized * SystemML config file. */ protected File getCurLocalTempDir() { if (null == curLocalTempDir) { throw new RuntimeException("Called getCurLocalTempDir() before calling loadTestConfiguration()"); } return curLocalTempDir; } /** * Subclasses must call {@link #loadTestConfiguration(TestConfiguration)} * before calling this method. * * @return the location of the current test case's SystemML config file */ protected File getCurConfigFile() { return new File(getCurLocalTempDir(), "SystemML-config.xml"); } protected MLContext getMLContextForTesting() throws DMLRuntimeException { synchronized (AutomatedTestBase.class) { RUNTIME_PLATFORM oldRT = DMLScript.rtplatform; try { DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; ExecutionContext ec = ExecutionContextFactory.createContext(); if (ec instanceof SparkExecutionContext) { MLContext mlCtx = new MLContext(((SparkExecutionContext) ec).getSparkContext()); return mlCtx; } } finally { DMLScript.rtplatform = oldRT; } throw new DMLRuntimeException("Cannot create MLContext"); } } /** * <p> * Generates a random matrix with the specified characteristics and returns * it as a two dimensional array. * </p> * * @param rows * number of rows * @param cols * number of columns * @param min * minimum value * @param max * maximum value * @param sparsity * sparsity * @param seed * seed * @return two dimensional array containing random matrix */ protected double[][] getRandomMatrix(int rows, int cols, double min, double max, double sparsity, long seed) { return TestUtils.generateTestMatrix(rows, cols, min, max, sparsity, seed); } /** * <p> * Generates a random matrix with the specified characteristics which does * not contain any zero values and returns it as a two dimensional array. * </p> * * @param rows * number of rows * @param cols * number of columns * @param min * minimum value * @param max * maximum value * @param seed * seed * @return two dimensional array containing random matrix */ protected double[][] getNonZeroRandomMatrix(int rows, int cols, double min, double max, long seed) { return TestUtils.generateNonZeroTestMatrix(rows, cols, min, max, seed); } /** * <p> * Generates a random matrix with the specified characteristics and writes * it to a file. * </p> * * @param name * directory name * @param rows * number of rows * @param cols * number of columns * @param min * minimum value * @param max * maximum value * @param sparsity * sparsity * @param seed * seed */ protected double[][] createRandomMatrix(String name, int rows, int cols, double min, double max, double sparsity, long seed) { return createRandomMatrix(name, rows, cols, min, max, sparsity, seed, false); } /** * <p> * Generates a random matrix with the specified characteristics and writes * it to a file. * </p> * * @param name * directory name * @param rows * number of rows * @param cols * number of columns * @param min * minimum value * @param max * maximum value * @param sparsity * sparsity * @param seed * seed * @param bIncludeR * If true, writes also a R matrix to disk */ protected double[][] createRandomMatrix(String name, int rows, int cols, double min, double max, double sparsity, long seed, boolean bIncludeR) { double[][] matrix = TestUtils.generateTestMatrix(rows, cols, min, max, sparsity, seed); String completePath = baseDirectory + INPUT_DIR + name + "/in"; TestUtils.writeTestMatrix(completePath, matrix, bIncludeR); if (DEBUG) TestUtils.writeTestMatrix(DEBUG_TEMP_DIR + completePath, matrix); inputDirectories.add(baseDirectory + INPUT_DIR + name); return matrix; } /** * <p> * Generates a random matrix with the specified characteristics and writes * it to a file. * </p> * * @param matrix * matrix characteristics */ protected void createRandomMatrix(TestMatrixCharacteristics matrix) { createRandomMatrix(matrix.getMatrixName(), matrix.getRows(), matrix.getCols(), matrix.getMinValue(), matrix.getMaxValue(), matrix.getSparsity(), matrix.getSeed()); } private void cleanupExistingData(String fname, boolean cleanupRData) throws IOException { MapReduceTool.deleteFileIfExistOnHDFS(fname); MapReduceTool.deleteFileIfExistOnHDFS(fname + ".mtd"); if (cleanupRData) MapReduceTool.deleteFileIfExistOnHDFS(fname + ".mtx"); } /** * <p> * Adds a matrix to the input path and writes it to a file. * </p> * * @param name * directory name * @param matrix * two dimensional matrix * @param bIncludeR * generates also the corresponding R matrix */ protected double[][] writeInputMatrix(String name, double[][] matrix, boolean bIncludeR) { String completePath = baseDirectory + INPUT_DIR + name + "/in"; String completeRPath = baseDirectory + INPUT_DIR + name + ".mtx"; try { cleanupExistingData(baseDirectory + INPUT_DIR + name, bIncludeR); } catch (IOException e) { e.printStackTrace(); throw new RuntimeException(e); } TestUtils.writeTestMatrix(completePath, matrix); if (bIncludeR) { TestUtils.writeTestMatrix(completeRPath, matrix, true); inputRFiles.add(completeRPath); } if (DEBUG) TestUtils.writeTestMatrix(DEBUG_TEMP_DIR + completePath, matrix); inputDirectories.add(baseDirectory + INPUT_DIR + name); return matrix; } protected double[][] writeInputMatrixWithMTD(String name, double[][] matrix, boolean bIncludeR) { MatrixCharacteristics mc = new MatrixCharacteristics(matrix.length, matrix[0].length, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, -1); return writeInputMatrixWithMTD(name, matrix, bIncludeR, mc); } protected double[][] writeInputMatrixWithMTD(String name, double[][] matrix, int nnz, boolean bIncludeR) { MatrixCharacteristics mc = new MatrixCharacteristics(matrix.length, matrix[0].length, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, nnz); return writeInputMatrixWithMTD(name, matrix, bIncludeR, mc); } protected double[][] writeInputMatrixWithMTD(String name, double[][] matrix, boolean bIncludeR, MatrixCharacteristics mc) { writeInputMatrix(name, matrix, bIncludeR); // write metadata file try { String completeMTDPath = baseDirectory + INPUT_DIR + name + ".mtd"; MapReduceTool.writeMetaDataFile(completeMTDPath, ValueType.DOUBLE, mc, OutputInfo.stringToOutputInfo("textcell")); } catch (IOException e) { e.printStackTrace(); throw new RuntimeException(e); } return matrix; } /** * <p> * Adds a matrix to the input path and writes it to a file. * </p> * * @param name * directory name * @param matrix * two dimensional matrix */ protected double[][] writeInputMatrix(String name, double[][] matrix) { return writeInputMatrix(name, matrix, false); } /** * <p> * Adds a matrix to the input path and writes it to a file in binary format. * </p> * * @param name * directory name * @param matrix * two dimensional matrix * @param rowsInBlock * rows in block * @param colsInBlock * columns in block * @param sparseFormat * sparse format */ protected void writeInputBinaryMatrix(String name, double[][] matrix, int rowsInBlock, int colsInBlock, boolean sparseFormat) { String completePath = baseDirectory + INPUT_DIR + name + "/in"; try { cleanupExistingData(baseDirectory + INPUT_DIR + name, false); } catch (IOException e) { e.printStackTrace(); throw new RuntimeException(e); } if (rowsInBlock == 1 && colsInBlock == 1) { TestUtils.writeBinaryTestMatrixCells(completePath, matrix); if (DEBUG) TestUtils.writeBinaryTestMatrixCells(DEBUG_TEMP_DIR + completePath, matrix); } else { TestUtils.writeBinaryTestMatrixBlocks(completePath, matrix, rowsInBlock, colsInBlock, sparseFormat); if (DEBUG) TestUtils.writeBinaryTestMatrixBlocks(DEBUG_TEMP_DIR + completePath, matrix, rowsInBlock, colsInBlock, sparseFormat); } inputDirectories.add(baseDirectory + INPUT_DIR + name); } /** * Writes the given matrix to input path, and writes the associated metadata file. * * @param name * @param matrix * @param rowsInBlock * @param colsInBlock * @param sparseFormat * @param mc * @throws IOException */ protected void writeInputBinaryMatrixWithMTD(String name, double[][] matrix, int rowsInBlock, int colsInBlock, boolean sparseFormat, MatrixCharacteristics mc) throws IOException { writeInputBinaryMatrix(name, matrix, rowsInBlock, colsInBlock, sparseFormat); // write metadata file String completeMTDPath = baseDirectory + INPUT_DIR + name + ".mtd"; MapReduceTool.writeMetaDataFile(completeMTDPath, ValueType.DOUBLE, mc, OutputInfo.stringToOutputInfo("binaryblock")); } /** * <p> * Adds a matrix to the expectation path and writes it to a file. * </p> * * @param name * directory name * @param matrix * two dimensional matrix */ protected void writeExpectedMatrix(String name, double[][] matrix) { TestUtils.writeTestMatrix(baseDirectory + EXPECTED_DIR + cacheDir + name, matrix); expectedFiles.add(baseDirectory + EXPECTED_DIR + cacheDir + name); } /** * <p> * Adds a matrix to the expectation path and writes it to a file. * </p> * * @param name * directory name * @param matrix * two dimensional matrix */ protected void writeExpectedMatrixMarket(String name, double[][] matrix) { File path = new File(baseDirectory, EXPECTED_DIR + cacheDir); path.mkdirs(); TestUtils.writeTestMatrix(baseDirectory + EXPECTED_DIR + cacheDir + name, matrix, true); expectedFiles.add(baseDirectory + EXPECTED_DIR + cacheDir + name); } /** * <p> * Adds a matrix to the expectation path and writes it to a file in binary * format. * </p> * * @param name * directory name * @param matrix * two dimensional matrix * @param rowsInBlock * rows in block * @param colsInBlock * columns in block * @param sparseFormat * sparse format */ protected void writeExpectedBinaryMatrix(String name, double[][] matrix, int rowsInBlock, int colsInBlock, boolean sparseFormat) { if (rowsInBlock == 1 && colsInBlock == 1) { TestUtils.writeBinaryTestMatrixCells(baseDirectory + EXPECTED_DIR + name + "/in", matrix); } else { TestUtils.writeBinaryTestMatrixBlocks(baseDirectory + EXPECTED_DIR + name + "/in", matrix, rowsInBlock, colsInBlock, sparseFormat); } inputDirectories.add(baseDirectory + EXPECTED_DIR + name); } /** * <p> * Creates a helper matrix which can be used for writing scalars to a file. * </p> */ protected void createHelperMatrix() { TestUtils.writeTestMatrix(baseDirectory + INPUT_DIR + "helper/in", new double[][] { { 1, 1 } }); inputDirectories.add(baseDirectory + INPUT_DIR + "helper"); } /** * <p> * Creates a expectation helper matrix which can be used to compare scalars. * </p> * * @param name * file name * @param value * scalar value */ protected void writeExpectedHelperMatrix(String name, double value) { TestUtils.writeTestMatrix(baseDirectory + EXPECTED_DIR + cacheDir + name, new double[][] { { value, value } }); expectedFiles.add(baseDirectory + EXPECTED_DIR + cacheDir + name); } protected void writeExpectedScalar(String name, double value) { TestUtils.writeTestScalar(baseDirectory + EXPECTED_DIR + cacheDir + name, value); expectedFiles.add(baseDirectory + EXPECTED_DIR + cacheDir + name); } @SuppressWarnings("deprecation") protected static HashMap<CellIndex, Double> readDMLMatrixFromHDFS(String fileName) { return TestUtils.readDMLMatrixFromHDFS(baseDirectory + OUTPUT_DIR + fileName); } @SuppressWarnings("deprecation") public HashMap<CellIndex, Double> readRMatrixFromFS(String fileName) { System.out.println("R script out: " + baseDirectory + EXPECTED_DIR + cacheDir + fileName); return TestUtils.readRMatrixFromFS(baseDirectory + EXPECTED_DIR + cacheDir + fileName); } protected static HashMap<CellIndex, Double> readDMLScalarFromHDFS(String fileName) { return TestUtils.readDMLScalarFromHDFS(baseDirectory + OUTPUT_DIR + fileName); } public HashMap<CellIndex, Double> readRScalarFromFS(String fileName) { System.out.println("R script out: " + baseDirectory + EXPECTED_DIR + cacheDir + fileName); return TestUtils.readRScalarFromFS(baseDirectory + EXPECTED_DIR + cacheDir + fileName); } /** * * @param fileName * @param mc */ public static void checkDMLMetaDataFile(String fileName, MatrixCharacteristics mc) { try { String fname = baseDirectory + OUTPUT_DIR + fileName + ".mtd"; JSONObject meta = new DataExpression().readMetadataFile(fname, false); long rlen = Long.parseLong(meta.get(DataExpression.READROWPARAM).toString()); long clen = Long.parseLong(meta.get(DataExpression.READCOLPARAM).toString()); Assert.assertEquals(mc.getRows(), rlen); Assert.assertEquals(mc.getCols(), clen); } catch (Exception ex) { throw new RuntimeException(ex); } } /** * <p> * Loads a test configuration with its parameters. Adds the output * directories to the output list as well as to the list of possible * comparison files. * </p> * * @param configurationName * test configuration name * */ protected void loadTestConfiguration(TestConfiguration config) { loadTestConfiguration(config, null); } /** * <p> * Loads a test configuration with its parameters. Adds the output * directories to the output list as well as to the list of possible * comparison files. * </p> * * @param configurationName * test configuration name * @param cacheDirectory * subdirectory for reusing R script expected results * if null, defaults to empty string (i.e., no cache) */ protected void loadTestConfiguration(TestConfiguration config, String cacheDirectory) { if (!availableTestConfigurations.containsValue(config)) fail("test configuration not available: " + config.getTestScript()); String testDirectory = config.getTestDirectory(); if (testDirectory != null) { if (isTargetTestDirectory(testDirectory)) { baseDirectory = TEST_DATA_DIR + testDirectory; sourceDirectory = SCRIPT_DIR + getSourceDirectory(testDirectory); } else { baseDirectory = SCRIPT_DIR + testDirectory; sourceDirectory = baseDirectory; } } setCacheDirectory(cacheDirectory); selectedTest = config.getTestScript(); String[] outputFiles = config.getOutputFiles(); if (outputFiles != null) { outputDirectories = new String[outputFiles.length]; comparisonFiles = new String[outputFiles.length]; for (int i = 0; i < outputFiles.length; i++) { outputDirectories[i] = baseDirectory + OUTPUT_DIR + outputFiles[i]; comparisonFiles[i] = baseDirectory + EXPECTED_DIR + cacheDir + outputFiles[i]; } } testVariables = config.getVariables(); testVariables.put("basedir", baseDirectory); testVariables.put("indir", baseDirectory + INPUT_DIR); testVariables.put("outdir", baseDirectory + OUTPUT_DIR); testVariables.put("readhelper", "Helper = read(\"" + baseDirectory + INPUT_DIR + "helper/in\", " + "rows=1, cols=2, format=\"text\");"); testVariables.put("Routdir", baseDirectory + EXPECTED_DIR + cacheDir); // Create a temporary directory for this test case. // Eventually all files written by the tests should go under here, but making // that change will take quite a bit of effort. try { if (null == testDirectory) { System.err.printf("Warning: Test configuration did not specify a test directory.\n"); curLocalTempDir = new File(LOCAL_TEMP_ROOT, String.format("unknownTest/%s", selectedTest)); } else { curLocalTempDir = new File(LOCAL_TEMP_ROOT, String.format("%s/%s", testDirectory, selectedTest)); } curLocalTempDir.mkdirs(); TestUtils.clearDirectory(curLocalTempDir.getPath()); // Create a SystemML config file for this test case. // Use the canned file under src/test/config as a template String configTemplate = FileUtils.readFileToString(CONFIG_TEMPLATE_FILE, "UTF-8"); // *** HACK ALERT *** HACK ALERT *** HACK ALERT *** // Nimble does not accept paths that use backslash as the separator character. // Since some of the tests use Nimble, we use forward slash in the paths that // we put into the config file. String localTempForwardSlash = curLocalTempDir.getPath().replace(File.separator, "/"); String configContents = configTemplate.replace("<scratch>scratch_space</scratch>", String.format("<scratch>%s/scratch_space</scratch>", localTempForwardSlash)); configContents = configContents.replace("<localtmpdir>/tmp/systemml</localtmpdir>", String.format("<localtmpdir>%s/localtmp</localtmpdir>", localTempForwardSlash)); configContents = configContents.replace("<NimbleScratch>nimbleoutput</NimbleScratch>", String.format("<NimbleScratch>%s/nimbleoutput</NimbleScratch>", localTempForwardSlash)); // *** END HACK *** FileUtils.write(getCurConfigFile(), configContents, "UTF-8"); System.out.printf("This test case will use SystemML config file %s\n", getCurConfigFile()); } catch (IOException e) { throw new RuntimeException(e); } if (DEBUG) TestUtils.clearDirectory(DEBUG_TEMP_DIR + baseDirectory + INPUT_DIR); } /** * <p> * Loads a test configuration with its parameters. Adds the output * directories to the output list as well as to the list of possible * comparison files. * </p> * * @param configurationName * test configuration name * */ @Deprecated protected void loadTestConfiguration(String configurationName) { if (!availableTestConfigurations.containsKey(configurationName)) fail("test configuration not available: " + configurationName); TestConfiguration config = availableTestConfigurations.get(configurationName); loadTestConfiguration(config); } /** * Runs an R script, default to the old way */ protected void runRScript() { runRScript(false); } /** * Runs an R script in the old or the new way */ protected void runRScript(boolean newWay) { String executionFile = sourceDirectory + selectedTest + ".R"; // *** HACK ALERT *** HACK ALERT *** HACK ALERT *** // Some of the R scripts will fail if the "expected" directory doesn't exist. // Make sure the directory exists. File expectedDir = new File(baseDirectory, "expected" + "/" + cacheDir); expectedDir.mkdirs(); // *** END HACK *** String cmd; if (!newWay) { executionFile = executionFile + "t"; cmd = "R -f " + executionFile; } else { // *** HACK ALERT *** HACK ALERT *** HACK ALERT *** // Rscript does *not* load the "methods" package by default // to save on start time. The "Matrix" package used in the // tests requires the "methods" package and should still // load and attach it, but in R 3.2 with the latest version // of the "Matrix" package, "methods" is loaded *but not // attached* when run with Rscript. Therefore, we need to // explicitly load it with Rscript. cmd = rCmd.replaceFirst("Rscript", "Rscript --default-packages=methods,datasets,graphics,grDevices,stats,utils"); // *** END HACK *** } if (System.getProperty("os.name").contains("Windows")) { cmd = cmd.replace('/', '\\'); executionFile = executionFile.replace('/', '\\'); } if (DEBUG) { if (!newWay) { // not sure why have this condition TestUtils.printRScript(executionFile); } } if (!newWay) { ParameterBuilder.setVariablesInScript(sourceDirectory, selectedTest + ".R", testVariables); } if (cacheDir.length() > 0) { File expectedFile = null; String[] outputFiles = null; TestConfiguration testConfig = getTestConfiguration(selectedTest); if (testConfig != null) { outputFiles = testConfig.getOutputFiles(); } if (outputFiles != null && outputFiles.length > 0) { expectedFile = new File(expectedDir.getPath() + "/" + outputFiles[0]); if (expectedFile.canRead()) { System.out.println("Skipping R script cmd: " + cmd); return; } } } try { long t0 = System.nanoTime(); System.out.println("starting R script"); System.out.println("cmd: " + cmd); Process child = Runtime.getRuntime().exec(cmd); String outputR = IOUtils.toString(child.getInputStream()); System.out.println("Standard Output from R:" + outputR); String errorString = IOUtils.toString(child.getErrorStream()); System.err.println("Standard Error from R:" + errorString); // // To give any stream enough time to print all data, otherwise there // are situations where the test case fails, even before everything // has been printed // child.waitFor(); // Thread.sleep(30000); try { if (child.exitValue() != 0) { throw new Exception( "ERROR: R has ended irregularly\n" + outputR + "\nscript file: " + executionFile); } } catch (IllegalThreadStateException ie) { // // In UNIX JVM does not seem to be able to close threads // correctly. However, give it a try, since R processed the // script, therefore we can terminate the process. // child.destroy(); } long t1 = System.nanoTime(); System.out.println("R is finished (in " + ((double) t1 - t0) / 1000000000 + " sec)"); } catch (Exception e) { e.printStackTrace(); StringBuilder errorMessage = new StringBuilder(); errorMessage.append("failed to run script " + executionFile); errorMessage.append("\nexception: " + e.toString()); errorMessage.append("\nmessage: " + e.getMessage()); errorMessage.append("\nstack trace:"); for (StackTraceElement ste : e.getStackTrace()) { errorMessage.append("\n>" + ste); } fail(errorMessage.toString()); } } /** * <p> * Runs a test for which no exception is expected. * </p> */ protected void runTest() { runTest(false, null); } /** * <p> * Runs a test for which no exception is expected. If SystemML executes more * MR jobs than specified in maxMRJobs this test will fail. * </p> * * @param maxMRJobs * specifies a maximum limit for the number of MR jobs. If set to * -1 there is no limit. */ protected void runTest(int maxMRJobs) { runTest(false, null, maxMRJobs); } /** * <p> * Runs a test for which the exception expectation can be specified. * </p> * * @param exceptionExpected * exception expected */ protected void runTest(boolean exceptionExpected) { runTest(exceptionExpected, null); } /** * <p> * Runs a test for which the exception expectation can be specified as well * as the specific expectation which is expected. * </p> * * @param exceptionExpected * exception expected * @param expectedException * expected exception */ protected void runTest(boolean exceptionExpected, Class<?> expectedException) { runTest(exceptionExpected, expectedException, -1); } /** * <p> * Runs a test for which the exception expectation can be specified as well * as the specific expectation which is expected. If SystemML executes more * MR jobs than specified in maxMRJobs this test will fail. * </p> * * @param exceptionExpected * exception expected * @param expectedException * expected exception * @param maxMRJobs * specifies a maximum limit for the number of MR jobs. If set to * -1 there is no limit. */ protected void runTest(boolean exceptionExpected, Class<?> expectedException, int maxMRJobs) { runTest(false, exceptionExpected, expectedException, maxMRJobs); } /** * <p> * Runs a test for which the exception expectation can be specified as well * as the specific expectation which is expected. If SystemML executes more * MR jobs than specified in maxMRJobs this test will fail. * </p> * @param newWay * in the new way if it is set to true * @param exceptionExpected * exception expected * @param expectedException * expected exception * @param maxMRJobs * specifies a maximum limit for the number of MR jobs. If set to * -1 there is no limit. */ protected void runTest(boolean newWay, boolean exceptionExpected, Class<?> expectedException, int maxMRJobs) { String executionFile = sourceDirectory + selectedTest + ".dml"; if (!newWay) { executionFile = executionFile + "t"; ParameterBuilder.setVariablesInScript(sourceDirectory, selectedTest + ".dml", testVariables); } //cleanup scratch folder (prevent side effect between tests) cleanupScratchSpace(); ArrayList<String> args = new ArrayList<String>(); // setup arguments to SystemML if (DEBUG) { args.add("-Dsystemml.logging=trace"); } if (scriptType != null) { // DML/PYDML tests have newWay==true and a non-null scriptType switch (scriptType) { case DML: // Need a null pointer check because some tests read DML from a string. if (null != fullDMLScriptName) { args.add("-f"); args.add(fullDMLScriptName); } break; case PYDML: if (null != fullDMLScriptName) { args.add("-f"); args.add(fullDMLScriptName); } break; } } else if (newWay) { // Need a null pointer check because some tests read DML from a string. if (null != fullDMLScriptName) { args.add("-f"); args.add(fullDMLScriptName); } } else { if (null != executionFile) { args.add("-f"); args.add(executionFile); } } // program-independent parameters if (VISUALIZE) args.add("-v"); args.add("-exec"); if (rtplatform == RUNTIME_PLATFORM.HADOOP) args.add("hadoop"); else if (rtplatform == RUNTIME_PLATFORM.HYBRID) args.add("hybrid"); else if (rtplatform == RUNTIME_PLATFORM.SINGLE_NODE) args.add("singlenode"); else if (rtplatform == RUNTIME_PLATFORM.SPARK) args.add("spark"); else if (rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK) args.add("hybrid_spark"); else { throw new RuntimeException("Unknown runtime platform: " + rtplatform); } //use optional config file since default under SystemML/DML args.add("-config=" + getCurConfigFile().getPath()); // program-specific parameters if (newWay) { for (int i = 0; i < programArgs.length; i++) args.add(programArgs[i]); } if (DEBUG) { if (!newWay) TestUtils.printDMLScript(executionFile); else { if (scriptType == null) { TestUtils.printDMLScript(fullDMLScriptName); } else if (scriptType == ScriptType.DML) { TestUtils.printDMLScript(fullDMLScriptName); } else if (scriptType == ScriptType.PYDML) { TestUtils.printPYDMLScript(fullDMLScriptName); } } } try { String[] dmlScriptArgs = args.toArray(new String[args.size()]); System.out.println("arguments to DMLScript: " + Arrays.toString(dmlScriptArgs)); DMLScript.main(dmlScriptArgs); /** check number of MR jobs */ if (maxMRJobs > -1 && maxMRJobs < Statistics.getNoOfCompiledMRJobs()) fail("Limit of MR jobs is exceeded: expected: " + maxMRJobs + ", occurred: " + Statistics.getNoOfCompiledMRJobs()); if (exceptionExpected) fail("expected exception which has not been raised: " + expectedException); } catch (Exception e) { if (!exceptionExpected || (expectedException != null && !(e.getClass().equals(expectedException)))) { e.printStackTrace(); StringBuilder errorMessage = new StringBuilder(); errorMessage.append("failed to run script " + executionFile); errorMessage.append("\nexception: " + e.toString()); errorMessage.append("\nmessage: " + e.getMessage()); errorMessage.append("\nstack trace:"); for (StackTraceElement ste : e.getStackTrace()) { errorMessage.append("\n>" + ste); } fail(errorMessage.toString()); } } } public void cleanupScratchSpace() { try { //parse config file DMLConfig conf = new DMLConfig(getCurConfigFile().getPath()); // delete the scratch_space and all contents // (prevent side effect between tests) String dir = conf.getTextValue(DMLConfig.SCRATCH_SPACE); MapReduceTool.deleteFileIfExistOnHDFS(dir); } catch (Exception ex) { //ex.printStackTrace(); return; //no effect on tests } } /** * <p> * Checks if a process-local temporary directory exists * in the current working directory. * </p> * * @return true if a process-local temp directory is present. */ public boolean checkForProcessLocalTemporaryDir() { try { DMLConfig conf = new DMLConfig(getCurConfigFile().getPath()); StringBuilder sb = new StringBuilder(); sb.append(conf.getTextValue(DMLConfig.SCRATCH_SPACE)); sb.append(Lop.FILE_SEPARATOR); sb.append(Lop.PROCESS_PREFIX); sb.append(DMLScript.getUUID()); String pLocalDir = sb.toString(); return MapReduceTool.existsFileOnHDFS(pLocalDir); } catch (Exception ex) { ex.printStackTrace(); return true; } } /** * <p> * Compares the results of the computation with the expected ones. * </p> */ protected void compareResults() { compareResults(0); } /** * <p> * Compares the results of the computation with the expected ones with a * specified tolerance. * </p> * * @param epsilon * tolerance */ protected void compareResultsWithR(double epsilon) { for (int i = 0; i < comparisonFiles.length; i++) { TestUtils.compareDMLHDFSFileWithRFile(comparisonFiles[i], outputDirectories[i], epsilon); } } /** * <p> * Compares the results of the computation with the Result calculated by R * </p> */ protected void compareResultsWithR() { compareResultsWithR(0); } protected void compareResultsWithMM() { TestUtils.compareMMMatrixWithJavaMatrix(comparisonFiles[0], outputDirectories[0], 0); } /** * <p> * Compares the results of the computation with the expected ones with a * specified tolerance. * </p> * * @param epsilon * tolerance */ protected void compareResults(double epsilon) { for (int i = 0; i < comparisonFiles.length; i++) { /* Note that DML scripts may generate a file with only scalar value */ if (outputDirectories[i].endsWith(".scalar")) { String javaFile = comparisonFiles[i].replace(".scalar", ""); String dmlFile = outputDirectories[i].replace(".scalar", ""); TestUtils.compareDMLScalarWithJavaScalar(javaFile, dmlFile, epsilon); } else { TestUtils.compareDMLMatrixWithJavaMatrix(comparisonFiles[i], outputDirectories[i], epsilon); } } } /** * Compare results of the computation with the expected results where rows may be permuted. * @param epsilon */ protected void compareResultsRowsOutOfOrder(double epsilon) { for (int i = 0; i < comparisonFiles.length; i++) { /* Note that DML scripts may generate a file with only scalar value */ if (outputDirectories[i].endsWith(".scalar")) { String javaFile = comparisonFiles[i].replace(".scalar", ""); String dmlFile = outputDirectories[i].replace(".scalar", ""); TestUtils.compareDMLScalarWithJavaScalar(javaFile, dmlFile, epsilon); } else { TestUtils.compareDMLMatrixWithJavaMatrixRowsOutOfOrder(comparisonFiles[i], outputDirectories[i], epsilon); } } } /** * Checks that the number of map-reduce jobs that the current test case has * compiled is equal to the expected number. Generates a JUnit error message * if the number is out of line. * * @param expectedNumCompiled * number of map-reduce jobs that the current test case is * expected to compile */ protected void checkNumCompiledMRJobs(int expectedNumCompiled) { if (OptimizerUtils.isSparkExecutionMode()) { // Skip MapReduce-related checks when running in Spark mode. return; } assertEquals("Unexpected number of compiled MR jobs.", expectedNumCompiled, Statistics.getNoOfCompiledMRJobs()); } /** * Checks that the number of map-reduce jobs that the current test case has * executed (as opposed to compiling into the execution plan) is equal to * the expected number. Generates a JUnit error message if the number is out * of line. * * @param expectedNumExecuted * number of map-reduce jobs that the current test case is * expected to run */ protected void checkNumExecutedMRJobs(int expectedNumExecuted) { if (OptimizerUtils.isSparkExecutionMode()) { // Skip MapReduce-related checks when running in Spark mode. return; } assertEquals("Unexpected number of executed MR jobs.", expectedNumExecuted, Statistics.getNoOfExecutedMRJobs()); } /** * <p> * Checks the results of a computation against a number of characteristics. * </p> * * @param rows * number of rows * @param cols * number of columns * @param min * minimum value * @param max * maximum value */ protected void checkResults(long rows, long cols, double min, double max) { for (int i = 0; i < outputDirectories.length; i++) { TestUtils.checkMatrix(outputDirectories[i], rows, cols, min, max); } } /** * <p> * Checks for the existence for all of the outputs. * </p> */ protected void checkForResultExistence() { for (int i = 0; i < outputDirectories.length; i++) { TestUtils.checkForOutputExistence(outputDirectories[i]); } } @After public void tearDown() { System.out.println("Duration: " + (System.currentTimeMillis() - lTimeBeforeTest) + "ms"); assertTrue("expected String did not occur: " + expectedStdOut, iExpectedStdOutState == 0 || iExpectedStdOutState == 2); assertTrue("expected String did not occur (stderr): " + expectedStdErr, iExpectedStdErrState == 0 || iExpectedStdErrState == 2); TestUtils.displayAssertionBuffer(); if (!isOutAndExpectedDeletionDisabled()) { TestUtils.removeHDFSDirectories(inputDirectories.toArray(new String[inputDirectories.size()])); TestUtils.removeFiles(inputRFiles.toArray(new String[inputRFiles.size()])); // The following cleanup code is disabled (see [SYSML-256]) until we can figure out // what test cases are creating temporary directories at the root of the project. //TestUtils.removeTemporaryFiles(); TestUtils.clearDirectory(baseDirectory + OUTPUT_DIR); TestUtils.removeHDFSFiles(expectedFiles.toArray(new String[expectedFiles.size()])); TestUtils.clearDirectory(baseDirectory + EXPECTED_DIR); TestUtils.removeFiles(new String[] { sourceDirectory + selectedTest + ".dmlt" }); TestUtils.removeFiles(new String[] { sourceDirectory + selectedTest + ".Rt" }); } TestUtils.clearAssertionInformation(); System.gc(); } /** * Disables the deletion of files and directories in the output and expected * folder for this test. */ public void disableOutAndExpectedDeletion() { setOutAndExpectedDeletionDisabled(true); } /** * Enables expection of a line in standard output stream. * * @param expected */ public void setExpectedStdOut(String expectedLine) { this.expectedStdOut = expectedLine; originalPrintStreamStd = System.out; iExpectedStdOutState = 1; System.setOut(new PrintStream(new ExpectedOutputStream())); } /** * This class is used to compare the standard output stream against an * expected string. * * * */ class ExpectedOutputStream extends OutputStream { private String line = ""; @Override public void write(int b) throws IOException { line += String.valueOf((char) b); if (((char) b) == '\n') { /** new line */ if (line.contains(expectedStdOut)) { iExpectedStdOutState = 2; System.setOut(originalPrintStreamStd); } else { // Reset buffer line = ""; } } originalPrintStreamStd.write(b); } } public void setExpectedStdErr(String expectedLine) { this.expectedStdErr = expectedLine; originalErrStreamStd = System.err; iExpectedStdErrState = 1; System.setErr(new PrintStream(new ExpectedErrorStream())); } /** * This class is used to compare the standard error stream against an * expected string. * * * */ class ExpectedErrorStream extends OutputStream { private String line = ""; @Override public void write(int b) throws IOException { line += String.valueOf((char) b); if (((char) b) == '\n') { /** new line */ if (line.contains(expectedStdErr)) { iExpectedStdErrState = 2; System.setErr(originalErrStreamStd); } else { // Reset buffer line = ""; } } originalErrStreamStd.write(b); } } /** * <p> * Generates a matrix containing easy to debug values in its cells. * </p> * * @param rows * @param cols * @param bContainsZeros * If true, the matrix contains zeros. If false, the matrix * contains only positive values. * @return */ protected double[][] createNonRandomMatrixValues(int rows, int cols, boolean bContainsZeros) { return TestUtils.createNonRandomMatrixValues(rows, cols, bContainsZeros); } /** * <p> * Generates a matrix containing easy to debug values in its cells. The * generated matrix contains zero values * </p> * * @param rows * @param cols * @return */ protected double[][] createNonRandomMatrixValues(int rows, int cols) { return TestUtils.createNonRandomMatrixValues(rows, cols, true); } /** * @return TRUE if the test harness is not deleting temporary files */ protected boolean isOutAndExpectedDeletionDisabled() { return isOutAndExpectedDeletionDisabled; } /** * Call this method from a subclass's setUp() method. * @param isOutAndExpectedDeletionDisabled * TRUE to disable code that deletes temporary files for this * test case */ protected void setOutAndExpectedDeletionDisabled(boolean isOutAndExpectedDeletionDisabled) { this.isOutAndExpectedDeletionDisabled = isOutAndExpectedDeletionDisabled; } protected String input(String input) { return baseDirectory + INPUT_DIR + input; } protected String inputDir() { return baseDirectory + INPUT_DIR; } protected String output(String output) { return baseDirectory + OUTPUT_DIR + output; } protected String outputDir() { return baseDirectory + OUTPUT_DIR; } protected String expected(String expected) { return baseDirectory + EXPECTED_DIR + cacheDir + expected; } protected String expectedDir() { return baseDirectory + EXPECTED_DIR + cacheDir; } protected String getScript() { return baseDirectory + selectedTest + "." + scriptType.lowerCase(); } protected String getRScript() { return baseDirectory + selectedTest + ".R"; } protected String getRCmd(String... args) { StringBuilder sb = new StringBuilder(); sb.append("Rscript "); sb.append(getRScript()); for (String arg : args) { sb.append(" "); sb.append(arg); } return sb.toString(); } private boolean isTargetTestDirectory(String path) { return (path != null && path.contains(getClass().getSimpleName())); } private void setCacheDirectory(String directory) { cacheDir = (directory != null) ? directory : ""; if (cacheDir.length() > 0 && !cacheDir.endsWith("/")) { cacheDir += "/"; } } private String getSourceDirectory(String testDirectory) { String sourceDirectory = ""; if (null != testDirectory) { if (testDirectory.endsWith("/")) { testDirectory = testDirectory.substring(0, testDirectory.length() - "/".length()); } sourceDirectory = testDirectory.substring(0, testDirectory.lastIndexOf("/") + "/".length()); } return sourceDirectory; } }