Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.util; import org.apache.commons.io.FileUtils; import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider; import org.junit.Assert; import java.io.BufferedReader; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.test.GenericTestUtils; import static org.apache.hadoop.util.Shell.*; import org.junit.Assume; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; import org.junit.rules.Timeout; public class TestShell extends Assert { /** * Set the timeout for every test */ @Rule public Timeout testTimeout = new Timeout(30000); @Rule public TestName methodName = new TestName(); private File rootTestDir = GenericTestUtils.getTestDir(); /** * A filename generated uniquely for each test method. The file * itself is neither created nor deleted during test setup/teardown. */ private File methodDir; private static class Command extends Shell { private int runCount = 0; private Command(long interval) { super(interval); } @Override protected String[] getExecString() { // There is no /bin/echo equivalent on Windows so just launch it as a // shell built-in. // return WINDOWS ? (new String[] { "cmd.exe", "/c", "echo", "hello" }) : (new String[] { "echo", "hello" }); } @Override protected void parseExecResult(BufferedReader lines) throws IOException { ++runCount; } public int getRunCount() { return runCount; } } @Before public void setup() { rootTestDir.mkdirs(); assertTrue("Not a directory " + rootTestDir, rootTestDir.isDirectory()); methodDir = new File(rootTestDir, methodName.getMethodName()); } @Test public void testInterval() throws IOException { testInterval(Long.MIN_VALUE / 60000); // test a negative interval testInterval(0L); // test a zero interval testInterval(10L); // interval equal to 10mins testInterval(Time.now() / 60000 + 60); // test a very big interval } /** * Assert that a string has a substring in it * @param string string to search * @param search what to search for it */ private void assertInString(String string, String search) { assertNotNull("Empty String", string); if (!string.contains(search)) { fail("Did not find \"" + search + "\" in " + string); } } @Test public void testShellCommandExecutorToString() throws Throwable { Shell.ShellCommandExecutor sce = new Shell.ShellCommandExecutor(new String[] { "ls", "..", "arg 2" }); String command = sce.toString(); assertInString(command, "ls"); assertInString(command, " .. "); assertInString(command, "\"arg 2\""); } @Test public void testShellCommandTimeout() throws Throwable { Assume.assumeFalse(WINDOWS); String rootDir = rootTestDir.getAbsolutePath(); File shellFile = new File(rootDir, "timeout.sh"); String timeoutCommand = "sleep 4; echo \"hello\""; Shell.ShellCommandExecutor shexc; try (PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile))) { writer.println(timeoutCommand); writer.close(); } FileUtil.setExecutable(shellFile, true); shexc = new Shell.ShellCommandExecutor(new String[] { shellFile.getAbsolutePath() }, null, null, 100); try { shexc.execute(); } catch (Exception e) { //When timing out exception is thrown. } shellFile.delete(); assertTrue("Script did not timeout", shexc.isTimedOut()); } @Test public void testEnvVarsWithInheritance() throws Exception { Assume.assumeFalse(WINDOWS); testEnvHelper(true); } @Test public void testEnvVarsWithoutInheritance() throws Exception { Assume.assumeFalse(WINDOWS); testEnvHelper(false); } private void testEnvHelper(boolean inheritParentEnv) throws Exception { Map<String, String> customEnv = Collections .singletonMap(AbstractJavaKeyStoreProvider.CREDENTIAL_PASSWORD_ENV_VAR, "foo"); Shell.ShellCommandExecutor command = new ShellCommandExecutor(new String[] { "env" }, null, customEnv, 0L, inheritParentEnv); command.execute(); String[] varsArr = command.getOutput().split("\n"); Map<String, String> vars = new HashMap<>(); for (String var : varsArr) { int eqIndex = var.indexOf('='); vars.put(var.substring(0, eqIndex), var.substring(eqIndex + 1)); } Map<String, String> expectedEnv = new HashMap<>(); expectedEnv.putAll(System.getenv()); if (inheritParentEnv) { expectedEnv.putAll(customEnv); } else { assertFalse( "child process environment should not have contained " + AbstractJavaKeyStoreProvider.CREDENTIAL_PASSWORD_ENV_VAR, vars.containsKey(AbstractJavaKeyStoreProvider.CREDENTIAL_PASSWORD_ENV_VAR)); } assertEquals(expectedEnv, vars); } private static int countTimerThreads() { ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); int count = 0; ThreadInfo[] infos = threadBean.getThreadInfo(threadBean.getAllThreadIds(), 20); for (ThreadInfo info : infos) { if (info == null) continue; for (StackTraceElement elem : info.getStackTrace()) { if (elem.getClassName().contains("Timer")) { count++; break; } } } return count; } @Test public void testShellCommandTimerLeak() throws Exception { String quickCommand[] = new String[] { "/bin/sleep", "100" }; int timersBefore = countTimerThreads(); System.err.println("before: " + timersBefore); for (int i = 0; i < 10; i++) { Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(quickCommand, null, null, 1); try { shexec.execute(); fail("Bad command should throw exception"); } catch (Exception e) { // expected } } Thread.sleep(1000); int timersAfter = countTimerThreads(); System.err.println("after: " + timersAfter); assertEquals(timersBefore, timersAfter); } @Test public void testGetCheckProcessIsAliveCommand() throws Exception { String anyPid = "9999"; String[] checkProcessAliveCommand = getCheckProcessIsAliveCommand(anyPid); String[] expectedCommand; if (Shell.WINDOWS) { expectedCommand = new String[] { getWinUtilsPath(), "task", "isAlive", anyPid }; } else if (Shell.isSetsidAvailable) { expectedCommand = new String[] { "bash", "-c", "kill -0 -- -'" + anyPid + "'" }; } else { expectedCommand = new String[] { "bash", "-c", "kill -0 '" + anyPid + "'" }; } Assert.assertArrayEquals(expectedCommand, checkProcessAliveCommand); } @Test public void testGetSignalKillCommand() throws Exception { String anyPid = "9999"; int anySignal = 9; String[] checkProcessAliveCommand = getSignalKillCommand(anySignal, anyPid); String[] expectedCommand; if (Shell.WINDOWS) { expectedCommand = new String[] { getWinUtilsPath(), "task", "kill", anyPid }; } else if (Shell.isSetsidAvailable) { expectedCommand = new String[] { "bash", "-c", "kill -9 -- -'" + anyPid + "'" }; } else { expectedCommand = new String[] { "bash", "-c", "kill -9 '" + anyPid + "'" }; } Assert.assertArrayEquals(expectedCommand, checkProcessAliveCommand); } private void testInterval(long interval) throws IOException { Command command = new Command(interval); command.run(); assertEquals(1, command.getRunCount()); command.run(); if (interval > 0) { assertEquals(1, command.getRunCount()); } else { assertEquals(2, command.getRunCount()); } } @Test public void testHadoopHomeUnset() throws Throwable { assertHomeResolveFailed(null, "unset"); } @Test public void testHadoopHomeEmpty() throws Throwable { assertHomeResolveFailed("", E_HADOOP_PROPS_EMPTY); } @Test public void testHadoopHomeEmptyDoubleQuotes() throws Throwable { assertHomeResolveFailed("\"\"", E_HADOOP_PROPS_EMPTY); } @Test public void testHadoopHomeEmptySingleQuote() throws Throwable { assertHomeResolveFailed("\"", E_HADOOP_PROPS_EMPTY); } @Test public void testHadoopHomeValid() throws Throwable { File f = checkHadoopHomeInner(rootTestDir.getCanonicalPath()); assertEquals(rootTestDir, f); } @Test public void testHadoopHomeValidQuoted() throws Throwable { File f = checkHadoopHomeInner('"' + rootTestDir.getCanonicalPath() + '"'); assertEquals(rootTestDir, f); } @Test public void testHadoopHomeNoDir() throws Throwable { assertHomeResolveFailed(methodDir.getCanonicalPath(), E_DOES_NOT_EXIST); } @Test public void testHadoopHomeNotADir() throws Throwable { File touched = touch(methodDir); try { assertHomeResolveFailed(touched.getCanonicalPath(), E_NOT_DIRECTORY); } finally { FileUtils.deleteQuietly(touched); } } @Test public void testHadoopHomeRelative() throws Throwable { assertHomeResolveFailed("./target", E_IS_RELATIVE); } @Test public void testBinDirMissing() throws Throwable { FileNotFoundException ex = assertWinutilsResolveFailed(methodDir, E_DOES_NOT_EXIST); assertInString(ex.toString(), "Hadoop bin directory"); } @Test public void testHadoopBinNotADir() throws Throwable { File bin = new File(methodDir, "bin"); touch(bin); try { assertWinutilsResolveFailed(methodDir, E_NOT_DIRECTORY); } finally { FileUtils.deleteQuietly(methodDir); } } @Test public void testBinWinUtilsFound() throws Throwable { try { File bin = new File(methodDir, "bin"); File winutils = new File(bin, WINUTILS_EXE); touch(winutils); assertEquals(winutils.getCanonicalPath(), getQualifiedBinInner(methodDir, WINUTILS_EXE).getCanonicalPath()); } finally { FileUtils.deleteQuietly(methodDir); } } @Test public void testBinWinUtilsNotAFile() throws Throwable { try { File bin = new File(methodDir, "bin"); File winutils = new File(bin, WINUTILS_EXE); winutils.mkdirs(); assertWinutilsResolveFailed(methodDir, E_NOT_EXECUTABLE_FILE); } finally { FileUtils.deleteDirectory(methodDir); } } /** * This test takes advantage of the invariant winutils path is valid * or access to it will raise an exception holds on Linux, and without * any winutils binary even if HADOOP_HOME points to a real hadoop * directory, the exception reporting can be validated */ @Test public void testNoWinutilsOnUnix() throws Throwable { Assume.assumeFalse(WINDOWS); try { getWinUtilsFile(); } catch (FileNotFoundException ex) { assertExContains(ex, E_NOT_A_WINDOWS_SYSTEM); } try { getWinUtilsPath(); } catch (RuntimeException ex) { assertExContains(ex, E_NOT_A_WINDOWS_SYSTEM); if (ex.getCause() == null || !(ex.getCause() instanceof FileNotFoundException)) { throw ex; } } } /** * Touch a file; creating parent dirs on demand. * @param path path of file * @return the file created * @throws IOException on any failure to write */ private File touch(File path) throws IOException { path.getParentFile().mkdirs(); FileUtils.writeByteArrayToFile(path, new byte[] {}); return path; } /** * Assert that an attept to resolve the hadoop home dir failed with * an expected text in the exception string value. * @param path input * @param expectedText expected exception text * @return the caught exception * @throws FileNotFoundException any FileNotFoundException that was thrown * but which did not contain the expected text */ private FileNotFoundException assertHomeResolveFailed(String path, String expectedText) throws Exception { try { File f = checkHadoopHomeInner(path); fail("Expected an exception with the text `" + expectedText + "`" + " -but got the path " + f); // unreachable return null; } catch (FileNotFoundException ex) { assertExContains(ex, expectedText); return ex; } } /** * Assert that an attept to resolve the {@code bin/winutils.exe} failed with * an expected text in the exception string value. * @param hadoopHome hadoop home directory * @param expectedText expected exception text * @return the caught exception * @throws Exception any Exception that was thrown * but which did not contain the expected text */ private FileNotFoundException assertWinutilsResolveFailed(File hadoopHome, String expectedText) throws Exception { try { File f = getQualifiedBinInner(hadoopHome, WINUTILS_EXE); fail("Expected an exception with the text `" + expectedText + "`" + " -but got the path " + f); // unreachable return null; } catch (FileNotFoundException ex) { assertExContains(ex, expectedText); return ex; } } private void assertExContains(Exception ex, String expectedText) throws Exception { if (!ex.toString().contains(expectedText)) { throw ex; } } @Test public void testBashQuote() { assertEquals("'foobar'", Shell.bashQuote("foobar")); assertEquals("'foo'\\''bar'", Shell.bashQuote("foo'bar")); assertEquals("''\\''foo'\\''bar'\\'''", Shell.bashQuote("'foo'bar'")); } }