Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.tez.client; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URL; import java.net.URLClassLoader; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.Records; import org.apache.tez.common.security.JobTokenIdentifier; import org.apache.tez.common.security.JobTokenSecretManager; import org.apache.tez.common.security.TokenCache; import org.apache.tez.dag.api.DAG; import org.apache.tez.dag.api.ProcessorDescriptor; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezConstants; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.dag.api.Vertex; import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto; import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair; import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor; import org.junit.Assert; import org.junit.Test; /** * */ public class TestTezClientUtils { private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR + TestTezClientUtils.class.getName() + "-tmpDir"; /** * */ @Test(timeout = 5000) public void validateSetTezJarLocalResourcesNotDefined() throws Exception { TezConfiguration conf = new TezConfiguration(false); Credentials credentials = new Credentials(); try { Map<String, LocalResource> resources = new HashMap<String, LocalResource>(); TezClientUtils.setupTezJarsLocalResources(conf, credentials, resources); Assert.fail("Expected TezUncheckedException"); } catch (TezUncheckedException e) { Assert.assertTrue(e.getMessage().contains("Invalid configuration of tez jars")); } } @Test(timeout = 5000) public void validateSetTezJarLocalResourcesDefinedButEmpty() throws Exception { File emptyDir = new File(TEST_ROOT_DIR, "emptyDir"); emptyDir.deleteOnExit(); Assert.assertTrue(emptyDir.mkdirs()); TezConfiguration conf = new TezConfiguration(); conf.set(TezConfiguration.TEZ_LIB_URIS, emptyDir.toURI().toURL().toString()); Credentials credentials = new Credentials(); try { Map<String, LocalResource> resources = new HashMap<String, LocalResource>(); TezClientUtils.setupTezJarsLocalResources(conf, credentials, resources); Assert.fail("Expected TezUncheckedException"); } catch (TezUncheckedException e) { Assert.assertTrue(e.getMessage().contains("No files found in locations")); } } /** * */ @Test(expected = FileNotFoundException.class, timeout = 5000) public void validateSetTezJarLocalResourcesDefinedNonExistingDirectory() throws Exception { TezConfiguration conf = new TezConfiguration(); conf.set(TezConfiguration.TEZ_LIB_URIS, "file:///foo"); Credentials credentials = new Credentials(); Map<String, LocalResource> resources = new HashMap<String, LocalResource>(); TezClientUtils.setupTezJarsLocalResources(conf, credentials, resources); } /** * */ @Test(timeout = 10000) public void validateSetTezJarLocalResourcesDefinedExistingDirectory() throws Exception { URL[] cp = ((URLClassLoader) ClassLoader.getSystemClassLoader()).getURLs(); StringBuffer buffer = new StringBuffer(); for (URL url : cp) { buffer.append(url.toExternalForm()); buffer.append(","); } TezConfiguration conf = new TezConfiguration(); conf.set(TezConfiguration.TEZ_LIB_URIS, buffer.toString()); Credentials credentials = new Credentials(); Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>(); boolean usingArchive = TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap); Assert.assertFalse(usingArchive); Set<String> resourceNames = localizedMap.keySet(); for (URL url : cp) { File file = FileUtils.toFile(url); if (file.isDirectory()) { String[] firList = file.list(); for (String fileNme : firList) { File innerFile = new File(file, fileNme); if (!innerFile.isDirectory()) { assertTrue(resourceNames.contains(innerFile.getName())); } // not supporting deep hierarchies } } else { assertTrue(resourceNames.contains(file.getName())); } } } /** * * @throws Exception */ @Test(timeout = 5000) public void validateSetTezJarLocalResourcesDefinedExistingDirectoryIgnored() throws Exception { URL[] cp = ((URLClassLoader) ClassLoader.getSystemClassLoader()).getURLs(); StringBuffer buffer = new StringBuffer(); for (URL url : cp) { buffer.append(url.toExternalForm()); buffer.append(","); } TezConfiguration conf = new TezConfiguration(); conf.set(TezConfiguration.TEZ_LIB_URIS, buffer.toString()); conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true); Credentials credentials = new Credentials(); Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>(); Assert.assertFalse(TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap)); assertTrue(localizedMap.isEmpty()); } /** * * @throws Exception */ @Test(timeout = 5000) public void validateSetTezJarLocalResourcesDefinedExistingDirectoryIgnoredSetToFalse() throws Exception { URL[] cp = ((URLClassLoader) ClassLoader.getSystemClassLoader()).getURLs(); StringBuffer buffer = new StringBuffer(); for (URL url : cp) { buffer.append(url.toExternalForm()); buffer.append(","); } TezConfiguration conf = new TezConfiguration(); conf.set(TezConfiguration.TEZ_LIB_URIS, buffer.toString()); conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false); Credentials credentials = new Credentials(); Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>(); Assert.assertFalse(TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap)); assertFalse(localizedMap.isEmpty()); } /** * */ @Test(timeout = 5000) public void testTezDefaultFS() throws Exception { FileSystem localFs = FileSystem.getLocal(new Configuration()); StringBuilder tezLibUris = new StringBuilder(); Path topDir = new Path(TEST_ROOT_DIR, "testTezDefaultFS"); if (localFs.exists(topDir)) { localFs.delete(topDir, true); } localFs.mkdirs(topDir); Path file = new Path(topDir, "file.jar"); Assert.assertTrue(localFs.createNewFile(file)); tezLibUris.append(localFs.makeQualified(file).toString()); TezConfiguration conf = new TezConfiguration(); conf.set(TezConfiguration.TEZ_LIB_URIS, tezLibUris.toString()); conf.set("fs.defaultFS", "hdfs:///localhost:1234"); Credentials credentials = new Credentials(); Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>(); TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap); Assert.assertTrue(localFs.delete(file, true)); Assert.assertTrue(localFs.delete(topDir, true)); } /** * */ @Test(timeout = 5000) public void validateSetTezJarLocalResourcesMultipleTarballs() throws Exception { FileSystem localFs = FileSystem.getLocal(new Configuration()); StringBuilder tezLibUris = new StringBuilder(); // Create 2 files Path topDir = new Path(TEST_ROOT_DIR, "validatemultipletarballs"); if (localFs.exists(topDir)) { localFs.delete(topDir, true); } localFs.mkdirs(topDir); Path tarFile1 = new Path(topDir, "f1.tar.gz"); Path tarFile2 = new Path(topDir, "f2.tar.gz"); Assert.assertTrue(localFs.createNewFile(tarFile1)); Assert.assertTrue(localFs.createNewFile(tarFile2)); tezLibUris.append(localFs.makeQualified(tarFile1).toString()).append("#tar1").append(","); tezLibUris.append(localFs.makeQualified(tarFile2).toString()).append("#tar2").append(","); TezConfiguration conf = new TezConfiguration(); conf.set(TezConfiguration.TEZ_LIB_URIS, tezLibUris.toString()); Credentials credentials = new Credentials(); Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>(); TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap); Set<String> resourceNames = localizedMap.keySet(); Assert.assertEquals(2, resourceNames.size()); Assert.assertTrue(resourceNames.contains("tar1")); Assert.assertTrue(resourceNames.contains("tar2")); Assert.assertFalse(resourceNames.contains("f1.tar.gz")); Assert.assertFalse(resourceNames.contains("f2.tar.gz")); Assert.assertTrue(localFs.delete(tarFile1, true)); Assert.assertTrue(localFs.delete(tarFile2, true)); Assert.assertTrue(localFs.delete(topDir, true)); } /** * */ @Test(timeout = 5000) public void validateSetTezJarLocalResourcesMixTarballAndJar() throws Exception { FileSystem localFs = FileSystem.getLocal(new Configuration()); StringBuilder tezLibUris = new StringBuilder(); // Create 2 jars and 1 archive Path topDir = new Path(TEST_ROOT_DIR, "validatetarballandjar"); if (localFs.exists(topDir)) { localFs.delete(topDir, true); } localFs.mkdirs(topDir); Path tarFile1 = new Path(topDir, "f1.tar.gz"); Path jarFile2 = new Path(topDir, "f2.jar"); Path jarFile3 = new Path(topDir, "f3.jar"); Assert.assertTrue(localFs.createNewFile(tarFile1)); Assert.assertTrue(localFs.createNewFile(jarFile2)); Assert.assertTrue(localFs.createNewFile(jarFile3)); tezLibUris.append(localFs.makeQualified(topDir).toString()).append(","); tezLibUris.append(localFs.makeQualified(tarFile1).toString()).append("#tar1").append(","); TezConfiguration conf = new TezConfiguration(); conf.set(TezConfiguration.TEZ_LIB_URIS, tezLibUris.toString()); Credentials credentials = new Credentials(); Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>(); TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap); Set<String> resourceNames = localizedMap.keySet(); Assert.assertEquals(4, resourceNames.size()); Assert.assertTrue(resourceNames.contains("tar1")); Assert.assertTrue(resourceNames.contains("f1.tar.gz")); Assert.assertTrue(resourceNames.contains("f2.jar")); Assert.assertTrue(resourceNames.contains("f3.jar")); Assert.assertTrue(localFs.delete(tarFile1, true)); Assert.assertTrue(localFs.delete(jarFile2, true)); Assert.assertTrue(localFs.delete(jarFile3, true)); Assert.assertTrue(localFs.delete(topDir, true)); } @Test(timeout = 2000) // this test checks if the priority field is set properly in the // ApplicationSubmissionContext public void testAppSubmissionContextForPriority() throws Exception { TezConfiguration tezConf = new TezConfiguration(); int testpriority = 999; ApplicationId appId = ApplicationId.newInstance(1000, 1); Credentials credentials = new Credentials(); TezClientUtils.createSessionToken(appId.toString(), new JobTokenSecretManager(), credentials); tezConf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true); Map<String, LocalResource> m = new HashMap<String, LocalResource>(); tezConf.setInt(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY, testpriority); AMConfiguration amConf = new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials); ApplicationSubmissionContext appcontext; appcontext = TezClientUtils.createApplicationSubmissionContext(appId, null, "dagname", amConf, m, credentials, false, new TezApiVersionInfo(), null, null); assertEquals(testpriority, appcontext.getPriority().getPriority()); } @Test(timeout = 1000) // when tez config property for app priority not set, // tez should not set app priority and let YARN deal with it. public void testSetApplicationPriority() { TezConfiguration conf = new TezConfiguration(false); AMConfiguration amconfig = new AMConfiguration(conf, null, null); ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); TezClientUtils.setApplicationPriority(appContext, amconfig); assertNull(appContext.getPriority()); } @Test(timeout = 1000) public void testSetApplicationTags() { TezConfiguration conf = new TezConfiguration(false); conf.set(TezConfiguration.TEZ_APPLICATION_TAGS, "foo,bar"); AMConfiguration amconfig = new AMConfiguration(conf, null, null); ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); Collection<String> tagsFromConf = amconfig.getTezConfiguration() .getTrimmedStringCollection(TezConfiguration.TEZ_APPLICATION_TAGS); appContext.setApplicationTags(new HashSet<String>(tagsFromConf)); assertTrue(appContext.getApplicationTags().contains("foo")); assertTrue(appContext.getApplicationTags().contains("bar")); } @Test(timeout = 5000) public void testSessionTokenInAmClc() throws IOException, YarnException { TezConfiguration tezConf = new TezConfiguration(); ApplicationId appId = ApplicationId.newInstance(1000, 1); DAG dag = DAG.create("testdag"); dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1) .setTaskLaunchCmdOpts("initialLaunchOpts")); Credentials credentials = new Credentials(); JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(); TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials); Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials); assertNotNull(jobToken); AMConfiguration amConf = new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials); ApplicationSubmissionContext appSubmissionContext = TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf, new HashMap<String, LocalResource>(), credentials, false, new TezApiVersionInfo(), null, null); ContainerLaunchContext amClc = appSubmissionContext.getAMContainerSpec(); Map<String, ByteBuffer> amServiceData = amClc.getServiceData(); assertNotNull(amServiceData); assertEquals(1, amServiceData.size()); DataInputByteBuffer dibb = new DataInputByteBuffer(); dibb.reset(amServiceData.values().iterator().next()); Token<JobTokenIdentifier> jtSent = new Token<JobTokenIdentifier>(); jtSent.readFields(dibb); assertTrue(Arrays.equals(jobToken.getIdentifier(), jtSent.getIdentifier())); } @Test(timeout = 5000) public void testAMLoggingOptsSimple() throws IOException, YarnException { TezConfiguration tezConf = new TezConfiguration(); tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN"); ApplicationId appId = ApplicationId.newInstance(1000, 1); Credentials credentials = new Credentials(); JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(); TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials); DAG dag = DAG.create("testdag"); dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1) .setTaskLaunchCmdOpts("initialLaunchOpts")); AMConfiguration amConf = new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials); ApplicationSubmissionContext appSubmissionContext = TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf, new HashMap<String, LocalResource>(), credentials, false, new TezApiVersionInfo(), null, null); List<String> expectedCommands = new LinkedList<String>(); expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator"); expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE); expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR); expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME); List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands(); assertEquals(1, commands.size()); for (String expectedCmd : expectedCommands) { assertTrue(commands.get(0).contains(expectedCmd)); } Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment(); String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS); assertNull(logEnv); } @Test(timeout = 5000) public void testAMLoggingOptsPerLogger() throws IOException, YarnException { TezConfiguration tezConf = new TezConfiguration(); tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG"); ApplicationId appId = ApplicationId.newInstance(1000, 1); Credentials credentials = new Credentials(); JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(); TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials); DAG dag = DAG.create("testdag"); dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1) .setTaskLaunchCmdOpts("initialLaunchOpts")); AMConfiguration amConf = new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials); ApplicationSubmissionContext appSubmissionContext = TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf, new HashMap<String, LocalResource>(), credentials, false, new TezApiVersionInfo(), null, null); List<String> expectedCommands = new LinkedList<String>(); expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator"); expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE); expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR); expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME); List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands(); assertEquals(1, commands.size()); for (String expectedCmd : expectedCommands) { assertTrue(commands.get(0).contains(expectedCmd)); } Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment(); String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS); assertEquals("org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG", logEnv); } @Test(timeout = 5000) public void testAMCommandOpts() { Path tmpDir = new Path(Environment.PWD.$(), YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR); String tmpOpts = "-Djava.io.tmpdir=" + tmpDir; TezConfiguration tezConf = new TezConfiguration(); String amCommandOpts = "-Xmx 200m -Dtest.property"; tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, amCommandOpts); // Test1: Rely on defaults for cluster-default opts String amOptsConstructed = TezClientUtils.constructAMLaunchOpts(tezConf, Resource.newInstance(1024, 1)); assertEquals(tmpOpts + " " + TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT + " " + amCommandOpts, amOptsConstructed); // Test2: Setup cluster-default command opts explicitly String clusterDefaultCommandOpts = "-server -Djava.net.preferIPv4Stack=true -XX:+PrintGCDetails -verbose:gc "; tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS, clusterDefaultCommandOpts); amOptsConstructed = TezClientUtils.constructAMLaunchOpts(tezConf, Resource.newInstance(1024, 1)); assertEquals(tmpOpts + " " + clusterDefaultCommandOpts + " " + amCommandOpts, amOptsConstructed); // Test3: Don't setup Xmx explicitly final double factor = 0.8; amCommandOpts = "-Dtest.property"; tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, amCommandOpts); amOptsConstructed = TezClientUtils.constructAMLaunchOpts(tezConf, Resource.newInstance(1024, 1)); // It's OK for the Xmx value to show up before cluster default options, since Xmx will not be replaced if it already exists. assertEquals(" -Xmx" + ((int) (1024 * factor)) + "m" + " " + tmpOpts + " " + clusterDefaultCommandOpts + " " + amCommandOpts, amOptsConstructed); // Test4: Ensure admin options with Xmx does not cause them to be overridden. This should almost never be done though. clusterDefaultCommandOpts = "-server -Djava.net.preferIPv4Stack=true -XX:+PrintGCDetails -verbose:gc -Xmx200m"; tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS, clusterDefaultCommandOpts); amOptsConstructed = TezClientUtils.constructAMLaunchOpts(tezConf, Resource.newInstance(1024, 1)); assertEquals(tmpOpts + " " + clusterDefaultCommandOpts + " " + amCommandOpts, amOptsConstructed); } @Test(timeout = 5000) public void testTaskCommandOpts() throws TezException { TezConfiguration tezConf = new TezConfiguration(); String taskCommandOpts = "-Xmx 200m -Dtest.property"; tezConf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, taskCommandOpts); String expected = null; // Test1: Rely on defaults for cluster default opts String taskOptsConstructed = TezClientUtils.addDefaultsToTaskLaunchCmdOpts("", tezConf); expected = TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT + " " + taskCommandOpts; assertTrue("Did not find Expected prefix: [" + expected + "] in string [" + taskOptsConstructed + "]", taskOptsConstructed.startsWith(expected)); // Test2: Setup cluster-default command opts explicitly String taskClusterDefaultCommandOpts = "-server -Djava.net.preferIPv4Stack=true -XX:+PrintGCDetails -verbose:gc "; tezConf.set(TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS, taskClusterDefaultCommandOpts); taskOptsConstructed = TezClientUtils.addDefaultsToTaskLaunchCmdOpts("", tezConf); expected = taskClusterDefaultCommandOpts + " " + taskCommandOpts; assertTrue("Did not find Expected prefix: [" + expected + "] in string [" + taskOptsConstructed + "]", taskOptsConstructed.startsWith(expected)); // Test3: Don't setup Xmx explicitly taskCommandOpts = "-Dtest.property"; tezConf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, taskCommandOpts); taskOptsConstructed = TezClientUtils.addDefaultsToTaskLaunchCmdOpts("", tezConf); expected = taskClusterDefaultCommandOpts + " " + taskCommandOpts; assertTrue("Did not find Expected prefix: [" + expected + "] in string [" + taskOptsConstructed + "]", taskOptsConstructed.startsWith(expected)); // Test4: Pass in a dag-configured value. String programmaticTaskOpts = "-Dset.programatically=true -Djava.net.preferIPv4Stack=false"; taskOptsConstructed = TezClientUtils.addDefaultsToTaskLaunchCmdOpts(programmaticTaskOpts, tezConf); // Container logging is always added at the end, if it's required. expected = taskClusterDefaultCommandOpts + " " + taskCommandOpts + " " + programmaticTaskOpts; assertTrue("Did not find Expected prefix: [" + expected + "] in string [" + taskOptsConstructed + "]", taskOptsConstructed.startsWith(expected)); } @Test(timeout = 5000) public void testDefaultMemoryJavaOpts() { final double factor = 0.8; String origJavaOpts = "-Xmx"; String javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts, Resource.newInstance(1000, 1), factor); Assert.assertEquals(origJavaOpts, javaOpts); origJavaOpts = ""; javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts, Resource.newInstance(1000, 1), factor); Assert.assertTrue(javaOpts.contains("-Xmx800m")); origJavaOpts = ""; javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts, Resource.newInstance(1, 1), factor); Assert.assertTrue(javaOpts.contains("-Xmx1m")); origJavaOpts = ""; javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts, Resource.newInstance(-1, 1), factor); Assert.assertEquals(origJavaOpts, javaOpts); origJavaOpts = ""; javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts, Resource.newInstance(355, 1), factor); Assert.assertTrue(javaOpts.contains("-Xmx284m")); origJavaOpts = " -Xms100m "; javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts, Resource.newInstance(355, 1), factor); Assert.assertFalse(javaOpts.contains("-Xmx284m")); Assert.assertTrue(javaOpts.contains("-Xms100m")); origJavaOpts = ""; javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts, Resource.newInstance(355, 1), 0); Assert.assertEquals(origJavaOpts, javaOpts); origJavaOpts = ""; javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts, Resource.newInstance(355, 1), 100); Assert.assertEquals(origJavaOpts, javaOpts); origJavaOpts = ""; javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts, Resource.newInstance(1000, 1), -1); Assert.assertTrue(javaOpts.contains("-Xmx700m")); origJavaOpts = ""; javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts, Resource.newInstance(5000, 1), -1); Assert.assertTrue(javaOpts.contains("-Xmx4000m")); } @Test(timeout = 5000) public void testDefaultLoggingJavaOpts() { String origJavaOpts = null; String javaOpts = TezClientUtils.maybeAddDefaultLoggingJavaOpts("FOOBAR", origJavaOpts); Assert.assertNotNull(javaOpts); Assert.assertTrue(javaOpts.contains("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=FOOBAR") && javaOpts.contains(TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE) && javaOpts.contains("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator")); } @Test(timeout = 5000) public void testConfSerializationForAm() { Configuration conf = new Configuration(false); String val1 = "fixedProperty"; String val2 = "parametrizedProperty/${user.name}"; String expVal2 = "parametrizedProperty/" + System.getProperty("user.name"); conf.set("property1", val1); conf.set("property2", val2); Map<String, String> expected = new HashMap<String, String>(); expected.put("property1", val1); expected.put("property2", expVal2); ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(conf, null); for (PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) { String v = expected.remove(kvPair.getKey()); assertEquals(v, kvPair.getValue()); } assertTrue(expected.isEmpty()); } // To run this test case see TestTezCommonUtils::testLocalResourceVisibility // We do not have much control over the directory structure, cannot mock as the functions are // static and do not want to spin up a minitez cluster just for this. public static void testLocalResourceVisibility(DistributedFileSystem remoteFs, Configuration conf) throws Exception { Path topLevelDir = null; try { FsPermission publicDirPerms = new FsPermission((short) 0755); // rwxr-xr-x FsPermission privateDirPerms = new FsPermission((short) 0754); // rwxr-xr-- FsPermission publicFilePerms = new FsPermission((short) 0554); // r-xr-xr-- FsPermission privateFilePerms = new FsPermission((short) 0550); // r-xr-x--- String fsURI = remoteFs.getUri().toString(); topLevelDir = new Path(fsURI, "/testLRVisibility"); Assert.assertTrue(remoteFs.mkdirs(topLevelDir, publicDirPerms)); Path publicSubDir = new Path(topLevelDir, "public_sub_dir"); Assert.assertTrue(remoteFs.mkdirs(publicSubDir, publicDirPerms)); Path privateSubDir = new Path(topLevelDir, "private_sub_dir"); Assert.assertTrue(remoteFs.mkdirs(privateSubDir, privateDirPerms)); Path publicFile = new Path(publicSubDir, "public_file"); Assert.assertTrue(remoteFs.createNewFile(publicFile)); remoteFs.setPermission(publicFile, publicFilePerms); Path privateFile = new Path(publicSubDir, "private_file"); Assert.assertTrue(remoteFs.createNewFile(privateFile)); remoteFs.setPermission(privateFile, privateFilePerms); Path publicFileInPrivateSubdir = new Path(privateSubDir, "public_file_in_private_subdir"); Assert.assertTrue(remoteFs.createNewFile(publicFileInPrivateSubdir)); remoteFs.setPermission(publicFileInPrivateSubdir, publicFilePerms); TezConfiguration tezConf = new TezConfiguration(conf); String tmpTezLibUris = String.format("%s,%s,%s,%s", topLevelDir, publicSubDir, privateSubDir, conf.get(TezConfiguration.TEZ_LIB_URIS, "")); tezConf.set(TezConfiguration.TEZ_LIB_URIS, tmpTezLibUris); Map<String, LocalResource> lrMap = new HashMap<String, LocalResource>(); TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap); Assert.assertEquals(publicFile.getName(), LocalResourceVisibility.PUBLIC, lrMap.get(publicFile.getName()).getVisibility()); Assert.assertEquals(privateFile.getName(), LocalResourceVisibility.PRIVATE, lrMap.get(privateFile.getName()).getVisibility()); Assert.assertEquals(publicFileInPrivateSubdir.getName(), LocalResourceVisibility.PRIVATE, lrMap.get(publicFileInPrivateSubdir.getName()).getVisibility()); // test tar.gz tezConf = new TezConfiguration(conf); Path tarFile = new Path(topLevelDir, "foo.tar.gz"); Assert.assertTrue(remoteFs.createNewFile(tarFile)); //public remoteFs.setPermission(tarFile, publicFilePerms); tezConf.set(TezConfiguration.TEZ_LIB_URIS, tarFile.toString()); lrMap.clear(); Assert.assertTrue(TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap)); Assert.assertEquals(LocalResourceVisibility.PUBLIC, lrMap.get(TezConstants.TEZ_TAR_LR_NAME).getVisibility()); //private remoteFs.setPermission(tarFile, privateFilePerms); lrMap.clear(); TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap); Assert.assertEquals(LocalResourceVisibility.PRIVATE, lrMap.get(TezConstants.TEZ_TAR_LR_NAME).getVisibility()); } finally { if (topLevelDir != null) { remoteFs.delete(topLevelDir, true); } } } @Test(timeout = 5000) public void testConfigurationAllowAll() { Configuration srcConf = new Configuration(false); Map<String, String> confMap = new HashMap<String, String>(); confMap.put("foo.property", "2000"); confMap.put("tez.property", "tezProperty"); confMap.put("yarn.property", "yarnProperty"); for (Map.Entry<String, String> entry : confMap.entrySet()) { srcConf.set(entry.getKey(), entry.getValue()); } ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(srcConf, null); for (PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) { String val = confMap.remove(kvPair.getKey()); assertNotNull(val); assertEquals(val, kvPair.getValue()); } assertTrue(confMap.isEmpty()); } private Path createFile(FileSystem fs, Path dir, String fileName) throws IOException { Path f1 = new Path(dir, fileName); FSDataOutputStream outputStream = fs.create(f1, true); outputStream.write(1); outputStream.close(); return fs.makeQualified(f1); } @Test(timeout = 5000) public void validateSetTezAuxLocalResourcesWithFilesAndFolders() throws Exception { FileSystem localFs = FileSystem.getLocal(new Configuration()); List<String> resources = new ArrayList<String>(); StringBuilder auxUriStr = new StringBuilder(); // Create 2 files Path topDir = new Path(TEST_ROOT_DIR, "validateauxwithfiles"); if (localFs.exists(topDir)) { localFs.delete(topDir, true); } localFs.mkdirs(topDir); resources.add(createFile(localFs, topDir, "f1.txt").toString()); auxUriStr.append(localFs.makeQualified(topDir).toString()).append(","); Path dir2 = new Path(topDir, "dir2"); localFs.mkdirs(dir2); Path nestedDir = new Path(dir2, "nestedDir"); localFs.mkdirs(nestedDir); createFile(localFs, nestedDir, "nested-f.txt"); resources.add(createFile(localFs, dir2, "dir2-f.txt").toString()); auxUriStr.append(localFs.makeQualified(dir2).toString()).append(","); Path dir3 = new Path(topDir, "dir3"); localFs.mkdirs(dir3); auxUriStr.append(localFs.makeQualified(dir3).toString()).append(","); TezConfiguration conf = new TezConfiguration(); conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true); conf.set(TezConfiguration.TEZ_AUX_URIS, auxUriStr.toString()); Credentials credentials = new Credentials(); Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>(); TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap); Set<String> resourceNames = localizedMap.keySet(); Assert.assertEquals(2, resourceNames.size()); Assert.assertTrue(resourceNames.contains("f1.txt")); Assert.assertTrue(resourceNames.contains("dir2-f.txt")); } @Test(timeout = 5000) public void testServiceDescriptorSerializationForAM() { Configuration conf = new Configuration(false); ServicePluginsDescriptor servicePluginsDescriptor = ServicePluginsDescriptor.create(true); ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(conf, servicePluginsDescriptor); assertTrue(confProto.hasAmPluginDescriptor()); assertTrue(confProto.getAmPluginDescriptor().getUberEnabled()); } @Test(timeout = 5000) public void testTaskLaunchCmdOptsSetup() throws TezException { Configuration conf = new Configuration(false); String vOpts = ""; String opts = TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vOpts, conf); Assert.assertEquals(opts, TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT + " " + TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT + " " + vOpts); vOpts = "foo"; opts = TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vOpts, conf); Assert.assertEquals(opts, TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT + " " + vOpts); String taskOpts = "taskOpts"; conf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, taskOpts); opts = TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vOpts, conf); Assert.assertEquals(opts, TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT + " " + taskOpts + " " + vOpts); } @Test(timeout = 5000) public void testClusterTaskLaunchCmdOptsSetup() throws TezException { Configuration conf = new Configuration(false); String adminOpts = "adminOpts"; conf.set(TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS, adminOpts); String vOpts = ""; String opts = TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vOpts, conf); Assert.assertEquals(opts, adminOpts + " " + TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT + " " + vOpts); vOpts = "foo"; opts = TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vOpts, conf); Assert.assertEquals(opts, adminOpts + " " + vOpts); String taskOpts = "taskOpts"; conf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, taskOpts); opts = TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vOpts, conf); Assert.assertEquals(opts, adminOpts + " " + taskOpts + " " + vOpts); } }