List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration
public YarnConfiguration()
From source file:org.apache.tez.dag.app.rm.TestTaskScheduler.java
License:Apache License
@SuppressWarnings("unchecked") @Test(timeout = 5000)/*from w w w. ja v a 2 s . c o m*/ public void testTaskSchedulerDetermineMinHeldContainers() throws Exception { RackResolver.init(new YarnConfiguration()); TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class); AppContext mockAppContext = mock(AppContext.class); when(mockAppContext.getAMState()).thenReturn(DAGAppMasterState.RUNNING); TezAMRMClientAsync<CookieContainerRequest> mockRMClient = mock(TezAMRMClientAsync.class); String appHost = "host"; int appPort = 0; String appUrl = "url"; TaskSchedulerWithDrainableAppCallback scheduler = new TaskSchedulerWithDrainableAppCallback(mockApp, new AlwaysMatchesContainerMatcher(), appHost, appPort, appUrl, mockRMClient, mockAppContext); Configuration conf = new Configuration(); scheduler.init(conf); RegisterApplicationMasterResponse mockRegResponse = mock(RegisterApplicationMasterResponse.class); Resource mockMaxResource = mock(Resource.class); Map<ApplicationAccessType, String> mockAcls = mock(Map.class); when(mockRegResponse.getMaximumResourceCapability()).thenReturn(mockMaxResource); when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls); when(mockRMClient.registerApplicationMaster(anyString(), anyInt(), anyString())) .thenReturn(mockRegResponse); Resource mockClusterResource = mock(Resource.class); when(mockRMClient.getAvailableResources()).thenReturn(mockClusterResource); scheduler.start(); String rack1 = "r1"; String rack2 = "r2"; String rack3 = "r3"; String node1Rack1 = "n1r1"; String node2Rack1 = "n2r1"; String node1Rack2 = "n1r2"; String node2Rack2 = "n2r2"; String node1Rack3 = "n1r3"; ApplicationAttemptId appId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0); ContainerId mockCId1 = ContainerId.newInstance(appId, 0); HeldContainer hc1 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc1.getNode()).thenReturn(node1Rack1); when(hc1.getRack()).thenReturn(rack1); when(hc1.getContainer().getId()).thenReturn(mockCId1); ContainerId mockCId2 = ContainerId.newInstance(appId, 1); HeldContainer hc2 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc2.getNode()).thenReturn(node2Rack1); when(hc2.getRack()).thenReturn(rack1); when(hc2.getContainer().getId()).thenReturn(mockCId2); ContainerId mockCId3 = ContainerId.newInstance(appId, 2); HeldContainer hc3 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc3.getNode()).thenReturn(node1Rack1); when(hc3.getRack()).thenReturn(rack1); when(hc3.getContainer().getId()).thenReturn(mockCId3); ContainerId mockCId4 = ContainerId.newInstance(appId, 3); HeldContainer hc4 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc4.getNode()).thenReturn(node2Rack1); when(hc4.getRack()).thenReturn(rack1); when(hc4.getContainer().getId()).thenReturn(mockCId4); ContainerId mockCId5 = ContainerId.newInstance(appId, 4); HeldContainer hc5 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc5.getNode()).thenReturn(node1Rack2); when(hc5.getRack()).thenReturn(rack2); when(hc5.getContainer().getId()).thenReturn(mockCId5); ContainerId mockCId6 = ContainerId.newInstance(appId, 5); HeldContainer hc6 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc6.getNode()).thenReturn(node2Rack2); when(hc6.getRack()).thenReturn(rack2); when(hc6.getContainer().getId()).thenReturn(mockCId6); ContainerId mockCId7 = ContainerId.newInstance(appId, 6); HeldContainer hc7 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc7.getNode()).thenReturn(node1Rack3); when(hc7.getRack()).thenReturn(rack3); when(hc7.getContainer().getId()).thenReturn(mockCId7); scheduler.heldContainers.put(mockCId1, hc1); scheduler.heldContainers.put(mockCId2, hc2); scheduler.heldContainers.put(mockCId3, hc3); scheduler.heldContainers.put(mockCId4, hc4); scheduler.heldContainers.put(mockCId5, hc5); scheduler.heldContainers.put(mockCId6, hc6); scheduler.heldContainers.put(mockCId7, hc7); // test empty case scheduler.sessionNumMinHeldContainers = 0; scheduler.determineMinHeldContainers(); Assert.assertEquals(0, scheduler.sessionMinHeldContainers.size()); // test min >= held scheduler.sessionNumMinHeldContainers = 7; scheduler.determineMinHeldContainers(); Assert.assertEquals(7, scheduler.sessionMinHeldContainers.size()); // test min < held scheduler.sessionNumMinHeldContainers = 5; scheduler.determineMinHeldContainers(); Assert.assertEquals(5, scheduler.sessionMinHeldContainers.size()); Set<HeldContainer> heldContainers = Sets.newHashSet(); for (ContainerId cId : scheduler.sessionMinHeldContainers) { heldContainers.add(scheduler.heldContainers.get(cId)); } Set<String> racks = Sets.newHashSet(); Set<String> nodes = Sets.newHashSet(); for (HeldContainer hc : heldContainers) { nodes.add(hc.getNode()); racks.add(hc.getRack()); } // 1 container from each node in rack1 and rack2. 1 container from rack3. // covers not enough containers in rack (rack 3) // covers just enough containers in rack (rack 2) // covers more than enough containers in rack (rack 1) Assert.assertEquals(5, nodes.size()); Assert.assertTrue(nodes.contains(node1Rack1) && nodes.contains(node2Rack1) && nodes.contains(node1Rack2) && nodes.contains(node2Rack2) && nodes.contains(node1Rack3)); Assert.assertEquals(3, racks.size()); Assert.assertTrue(racks.contains(rack1) && racks.contains(rack2) && racks.contains(rack3)); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.stop(); scheduler.close(); }
From source file:org.apache.tez.dag.app.rm.TestTaskScheduler.java
License:Apache License
@SuppressWarnings("unchecked") @Test(timeout = 5000)/*from www . j a v a2 s . c o m*/ public void testTaskSchedulerRandomReuseExpireTime() throws Exception { RackResolver.init(new YarnConfiguration()); TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class); AppContext mockAppContext = mock(AppContext.class); when(mockAppContext.getAMState()).thenReturn(DAGAppMasterState.RUNNING); TezAMRMClientAsync<CookieContainerRequest> mockRMClient = mock(TezAMRMClientAsync.class); String appHost = "host"; int appPort = 0; String appUrl = "url"; TaskSchedulerWithDrainableAppCallback scheduler1 = new TaskSchedulerWithDrainableAppCallback(mockApp, new AlwaysMatchesContainerMatcher(), appHost, appPort, appUrl, mockRMClient, mockAppContext); TaskSchedulerWithDrainableAppCallback scheduler2 = new TaskSchedulerWithDrainableAppCallback(mockApp, new AlwaysMatchesContainerMatcher(), appHost, appPort, appUrl, mockRMClient, mockAppContext); long minTime = 1000l; long maxTime = 100000l; Configuration conf1 = new Configuration(); conf1.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, minTime); conf1.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, minTime); scheduler1.init(conf1); Configuration conf2 = new Configuration(); conf2.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, minTime); conf2.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, maxTime); scheduler2.init(conf2); RegisterApplicationMasterResponse mockRegResponse = mock(RegisterApplicationMasterResponse.class); Resource mockMaxResource = mock(Resource.class); Map<ApplicationAccessType, String> mockAcls = mock(Map.class); when(mockRegResponse.getMaximumResourceCapability()).thenReturn(mockMaxResource); when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls); when(mockRMClient.registerApplicationMaster(anyString(), anyInt(), anyString())) .thenReturn(mockRegResponse); Resource mockClusterResource = mock(Resource.class); when(mockRMClient.getAvailableResources()).thenReturn(mockClusterResource); scheduler1.start(); scheduler2.start(); // when min == max the expire time is always min for (int i = 0; i < 10; ++i) { Assert.assertEquals(minTime, scheduler1.getHeldContainerExpireTime(0)); } long lastExpireTime = 0; // when min < max the expire time is random in between min and max for (int i = 0; i < 10; ++i) { long currExpireTime = scheduler2.getHeldContainerExpireTime(0); Assert.assertTrue("min: " + minTime + " curr: " + currExpireTime + " max: " + maxTime, (minTime <= currExpireTime && currExpireTime <= maxTime)); Assert.assertNotEquals(lastExpireTime, currExpireTime); lastExpireTime = currExpireTime; } String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler1.stop(); scheduler1.close(); scheduler2.stop(); scheduler2.close(); }
From source file:org.apache.tez.dag.app.rm.TestTaskScheduler.java
License:Apache License
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test(timeout = 5000)//from w ww .jav a 2s. c o m
public void testTaskSchedulerPreemption() throws Exception {
RackResolver.init(new YarnConfiguration());
TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class);
AppContext mockAppContext = mock(AppContext.class);
when(mockAppContext.getAMState()).thenReturn(DAGAppMasterState.RUNNING);
TezAMRMClientAsync<CookieContainerRequest> mockRMClient = mock(TezAMRMClientAsync.class);
String appHost = "host";
int appPort = 0;
String appUrl = "url";
final TaskSchedulerWithDrainableAppCallback scheduler = new TaskSchedulerWithDrainableAppCallback(mockApp,
new PreemptionMatcher(), appHost, appPort, appUrl, mockRMClient, mockAppContext);
TaskSchedulerAppCallbackDrainable drainableAppCallback = scheduler.getDrainableAppCallback();
Configuration conf = new Configuration();
conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false);
scheduler.init(conf);
RegisterApplicationMasterResponse mockRegResponse = mock(RegisterApplicationMasterResponse.class);
when(mockRMClient.registerApplicationMaster(anyString(), anyInt(), anyString()))
.thenReturn(mockRegResponse);
scheduler.start();
Resource totalResource = Resource.newInstance(4000, 4);
when(mockRMClient.getAvailableResources()).thenReturn(totalResource);
// no preemption
scheduler.getProgress();
drainableAppCallback.drain();
Assert.assertEquals(totalResource, scheduler.getTotalResources());
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
// allocate task
Object mockTask1 = mock(Object.class);
Object mockTask2 = mock(Object.class);
Object mockTask3 = mock(Object.class);
Object mockTask3Wait = mock(Object.class);
Object mockTask3Retry = mock(Object.class);
Object mockTask3KillA = mock(Object.class);
Object mockTask3KillB = mock(Object.class);
Object obj3 = new Object();
Priority pri2 = Priority.newInstance(2);
Priority pri4 = Priority.newInstance(4);
Priority pri5 = Priority.newInstance(5);
Priority pri6 = Priority.newInstance(6);
ArgumentCaptor<CookieContainerRequest> requestCaptor = ArgumentCaptor
.forClass(CookieContainerRequest.class);
final ArrayList<CookieContainerRequest> anyContainers = new ArrayList<CookieContainerRequest>();
Resource taskAsk = Resource.newInstance(1024, 1);
scheduler.allocateTask(mockTask1, taskAsk, null, null, pri2, null, null);
drainableAppCallback.drain();
verify(mockRMClient, times(1)).addContainerRequest(requestCaptor.capture());
anyContainers.add(requestCaptor.getValue());
scheduler.allocateTask(mockTask3, taskAsk, null, null, pri6, obj3, null);
drainableAppCallback.drain();
verify(mockRMClient, times(2)).addContainerRequest(requestCaptor.capture());
anyContainers.add(requestCaptor.getValue());
// later one in the allocation gets killed between the two task3's
scheduler.allocateTask(mockTask3KillA, taskAsk, null, null, pri6, obj3, null);
drainableAppCallback.drain();
verify(mockRMClient, times(3)).addContainerRequest(requestCaptor.capture());
anyContainers.add(requestCaptor.getValue());
// later one in the allocation gets killed between the two task3's
scheduler.allocateTask(mockTask3KillB, taskAsk, null, null, pri6, obj3, null);
drainableAppCallback.drain();
verify(mockRMClient, times(4)).addContainerRequest(requestCaptor.capture());
anyContainers.add(requestCaptor.getValue());
Resource freeResource = Resource.newInstance(500, 0);
when(mockRMClient.getAvailableResources()).thenReturn(freeResource);
scheduler.getProgress();
drainableAppCallback.drain();
Assert.assertEquals(totalResource, scheduler.getTotalResources());
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
final List<ArrayList<CookieContainerRequest>> anyList = new LinkedList<ArrayList<CookieContainerRequest>>();
final List<ArrayList<CookieContainerRequest>> emptyList = new LinkedList<ArrayList<CookieContainerRequest>>();
anyList.add(anyContainers);
List<Container> containers = new ArrayList<Container>();
Container mockContainer1 = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer1.getNodeId().getHost()).thenReturn("host1");
when(mockContainer1.getResource()).thenReturn(taskAsk);
when(mockContainer1.getPriority()).thenReturn(pri2);
ContainerId mockCId1 = mock(ContainerId.class);
when(mockContainer1.getId()).thenReturn(mockCId1);
containers.add(mockContainer1);
Container mockContainer2 = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer2.getNodeId().getHost()).thenReturn("host1");
when(mockContainer2.getResource()).thenReturn(taskAsk);
when(mockContainer2.getPriority()).thenReturn(pri6);
ContainerId mockCId2 = mock(ContainerId.class);
when(mockContainer2.getId()).thenReturn(mockCId2);
containers.add(mockContainer2);
Container mockContainer3A = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer3A.getNodeId().getHost()).thenReturn("host1");
when(mockContainer3A.getResource()).thenReturn(taskAsk);
when(mockContainer3A.getPriority()).thenReturn(pri6);
ContainerId mockCId3A = mock(ContainerId.class);
when(mockContainer3A.getId()).thenReturn(mockCId3A);
containers.add(mockContainer3A);
Container mockContainer3B = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer3B.getNodeId().getHost()).thenReturn("host1");
when(mockContainer3B.getResource()).thenReturn(taskAsk);
when(mockContainer3B.getPriority()).thenReturn(pri2); // high priority container
ContainerId mockCId3B = mock(ContainerId.class);
when(mockContainer3B.getId()).thenReturn(mockCId3B);
containers.add(mockContainer3B);
when(mockRMClient.getMatchingRequests((Priority) any(), eq("host1"), (Resource) any()))
.thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() {
@Override
public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation)
throws Throwable {
return emptyList;
}
});
// RackResolver by default puts hosts in default-rack
when(mockRMClient.getMatchingRequests((Priority) any(), eq("/default-rack"), (Resource) any()))
.thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() {
@Override
public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation)
throws Throwable {
return emptyList;
}
});
when(mockRMClient.getMatchingRequests((Priority) any(), eq(ResourceRequest.ANY), (Resource) any()))
.thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() {
int calls = 0;
@Override
public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation)
throws Throwable {
if (calls > 0) {
anyContainers.remove(0);
}
calls++;
return anyList;
}
});
Mockito.doAnswer(new Answer() {
public Object answer(InvocationOnMock invocation) {
Object[] args = invocation.getArguments();
ContainerId cId = (ContainerId) args[0];
scheduler.deallocateContainer(cId);
return null;
}
}).when(mockApp).preemptContainer((ContainerId) any());
scheduler.onContainersAllocated(containers);
drainableAppCallback.drain();
Assert.assertEquals(4, scheduler.taskAllocations.size());
Assert.assertEquals(4096, scheduler.allocatedResources.getMemory());
Assert.assertEquals(mockCId1, scheduler.taskAllocations.get(mockTask1).getId());
Assert.assertEquals(mockCId2, scheduler.taskAllocations.get(mockTask3).getId());
Assert.assertEquals(mockCId3A, scheduler.taskAllocations.get(mockTask3KillA).getId());
// high priority container assigned to lower pri task. This task should still be preempted
// because the task priority is relevant for preemption and not the container priority
Assert.assertEquals(mockCId3B, scheduler.taskAllocations.get(mockTask3KillB).getId());
// no preemption
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
Object mockTask3WaitCookie = new Object();
scheduler.allocateTask(mockTask3Wait, taskAsk, null, null, pri6, obj3, mockTask3WaitCookie);
// no preemption - same pri
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
Priority pri8 = Priority.newInstance(8);
Container mockContainer4 = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer4.getNodeId().getHost()).thenReturn("host1");
when(mockContainer4.getResource()).thenReturn(taskAsk);
when(mockContainer4.getPriority()).thenReturn(pri8);
ContainerId mockCId4 = mock(ContainerId.class);
when(mockContainer4.getId()).thenReturn(mockCId4);
containers.clear();
containers.add(mockContainer4);
// Fudge new container being present in delayed allocation list due to race
HeldContainer heldContainer = new HeldContainer(mockContainer4, -1, -1, null);
scheduler.delayedContainerManager.delayedContainers.add(heldContainer);
// no preemption - container assignment attempts < 3
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
heldContainer.incrementAssignmentAttempts();
// no preemption - container assignment attempts < 3
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
heldContainer.incrementAssignmentAttempts();
heldContainer.incrementAssignmentAttempts();
// preemption - container released and resource asked again
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any());
verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId4);
verify(mockRMClient, times(5)).addContainerRequest(requestCaptor.capture());
CookieContainerRequest reAdded = requestCaptor.getValue();
Assert.assertEquals(pri6, reAdded.getPriority());
Assert.assertEquals(taskAsk, reAdded.getCapability());
Assert.assertEquals(mockTask3WaitCookie, reAdded.getCookie().getAppCookie());
// remove fudging.
scheduler.delayedContainerManager.delayedContainers.clear();
scheduler.allocateTask(mockTask3Retry, taskAsk, null, null, pri5, obj3, null);
// no preemption - higher pri. exact match
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any());
for (int i = 0; i < 11; ++i) {
scheduler.allocateTask(mockTask2, taskAsk, null, null, pri4, null, null);
}
drainableAppCallback.drain();
// mockTaskPri3KillB gets preempted to clear 10% of outstanding running preemptable tasks
// this is also a higher priority container than the pending task priority but was running a
// lower priority task. Task priority is relevant for preemption and not container priority as
// containers can run tasks of different priorities
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId) any());
verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId3B);
// next 3 heartbeats do nothing, waiting for the RM to act on the last released resources
scheduler.getProgress();
scheduler.getProgress();
scheduler.getProgress();
verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId) any());
scheduler.getProgress();
drainableAppCallback.drain();
// Next oldest mockTaskPri3KillA gets preempted to clear 10% of outstanding running preemptable tasks
verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any());
verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId3A);
AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, "", appUrl);
when(mockApp.getFinalAppStatus()).thenReturn(finalStatus);
scheduler.stop();
drainableAppCallback.drain();
scheduler.close();
}
From source file:org.apache.tez.history.ATSImportTool_V2.java
License:Apache License
@VisibleForTesting static int process(String[] args) { Options options = buildOptions();/* w w w . j a v a 2 s . c o m*/ LOG.info("Processing.."); int result = -1; try { YarnConfiguration conf = new YarnConfiguration(); CommandLine cmdLine = new GnuParser().parse(options, args); String appId = cmdLine.getOptionValue(APP_ID); Preconditions.checkArgument(!Strings.isNullOrEmpty(appId), "Please provide appId"); String[] dagIds = cmdLine.getOptionValues(DAG_ID); File downloadDir = new File(cmdLine.getOptionValue(BASE_DOWNLOAD_DIR)); String yarnTimelineAddress = cmdLine.getOptionValue(YARN_TIMELINE_SERVICE_ADDRESS); String baseTimelineURL = getBaseTimelineURL(yarnTimelineAddress, conf); String yarnRMStatusUrl = null; if (yarnTimelineAddress != null) { yarnRMStatusUrl = cmdLine.getOptionValue(YARN_RM_STATUS_ADDRESS); Preconditions.checkArgument(!Strings.isNullOrEmpty(yarnRMStatusUrl), "Please provide a " + "valid " + YARN_RM_STATUS_ADDRESS + " e.g --yarnRMStatusAddress " + "\"http://RMNode:8088\""); yarnRMStatusUrl = yarnRMStatusUrl + YARN_RM_STATUS_QUERY; } else { //Get it from local conf yarnRMStatusUrl = getYarnRMWebAppUrl(conf); } int batchSize = (cmdLine.hasOption(BATCH_SIZE)) ? (Integer.parseInt(cmdLine.getOptionValue(BATCH_SIZE))) : BATCH_SIZE_DEFAULT; result = ToolRunner.run(conf, new ATSImportTool_V2(yarnRMStatusUrl, baseTimelineURL, appId, dagIds, downloadDir, batchSize), args); return result; } catch (MissingOptionException missingOptionException) { LOG.error("Error in parsing options ", missingOptionException); printHelp(options); } catch (ParseException e) { LOG.error("Error in parsing options ", e); printHelp(options); } catch (Throwable e) { LOG.error("Error in processing ", e); throw e; } finally { return result; } }
From source file:org.apache.tez.test.FaultToleranceTestRunner.java
License:Apache License
void setup() throws Exception { TezConfiguration tezConf = null;/*from w ww. j a v a 2 s . com*/ if (conf == null) { tezConf = new TezConfiguration(new YarnConfiguration()); } else { tezConf = new TezConfiguration(new YarnConfiguration(this.conf)); } FileSystem defaultFs = FileSystem.get(tezConf); Path remoteStagingDir = defaultFs .makeQualified(new Path(TEST_ROOT_DIR, String.valueOf(new Random().nextInt(100000)))); TezClientUtils.ensureStagingDirExists(tezConf, remoteStagingDir); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); tezSession = TezClient.create("FaultToleranceTestRunner", tezConf); tezSession.start(); }
From source file:org.apache.twill.example.yarn.BundledJarExample.java
License:Apache License
public static void main(String[] args) { if (args.length < 3) { System.err.println("Arguments format: <host:port of zookeeper server>" + " <bundle jar path> <main class name> <extra args>"); System.exit(1);//from w ww .j a va 2 s. c o m } String zkStr = args[0]; BundledJarRunner.Arguments arguments = new BundledJarRunner.Arguments(args[1], "/lib", args[2], Arrays.copyOfRange(args, 3, args.length)); File jarFile = new File(arguments.getJarFileName()); Preconditions.checkState(jarFile.exists()); Preconditions.checkState(jarFile.canRead()); final TwillRunnerService twillRunner = new YarnTwillRunnerService(new YarnConfiguration(), zkStr); twillRunner.start(); final TwillController controller = twillRunner .prepare(new ExampleBundledJarApp(jarFile.getName(), jarFile.toURI())) .withArguments("BundledJarRunnable", arguments.toArray()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))).start(); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { Futures.getUnchecked(controller.terminate()); } finally { twillRunner.stop(); } } }); try { controller.awaitTerminated(); } catch (ExecutionException e) { LOG.error("Error", e); } }
From source file:org.apache.twill.example.yarn.HelloWorld.java
License:Apache License
public static void main(String[] args) { if (args.length < 1) { System.err.println("Arguments format: <host:port of zookeeper server>"); System.exit(1);/* w w w .j a va 2s.c om*/ } String zkStr = args[0]; YarnConfiguration yarnConfiguration = new YarnConfiguration(); final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr); twillRunner.start(); String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)); List<String> applicationClassPaths = Lists.newArrayList(); Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath)); final TwillController controller = twillRunner.prepare(new HelloWorldRunnable()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .withApplicationClassPaths(applicationClassPaths) .withBundlerClassAcceptor(new HadoopClassExcluder()).start(); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { Futures.getUnchecked(controller.terminate()); } finally { twillRunner.stop(); } } }); try { controller.awaitTerminated(); } catch (ExecutionException e) { e.printStackTrace(); } }
From source file:org.chenchun.ApplicationMaster.java
License:Apache License
public static void main(String[] args) throws IOException, YarnException, InterruptedException { final String params = args[0]; final int containerNum = Integer.valueOf(args[1]); // Initialize clients to ResourceManager and NodeManagers Configuration conf = new YarnConfiguration(); AMRMClient<AMRMClient.ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf);//from w ww . ja v a 2 s . co m rmClient.start(); NMClient nmClient = NMClient.createNMClient(); nmClient.init(conf); nmClient.start(); // Register with ResourceManager System.out.println("registerApplicationMaster 0"); rmClient.registerApplicationMaster("", 0, ""); System.out.println("registerApplicationMaster 1"); // Priority for worker containers - priorities are intra-application Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); // Resource requirements for worker containers Resource capability = Records.newRecord(Resource.class); capability.setMemory(128); capability.setVirtualCores(1); // Make container requests to ResourceManager for (int i = 0; i < containerNum; ++i) { AMRMClient.ContainerRequest containerAsk = new AMRMClient.ContainerRequest(capability, null, null, priority); System.out.println("Making res-req " + i); rmClient.addContainerRequest(containerAsk); } // Obtain allocated containers, launch and check for responses int responseId = 0; int completedContainers = 0; while (completedContainers < containerNum) { AllocateResponse response = rmClient.allocate(responseId++); System.out.println("Allocate response " + response.getAMCommand() + " " + "allocate " + response.getAllocatedContainers().size() + "contains"); for (Container container : response.getAllocatedContainers()) { // Launch container by create ContainerLaunchContext ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); ctx.setCommands( Collections.singletonList(params + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); System.out.println("Launching container " + container.getId() + " on " + container.getNodeId()); nmClient.startContainer(container, ctx); } for (ContainerStatus status : response.getCompletedContainersStatuses()) { ++completedContainers; System.out.println("Completed container " + status.getContainerId()); } Thread.sleep(1000); } System.out.println("Unregister ApplicationMaster"); // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); }
From source file:org.chenchun.twill.EchoServer.java
License:Apache License
public static void main(String[] args) { if (args.length < 1) { System.err.println("Arguments format: <host:port of zookeeper server>"); System.exit(1);/*from ww w . jav a 2s. co m*/ } String zkStr = args[0]; final TwillRunnerService twillRunner = new YarnTwillRunnerService(new YarnConfiguration(), zkStr); twillRunner.startAndWait(); final TwillController controller = twillRunner.prepare(new EchoServer()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))).start(); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { controller.stopAndWait(); twillRunner.stopAndWait(); } }); try { Services.getCompletionFuture(controller).get(); } catch (InterruptedException e) { e.printStackTrace(); } catch (ExecutionException e) { e.printStackTrace(); } }
From source file:org.chenchun.twill.TwillMultiTask.java
License:Apache License
public static void main(String[] args) throws ExecutionException, InterruptedException { String zkStr = "localhost:2181"; final TwillRunnerService twillRunner = new YarnTwillRunnerService(new YarnConfiguration(), zkStr); twillRunner.startAndWait();//from ww w.j a va 2 s . c o m final TwillController controller = twillRunner.prepare(new TwillMultiTask()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))).start(); Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { @Override public void run() { controller.stopAndWait(); twillRunner.stopAndWait(); } })); Services.getCompletionFuture(controller).get(); }