List of usage examples for java.util Random setSeed
public synchronized void setSeed(long seed)
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test @SuppressWarnings("unchecked") // mocked generics public void testResourceRelease() throws Exception { List<Path> localDirs = new ArrayList<Path>(); String[] sDirs = new String[4]; for (int i = 0; i < 4; ++i) { localDirs.add(lfs.makeQualified(new Path(basedir, i + ""))); sDirs[i] = localDirs.get(i).toString(); }//w w w .ja v a 2s . c o m conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); LocalizerTracker mockLocallilzerTracker = mock(LocalizerTracker.class); DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); //Ignore actual localization EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerBus); ContainerExecutor exec = mock(ContainerExecutor.class); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); DeletionService delService = new DeletionService(exec); delService.init(new Configuration()); delService.start(); ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(mockLocallilzerTracker).when(spyService).createLocalizerTracker(isA(Configuration.class)); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); try { spyService.init(conf); spyService.start(); final String user = "user0"; final String userFolder = "user0Folder"; // init application final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); when(app.getUser()).thenReturn(user); when(app.getUserFolder()).thenReturn(userFolder); when(app.getAppId()).thenReturn(appId); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); dispatcher.await(); //Get a handle on the trackers after they're setup with INIT_APP_RESOURCES LocalResourcesTracker appTracker = spyService .getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user, appId); LocalResourcesTracker privTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user, appId); LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId); // init container. final Container c = getMockContainer(appId, 42, user, userFolder); // init resources Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); // Send localization requests for one resource of each type. final LocalResource privResource = getPrivateMockedResource(r); final LocalResourceRequest privReq = new LocalResourceRequest(privResource); final LocalResource pubResource = getPublicMockedResource(r); final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource); final LocalResource pubResource2 = getPublicMockedResource(r); final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2); final LocalResource appResource = getAppMockedResource(r); final LocalResourceRequest appReq = new LocalResourceRequest(appResource); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq)); req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq)); req.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq)); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req2.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq)); req2.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq2)); Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>(); pubRsrcs.add(pubReq); pubRsrcs.add(pubReq2); // Send Request event spyService.handle(new ContainerLocalizationRequestEvent(c, req)); spyService.handle(new ContainerLocalizationRequestEvent(c, req2)); dispatcher.await(); int privRsrcCount = 0; for (LocalizedResource lr : privTracker) { privRsrcCount++; Assert.assertEquals("Incorrect reference count", 2, lr.getRefCount()); Assert.assertEquals(privReq, lr.getRequest()); } Assert.assertEquals(1, privRsrcCount); int pubRsrcCount = 0; for (LocalizedResource lr : pubTracker) { pubRsrcCount++; Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount()); pubRsrcs.remove(lr.getRequest()); } Assert.assertEquals(0, pubRsrcs.size()); Assert.assertEquals(2, pubRsrcCount); int appRsrcCount = 0; for (LocalizedResource lr : appTracker) { appRsrcCount++; Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount()); Assert.assertEquals(appReq, lr.getRequest()); } Assert.assertEquals(1, appRsrcCount); //Send Cleanup Event spyService.handle(new ContainerLocalizationCleanupEvent(c, req)); verify(mockLocallilzerTracker).cleanupPrivLocalizers("container_314159265358979_0003_01_000042"); req2.remove(LocalResourceVisibility.PRIVATE); spyService.handle(new ContainerLocalizationCleanupEvent(c, req2)); dispatcher.await(); pubRsrcs.add(pubReq); pubRsrcs.add(pubReq2); privRsrcCount = 0; for (LocalizedResource lr : privTracker) { privRsrcCount++; Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount()); Assert.assertEquals(privReq, lr.getRequest()); } Assert.assertEquals(1, privRsrcCount); pubRsrcCount = 0; for (LocalizedResource lr : pubTracker) { pubRsrcCount++; Assert.assertEquals("Incorrect reference count", 0, lr.getRefCount()); pubRsrcs.remove(lr.getRequest()); } Assert.assertEquals(0, pubRsrcs.size()); Assert.assertEquals(2, pubRsrcCount); appRsrcCount = 0; for (LocalizedResource lr : appTracker) { appRsrcCount++; } Assert.assertEquals(0, appRsrcCount); } finally { dispatcher.stop(); delService.stop(); } }
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test @SuppressWarnings("unchecked") // mocked generics public void testFailedDirsResourceRelease() throws Exception { // setup components File f = new File(basedir.toString()); String[] sDirs = new String[4]; List<Path> localDirs = new ArrayList<Path>(sDirs.length); for (int i = 0; i < 4; ++i) { sDirs[i] = f.getAbsolutePath() + i; localDirs.add(new Path(sDirs[i])); }/*from w w w. j a v a 2s . c o m*/ List<Path> containerLocalDirs = new ArrayList<Path>(localDirs.size()); List<Path> appLocalDirs = new ArrayList<Path>(localDirs.size()); List<Path> nmLocalContainerDirs = new ArrayList<Path>(localDirs.size()); List<Path> nmLocalAppDirs = new ArrayList<Path>(localDirs.size()); conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); conf.setLong(YarnConfiguration.NM_DISK_HEALTH_CHECK_INTERVAL_MS, 500); LocalizerTracker mockLocallilzerTracker = mock(LocalizerTracker.class); DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); // Ignore actual localization EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerBus); ContainerExecutor exec = mock(ContainerExecutor.class); LocalDirsHandlerService mockDirsHandler = mock(LocalDirsHandlerService.class); doReturn(new ArrayList<String>(Arrays.asList(sDirs))).when(mockDirsHandler).getLocalDirsForCleanup(); DeletionService delService = mock(DeletionService.class); // setup mocks ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, mockDirsHandler, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(mockLocallilzerTracker).when(spyService).createLocalizerTracker(isA(Configuration.class)); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask()); FsPermission nmPermission = ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask()); final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", localDirs.get(0)); final FileStatus nmFs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, nmPermission, "", "", localDirs.get(0)); final String user = "user0"; final String userFolder = "user0Folder"; // init application final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); when(app.getUser()).thenReturn(user); when(app.getUserFolder()).thenReturn(userFolder); when(app.getAppId()).thenReturn(appId); when(app.toString()).thenReturn(appId.toString()); // init container. final Container c = getMockContainer(appId, 42, user, userFolder); // setup local app dirs List<String> tmpDirs = mockDirsHandler.getLocalDirs(); for (int i = 0; i < tmpDirs.size(); ++i) { Path usersdir = new Path(tmpDirs.get(i), ContainerLocalizer.USERCACHE); Path userdir = new Path(usersdir, user); Path allAppsdir = new Path(userdir, ContainerLocalizer.APPCACHE); Path appDir = new Path(allAppsdir, appId.toString()); Path containerDir = new Path(appDir, c.getContainerId().toString()); containerLocalDirs.add(containerDir); appLocalDirs.add(appDir); Path sysDir = new Path(tmpDirs.get(i), ResourceLocalizationService.NM_PRIVATE_DIR); Path appSysDir = new Path(sysDir, appId.toString()); Path containerSysDir = new Path(appSysDir, c.getContainerId().toString()); nmLocalContainerDirs.add(containerSysDir); nmLocalAppDirs.add(appSysDir); } try { spyService.init(conf); spyService.start(); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); dispatcher.await(); // Get a handle on the trackers after they're setup with // INIT_APP_RESOURCES LocalResourcesTracker appTracker = spyService .getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user, appId); LocalResourcesTracker privTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user, appId); LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId); // init resources Random r = new Random(); long seed = r.nextLong(); r.setSeed(seed); // Send localization requests, one for each type of resource final LocalResource privResource = getPrivateMockedResource(r); final LocalResourceRequest privReq = new LocalResourceRequest(privResource); final LocalResource appResource = getAppMockedResource(r); final LocalResourceRequest appReq = new LocalResourceRequest(appResource); final LocalResource pubResource = getPublicMockedResource(r); final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq)); req.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq)); req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq)); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req2.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq)); // Send Request event spyService.handle(new ContainerLocalizationRequestEvent(c, req)); spyService.handle(new ContainerLocalizationRequestEvent(c, req2)); dispatcher.await(); int privRsrcCount = 0; for (LocalizedResource lr : privTracker) { privRsrcCount++; Assert.assertEquals("Incorrect reference count", 2, lr.getRefCount()); Assert.assertEquals(privReq, lr.getRequest()); } Assert.assertEquals(1, privRsrcCount); int appRsrcCount = 0; for (LocalizedResource lr : appTracker) { appRsrcCount++; Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount()); Assert.assertEquals(appReq, lr.getRequest()); } Assert.assertEquals(1, appRsrcCount); int pubRsrcCount = 0; for (LocalizedResource lr : pubTracker) { pubRsrcCount++; Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount()); Assert.assertEquals(pubReq, lr.getRequest()); } Assert.assertEquals(1, pubRsrcCount); // setup mocks for test, a set of dirs with IOExceptions and let the rest // go through for (int i = 0; i < containerLocalDirs.size(); ++i) { if (i == 2) { Mockito.doThrow(new IOException()).when(spylfs).getFileStatus(eq(containerLocalDirs.get(i))); Mockito.doThrow(new IOException()).when(spylfs).getFileStatus(eq(nmLocalContainerDirs.get(i))); } else { doReturn(fs).when(spylfs).getFileStatus(eq(containerLocalDirs.get(i))); doReturn(nmFs).when(spylfs).getFileStatus(eq(nmLocalContainerDirs.get(i))); } } // Send Cleanup Event spyService.handle(new ContainerLocalizationCleanupEvent(c, req)); verify(mockLocallilzerTracker).cleanupPrivLocalizers("container_314159265358979_0003_01_000042"); // match cleanup events with the mocks we setup earlier for (int i = 0; i < containerLocalDirs.size(); ++i) { if (i == 2) { try { verify(delService).delete(user, containerLocalDirs.get(i)); verify(delService).delete(null, nmLocalContainerDirs.get(i)); Assert.fail("deletion attempts for invalid dirs"); } catch (Throwable e) { continue; } } else { verify(delService).delete(user, containerLocalDirs.get(i)); verify(delService).delete(null, nmLocalContainerDirs.get(i)); } } ArgumentMatcher<ApplicationEvent> matchesAppDestroy = new ArgumentMatcher<ApplicationEvent>() { @Override public boolean matches(Object o) { ApplicationEvent evt = (ApplicationEvent) o; return (evt.getType() == ApplicationEventType.APPLICATION_RESOURCES_CLEANEDUP) && appId == evt.getApplicationID(); } }; dispatcher.await(); // setup mocks again, this time throw UnsupportedFileSystemException and // IOExceptions for (int i = 0; i < containerLocalDirs.size(); ++i) { if (i == 3) { Mockito.doThrow(new IOException()).when(spylfs).getFileStatus(eq(appLocalDirs.get(i))); Mockito.doThrow(new UnsupportedFileSystemException("test")).when(spylfs) .getFileStatus(eq(nmLocalAppDirs.get(i))); } else { doReturn(fs).when(spylfs).getFileStatus(eq(appLocalDirs.get(i))); doReturn(nmFs).when(spylfs).getFileStatus(eq(nmLocalAppDirs.get(i))); } } LocalizationEvent destroyApp = new ApplicationLocalizationEvent( LocalizationEventType.DESTROY_APPLICATION_RESOURCES, app); spyService.handle(destroyApp); // Waits for APPLICATION_RESOURCES_CLEANEDUP event to be handled. dispatcher.await(); verify(applicationBus).handle(argThat(matchesAppDestroy)); // verify we got the right delete calls for (int i = 0; i < containerLocalDirs.size(); ++i) { if (i == 3) { try { verify(delService).delete(user, containerLocalDirs.get(i)); verify(delService).delete(null, nmLocalContainerDirs.get(i)); Assert.fail("deletion attempts for invalid dirs"); } catch (Throwable e) { continue; } } else { verify(delService).delete(user, appLocalDirs.get(i)); verify(delService).delete(null, nmLocalAppDirs.get(i)); } } } finally { dispatcher.stop(); delService.stop(); } }
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test @SuppressWarnings("unchecked") // mocked generics public void testRecovery() throws Exception { final String user1 = "user1"; final String user2 = "user2"; final String user1Folder = "user1Folder"; final String user2Folder = "user2Folder"; final ApplicationId appId1 = ApplicationId.newInstance(1, 1); final ApplicationId appId2 = ApplicationId.newInstance(1, 2); List<Path> localDirs = new ArrayList<Path>(); String[] sDirs = new String[4]; for (int i = 0; i < 4; ++i) { localDirs.add(lfs.makeQualified(new Path(basedir, i + ""))); sDirs[i] = localDirs.get(i).toString(); }/*from w w w . j a v a 2s. c om*/ conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true); NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService(); stateStore.init(conf); stateStore.start(); DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); //Ignore actual localization EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerBus); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); ResourceLocalizationService spyService = createSpyService(dispatcher, dirsHandler, stateStore); try { spyService.init(conf); spyService.start(); final Application app1 = mock(Application.class); when(app1.getUser()).thenReturn(user1); when(app1.getUserFolder()).thenReturn(user1Folder); when(app1.getAppId()).thenReturn(appId1); final Application app2 = mock(Application.class); when(app2.getUser()).thenReturn(user2); when(app2.getUserFolder()).thenReturn(user2Folder); when(app2.getAppId()).thenReturn(appId2); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app1)); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app2)); dispatcher.await(); //Get a handle on the trackers after they're setup with INIT_APP_RESOURCES LocalResourcesTracker appTracker1 = spyService .getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user1, appId1); LocalResourcesTracker privTracker1 = spyService .getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user1, null); LocalResourcesTracker appTracker2 = spyService .getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user2, appId2); LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, null, null); // init containers final Container c1 = getMockContainer(appId1, 1, user1, user1Folder); final Container c2 = getMockContainer(appId2, 2, user2, user2Folder); // init resources Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); // Send localization requests of each type. final LocalResource privResource1 = getPrivateMockedResource(r); final LocalResourceRequest privReq1 = new LocalResourceRequest(privResource1); final LocalResource privResource2 = getPrivateMockedResource(r); final LocalResourceRequest privReq2 = new LocalResourceRequest(privResource2); final LocalResource pubResource1 = getPublicMockedResource(r); final LocalResourceRequest pubReq1 = new LocalResourceRequest(pubResource1); final LocalResource pubResource2 = getPublicMockedResource(r); final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2); final LocalResource appResource1 = getAppMockedResource(r); final LocalResourceRequest appReq1 = new LocalResourceRequest(appResource1); final LocalResource appResource2 = getAppMockedResource(r); final LocalResourceRequest appReq2 = new LocalResourceRequest(appResource2); final LocalResource appResource3 = getAppMockedResource(r); final LocalResourceRequest appReq3 = new LocalResourceRequest(appResource3); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req1.put(LocalResourceVisibility.PRIVATE, Arrays.asList(new LocalResourceRequest[] { privReq1, privReq2 })); req1.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq1)); req1.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq1)); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req2.put(LocalResourceVisibility.APPLICATION, Arrays.asList(new LocalResourceRequest[] { appReq2, appReq3 })); req2.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq2)); // Send Request event spyService.handle(new ContainerLocalizationRequestEvent(c1, req1)); spyService.handle(new ContainerLocalizationRequestEvent(c2, req2)); dispatcher.await(); // Simulate start of localization for all resources privTracker1.getPathForLocalization(privReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE + user1), null); privTracker1.getPathForLocalization(privReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE + user1), null); LocalizedResource privLr1 = privTracker1.getLocalizedResource(privReq1); LocalizedResource privLr2 = privTracker1.getLocalizedResource(privReq2); appTracker1.getPathForLocalization(appReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId1), null); LocalizedResource appLr1 = appTracker1.getLocalizedResource(appReq1); appTracker2.getPathForLocalization(appReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId2), null); LocalizedResource appLr2 = appTracker2.getLocalizedResource(appReq2); appTracker2.getPathForLocalization(appReq3, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId2), null); LocalizedResource appLr3 = appTracker2.getLocalizedResource(appReq3); pubTracker.getPathForLocalization(pubReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.FILECACHE), null); LocalizedResource pubLr1 = pubTracker.getLocalizedResource(pubReq1); pubTracker.getPathForLocalization(pubReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.FILECACHE), null); LocalizedResource pubLr2 = pubTracker.getLocalizedResource(pubReq2); // Simulate completion of localization for most resources with // possibly different sizes than in the request assertNotNull("Localization not started", privLr1.getLocalPath()); privTracker1 .handle(new ResourceLocalizedEvent(privReq1, privLr1.getLocalPath(), privLr1.getSize() + 5)); assertNotNull("Localization not started", privLr2.getLocalPath()); privTracker1 .handle(new ResourceLocalizedEvent(privReq2, privLr2.getLocalPath(), privLr2.getSize() + 10)); assertNotNull("Localization not started", appLr1.getLocalPath()); appTracker1.handle(new ResourceLocalizedEvent(appReq1, appLr1.getLocalPath(), appLr1.getSize())); assertNotNull("Localization not started", appLr3.getLocalPath()); appTracker2.handle(new ResourceLocalizedEvent(appReq3, appLr3.getLocalPath(), appLr3.getSize() + 7)); assertNotNull("Localization not started", pubLr1.getLocalPath()); pubTracker.handle(new ResourceLocalizedEvent(pubReq1, pubLr1.getLocalPath(), pubLr1.getSize() + 1000)); assertNotNull("Localization not started", pubLr2.getLocalPath()); pubTracker.handle(new ResourceLocalizedEvent(pubReq2, pubLr2.getLocalPath(), pubLr2.getSize() + 99999)); dispatcher.await(); assertEquals(ResourceState.LOCALIZED, privLr1.getState()); assertEquals(ResourceState.LOCALIZED, privLr2.getState()); assertEquals(ResourceState.LOCALIZED, appLr1.getState()); assertEquals(ResourceState.DOWNLOADING, appLr2.getState()); assertEquals(ResourceState.LOCALIZED, appLr3.getState()); assertEquals(ResourceState.LOCALIZED, pubLr1.getState()); assertEquals(ResourceState.LOCALIZED, pubLr2.getState()); // restart and recover spyService = createSpyService(dispatcher, dirsHandler, stateStore); spyService.init(conf); spyService.recoverLocalizedResources(stateStore.loadLocalizationState()); dispatcher.await(); appTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user1, appId1); privTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user1, null); appTracker2 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user2, appId2); pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, null, null); LocalizedResource recoveredRsrc = privTracker1.getLocalizedResource(privReq1); assertEquals(privReq1, recoveredRsrc.getRequest()); assertEquals(privLr1.getLocalPath(), recoveredRsrc.getLocalPath()); assertEquals(privLr1.getSize(), recoveredRsrc.getSize()); assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState()); recoveredRsrc = privTracker1.getLocalizedResource(privReq2); assertEquals(privReq2, recoveredRsrc.getRequest()); assertEquals(privLr2.getLocalPath(), recoveredRsrc.getLocalPath()); assertEquals(privLr2.getSize(), recoveredRsrc.getSize()); assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState()); recoveredRsrc = appTracker1.getLocalizedResource(appReq1); assertEquals(appReq1, recoveredRsrc.getRequest()); assertEquals(appLr1.getLocalPath(), recoveredRsrc.getLocalPath()); assertEquals(appLr1.getSize(), recoveredRsrc.getSize()); assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState()); recoveredRsrc = appTracker2.getLocalizedResource(appReq2); assertNull("in-progress resource should not be present", recoveredRsrc); recoveredRsrc = appTracker2.getLocalizedResource(appReq3); assertEquals(appReq3, recoveredRsrc.getRequest()); assertEquals(appLr3.getLocalPath(), recoveredRsrc.getLocalPath()); assertEquals(appLr3.getSize(), recoveredRsrc.getSize()); assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState()); } finally { dispatcher.stop(); stateStore.close(); } }
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test(timeout = 20000) @SuppressWarnings("unchecked") public void testDownloadingResourcesOnContainerKill() throws Exception { List<Path> localDirs = new ArrayList<Path>(); String[] sDirs = new String[1]; localDirs.add(lfs.makeQualified(new Path(basedir, 0 + ""))); sDirs[0] = localDirs.get(0).toString(); conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf);//from www . j a va2 s. c om dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); DummyExecutor exec = new DummyExecutor(); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); DeletionService delServiceReal = new DeletionService(exec); DeletionService delService = spy(delServiceReal); delService.init(new Configuration()); delService.start(); ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask()); FsPermission nmPermission = ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask()); final FsPermission securePermission = new FsPermission((short) 0711); final Path userDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.USERCACHE); final Path fileDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.FILECACHE); final Path sysDir = new Path(sDirs[0].substring("file:".length()), ResourceLocalizationService.NM_PRIVATE_DIR); final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", new Path(sDirs[0])); final FileStatus nmFs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, nmPermission, "", "", sysDir); final FileStatus ufs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, securePermission, "", "", new Path(sDirs[0])); doAnswer(new Answer<FileStatus>() { @Override public FileStatus answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); if (args.length > 0) { if (args[0].equals(fileDir)) { return fs; } else if (args[0].equals(userDir)) { return ufs; } } return nmFs; } }).when(spylfs).getFileStatus(isA(Path.class)); try { spyService.init(conf); spyService.start(); final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); String user = "user0"; when(app.getUser()).thenReturn(user); when(app.getAppId()).thenReturn(appId); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); ArgumentMatcher<ApplicationEvent> matchesAppInit = new ArgumentMatcher<ApplicationEvent>() { @Override public boolean matches(Object o) { ApplicationEvent evt = (ApplicationEvent) o; return evt.getType() == ApplicationEventType.APPLICATION_INITED && appId == evt.getApplicationID(); } }; dispatcher.await(); verify(applicationBus).handle(argThat(matchesAppInit)); // Initialize localizer. Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); final Container c1 = getMockContainer(appId, 42, "user0", "user0Folder"); final Container c2 = getMockContainer(appId, 43, "user0", "user0Folder"); FSDataOutputStream out = new FSDataOutputStream(new DataOutputBuffer(), null); doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean()); final LocalResource resource1 = getPrivateMockedResource(r); LocalResource resource2 = null; do { resource2 = getPrivateMockedResource(r); } while (resource2 == null || resource2.equals(resource1)); LocalResource resource3 = null; do { resource3 = getPrivateMockedResource(r); } while (resource3 == null || resource3.equals(resource1) || resource3.equals(resource2)); // Send localization requests for container c1 and c2. final LocalResourceRequest req1 = new LocalResourceRequest(resource1); final LocalResourceRequest req2 = new LocalResourceRequest(resource2); final LocalResourceRequest req3 = new LocalResourceRequest(resource3); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>(); privateResourceList.add(req1); privateResourceList.add(req2); privateResourceList.add(req3); rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList); spyService.handle(new ContainerLocalizationRequestEvent(c1, rsrcs)); final LocalResourceRequest req1_1 = new LocalResourceRequest(resource2); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); List<LocalResourceRequest> privateResourceList1 = new ArrayList<LocalResourceRequest>(); privateResourceList1.add(req1_1); rsrcs1.put(LocalResourceVisibility.PRIVATE, privateResourceList1); spyService.handle(new ContainerLocalizationRequestEvent(c2, rsrcs1)); dispatcher.await(); // Wait for localizers of both container c1 and c2 to begin. exec.waitForLocalizers(2); LocalizerRunner locC1 = spyService.getLocalizerRunner(c1.getContainerId().toString()); final String containerIdStr = c1.getContainerId().toString(); // Heartbeats from container localizer LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class); LocalResourceStatus rsrc2pending = mock(LocalResourceStatus.class); LocalizerStatus stat = mock(LocalizerStatus.class); when(stat.getLocalizerId()).thenReturn(containerIdStr); when(rsrc1success.getResource()).thenReturn(resource1); when(rsrc2pending.getResource()).thenReturn(resource2); when(rsrc1success.getLocalSize()).thenReturn(4344L); URL locPath = getPath("/some/path"); when(rsrc1success.getLocalPath()).thenReturn(locPath); when(rsrc1success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS); when(rsrc2pending.getStatus()).thenReturn(ResourceStatusType.FETCH_PENDING); when(stat.getResources()).thenReturn(Collections.<LocalResourceStatus>emptyList()) .thenReturn(Collections.singletonList(rsrc1success)) .thenReturn(Collections.singletonList(rsrc2pending)) .thenReturn(Collections.singletonList(rsrc2pending)) .thenReturn(Collections.<LocalResourceStatus>emptyList()); // First heartbeat which schedules first resource. LocalizerHeartbeatResponse response = spyService.heartbeat(stat); assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); // Second heartbeat which reports first resource as success. // Second resource is scheduled. response = spyService.heartbeat(stat); assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); final String locPath1 = response.getResourceSpecs().get(0).getDestinationDirectory().getFile(); // Third heartbeat which reports second resource as pending. // Third resource is scheduled. response = spyService.heartbeat(stat); assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); final String locPath2 = response.getResourceSpecs().get(0).getDestinationDirectory().getFile(); // Container c1 is killed which leads to cleanup spyService.handle(new ContainerLocalizationCleanupEvent(c1, rsrcs)); // This heartbeat will indicate to container localizer to die as localizer // runner has stopped. response = spyService.heartbeat(stat); assertEquals(LocalizerAction.DIE, response.getLocalizerAction()); exec.setStopLocalization(); dispatcher.await(); // verify container notification ArgumentMatcher<ContainerEvent> successContainerLoc = new ArgumentMatcher<ContainerEvent>() { @Override public boolean matches(Object o) { ContainerEvent evt = (ContainerEvent) o; return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED && c1.getContainerId() == evt.getContainerID(); } }; // Only one resource gets localized for container c1. verify(containerBus).handle(argThat(successContainerLoc)); Set<Path> paths = Sets.newHashSet(new Path(locPath1), new Path(locPath1 + "_tmp"), new Path(locPath2), new Path(locPath2 + "_tmp")); // Wait for localizer runner thread for container c1 to finish. while (locC1.getState() != Thread.State.TERMINATED) { Thread.sleep(50); } // Verify if downloading resources were submitted for deletion. verify(delService).delete(eq(user), (Path) eq(null), argThat(new DownloadingPathsMatcher(paths))); LocalResourcesTracker tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, "user0", appId); // Container c1 was killed but this resource was localized before kill // hence its not removed despite ref cnt being 0. LocalizedResource rsrc1 = tracker.getLocalizedResource(req1); assertNotNull(rsrc1); assertEquals(rsrc1.getState(), ResourceState.LOCALIZED); assertEquals(rsrc1.getRefCount(), 0); // Container c1 was killed but this resource is referenced by container c2 // as well hence its ref cnt is 1. LocalizedResource rsrc2 = tracker.getLocalizedResource(req2); assertNotNull(rsrc2); assertEquals(rsrc2.getState(), ResourceState.DOWNLOADING); assertEquals(rsrc2.getRefCount(), 1); // As container c1 was killed and this resource was not referenced by any // other container, hence its removed. LocalizedResource rsrc3 = tracker.getLocalizedResource(req3); assertNull(rsrc3); } finally { spyService.stop(); dispatcher.stop(); delService.stop(); } }
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test(timeout = 1000000) @SuppressWarnings("unchecked") // mocked generics public void testLocalizationHeartbeat() throws Exception { List<Path> localDirs = new ArrayList<Path>(); String[] sDirs = new String[1]; // Making sure that we have only one local disk so that it will only be // selected for consecutive resource localization calls. This is required // to test LocalCacheDirectoryManager. localDirs.add(lfs.makeQualified(new Path(basedir, 0 + ""))); sDirs[0] = localDirs.get(0).toString(); conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); // Adding configuration to make sure there is only one file per // directory//w w w .j av a2 s . com conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "37"); DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); ContainerExecutor exec = mock(ContainerExecutor.class); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); DeletionService delServiceReal = new DeletionService(exec); DeletionService delService = spy(delServiceReal); delService.init(new Configuration()); delService.start(); ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask()); final FsPermission securePermission = new FsPermission((short) 0711); FsPermission nmPermission = ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask()); final Path userDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.USERCACHE); final Path fileDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.FILECACHE); final Path sysDir = new Path(sDirs[0].substring("file:".length()), ResourceLocalizationService.NM_PRIVATE_DIR); final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", new Path(sDirs[0])); final FileStatus ufs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, securePermission, "", "", new Path(sDirs[0])); final FileStatus nmFs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, nmPermission, "", "", sysDir); doAnswer(new Answer<FileStatus>() { @Override public FileStatus answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); if (args.length > 0) { if (args[0].equals(fileDir)) { return fs; } else if (args[0].equals(userDir)) { return ufs; } } return nmFs; } }).when(spylfs).getFileStatus(isA(Path.class)); try { spyService.init(conf); spyService.start(); // init application final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); when(app.getUser()).thenReturn("user0"); when(app.getUserFolder()).thenReturn("user0Folder"); when(app.getAppId()).thenReturn(appId); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); ArgumentMatcher<ApplicationEvent> matchesAppInit = new ArgumentMatcher<ApplicationEvent>() { @Override public boolean matches(Object o) { ApplicationEvent evt = (ApplicationEvent) o; return evt.getType() == ApplicationEventType.APPLICATION_INITED && appId == evt.getApplicationID(); } }; dispatcher.await(); verify(applicationBus).handle(argThat(matchesAppInit)); // init container rsrc, localizer Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); final Container c = getMockContainer(appId, 42, "user0", "user0Folder"); FSDataOutputStream out = new FSDataOutputStream(new DataOutputBuffer(), null); doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean()); final LocalResource resource1 = getPrivateMockedResource(r); LocalResource resource2 = null; do { resource2 = getPrivateMockedResource(r); } while (resource2 == null || resource2.equals(resource1)); LocalResource resource3 = null; do { resource3 = getPrivateMockedResource(r); } while (resource3 == null || resource3.equals(resource1) || resource3.equals(resource2)); // above call to make sure we don't get identical resources. final LocalResourceRequest req1 = new LocalResourceRequest(resource1); final LocalResourceRequest req2 = new LocalResourceRequest(resource2); final LocalResourceRequest req3 = new LocalResourceRequest(resource3); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>(); privateResourceList.add(req1); privateResourceList.add(req2); privateResourceList.add(req3); rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList); spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs)); // Sigh. Thread init of private localizer not accessible Thread.sleep(1000); dispatcher.await(); String appStr = appId.toString(); String ctnrStr = c.getContainerId().toString(); ArgumentCaptor<LocalizerStartContext> contextCaptor = ArgumentCaptor .forClass(LocalizerStartContext.class); verify(exec).startLocalizer(contextCaptor.capture()); LocalizerStartContext context = contextCaptor.getValue(); Path localizationTokenPath = context.getNmPrivateContainerTokens(); assertEquals("user0", context.getUser()); assertEquals(appStr, context.getAppId()); assertEquals(ctnrStr, context.getLocId()); assertEquals("user0Folder", context.getUserFolder()); // heartbeat from localizer LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class); LocalResourceStatus rsrc2pending = mock(LocalResourceStatus.class); LocalResourceStatus rsrc2success = mock(LocalResourceStatus.class); LocalResourceStatus rsrc3success = mock(LocalResourceStatus.class); LocalizerStatus stat = mock(LocalizerStatus.class); when(stat.getLocalizerId()).thenReturn(ctnrStr); when(rsrc1success.getResource()).thenReturn(resource1); when(rsrc2pending.getResource()).thenReturn(resource2); when(rsrc2success.getResource()).thenReturn(resource2); when(rsrc3success.getResource()).thenReturn(resource3); when(rsrc1success.getLocalSize()).thenReturn(4344L); when(rsrc2success.getLocalSize()).thenReturn(2342L); when(rsrc3success.getLocalSize()).thenReturn(5345L); URL locPath = getPath("/cache/private/blah"); when(rsrc1success.getLocalPath()).thenReturn(locPath); when(rsrc2success.getLocalPath()).thenReturn(locPath); when(rsrc3success.getLocalPath()).thenReturn(locPath); when(rsrc1success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS); when(rsrc2pending.getStatus()).thenReturn(ResourceStatusType.FETCH_PENDING); when(rsrc2success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS); when(rsrc3success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS); // Four heartbeats with sending: // 1 - empty // 2 - resource1 FETCH_SUCCESS // 3 - resource2 FETCH_PENDING // 4 - resource2 FETCH_SUCCESS, resource3 FETCH_SUCCESS List<LocalResourceStatus> rsrcs4 = new ArrayList<LocalResourceStatus>(); rsrcs4.add(rsrc2success); rsrcs4.add(rsrc3success); when(stat.getResources()).thenReturn(Collections.<LocalResourceStatus>emptyList()) .thenReturn(Collections.singletonList(rsrc1success)) .thenReturn(Collections.singletonList(rsrc2pending)).thenReturn(rsrcs4) .thenReturn(Collections.<LocalResourceStatus>emptyList()); String localPath = Path.SEPARATOR + ContainerLocalizer.USERCACHE + Path.SEPARATOR + "user0Folder" + Path.SEPARATOR + ContainerLocalizer.FILECACHE; // First heartbeat LocalizerHeartbeatResponse response = spyService.heartbeat(stat); assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); assertEquals(1, response.getResourceSpecs().size()); assertEquals(req1, new LocalResourceRequest(response.getResourceSpecs().get(0).getResource())); URL localizedPath = response.getResourceSpecs().get(0).getDestinationDirectory(); // Appending to local path unique number(10) generated as a part of // LocalResourcesTracker assertTrue(localizedPath.getFile().endsWith(localPath + Path.SEPARATOR + "10")); // Second heartbeat response = spyService.heartbeat(stat); assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); assertEquals(1, response.getResourceSpecs().size()); assertEquals(req2, new LocalResourceRequest(response.getResourceSpecs().get(0).getResource())); localizedPath = response.getResourceSpecs().get(0).getDestinationDirectory(); // Resource's destination path should be now inside sub directory 0 as // LocalCacheDirectoryManager will be used and we have restricted number // of files per directory to 1. assertTrue(localizedPath.getFile().endsWith(localPath + Path.SEPARATOR + "0" + Path.SEPARATOR + "11")); // Third heartbeat response = spyService.heartbeat(stat); assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); assertEquals(1, response.getResourceSpecs().size()); assertEquals(req3, new LocalResourceRequest(response.getResourceSpecs().get(0).getResource())); localizedPath = response.getResourceSpecs().get(0).getDestinationDirectory(); assertTrue(localizedPath.getFile().endsWith(localPath + Path.SEPARATOR + "1" + Path.SEPARATOR + "12")); response = spyService.heartbeat(stat); assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); spyService .handle(new ContainerLocalizationEvent(LocalizationEventType.CONTAINER_RESOURCES_LOCALIZED, c)); // get shutdown after receive CONTAINER_RESOURCES_LOCALIZED event response = spyService.heartbeat(stat); assertEquals(LocalizerAction.DIE, response.getLocalizerAction()); dispatcher.await(); // verify container notification ArgumentMatcher<ContainerEvent> matchesContainerLoc = new ArgumentMatcher<ContainerEvent>() { @Override public boolean matches(Object o) { ContainerEvent evt = (ContainerEvent) o; return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED && c.getContainerId() == evt.getContainerID(); } }; // total 3 resource localzation calls. one for each resource. verify(containerBus, times(3)).handle(argThat(matchesContainerLoc)); // Verify deletion of localization token. verify(delService).delete((String) isNull(), eq(localizationTokenPath)); } finally { spyService.stop(); dispatcher.stop(); delService.stop(); } }
From source file:org.apache.pig.test.TestBuiltin.java
@Test public void testMathFuncs() throws Exception { Random generator = new Random(); generator.setSeed(System.currentTimeMillis()); Double delta = 0.1;//from www .j a v a2s . c o m // We assume that UDFs are stored in org.apache.pig.builtin // Change this test case if we add more hierarchy later\ // Also, we assume that we have a function with math function // associated with these UDF with a lowercase name String[] mathFuncs = { "SIN", "SINH", "ASIN", "COS", "COSH", "ACOS", "TAN", "TANH", "ATAN", "LOG", "LOG10", "SQRT", "CEIL", "EXP", "FLOOR", "CBRT" }; String udfPackage = "org.apache.pig.builtin."; //String[] mathNonStdFuncs = {}; EvalFunc<Double> evalFunc; Tuple tup; Double input, actual, expected; Method mathMethod; String msg; for (String func : mathFuncs) { evalFunc = (EvalFunc<Double>) Class.forName(udfPackage + func).newInstance(); tup = TupleFactory.getInstance().newTuple(1); // double value between 0.0 and 1.0 input = generator.nextDouble(); tup.set(0, input); mathMethod = Math.class.getDeclaredMethod(func.toLowerCase(), double.class); actual = evalFunc.exec(tup); expected = (Double) mathMethod.invoke(null, input); msg = "[Testing " + func + " on input: " + input + " ( (actual) " + actual + " == " + expected + " (expected) )]"; assertEquals(msg, actual, expected, delta); } }
From source file:es.sm2.openppm.front.servlets.ProjectControlServlet.java
/** * Chart kpis//from w w w .j av a2s .c o m * * @param req * @param resp * @throws IOException */ private void kpiChartJX(HttpServletRequest req, HttpServletResponse resp) throws IOException { int idProject = ParamUtil.getInteger(req, "id"); PrintWriter out = resp.getWriter(); try { Date minDate = null; Date maxDate = null; SimpleDateFormat date = new SimpleDateFormat("MM/dd/yyyy"); SimpleDateFormat tempDate = new SimpleDateFormat("MM/dd/yyyy"); Random random = new Random(); Float r = null; Float g = null; Float b = null; String[] defaultColors = new String[] { "#4bb2c5", "#c5b47f", "#EAA228", "#579575", "#839557", "#958c12", "#953579", "#4b5de4", "#d8b83f", "#ff5800", "#0085cc" }; Boolean insufficientDates = true; HistorickpiLogic historickpiLogic = new HistorickpiLogic(); ProjectLogic projectLogic = new ProjectLogic(getSettings(req), getResourceBundle(req)); JSONArray kpisValuesJSON = new JSONArray(); JSONArray kpisNamesJSON = new JSONArray(); JSONArray kpisColorsJSON = new JSONArray(); JSONObject updateJSON = new JSONObject(); List<String> joins = new ArrayList<String>(); joins.add(Projectkpi.METRICKPI); Project project = projectLogic.findById(idProject); ProjectKpiLogic projectKpiLogic = new ProjectKpiLogic(); List<Projectkpi> projectKpis = projectKpiLogic.findByRelation(Projectkpi.PROJECT, project, joins); //colors for (String defaultColor : defaultColors) { kpisColorsJSON.add(defaultColor); } int varianceColors = projectKpis.size() > defaultColors.length ? projectKpis.size() - defaultColors.length : 0; for (int i = 0; i < varianceColors; i++) { random.setSeed(new Date().getTime()); r = random.nextFloat(); g = random.nextFloat(); b = random.nextFloat(); Color randomColor = new Color(r, g, b); String rgb = Integer.toHexString(randomColor.getRGB()); rgb = rgb.substring(2, rgb.length()); rgb = StringPool.POUND.concat(rgb); kpisColorsJSON.add(rgb); } List<String> joinsHistoricKpi = new ArrayList<String>(); joinsHistoricKpi.add(Historickpi.EMPLOYEE); joinsHistoricKpi.add(Historickpi.EMPLOYEE + "." + Employee.CONTACT); for (Projectkpi kpi : projectKpis) { JSONArray kpiJSON = new JSONArray(); List<Historickpi> historic = historickpiLogic.findByRelation(Historickpi.PROJECTKPI, new Projectkpi(kpi.getIdProjectKpi()), Historickpi.ACTUALDATE, Constants.ASCENDENT, joinsHistoricKpi); if (!historic.isEmpty()) { String tempDateStr = tempDate.format(historic.get(0).getActualDate()); for (Historickpi hKpi : historic) { JSONArray hKpiJSON = new JSONArray(); hKpiJSON.add(date.format(hKpi.getActualDate())); hKpiJSON.add(hKpi.getValueKpi() == null ? 0 : hKpi.getValueKpi()); if (!tempDateStr.equals(date.format(hKpi.getActualDate()))) { insufficientDates = false; } kpiJSON.add(hKpiJSON); } } kpisValuesJSON.add(kpiJSON); if (kpi.getMetrickpi() != null) { kpisNamesJSON.add(kpi.getMetrickpi().getName()); } else { kpisNamesJSON.add(kpi.getSpecificKpi()); } /* min and max dates*/ if (!insufficientDates) { Date tempMinDate = historic.get(0).getActualDate(); Date tempMaxDate = historic.get(historic.size() - 1).getActualDate(); if (minDate == null && maxDate == null) { minDate = tempMinDate; maxDate = tempMaxDate; } else { if (tempMinDate.compareTo(minDate) < 0) { minDate = tempMinDate; } if (tempMaxDate.compareTo(maxDate) > 0) { maxDate = tempMaxDate; } } } } ChartJQPLOT chart = new ChartJQPLOT(minDate, maxDate); updateJSON.put("tickInterval", chart.getTickInterval()); updateJSON.put("minDate", chart.addDate(minDate, -1)); updateJSON.put("maxDate", chart.addDate(maxDate, 1)); updateJSON.put("kpisColors", kpisColorsJSON); updateJSON.put("kpisNames", kpisNamesJSON); updateJSON.put("kpisValues", kpisValuesJSON); updateJSON.put("insufficientDates", insufficientDates); out.print(updateJSON); } catch (Exception e) { ExceptionUtil.evalueExceptionJX(out, req, getResourceBundle(req), LOGGER, e); } finally { out.close(); } }