List of usage examples for java.util Random nextLong
public long nextLong()
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test @SuppressWarnings("unchecked") // mocked generics public void testRecovery() throws Exception { final String user1 = "user1"; final String user2 = "user2"; final String user1Folder = "user1Folder"; final String user2Folder = "user2Folder"; final ApplicationId appId1 = ApplicationId.newInstance(1, 1); final ApplicationId appId2 = ApplicationId.newInstance(1, 2); List<Path> localDirs = new ArrayList<Path>(); String[] sDirs = new String[4]; for (int i = 0; i < 4; ++i) { localDirs.add(lfs.makeQualified(new Path(basedir, i + ""))); sDirs[i] = localDirs.get(i).toString(); }//from w w w . j av a 2 s. co m conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true); NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService(); stateStore.init(conf); stateStore.start(); DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); //Ignore actual localization EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerBus); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); ResourceLocalizationService spyService = createSpyService(dispatcher, dirsHandler, stateStore); try { spyService.init(conf); spyService.start(); final Application app1 = mock(Application.class); when(app1.getUser()).thenReturn(user1); when(app1.getUserFolder()).thenReturn(user1Folder); when(app1.getAppId()).thenReturn(appId1); final Application app2 = mock(Application.class); when(app2.getUser()).thenReturn(user2); when(app2.getUserFolder()).thenReturn(user2Folder); when(app2.getAppId()).thenReturn(appId2); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app1)); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app2)); dispatcher.await(); //Get a handle on the trackers after they're setup with INIT_APP_RESOURCES LocalResourcesTracker appTracker1 = spyService .getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user1, appId1); LocalResourcesTracker privTracker1 = spyService .getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user1, null); LocalResourcesTracker appTracker2 = spyService .getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user2, appId2); LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, null, null); // init containers final Container c1 = getMockContainer(appId1, 1, user1, user1Folder); final Container c2 = getMockContainer(appId2, 2, user2, user2Folder); // init resources Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); // Send localization requests of each type. final LocalResource privResource1 = getPrivateMockedResource(r); final LocalResourceRequest privReq1 = new LocalResourceRequest(privResource1); final LocalResource privResource2 = getPrivateMockedResource(r); final LocalResourceRequest privReq2 = new LocalResourceRequest(privResource2); final LocalResource pubResource1 = getPublicMockedResource(r); final LocalResourceRequest pubReq1 = new LocalResourceRequest(pubResource1); final LocalResource pubResource2 = getPublicMockedResource(r); final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2); final LocalResource appResource1 = getAppMockedResource(r); final LocalResourceRequest appReq1 = new LocalResourceRequest(appResource1); final LocalResource appResource2 = getAppMockedResource(r); final LocalResourceRequest appReq2 = new LocalResourceRequest(appResource2); final LocalResource appResource3 = getAppMockedResource(r); final LocalResourceRequest appReq3 = new LocalResourceRequest(appResource3); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req1.put(LocalResourceVisibility.PRIVATE, Arrays.asList(new LocalResourceRequest[] { privReq1, privReq2 })); req1.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq1)); req1.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq1)); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req2.put(LocalResourceVisibility.APPLICATION, Arrays.asList(new LocalResourceRequest[] { appReq2, appReq3 })); req2.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq2)); // Send Request event spyService.handle(new ContainerLocalizationRequestEvent(c1, req1)); spyService.handle(new ContainerLocalizationRequestEvent(c2, req2)); dispatcher.await(); // Simulate start of localization for all resources privTracker1.getPathForLocalization(privReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE + user1), null); privTracker1.getPathForLocalization(privReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE + user1), null); LocalizedResource privLr1 = privTracker1.getLocalizedResource(privReq1); LocalizedResource privLr2 = privTracker1.getLocalizedResource(privReq2); appTracker1.getPathForLocalization(appReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId1), null); LocalizedResource appLr1 = appTracker1.getLocalizedResource(appReq1); appTracker2.getPathForLocalization(appReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId2), null); LocalizedResource appLr2 = appTracker2.getLocalizedResource(appReq2); appTracker2.getPathForLocalization(appReq3, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId2), null); LocalizedResource appLr3 = appTracker2.getLocalizedResource(appReq3); pubTracker.getPathForLocalization(pubReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.FILECACHE), null); LocalizedResource pubLr1 = pubTracker.getLocalizedResource(pubReq1); pubTracker.getPathForLocalization(pubReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.FILECACHE), null); LocalizedResource pubLr2 = pubTracker.getLocalizedResource(pubReq2); // Simulate completion of localization for most resources with // possibly different sizes than in the request assertNotNull("Localization not started", privLr1.getLocalPath()); privTracker1 .handle(new ResourceLocalizedEvent(privReq1, privLr1.getLocalPath(), privLr1.getSize() + 5)); assertNotNull("Localization not started", privLr2.getLocalPath()); privTracker1 .handle(new ResourceLocalizedEvent(privReq2, privLr2.getLocalPath(), privLr2.getSize() + 10)); assertNotNull("Localization not started", appLr1.getLocalPath()); appTracker1.handle(new ResourceLocalizedEvent(appReq1, appLr1.getLocalPath(), appLr1.getSize())); assertNotNull("Localization not started", appLr3.getLocalPath()); appTracker2.handle(new ResourceLocalizedEvent(appReq3, appLr3.getLocalPath(), appLr3.getSize() + 7)); assertNotNull("Localization not started", pubLr1.getLocalPath()); pubTracker.handle(new ResourceLocalizedEvent(pubReq1, pubLr1.getLocalPath(), pubLr1.getSize() + 1000)); assertNotNull("Localization not started", pubLr2.getLocalPath()); pubTracker.handle(new ResourceLocalizedEvent(pubReq2, pubLr2.getLocalPath(), pubLr2.getSize() + 99999)); dispatcher.await(); assertEquals(ResourceState.LOCALIZED, privLr1.getState()); assertEquals(ResourceState.LOCALIZED, privLr2.getState()); assertEquals(ResourceState.LOCALIZED, appLr1.getState()); assertEquals(ResourceState.DOWNLOADING, appLr2.getState()); assertEquals(ResourceState.LOCALIZED, appLr3.getState()); assertEquals(ResourceState.LOCALIZED, pubLr1.getState()); assertEquals(ResourceState.LOCALIZED, pubLr2.getState()); // restart and recover spyService = createSpyService(dispatcher, dirsHandler, stateStore); spyService.init(conf); spyService.recoverLocalizedResources(stateStore.loadLocalizationState()); dispatcher.await(); appTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user1, appId1); privTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user1, null); appTracker2 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user2, appId2); pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, null, null); LocalizedResource recoveredRsrc = privTracker1.getLocalizedResource(privReq1); assertEquals(privReq1, recoveredRsrc.getRequest()); assertEquals(privLr1.getLocalPath(), recoveredRsrc.getLocalPath()); assertEquals(privLr1.getSize(), recoveredRsrc.getSize()); assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState()); recoveredRsrc = privTracker1.getLocalizedResource(privReq2); assertEquals(privReq2, recoveredRsrc.getRequest()); assertEquals(privLr2.getLocalPath(), recoveredRsrc.getLocalPath()); assertEquals(privLr2.getSize(), recoveredRsrc.getSize()); assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState()); recoveredRsrc = appTracker1.getLocalizedResource(appReq1); assertEquals(appReq1, recoveredRsrc.getRequest()); assertEquals(appLr1.getLocalPath(), recoveredRsrc.getLocalPath()); assertEquals(appLr1.getSize(), recoveredRsrc.getSize()); assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState()); recoveredRsrc = appTracker2.getLocalizedResource(appReq2); assertNull("in-progress resource should not be present", recoveredRsrc); recoveredRsrc = appTracker2.getLocalizedResource(appReq3); assertEquals(appReq3, recoveredRsrc.getRequest()); assertEquals(appLr3.getLocalPath(), recoveredRsrc.getLocalPath()); assertEquals(appLr3.getSize(), recoveredRsrc.getSize()); assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState()); } finally { dispatcher.stop(); stateStore.close(); } }
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test(timeout = 20000) @SuppressWarnings("unchecked") public void testDownloadingResourcesOnContainerKill() throws Exception { List<Path> localDirs = new ArrayList<Path>(); String[] sDirs = new String[1]; localDirs.add(lfs.makeQualified(new Path(basedir, 0 + ""))); sDirs[0] = localDirs.get(0).toString(); conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf);//from w w w. j a v a 2s . co m dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); DummyExecutor exec = new DummyExecutor(); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); DeletionService delServiceReal = new DeletionService(exec); DeletionService delService = spy(delServiceReal); delService.init(new Configuration()); delService.start(); ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask()); FsPermission nmPermission = ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask()); final FsPermission securePermission = new FsPermission((short) 0711); final Path userDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.USERCACHE); final Path fileDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.FILECACHE); final Path sysDir = new Path(sDirs[0].substring("file:".length()), ResourceLocalizationService.NM_PRIVATE_DIR); final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", new Path(sDirs[0])); final FileStatus nmFs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, nmPermission, "", "", sysDir); final FileStatus ufs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, securePermission, "", "", new Path(sDirs[0])); doAnswer(new Answer<FileStatus>() { @Override public FileStatus answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); if (args.length > 0) { if (args[0].equals(fileDir)) { return fs; } else if (args[0].equals(userDir)) { return ufs; } } return nmFs; } }).when(spylfs).getFileStatus(isA(Path.class)); try { spyService.init(conf); spyService.start(); final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); String user = "user0"; when(app.getUser()).thenReturn(user); when(app.getAppId()).thenReturn(appId); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); ArgumentMatcher<ApplicationEvent> matchesAppInit = new ArgumentMatcher<ApplicationEvent>() { @Override public boolean matches(Object o) { ApplicationEvent evt = (ApplicationEvent) o; return evt.getType() == ApplicationEventType.APPLICATION_INITED && appId == evt.getApplicationID(); } }; dispatcher.await(); verify(applicationBus).handle(argThat(matchesAppInit)); // Initialize localizer. Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); final Container c1 = getMockContainer(appId, 42, "user0", "user0Folder"); final Container c2 = getMockContainer(appId, 43, "user0", "user0Folder"); FSDataOutputStream out = new FSDataOutputStream(new DataOutputBuffer(), null); doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean()); final LocalResource resource1 = getPrivateMockedResource(r); LocalResource resource2 = null; do { resource2 = getPrivateMockedResource(r); } while (resource2 == null || resource2.equals(resource1)); LocalResource resource3 = null; do { resource3 = getPrivateMockedResource(r); } while (resource3 == null || resource3.equals(resource1) || resource3.equals(resource2)); // Send localization requests for container c1 and c2. final LocalResourceRequest req1 = new LocalResourceRequest(resource1); final LocalResourceRequest req2 = new LocalResourceRequest(resource2); final LocalResourceRequest req3 = new LocalResourceRequest(resource3); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>(); privateResourceList.add(req1); privateResourceList.add(req2); privateResourceList.add(req3); rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList); spyService.handle(new ContainerLocalizationRequestEvent(c1, rsrcs)); final LocalResourceRequest req1_1 = new LocalResourceRequest(resource2); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); List<LocalResourceRequest> privateResourceList1 = new ArrayList<LocalResourceRequest>(); privateResourceList1.add(req1_1); rsrcs1.put(LocalResourceVisibility.PRIVATE, privateResourceList1); spyService.handle(new ContainerLocalizationRequestEvent(c2, rsrcs1)); dispatcher.await(); // Wait for localizers of both container c1 and c2 to begin. exec.waitForLocalizers(2); LocalizerRunner locC1 = spyService.getLocalizerRunner(c1.getContainerId().toString()); final String containerIdStr = c1.getContainerId().toString(); // Heartbeats from container localizer LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class); LocalResourceStatus rsrc2pending = mock(LocalResourceStatus.class); LocalizerStatus stat = mock(LocalizerStatus.class); when(stat.getLocalizerId()).thenReturn(containerIdStr); when(rsrc1success.getResource()).thenReturn(resource1); when(rsrc2pending.getResource()).thenReturn(resource2); when(rsrc1success.getLocalSize()).thenReturn(4344L); URL locPath = getPath("/some/path"); when(rsrc1success.getLocalPath()).thenReturn(locPath); when(rsrc1success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS); when(rsrc2pending.getStatus()).thenReturn(ResourceStatusType.FETCH_PENDING); when(stat.getResources()).thenReturn(Collections.<LocalResourceStatus>emptyList()) .thenReturn(Collections.singletonList(rsrc1success)) .thenReturn(Collections.singletonList(rsrc2pending)) .thenReturn(Collections.singletonList(rsrc2pending)) .thenReturn(Collections.<LocalResourceStatus>emptyList()); // First heartbeat which schedules first resource. LocalizerHeartbeatResponse response = spyService.heartbeat(stat); assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); // Second heartbeat which reports first resource as success. // Second resource is scheduled. response = spyService.heartbeat(stat); assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); final String locPath1 = response.getResourceSpecs().get(0).getDestinationDirectory().getFile(); // Third heartbeat which reports second resource as pending. // Third resource is scheduled. response = spyService.heartbeat(stat); assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); final String locPath2 = response.getResourceSpecs().get(0).getDestinationDirectory().getFile(); // Container c1 is killed which leads to cleanup spyService.handle(new ContainerLocalizationCleanupEvent(c1, rsrcs)); // This heartbeat will indicate to container localizer to die as localizer // runner has stopped. response = spyService.heartbeat(stat); assertEquals(LocalizerAction.DIE, response.getLocalizerAction()); exec.setStopLocalization(); dispatcher.await(); // verify container notification ArgumentMatcher<ContainerEvent> successContainerLoc = new ArgumentMatcher<ContainerEvent>() { @Override public boolean matches(Object o) { ContainerEvent evt = (ContainerEvent) o; return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED && c1.getContainerId() == evt.getContainerID(); } }; // Only one resource gets localized for container c1. verify(containerBus).handle(argThat(successContainerLoc)); Set<Path> paths = Sets.newHashSet(new Path(locPath1), new Path(locPath1 + "_tmp"), new Path(locPath2), new Path(locPath2 + "_tmp")); // Wait for localizer runner thread for container c1 to finish. while (locC1.getState() != Thread.State.TERMINATED) { Thread.sleep(50); } // Verify if downloading resources were submitted for deletion. verify(delService).delete(eq(user), (Path) eq(null), argThat(new DownloadingPathsMatcher(paths))); LocalResourcesTracker tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, "user0", appId); // Container c1 was killed but this resource was localized before kill // hence its not removed despite ref cnt being 0. LocalizedResource rsrc1 = tracker.getLocalizedResource(req1); assertNotNull(rsrc1); assertEquals(rsrc1.getState(), ResourceState.LOCALIZED); assertEquals(rsrc1.getRefCount(), 0); // Container c1 was killed but this resource is referenced by container c2 // as well hence its ref cnt is 1. LocalizedResource rsrc2 = tracker.getLocalizedResource(req2); assertNotNull(rsrc2); assertEquals(rsrc2.getState(), ResourceState.DOWNLOADING); assertEquals(rsrc2.getRefCount(), 1); // As container c1 was killed and this resource was not referenced by any // other container, hence its removed. LocalizedResource rsrc3 = tracker.getLocalizedResource(req3); assertNull(rsrc3); } finally { spyService.stop(); dispatcher.stop(); delService.stop(); } }
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test(timeout = 1000000) @SuppressWarnings("unchecked") // mocked generics public void testLocalizationHeartbeat() throws Exception { List<Path> localDirs = new ArrayList<Path>(); String[] sDirs = new String[1]; // Making sure that we have only one local disk so that it will only be // selected for consecutive resource localization calls. This is required // to test LocalCacheDirectoryManager. localDirs.add(lfs.makeQualified(new Path(basedir, 0 + ""))); sDirs[0] = localDirs.get(0).toString(); conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); // Adding configuration to make sure there is only one file per // directory/*from w ww. j a va 2s . c o m*/ conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "37"); DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); ContainerExecutor exec = mock(ContainerExecutor.class); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); DeletionService delServiceReal = new DeletionService(exec); DeletionService delService = spy(delServiceReal); delService.init(new Configuration()); delService.start(); ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask()); final FsPermission securePermission = new FsPermission((short) 0711); FsPermission nmPermission = ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask()); final Path userDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.USERCACHE); final Path fileDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.FILECACHE); final Path sysDir = new Path(sDirs[0].substring("file:".length()), ResourceLocalizationService.NM_PRIVATE_DIR); final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", new Path(sDirs[0])); final FileStatus ufs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, securePermission, "", "", new Path(sDirs[0])); final FileStatus nmFs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, nmPermission, "", "", sysDir); doAnswer(new Answer<FileStatus>() { @Override public FileStatus answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); if (args.length > 0) { if (args[0].equals(fileDir)) { return fs; } else if (args[0].equals(userDir)) { return ufs; } } return nmFs; } }).when(spylfs).getFileStatus(isA(Path.class)); try { spyService.init(conf); spyService.start(); // init application final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); when(app.getUser()).thenReturn("user0"); when(app.getUserFolder()).thenReturn("user0Folder"); when(app.getAppId()).thenReturn(appId); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); ArgumentMatcher<ApplicationEvent> matchesAppInit = new ArgumentMatcher<ApplicationEvent>() { @Override public boolean matches(Object o) { ApplicationEvent evt = (ApplicationEvent) o; return evt.getType() == ApplicationEventType.APPLICATION_INITED && appId == evt.getApplicationID(); } }; dispatcher.await(); verify(applicationBus).handle(argThat(matchesAppInit)); // init container rsrc, localizer Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); final Container c = getMockContainer(appId, 42, "user0", "user0Folder"); FSDataOutputStream out = new FSDataOutputStream(new DataOutputBuffer(), null); doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean()); final LocalResource resource1 = getPrivateMockedResource(r); LocalResource resource2 = null; do { resource2 = getPrivateMockedResource(r); } while (resource2 == null || resource2.equals(resource1)); LocalResource resource3 = null; do { resource3 = getPrivateMockedResource(r); } while (resource3 == null || resource3.equals(resource1) || resource3.equals(resource2)); // above call to make sure we don't get identical resources. final LocalResourceRequest req1 = new LocalResourceRequest(resource1); final LocalResourceRequest req2 = new LocalResourceRequest(resource2); final LocalResourceRequest req3 = new LocalResourceRequest(resource3); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>(); privateResourceList.add(req1); privateResourceList.add(req2); privateResourceList.add(req3); rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList); spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs)); // Sigh. Thread init of private localizer not accessible Thread.sleep(1000); dispatcher.await(); String appStr = appId.toString(); String ctnrStr = c.getContainerId().toString(); ArgumentCaptor<LocalizerStartContext> contextCaptor = ArgumentCaptor .forClass(LocalizerStartContext.class); verify(exec).startLocalizer(contextCaptor.capture()); LocalizerStartContext context = contextCaptor.getValue(); Path localizationTokenPath = context.getNmPrivateContainerTokens(); assertEquals("user0", context.getUser()); assertEquals(appStr, context.getAppId()); assertEquals(ctnrStr, context.getLocId()); assertEquals("user0Folder", context.getUserFolder()); // heartbeat from localizer LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class); LocalResourceStatus rsrc2pending = mock(LocalResourceStatus.class); LocalResourceStatus rsrc2success = mock(LocalResourceStatus.class); LocalResourceStatus rsrc3success = mock(LocalResourceStatus.class); LocalizerStatus stat = mock(LocalizerStatus.class); when(stat.getLocalizerId()).thenReturn(ctnrStr); when(rsrc1success.getResource()).thenReturn(resource1); when(rsrc2pending.getResource()).thenReturn(resource2); when(rsrc2success.getResource()).thenReturn(resource2); when(rsrc3success.getResource()).thenReturn(resource3); when(rsrc1success.getLocalSize()).thenReturn(4344L); when(rsrc2success.getLocalSize()).thenReturn(2342L); when(rsrc3success.getLocalSize()).thenReturn(5345L); URL locPath = getPath("/cache/private/blah"); when(rsrc1success.getLocalPath()).thenReturn(locPath); when(rsrc2success.getLocalPath()).thenReturn(locPath); when(rsrc3success.getLocalPath()).thenReturn(locPath); when(rsrc1success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS); when(rsrc2pending.getStatus()).thenReturn(ResourceStatusType.FETCH_PENDING); when(rsrc2success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS); when(rsrc3success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS); // Four heartbeats with sending: // 1 - empty // 2 - resource1 FETCH_SUCCESS // 3 - resource2 FETCH_PENDING // 4 - resource2 FETCH_SUCCESS, resource3 FETCH_SUCCESS List<LocalResourceStatus> rsrcs4 = new ArrayList<LocalResourceStatus>(); rsrcs4.add(rsrc2success); rsrcs4.add(rsrc3success); when(stat.getResources()).thenReturn(Collections.<LocalResourceStatus>emptyList()) .thenReturn(Collections.singletonList(rsrc1success)) .thenReturn(Collections.singletonList(rsrc2pending)).thenReturn(rsrcs4) .thenReturn(Collections.<LocalResourceStatus>emptyList()); String localPath = Path.SEPARATOR + ContainerLocalizer.USERCACHE + Path.SEPARATOR + "user0Folder" + Path.SEPARATOR + ContainerLocalizer.FILECACHE; // First heartbeat LocalizerHeartbeatResponse response = spyService.heartbeat(stat); assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); assertEquals(1, response.getResourceSpecs().size()); assertEquals(req1, new LocalResourceRequest(response.getResourceSpecs().get(0).getResource())); URL localizedPath = response.getResourceSpecs().get(0).getDestinationDirectory(); // Appending to local path unique number(10) generated as a part of // LocalResourcesTracker assertTrue(localizedPath.getFile().endsWith(localPath + Path.SEPARATOR + "10")); // Second heartbeat response = spyService.heartbeat(stat); assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); assertEquals(1, response.getResourceSpecs().size()); assertEquals(req2, new LocalResourceRequest(response.getResourceSpecs().get(0).getResource())); localizedPath = response.getResourceSpecs().get(0).getDestinationDirectory(); // Resource's destination path should be now inside sub directory 0 as // LocalCacheDirectoryManager will be used and we have restricted number // of files per directory to 1. assertTrue(localizedPath.getFile().endsWith(localPath + Path.SEPARATOR + "0" + Path.SEPARATOR + "11")); // Third heartbeat response = spyService.heartbeat(stat); assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); assertEquals(1, response.getResourceSpecs().size()); assertEquals(req3, new LocalResourceRequest(response.getResourceSpecs().get(0).getResource())); localizedPath = response.getResourceSpecs().get(0).getDestinationDirectory(); assertTrue(localizedPath.getFile().endsWith(localPath + Path.SEPARATOR + "1" + Path.SEPARATOR + "12")); response = spyService.heartbeat(stat); assertEquals(LocalizerAction.LIVE, response.getLocalizerAction()); spyService .handle(new ContainerLocalizationEvent(LocalizationEventType.CONTAINER_RESOURCES_LOCALIZED, c)); // get shutdown after receive CONTAINER_RESOURCES_LOCALIZED event response = spyService.heartbeat(stat); assertEquals(LocalizerAction.DIE, response.getLocalizerAction()); dispatcher.await(); // verify container notification ArgumentMatcher<ContainerEvent> matchesContainerLoc = new ArgumentMatcher<ContainerEvent>() { @Override public boolean matches(Object o) { ContainerEvent evt = (ContainerEvent) o; return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED && c.getContainerId() == evt.getContainerID(); } }; // total 3 resource localzation calls. one for each resource. verify(containerBus, times(3)).handle(argThat(matchesContainerLoc)); // Verify deletion of localization token. verify(delService).delete((String) isNull(), eq(localizationTokenPath)); } finally { spyService.stop(); dispatcher.stop(); delService.stop(); } }
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test @SuppressWarnings("unchecked") // mocked generics public void testFailedDirsResourceRelease() throws Exception { // setup components File f = new File(basedir.toString()); String[] sDirs = new String[4]; List<Path> localDirs = new ArrayList<Path>(sDirs.length); for (int i = 0; i < 4; ++i) { sDirs[i] = f.getAbsolutePath() + i; localDirs.add(new Path(sDirs[i])); }// w w w . jav a 2 s. co m List<Path> containerLocalDirs = new ArrayList<Path>(localDirs.size()); List<Path> appLocalDirs = new ArrayList<Path>(localDirs.size()); List<Path> nmLocalContainerDirs = new ArrayList<Path>(localDirs.size()); List<Path> nmLocalAppDirs = new ArrayList<Path>(localDirs.size()); conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); conf.setLong(YarnConfiguration.NM_DISK_HEALTH_CHECK_INTERVAL_MS, 500); LocalizerTracker mockLocallilzerTracker = mock(LocalizerTracker.class); DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); // Ignore actual localization EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerBus); ContainerExecutor exec = mock(ContainerExecutor.class); LocalDirsHandlerService mockDirsHandler = mock(LocalDirsHandlerService.class); doReturn(new ArrayList<String>(Arrays.asList(sDirs))).when(mockDirsHandler).getLocalDirsForCleanup(); DeletionService delService = mock(DeletionService.class); // setup mocks ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, mockDirsHandler, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(mockLocallilzerTracker).when(spyService).createLocalizerTracker(isA(Configuration.class)); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask()); FsPermission nmPermission = ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask()); final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", localDirs.get(0)); final FileStatus nmFs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, nmPermission, "", "", localDirs.get(0)); final String user = "user0"; final String userFolder = "user0Folder"; // init application final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); when(app.getUser()).thenReturn(user); when(app.getUserFolder()).thenReturn(userFolder); when(app.getAppId()).thenReturn(appId); when(app.toString()).thenReturn(appId.toString()); // init container. final Container c = getMockContainer(appId, 42, user, userFolder); // setup local app dirs List<String> tmpDirs = mockDirsHandler.getLocalDirs(); for (int i = 0; i < tmpDirs.size(); ++i) { Path usersdir = new Path(tmpDirs.get(i), ContainerLocalizer.USERCACHE); Path userdir = new Path(usersdir, user); Path allAppsdir = new Path(userdir, ContainerLocalizer.APPCACHE); Path appDir = new Path(allAppsdir, appId.toString()); Path containerDir = new Path(appDir, c.getContainerId().toString()); containerLocalDirs.add(containerDir); appLocalDirs.add(appDir); Path sysDir = new Path(tmpDirs.get(i), ResourceLocalizationService.NM_PRIVATE_DIR); Path appSysDir = new Path(sysDir, appId.toString()); Path containerSysDir = new Path(appSysDir, c.getContainerId().toString()); nmLocalContainerDirs.add(containerSysDir); nmLocalAppDirs.add(appSysDir); } try { spyService.init(conf); spyService.start(); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); dispatcher.await(); // Get a handle on the trackers after they're setup with // INIT_APP_RESOURCES LocalResourcesTracker appTracker = spyService .getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user, appId); LocalResourcesTracker privTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user, appId); LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId); // init resources Random r = new Random(); long seed = r.nextLong(); r.setSeed(seed); // Send localization requests, one for each type of resource final LocalResource privResource = getPrivateMockedResource(r); final LocalResourceRequest privReq = new LocalResourceRequest(privResource); final LocalResource appResource = getAppMockedResource(r); final LocalResourceRequest appReq = new LocalResourceRequest(appResource); final LocalResource pubResource = getPublicMockedResource(r); final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq)); req.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq)); req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq)); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req2.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq)); // Send Request event spyService.handle(new ContainerLocalizationRequestEvent(c, req)); spyService.handle(new ContainerLocalizationRequestEvent(c, req2)); dispatcher.await(); int privRsrcCount = 0; for (LocalizedResource lr : privTracker) { privRsrcCount++; Assert.assertEquals("Incorrect reference count", 2, lr.getRefCount()); Assert.assertEquals(privReq, lr.getRequest()); } Assert.assertEquals(1, privRsrcCount); int appRsrcCount = 0; for (LocalizedResource lr : appTracker) { appRsrcCount++; Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount()); Assert.assertEquals(appReq, lr.getRequest()); } Assert.assertEquals(1, appRsrcCount); int pubRsrcCount = 0; for (LocalizedResource lr : pubTracker) { pubRsrcCount++; Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount()); Assert.assertEquals(pubReq, lr.getRequest()); } Assert.assertEquals(1, pubRsrcCount); // setup mocks for test, a set of dirs with IOExceptions and let the rest // go through for (int i = 0; i < containerLocalDirs.size(); ++i) { if (i == 2) { Mockito.doThrow(new IOException()).when(spylfs).getFileStatus(eq(containerLocalDirs.get(i))); Mockito.doThrow(new IOException()).when(spylfs).getFileStatus(eq(nmLocalContainerDirs.get(i))); } else { doReturn(fs).when(spylfs).getFileStatus(eq(containerLocalDirs.get(i))); doReturn(nmFs).when(spylfs).getFileStatus(eq(nmLocalContainerDirs.get(i))); } } // Send Cleanup Event spyService.handle(new ContainerLocalizationCleanupEvent(c, req)); verify(mockLocallilzerTracker).cleanupPrivLocalizers("container_314159265358979_0003_01_000042"); // match cleanup events with the mocks we setup earlier for (int i = 0; i < containerLocalDirs.size(); ++i) { if (i == 2) { try { verify(delService).delete(user, containerLocalDirs.get(i)); verify(delService).delete(null, nmLocalContainerDirs.get(i)); Assert.fail("deletion attempts for invalid dirs"); } catch (Throwable e) { continue; } } else { verify(delService).delete(user, containerLocalDirs.get(i)); verify(delService).delete(null, nmLocalContainerDirs.get(i)); } } ArgumentMatcher<ApplicationEvent> matchesAppDestroy = new ArgumentMatcher<ApplicationEvent>() { @Override public boolean matches(Object o) { ApplicationEvent evt = (ApplicationEvent) o; return (evt.getType() == ApplicationEventType.APPLICATION_RESOURCES_CLEANEDUP) && appId == evt.getApplicationID(); } }; dispatcher.await(); // setup mocks again, this time throw UnsupportedFileSystemException and // IOExceptions for (int i = 0; i < containerLocalDirs.size(); ++i) { if (i == 3) { Mockito.doThrow(new IOException()).when(spylfs).getFileStatus(eq(appLocalDirs.get(i))); Mockito.doThrow(new UnsupportedFileSystemException("test")).when(spylfs) .getFileStatus(eq(nmLocalAppDirs.get(i))); } else { doReturn(fs).when(spylfs).getFileStatus(eq(appLocalDirs.get(i))); doReturn(nmFs).when(spylfs).getFileStatus(eq(nmLocalAppDirs.get(i))); } } LocalizationEvent destroyApp = new ApplicationLocalizationEvent( LocalizationEventType.DESTROY_APPLICATION_RESOURCES, app); spyService.handle(destroyApp); // Waits for APPLICATION_RESOURCES_CLEANEDUP event to be handled. dispatcher.await(); verify(applicationBus).handle(argThat(matchesAppDestroy)); // verify we got the right delete calls for (int i = 0; i < containerLocalDirs.size(); ++i) { if (i == 3) { try { verify(delService).delete(user, containerLocalDirs.get(i)); verify(delService).delete(null, nmLocalContainerDirs.get(i)); Assert.fail("deletion attempts for invalid dirs"); } catch (Throwable e) { continue; } } else { verify(delService).delete(user, appLocalDirs.get(i)); verify(delService).delete(null, nmLocalAppDirs.get(i)); } } } finally { dispatcher.stop(); delService.stop(); } }
From source file:com.blockwithme.longdb.test.BETableJUnit.java
/** Testing BETable.get(long key, LongArrayList columns) Test is performed by * retrieving all Test-rows-ids from test data file. Getting all the column * ids for each row in the test data, creating multiple LongArrayList of * column ids by adding one column id at a time to the LongArrayList and * Calling the get(long key, LongArrayList columns) method. Comparing result * with the Test data. *//*from www.j a va2s . c o m*/ @Test public void testGetLongLongArrayList() throws Exception { final LongArrayList rowIDs = J_UTIL.getRowIDs(); final Random rndm = new Random(); for (final LongCursor rowCursor : rowIDs) { final LongObjectOpenHashMap<Bytes> jColumnsData = J_UTIL.getColumnData(rowCursor.value); final Iterator<LongCursor> jColCursor = jColumnsData.keys().iterator(); final LongArrayList colIDList = new LongArrayList(); while (jColCursor.hasNext()) { final long colkey = jColCursor.next().value; colIDList.add(colkey); // the following method (BETable.get(long row,LongArrayList // columns)) is being tested final Columns dbColumns = table.get(rowCursor.value, colIDList); assertNotNull("No Columns found for row ID:" + rowCursor.value, dbColumns); assertEquals("Incorrect size of columns returned row ID:" + +rowCursor.value, colIDList.size(), dbColumns.size()); assertTrue("Test column ID:" + colkey + " not found in DB row ID:" + rowCursor.value, dbColumns.containsColumn(colkey)); for (final LongHolder longHolder : dbColumns) { final long colkey2 = longHolder.value(); final Bytes dbBytes = dbColumns.getBytes(colkey2); assertNotNull("No bytes found for row ID:" + rowCursor.value + " Col ID: " + colkey2, dbBytes); final Bytes jBytes = jColumnsData.get(colkey2); assertEquals( "Column Values do not match for row ID:" + rowCursor.value + " column ID:" + colkey2, dbBytes, jBytes); } } // Other tests: // pass some extra ids to column lists. final int currentListSize = colIDList.size(); Long rnd = rndm.nextLong(); for (int i = 0; i < 4; i++) { if (!colIDList.contains(rnd)) colIDList.add(rnd); rnd = rndm.nextLong(); } final Columns dbColumns2 = table.get(rowCursor.value, colIDList); assertNotNull("Colmns not found for row id: " + rowCursor.value, dbColumns2); assertEquals("Number of columns mismatch for row id: " + rowCursor.value, dbColumns2.size(), currentListSize); // pass empty list. final Columns dbColumns3 = table.get(rowCursor.value, new LongArrayList()); assertNull("Setting List to zero element list, should return null ", dbColumns3); } // pass zero size list final Columns dbColumns5 = table.get(0, new LongArrayList()); assertNull("Passing zero as rowId should return null", dbColumns5); }
From source file:com.blockwithme.longdb.test.BETableJUnit.java
/** Testing BETable.get(long key, long... columns) Test is performed by * retrieving all Test-rows-ids from test data file. Getting all the column * ids for each row in the test data, creating multiple arrays of column ids * by adding one column id at a time to the array and Calling the get(long * key, long... columns) method. Comparing result with the Test data. */ @Test// www . j a v a 2s . c om public void testGetLongLongArray() throws Exception { final LongArrayList rowIDs = J_UTIL.getRowIDs(); final Random rndm = new Random(); for (final LongCursor rowCursor : rowIDs) { final LongObjectOpenHashMap<Bytes> jColumnsData = J_UTIL.getColumnData(rowCursor.value); final Iterator<LongCursor> jKeyCursor = jColumnsData.keys().iterator(); final List<Long> colIDList = new ArrayList<Long>(); while (jKeyCursor.hasNext()) { final long colkey = jKeyCursor.next().value; colIDList.add(colkey); final long[] currentKeys = listToArray(colIDList); final Columns dbColumns = table.get(rowCursor.value, currentKeys); assertNotNull("No Columns found for row ID:" + rowCursor.value, dbColumns); assertEquals("Incorrect size of columns returned row ID:" + rowCursor.value, colIDList.size(), dbColumns.size()); assertTrue("Test column ID:" + colkey + " not found in DB row ID:" + rowCursor.value, dbColumns.containsColumn(colkey)); for (final LongHolder longHolder : dbColumns) { final long colkey2 = longHolder.value(); final Bytes dbBytes = dbColumns.getBytes(colkey2); assertNotNull("No bytes found for row ID:" + rowCursor.value + " Col ID: " + colkey2, dbBytes); final Bytes jBytes = jColumnsData.get(colkey2); assertEquals( "Column Values do not match for row ID:" + rowCursor.value + " column ID:" + colkey2, Arrays.toString(jBytes.toArray(false)), Arrays.toString(dbBytes.toArray(false))); assertEquals( "Column Values do not match for row ID:" + rowCursor.value + " column ID:" + colkey2, dbBytes, jBytes); } } // Other tests: // pass some extra ids to column array. final int currentListSize = colIDList.size(); Long rnd = rndm.nextLong(); if (!colIDList.contains(rnd)) colIDList.add(rnd); rnd = rndm.nextLong(); if (!colIDList.contains(rnd)) colIDList.add(rnd); rnd = rndm.nextLong(); if (!colIDList.contains(rnd)) colIDList.add(rnd); final Columns dbColumns2 = table.get(rowCursor.value, listToArray(colIDList)); assertNotNull("Columns not found for row id: " + rowCursor.value, dbColumns2); assertEquals("Column Size mismatch for row id: " + rowCursor.value, dbColumns2.size(), currentListSize); // pass empty list. final Columns dbColumns3 = table.get(rowCursor.value, ZERO_LONG_ARRAY); assertNull("Setting Empty Array should return null ", dbColumns3); // final Columns dbColumns4 = table.get(rowCursor.value, // (long[]) null); // assertNull("Setting Empty array should return null", dbColumns4); } // pass zero length array final Columns dbColumns5 = table.get(0, ZERO_LONG_ARRAY); assertNull("Passing zero as rowId should return null", dbColumns5); // // pass zero as rowId // final Columns dbColumns6 = table.get(0, (Range) null); // assertNull("Passing zero as rowId should return null", dbColumns6); }
From source file:org.lilyproject.hadooptestfw.fork.HBaseTestingUtility.java
/** * Creates a random table with the given parameters *///from w w w.ja v a2 s .c o m public HTable createRandomTable(String tableName, final Collection<String> families, final int maxVersions, final int numColsPerRow, final int numFlushes, final int numRegions, final int numRowsPerFlush) throws IOException, InterruptedException { LOG.info("\n\nCreating random table " + tableName + " with " + numRegions + " regions, " + numFlushes + " storefiles per region, " + numRowsPerFlush + " rows per flush, maxVersions=" + maxVersions + "\n"); final Random rand = new Random(tableName.hashCode() * 17L + 12938197137L); final int numCF = families.size(); final byte[][] cfBytes = new byte[numCF][]; final byte[] tableNameBytes = Bytes.toBytes(tableName); { int cfIndex = 0; for (String cf : families) { cfBytes[cfIndex++] = Bytes.toBytes(cf); } } final int actualStartKey = 0; final int actualEndKey = Integer.MAX_VALUE; final int keysPerRegion = (actualEndKey - actualStartKey) / numRegions; final int splitStartKey = actualStartKey + keysPerRegion; final int splitEndKey = actualEndKey - keysPerRegion; final String keyFormat = "%08x"; final HTable table = createTable(tableNameBytes, cfBytes, maxVersions, Bytes.toBytes(String.format(keyFormat, splitStartKey)), Bytes.toBytes(String.format(keyFormat, splitEndKey)), numRegions); if (hbaseCluster != null) { getMiniHBaseCluster().flushcache(HConstants.META_TABLE_NAME); } for (int iFlush = 0; iFlush < numFlushes; ++iFlush) { for (int iRow = 0; iRow < numRowsPerFlush; ++iRow) { final byte[] row = Bytes.toBytes( String.format(keyFormat, actualStartKey + rand.nextInt(actualEndKey - actualStartKey))); Put put = new Put(row); Delete del = new Delete(row); for (int iCol = 0; iCol < numColsPerRow; ++iCol) { final byte[] cf = cfBytes[rand.nextInt(numCF)]; final long ts = rand.nextInt(); final byte[] qual = Bytes.toBytes("col" + iCol); if (rand.nextBoolean()) { final byte[] value = Bytes .toBytes("value_for_row_" + iRow + "_cf_" + Bytes.toStringBinary(cf) + "_col_" + iCol + "_ts_" + ts + "_random_" + rand.nextLong()); put.add(cf, qual, ts, value); } else if (rand.nextDouble() < 0.8) { del.deleteColumn(cf, qual, ts); } else { del.deleteColumns(cf, qual, ts); } } if (!put.isEmpty()) { table.put(put); } if (!del.isEmpty()) { table.delete(del); } } LOG.info("Initiating flush #" + iFlush + " for table " + tableName); table.flushCommits(); if (hbaseCluster != null) { getMiniHBaseCluster().flushcache(tableNameBytes); } } return table; }
From source file:org.ejbca.core.ejb.ca.caadmin.CAAdminSessionBean.java
@Override @TransactionAttribute(TransactionAttributeType.REQUIRED) public void updateCAIds(AuthenticationToken authenticationToken, int fromId, int toId, String toDN) throws AuthorizationDeniedException { log.info("Updating CAIds in relations from " + fromId + " to " + toId + "\n"); // Update Certificate Profiles final Map<Integer, String> certProfiles = certificateProfileSession.getCertificateProfileIdToNameMap(); for (Integer certProfId : certProfiles.keySet()) { boolean changed = false; final CertificateProfile certProfile = certificateProfileSession.getCertificateProfile(certProfId); final List<Integer> availableCAs = new ArrayList<Integer>(certProfile.getAvailableCAs()); // The list is modified so we can't use an iterator for (int i = 0; i < availableCAs.size(); i++) { int value = availableCAs.get(i); if (value == fromId) { availableCAs.set(i, toId); changed = true;//from ww w .j a v a 2s.co m } } if (changed) { certProfile.setAvailableCAs(availableCAs); String name = certProfiles.get(certProfId); certificateProfileSession.changeCertificateProfile(authenticationToken, name, certProfile); } } // Update End-Entity Profiles final Map<Integer, String> endEntityProfiles = endEntityProfileSession.getEndEntityProfileIdToNameMap(); for (Integer endEntityProfId : endEntityProfiles.keySet()) { boolean changed = false; final EndEntityProfile endEntityProfile = endEntityProfileSession.getEndEntityProfile(endEntityProfId); if (endEntityProfile.getDefaultCA() == fromId) { endEntityProfile.setValue(EndEntityProfile.DEFAULTCA, 0, String.valueOf(toId)); changed = true; } final Collection<String> original = endEntityProfile.getAvailableCAs(); final List<Integer> updated = new ArrayList<Integer>(); for (String oldvalueStr : original) { int oldvalue = Integer.valueOf(oldvalueStr); int newvalue; if (oldvalue == fromId) { newvalue = toId; changed = true; } else { newvalue = oldvalue; } updated.add(newvalue); } if (changed) { endEntityProfile.setAvailableCAs(updated); String name = endEntityProfiles.get(endEntityProfId); try { endEntityProfileSession.changeEndEntityProfile(authenticationToken, name, endEntityProfile); } catch (EndEntityProfileNotFoundException e) { log.error("End-entity profile " + name + " could no longer be found", e); } } } // Update End-Entities (only if it's possible to get the session bean) EndEntityManagementSessionLocal endEntityManagementSession = getEndEntityManagementSession(); if (endEntityManagementSession != null) { final Collection<EndEntityInformation> endEntities = endEntityManagementSession .findAllUsersByCaId(authenticationToken, fromId); for (EndEntityInformation endEntityInfo : endEntities) { endEntityInfo.setCAId(toId); try { endEntityManagementSession.updateCAId(authenticationToken, endEntityInfo.getUsername(), toId); } catch (NoSuchEndEntityException e) { log.error("End entity " + endEntityInfo.getUsername() + " could no longer be found", e); } } } else { log.info("Can not update CAIds of end-entities (this requires EJB 3.1 support in the appserver)"); } // Update Data Sources final Map<Integer, String> dataSources = userDataSourceSession .getUserDataSourceIdToNameMap(authenticationToken); for (Integer dataSourceId : dataSources.keySet()) { boolean changed = false; final BaseUserDataSource dataSource = userDataSourceSession.getUserDataSource(authenticationToken, dataSourceId); dataSource.getApplicableCAs(); final List<Integer> applicableCAs = new ArrayList<Integer>(dataSource.getApplicableCAs()); // The list is modified so we can't use an iterator for (int i = 0; i < applicableCAs.size(); i++) { int value = applicableCAs.get(i); if (value == fromId) { applicableCAs.set(i, toId); changed = true; } } if (changed) { dataSource.setApplicableCAs(applicableCAs); String name = dataSources.get(dataSourceId); userDataSourceSession.changeUserDataSource(authenticationToken, name, dataSource); } } // Update Services ServiceSessionLocal serviceSession = getServiceSession(); if (serviceSession != null) { final Map<Integer, String> services = serviceSession.getServiceIdToNameMap(); for (String serviceName : services.values()) { final ServiceConfiguration serviceConf = serviceSession.getService(serviceName); final Properties workerProps = serviceConf.getWorkerProperties(); final String idsToCheckStr = workerProps.getProperty(BaseWorker.PROP_CAIDSTOCHECK); if (!StringUtils.isEmpty(idsToCheckStr)) { boolean changed = false; final String[] caIds = idsToCheckStr.split(";"); for (int i = 0; i < caIds.length; i++) { if (Integer.parseInt(caIds[i]) == fromId) { caIds[i] = String.valueOf(toId); changed = true; } } if (changed) { workerProps.setProperty(BaseWorker.PROP_CAIDSTOCHECK, StringUtils.join(caIds, ';')); serviceConf.setWorkerProperties(workerProps); serviceSession.changeService(authenticationToken, serviceName, serviceConf, false); } } } } // Update Internal Key Bindings Map<String, Map<String, InternalKeyBindingProperty<?>>> keyBindTypes = keyBindMgmtSession .getAvailableTypesAndProperties(); Map<String, List<Integer>> typesKeybindings = new HashMap<String, List<Integer>>(); for (String type : keyBindTypes.keySet()) { typesKeybindings.put(type, keyBindMgmtSession.getInternalKeyBindingIds(authenticationToken, type)); } for (Map.Entry<String, List<Integer>> entry : typesKeybindings.entrySet()) { final List<Integer> keybindIds = entry.getValue(); for (int keybindId : keybindIds) { final InternalKeyBinding keybind = keyBindMgmtSession.getInternalKeyBinding(authenticationToken, keybindId); boolean changed = false; List<InternalKeyBindingTrustEntry> trustentries = new ArrayList<InternalKeyBindingTrustEntry>(); for (InternalKeyBindingTrustEntry trustentry : keybind.getTrustedCertificateReferences()) { int trustCaId = trustentry.getCaId(); if (trustCaId == fromId) { trustCaId = toId; changed = true; } trustentries.add( new InternalKeyBindingTrustEntry(trustCaId, trustentry.fetchCertificateSerialNumber())); } if (changed) { keybind.setTrustedCertificateReferences(trustentries); try { keyBindMgmtSession.persistInternalKeyBinding(authenticationToken, keybind); } catch (InternalKeyBindingNameInUseException e) { // Should never happen log.error("Name existed when trying to update keybinding", e); } } } } // Update System Configuration GlobalConfiguration globalConfig = (GlobalConfiguration) globalConfigurationSession .getCachedConfiguration(GlobalConfiguration.GLOBAL_CONFIGURATION_ID); if (globalConfig != null) { boolean changed = false; if (globalConfig.getAutoEnrollCA() == fromId) { globalConfig.setAutoEnrollCA(toId); changed = true; } if (changed) { globalConfigurationSession.saveConfiguration(authenticationToken, globalConfig); } } // Update CMP Configuration // Only "Default CA" contains a reference to the Subject DN. All other fields reference the CAs by CA name. CmpConfiguration cmpConfig = (CmpConfiguration) globalConfigurationSession .getCachedConfiguration(CmpConfiguration.CMP_CONFIGURATION_ID); if (cmpConfig != null) { boolean changed = false; for (String alias : cmpConfig.getAliasList()) { final String defaultCaDN = cmpConfig.getCMPDefaultCA(alias); if (defaultCaDN != null && defaultCaDN.hashCode() == fromId) { cmpConfig.setCMPDefaultCA(alias, toDN); changed = true; } } if (changed) { globalConfigurationSession.saveConfiguration(authenticationToken, cmpConfig); } } // Update Roles final Random random = new Random(System.nanoTime()); for (RoleData role : roleManagementSession.getAllRolesAuthorizedToEdit(authenticationToken)) { final String roleName = role.getRoleName(); final Map<Integer, AccessUserAspectData> users = new HashMap<Integer, AccessUserAspectData>( role.getAccessUsers()); boolean changed = false; for (int id : new ArrayList<Integer>(users.keySet())) { AccessUserAspectData user = users.get(id); if (user.getCaId() == fromId) { user = new AccessUserAspectData(roleName, toId, user.getMatchWith(), user.getTokenType(), user.getMatchTypeAsType(), user.getMatchValue()); users.put(id, user); changed = true; } } if (changed) { final Map<Integer, AccessRuleData> rules = role.getAccessRules(); // Contains no CAIds. Used as-is try { // Rename old role so we can replace it without getting locked out final String oldTempName = roleName + "_CAIdUpdateOld" + random.nextLong(); roleManagementSession.renameRole(authenticationToken, roleName, oldTempName); RoleData newRole = roleManagementSession.create(authenticationToken, roleName); // Rights are unchanged because they don't reference CAs newRole = roleManagementSession.addAccessRulesToRole(authenticationToken, newRole, rules.values()); newRole = roleManagementSession.addSubjectsToRole(authenticationToken, newRole, users.values()); roleManagementSession.remove(authenticationToken, oldTempName); } catch (RoleNotFoundException e) { throw new IllegalStateException("Newly created temporary role was not found", e); } catch (RoleExistsException e) { throw new IllegalStateException("Temporary role name already exists", e); } } } final String detailsMsg = intres.getLocalizedMessage("caadmin.updatedcaid", fromId, toId, toDN); auditSession.log(EventTypes.CA_EDITING, EventStatus.SUCCESS, ModuleTypes.CA, ServiceTypes.CORE, authenticationToken.toString(), String.valueOf(toId), null, null, detailsMsg); }
From source file:org.structr.core.script.ScriptingTest.java
@Test public void testScriptedFindWithJSONObject() { final Random random = new Random(); final long long1 = 13475233523455L; final long long2 = 327326252322L; final double double1 = 1234.56789; final double double2 = 5678.975321; List<TestSix> testSixs = null; TestOne testOne1 = null;// w ww . j ava 2 s.c om TestOne testOne2 = null; TestTwo testTwo1 = null; TestTwo testTwo2 = null; TestThree testThree1 = null; TestThree testThree2 = null; TestFour testFour1 = null; TestFour testFour2 = null; Date date1 = null; Date date2 = null; // setup phase try (final Tx tx = app.tx()) { testSixs = createTestNodes(TestSix.class, 10); testOne1 = app.create(TestOne.class); testOne2 = app.create(TestOne.class); testTwo1 = app.create(TestTwo.class); testTwo2 = app.create(TestTwo.class); testThree1 = app.create(TestThree.class); testThree2 = app.create(TestThree.class); testFour1 = app.create(TestFour.class); testFour2 = app.create(TestFour.class); date1 = new Date(random.nextLong()); date2 = new Date(); testOne1.setProperty(TestOne.anInt, 42); testOne1.setProperty(TestOne.aLong, long1); testOne1.setProperty(TestOne.aDouble, double1); testOne1.setProperty(TestOne.aDate, date1); testOne1.setProperty(TestOne.anEnum, Status.One); testOne1.setProperty(TestOne.aString, "aString1"); testOne1.setProperty(TestOne.aBoolean, true); testOne1.setProperty(TestOne.testTwo, testTwo1); testOne1.setProperty(TestOne.testThree, testThree1); testOne1.setProperty(TestOne.testFour, testFour1); testOne1.setProperty(TestOne.manyToManyTestSixs, testSixs.subList(0, 5)); testOne2.setProperty(TestOne.anInt, 33); testOne2.setProperty(TestOne.aLong, long2); testOne2.setProperty(TestOne.aDouble, double2); testOne2.setProperty(TestOne.aDate, date2); testOne2.setProperty(TestOne.anEnum, Status.Two); testOne2.setProperty(TestOne.aString, "aString2"); testOne2.setProperty(TestOne.aBoolean, false); testOne2.setProperty(TestOne.testTwo, testTwo2); testOne2.setProperty(TestOne.testThree, testThree2); testOne2.setProperty(TestOne.testFour, testFour2); testOne2.setProperty(TestOne.manyToManyTestSixs, testSixs.subList(5, 10)); tx.success(); } catch (FrameworkException fex) { logger.warn("", fex); fail("Unexpected exception."); } // test phase, find all the things using scripting try (final Tx tx = app.tx()) { final ActionContext actionContext = new ActionContext(securityContext); assertEquals("Invalid scripted find() result", testOne1, Scripting.evaluate(actionContext, testOne1, "${{ return Structr.find('TestOne', { anInt: 42 })[0]; }}", "test")); assertEquals("Invalid scripted find() result", testOne2, Scripting.evaluate(actionContext, testOne1, "${{ return Structr.find('TestOne', { anInt: 33 })[0]; }}", "test")); assertEquals("Invalid scripted find() result", testOne1, Scripting.evaluate(actionContext, testOne1, "${{ return Structr.find('TestOne', { aLong: " + long1 + " })[0]; }}", "test")); assertEquals("Invalid scripted find() result", testOne2, Scripting.evaluate(actionContext, testOne1, "${{ return Structr.find('TestOne', { aLong: " + long2 + " })[0]; }}", "test")); assertEquals("Invalid scripted find() result", testOne1, Scripting.evaluate(actionContext, testOne1, "${{ return Structr.find('TestOne', { aDouble: " + double1 + " })[0]; }}", "test")); assertEquals("Invalid scripted find() result", testOne2, Scripting.evaluate(actionContext, testOne1, "${{ return Structr.find('TestOne', { aDouble: " + double2 + " })[0]; }}", "test")); assertEquals("Invalid scripted find() result", testOne1, Scripting.evaluate(actionContext, testOne1, "${{ return Structr.find('TestOne', { anEnum: 'One' })[0]; }}", "test")); assertEquals("Invalid scripted find() result", testOne2, Scripting.evaluate(actionContext, testOne1, "${{ return Structr.find('TestOne', { anEnum: 'Two' })[0]; }}", "test")); assertEquals("Invalid scripted find() result", testOne1, Scripting.evaluate(actionContext, testOne1, "${{ return Structr.find('TestOne', { aBoolean: true })[0]; }}", "test")); assertEquals("Invalid scripted find() result", testOne2, Scripting.evaluate(actionContext, testOne1, "${{ return Structr.find('TestOne', { aBoolean: false })[0]; }}", "test")); tx.success(); } catch (UnlicensedScriptException | FrameworkException fex) { logger.warn("", fex); fail("Unexpected exception."); } }
From source file:org.apache.hadoop.hbase.HBaseTestingUtility.java
/** Creates a random table with the given parameters */ public HTable createRandomTable(String tableName, final Collection<String> families, final int maxVersions, final int numColsPerRow, final int numFlushes, final int numRegions, final int numRowsPerFlush) throws IOException, InterruptedException { LOG.info("\n\nCreating random table " + tableName + " with " + numRegions + " regions, " + numFlushes + " storefiles per region, " + numRowsPerFlush + " rows per flush, maxVersions=" + maxVersions + "\n"); final Random rand = new Random(tableName.hashCode() * 17L + 12938197137L); final int numCF = families.size(); final byte[][] cfBytes = new byte[numCF][]; {/*from w ww . j a v a2 s . c o m*/ int cfIndex = 0; for (String cf : families) { cfBytes[cfIndex++] = Bytes.toBytes(cf); } } final int actualStartKey = 0; final int actualEndKey = Integer.MAX_VALUE; final int keysPerRegion = (actualEndKey - actualStartKey) / numRegions; final int splitStartKey = actualStartKey + keysPerRegion; final int splitEndKey = actualEndKey - keysPerRegion; final String keyFormat = "%08x"; final HTable table = createTable(tableName, cfBytes, maxVersions, Bytes.toBytes(String.format(keyFormat, splitStartKey)), Bytes.toBytes(String.format(keyFormat, splitEndKey)), numRegions); if (hbaseCluster != null) { getMiniHBaseCluster().flushcache(TableName.META_TABLE_NAME); } for (int iFlush = 0; iFlush < numFlushes; ++iFlush) { for (int iRow = 0; iRow < numRowsPerFlush; ++iRow) { final byte[] row = Bytes.toBytes( String.format(keyFormat, actualStartKey + rand.nextInt(actualEndKey - actualStartKey))); Put put = new Put(row); Delete del = new Delete(row); for (int iCol = 0; iCol < numColsPerRow; ++iCol) { final byte[] cf = cfBytes[rand.nextInt(numCF)]; final long ts = rand.nextInt(); final byte[] qual = Bytes.toBytes("col" + iCol); if (rand.nextBoolean()) { final byte[] value = Bytes .toBytes("value_for_row_" + iRow + "_cf_" + Bytes.toStringBinary(cf) + "_col_" + iCol + "_ts_" + ts + "_random_" + rand.nextLong()); put.add(cf, qual, ts, value); } else if (rand.nextDouble() < 0.8) { del.deleteColumn(cf, qual, ts); } else { del.deleteColumns(cf, qual, ts); } } if (!put.isEmpty()) { table.put(put); } if (!del.isEmpty()) { table.delete(del); } } LOG.info("Initiating flush #" + iFlush + " for table " + tableName); table.flushCommits(); if (hbaseCluster != null) { getMiniHBaseCluster().flushcache(table.getName()); } } return table; }