List of usage examples for java.util Random nextLong
public long nextLong()
From source file:org.apache.hadoop.raid.RaidNode.java
private void singleHar(Codec codec, FileSystem destFs, FileStatus dest, String tmpHarPath, long harBlockSize, short harReplication) throws IOException { Random rand = new Random(); Path root = new Path("/"); Path qualifiedPath = dest.getPath().makeQualified(destFs); String harFileDst = qualifiedPath.getName() + HAR_SUFFIX; String harFileSrc = qualifiedPath.getName() + "-" + rand.nextLong() + "-" + HAR_SUFFIX; // HadoopArchives.HAR_PARTFILE_LABEL is private, so hard-coding the label. conf.setLong("har.partfile.size", configMgr.getHarPartfileSize()); conf.setLong("har.block.size", harBlockSize); HadoopArchives har = new HadoopArchives(conf); String[] args = new String[7]; args[0] = "-Ddfs.replication=" + harReplication; args[1] = "-archiveName"; args[2] = harFileSrc;//from w ww.ja v a2 s .c o m args[3] = "-p"; args[4] = root.makeQualified(destFs).toString(); args[5] = qualifiedPath.toUri().getPath().substring(1); args[6] = tmpHarPath.toString(); int ret = 0; Path tmpHar = new Path(tmpHarPath + "/" + harFileSrc); try { ret = ToolRunner.run(har, args); if (ret == 0 && !destFs.rename(tmpHar, new Path(qualifiedPath, harFileDst))) { LOG.info("HAR rename didn't succeed from " + tmpHarPath + "/" + harFileSrc + " to " + qualifiedPath + "/" + harFileDst); ret = -2; } } catch (Exception exc) { throw new IOException("Error while creating archive " + ret, exc); } finally { destFs.delete(tmpHar, true); } if (ret != 0) { throw new IOException("Error while creating archive " + ret); } return; }
From source file:com.linkedin.pinot.common.utils.DataTableBuilderTest.java
@Test public void testSimple() throws Exception { final DataType[] columnTypes = DataType.values(); final String[] columnNames = new String[columnTypes.length]; for (int i = 0; i < columnTypes.length; i++) { columnNames[i] = columnTypes[i].toString(); }/*from www . j av a 2 s .c o m*/ final DataSchema schema = new DataSchema(columnNames, columnTypes); final DataTableBuilder builder = new DataTableBuilder(schema); builder.open(); final Random r = new Random(); final int NUM_ROWS = 100; final boolean[] boolArr = new boolean[NUM_ROWS]; final char[] cArr = new char[NUM_ROWS]; final byte[] bArr = new byte[NUM_ROWS]; final short[] sArr = new short[NUM_ROWS]; final int[] iArr = new int[NUM_ROWS]; final float[] fArr = new float[NUM_ROWS]; final long[] lArr = new long[NUM_ROWS]; final double[] dArr = new double[NUM_ROWS]; final String[] strArr = new String[NUM_ROWS]; final Object[] oArr = new Object[NUM_ROWS]; for (int rowId = 0; rowId < NUM_ROWS; rowId++) { builder.startRow(); for (int colId = 0; colId < schema.columnNames.length; colId++) { final DataType type = columnTypes[colId]; switch (type) { case BOOLEAN: final boolean bool = r.nextBoolean(); boolArr[rowId] = bool; builder.setColumn(colId, bool); break; case CHAR: final char ch = (char) (r.nextInt(26) + 'a'); cArr[rowId] = ch; builder.setColumn(colId, ch); break; case BYTE: final byte b = (byte) (r.nextInt((int) Math.pow(2, 8))); bArr[rowId] = b; builder.setColumn(colId, b); break; case SHORT: final short s = (short) (r.nextInt((int) Math.pow(2, 16))); sArr[rowId] = s; builder.setColumn(colId, s); break; case INT: final int i = (r.nextInt()); iArr[rowId] = i; builder.setColumn(colId, i); break; case LONG: final long l = (r.nextLong()); lArr[rowId] = l; builder.setColumn(colId, l); break; case FLOAT: final float f = (r.nextFloat()); fArr[rowId] = f; builder.setColumn(colId, f); break; case DOUBLE: final double d = (r.nextDouble()); dArr[rowId] = d; builder.setColumn(colId, d); break; case STRING: final String str = new BigInteger(130, r).toString(32); strArr[rowId] = str; builder.setColumn(colId, str); break; case OBJECT: final A obj = new A(r.nextInt()); oArr[rowId] = obj; builder.setColumn(colId, obj); break; default: break; } } builder.finishRow(); } builder.seal(); final DataTable dataTable = builder.build(); //System.out.println(dataTable); validate(dataTable, NUM_ROWS, schema, boolArr, cArr, bArr, sArr, iArr, fArr, lArr, dArr, strArr, oArr); final byte[] bytes = dataTable.toBytes(); final DataTable newDataTable = new DataTable(bytes); validate(newDataTable, NUM_ROWS, schema, boolArr, cArr, bArr, sArr, iArr, fArr, lArr, dArr, strArr, oArr); }
From source file:org.apache.flink.streaming.connectors.kafka.KafkaConsumerTestBase.java
public void runKeyValueTest() throws Exception { final String topic = "keyvaluetest"; createTestTopic(topic, 1, 1);//from w w w . j av a 2s . c om final int ELEMENT_COUNT = 5000; // ----------- Write some data into Kafka ------------------- StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort); env.setParallelism(1); env.setRestartStrategy(RestartStrategies.noRestart()); env.getConfig().disableSysoutLogging(); DataStream<Tuple2<Long, PojoValue>> kvStream = env.addSource(new SourceFunction<Tuple2<Long, PojoValue>>() { @Override public void run(SourceContext<Tuple2<Long, PojoValue>> ctx) throws Exception { Random rnd = new Random(1337); for (long i = 0; i < ELEMENT_COUNT; i++) { PojoValue pojo = new PojoValue(); pojo.when = new Date(rnd.nextLong()); pojo.lon = rnd.nextLong(); pojo.lat = i; // make every second key null to ensure proper "null" serialization Long key = (i % 2 == 0) ? null : i; ctx.collect(new Tuple2<>(key, pojo)); } } @Override public void cancel() { } }); KeyedSerializationSchema<Tuple2<Long, PojoValue>> schema = new TypeInformationKeyValueSerializationSchema<>( Long.class, PojoValue.class, env.getConfig()); Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings); producerProperties.setProperty("retries", "3"); kafkaServer.produceIntoKafka(kvStream, topic, schema, producerProperties, null); env.execute("Write KV to Kafka"); // ----------- Read the data again ------------------- env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort); env.setParallelism(1); env.setRestartStrategy(RestartStrategies.noRestart()); env.getConfig().disableSysoutLogging(); KeyedDeserializationSchema<Tuple2<Long, PojoValue>> readSchema = new TypeInformationKeyValueSerializationSchema<>( Long.class, PojoValue.class, env.getConfig()); Properties props = new Properties(); props.putAll(standardProps); props.putAll(secureProps); DataStream<Tuple2<Long, PojoValue>> fromKafka = env .addSource(kafkaServer.getConsumer(topic, readSchema, props)); fromKafka.flatMap(new RichFlatMapFunction<Tuple2<Long, PojoValue>, Object>() { long counter = 0; @Override public void flatMap(Tuple2<Long, PojoValue> value, Collector<Object> out) throws Exception { // the elements should be in order. Assert.assertTrue("Wrong value " + value.f1.lat, value.f1.lat == counter); if (value.f1.lat % 2 == 0) { assertNull("key was not null", value.f0); } else { Assert.assertTrue("Wrong value " + value.f0, value.f0 == counter); } counter++; if (counter == ELEMENT_COUNT) { // we got the right number of elements throw new SuccessException(); } } }); tryExecute(env, "Read KV from Kafka"); deleteTestTopic(topic); }
From source file:org.encuestame.test.config.AbstractBase.java
/** * Generate a Long random value.// w ww . j a v a 2 s . c o m * @return */ private Long randomLongGenerator() { Random randomno = new Random(); long value = randomno.nextLong(); return value; }
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test(timeout = 10000) @SuppressWarnings("unchecked") // mocked generics public void testLocalizerRunnerException() throws Exception { DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf);/*from w w w . ja v a 2s . co m*/ dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); ContainerExecutor exec = mock(ContainerExecutor.class); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler); dirsHandlerSpy.init(conf); DeletionService delServiceReal = new DeletionService(exec); DeletionService delService = spy(delServiceReal); delService.init(new Configuration()); delService.start(); ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandlerSpy, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); try { spyService.init(conf); spyService.start(); // init application final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); when(app.getUser()).thenReturn("user0"); when(app.getAppId()).thenReturn(appId); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); dispatcher.await(); Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); final Container c = getMockContainer(appId, 42, "user0", "user0Folder"); final LocalResource resource1 = getPrivateMockedResource(r); System.out.println("Here 4"); final LocalResourceRequest req1 = new LocalResourceRequest(resource1); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>(); privateResourceList.add(req1); rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList); final Constructor<?>[] constructors = FSError.class.getDeclaredConstructors(); constructors[0].setAccessible(true); FSError fsError = (FSError) constructors[0].newInstance(new IOException("Disk Error")); Mockito.doThrow(fsError).when(dirsHandlerSpy).getLocalPathForWrite(isA(String.class)); spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs)); Thread.sleep(1000); dispatcher.await(); // Verify if ContainerResourceFailedEvent is invoked on FSError verify(containerBus).handle(isA(ContainerResourceFailedEvent.class)); } finally { spyService.stop(); dispatcher.stop(); delService.stop(); } }
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test(timeout = 20000) @SuppressWarnings("unchecked") // mocked generics public void testFailedPublicResource() throws Exception { List<Path> localDirs = new ArrayList<Path>(); String[] sDirs = new String[4]; for (int i = 0; i < 4; ++i) { localDirs.add(lfs.makeQualified(new Path(basedir, i + ""))); sDirs[i] = localDirs.get(i).toString(); }//from w w w.j a va 2 s . c o m conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); DrainDispatcher dispatcher = new DrainDispatcher(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); ContainerExecutor exec = mock(ContainerExecutor.class); DeletionService delService = mock(DeletionService.class); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); dispatcher.init(conf); dispatcher.start(); try { ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); spyService.init(conf); spyService.start(); final String user = "user0"; final String userFolder = "user0Folder"; // init application final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); when(app.getUser()).thenReturn(user); when(app.getUserFolder()).thenReturn(userFolder); when(app.getAppId()).thenReturn(appId); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); dispatcher.await(); // init container. final Container c = getMockContainer(appId, 42, user, userFolder); // init resources Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); // cause chmod to fail after a delay final CyclicBarrier barrier = new CyclicBarrier(2); doAnswer(new Answer<Void>() { public Void answer(InvocationOnMock invocation) throws IOException { try { barrier.await(); } catch (InterruptedException e) { } catch (BrokenBarrierException e) { } throw new IOException("forced failure"); } }).when(spylfs).setPermission(isA(Path.class), isA(FsPermission.class)); // Queue up two localization requests for the same public resource final LocalResource pubResource = getPublicMockedResource(r); final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq)); Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>(); pubRsrcs.add(pubReq); spyService.handle(new ContainerLocalizationRequestEvent(c, req)); spyService.handle(new ContainerLocalizationRequestEvent(c, req)); dispatcher.await(); // allow the chmod to fail now that both requests have been queued barrier.await(); verify(containerBus, timeout(5000).times(2)).handle(isA(ContainerResourceFailedEvent.class)); } finally { dispatcher.stop(); } }
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test(timeout = 20000) @SuppressWarnings("unchecked") public void testLocalizerHeartbeatWhenAppCleaningUp() throws Exception { conf.set(YarnConfiguration.NM_LOCAL_DIRS, lfs.makeQualified(new Path(basedir, 0 + "")).toString()); // Start dispatcher. DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf);/*from ww w.java 2 s . c o m*/ dispatcher.start(); dispatcher.register(ApplicationEventType.class, mock(EventHandler.class)); dispatcher.register(ContainerEventType.class, mock(EventHandler.class)); DummyExecutor exec = new DummyExecutor(); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); // Start resource localization service. ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, mock(DeletionService.class), dirsHandler, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); try { spyService.init(conf); spyService.start(); // Init application resources. final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(1234567890L, 3); when(app.getUser()).thenReturn("user0"); when(app.getUserFolder()).thenReturn("user0Folder"); when(app.getAppId()).thenReturn(appId); when(app.toString()).thenReturn(appId.toString()); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); dispatcher.await(); // Initialize localizer. Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); final Container c = getMockContainer(appId, 46, "user0", "user0Folder"); FSDataOutputStream out = new FSDataOutputStream(new DataOutputBuffer(), null); doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean()); final LocalResource resource1 = getAppMockedResource(r); final LocalResource resource2 = getAppMockedResource(r); // Send localization requests for container. // 2 resources generated with APPLICATION visibility. final LocalResourceRequest req1 = new LocalResourceRequest(resource1); final LocalResourceRequest req2 = new LocalResourceRequest(resource2); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); List<LocalResourceRequest> appResourceList = Arrays.asList(req1, req2); rsrcs.put(LocalResourceVisibility.APPLICATION, appResourceList); spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs)); dispatcher.await(); // Wait for localization to begin. exec.waitForLocalizers(1); final String containerIdStr = c.getContainerId().toString(); LocalizerRunner locRunnerForContainer = spyService.getLocalizerRunner(containerIdStr); // Heartbeats from container localizer LocalResourceStatus rsrcSuccess = mock(LocalResourceStatus.class); LocalizerStatus stat = mock(LocalizerStatus.class); when(stat.getLocalizerId()).thenReturn(containerIdStr); when(rsrcSuccess.getResource()).thenReturn(resource1); when(rsrcSuccess.getLocalSize()).thenReturn(4344L); when(rsrcSuccess.getLocalPath()).thenReturn(getPath("/some/path")); when(rsrcSuccess.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS); when(stat.getResources()).thenReturn(Collections.<LocalResourceStatus>emptyList()); // First heartbeat which schedules first resource. LocalizerHeartbeatResponse response = spyService.heartbeat(stat); assertEquals("NM should tell localizer to be LIVE in Heartbeat.", LocalizerAction.LIVE, response.getLocalizerAction()); // Cleanup application. spyService.handle(new ContainerLocalizationCleanupEvent(c, rsrcs)); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.DESTROY_APPLICATION_RESOURCES, app)); dispatcher.await(); try { // Directly send heartbeat to introduce race as app is being cleaned up. locRunnerForContainer.processHeartbeat(Collections.singletonList(rsrcSuccess)); } catch (Exception e) { fail("Exception should not have been thrown on processing heartbeat"); } // Send another heartbeat. response = spyService.heartbeat(stat); assertEquals("NM should tell localizer to DIE in Heartbeat.", LocalizerAction.DIE, response.getLocalizerAction()); exec.setStopLocalization(); } finally { spyService.stop(); dispatcher.stop(); } }
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test @SuppressWarnings("unchecked") public void testPublicResourceAddResourceExceptions() throws Exception { List<Path> localDirs = new ArrayList<Path>(); String[] sDirs = new String[4]; for (int i = 0; i < 4; ++i) { localDirs.add(lfs.makeQualified(new Path(basedir, i + ""))); sDirs[i] = localDirs.get(i).toString(); }//from w w w . j a v a2s.co m conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); DrainDispatcher dispatcher = new DrainDispatcher(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); ContainerExecutor exec = mock(ContainerExecutor.class); DeletionService delService = mock(DeletionService.class); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler); dirsHandlerSpy.init(conf); dispatcher.init(conf); dispatcher.start(); try { ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandlerSpy, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); spyService.init(conf); spyService.start(); final String user = "user0"; final String userFolder = "user0Folder"; // init application final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); when(app.getUser()).thenReturn(user); when(app.getUserFolder()).thenReturn(userFolder); when(app.getAppId()).thenReturn(appId); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); dispatcher.await(); // init resources Random r = new Random(); r.setSeed(r.nextLong()); // Queue localization request for the public resource final LocalResource pubResource = getPublicMockedResource(r); final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq)); // init container. final Container c = getMockContainer(appId, 42, user, userFolder); // first test ioexception Mockito.doThrow(new IOException()).when(dirsHandlerSpy).getLocalPathForWrite(isA(String.class), Mockito.anyLong(), Mockito.anyBoolean()); // send request spyService.handle(new ContainerLocalizationRequestEvent(c, req)); dispatcher.await(); LocalResourcesTracker tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId); Assert.assertNull(tracker.getLocalizedResource(pubReq)); // test IllegalArgumentException String name = Long.toHexString(r.nextLong()); URL url = getPath("/local/PRIVATE/" + name + "/"); final LocalResource rsrc = BuilderUtils.newLocalResource(url, LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false); final LocalResourceRequest pubReq1 = new LocalResourceRequest(rsrc); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req1.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq1)); Mockito.doCallRealMethod().when(dirsHandlerSpy).getLocalPathForWrite(isA(String.class), Mockito.anyLong(), Mockito.anyBoolean()); // send request spyService.handle(new ContainerLocalizationRequestEvent(c, req1)); dispatcher.await(); tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId); Assert.assertNull(tracker.getLocalizedResource(pubReq)); // test RejectedExecutionException by shutting down the thread pool PublicLocalizer publicLocalizer = spyService.getPublicLocalizer(); publicLocalizer.threadPool.shutdown(); spyService.handle(new ContainerLocalizationRequestEvent(c, req)); dispatcher.await(); tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId); Assert.assertNull(tracker.getLocalizedResource(pubReq)); } finally { // if we call stop with events in the queue, an InterruptedException gets // thrown resulting in the dispatcher thread causing a system exit dispatcher.await(); dispatcher.stop(); } }
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test @SuppressWarnings("unchecked") public void testPublicResourceInitializesLocalDir() throws Exception { // Setup state to simulate restart NM with existing state meaning no // directory creation during initialization NMStateStoreService spyStateStore = spy(nmContext.getNMStateStore()); when(spyStateStore.canRecover()).thenReturn(true); NMContext spyContext = spy(nmContext); when(spyContext.getNMStateStore()).thenReturn(spyStateStore); List<Path> localDirs = new ArrayList<Path>(); String[] sDirs = new String[4]; for (int i = 0; i < 4; ++i) { localDirs.add(lfs.makeQualified(new Path(basedir, i + ""))); sDirs[i] = localDirs.get(i).toString(); }//from w w w . j a v a 2s . com conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); DrainDispatcher dispatcher = new DrainDispatcher(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); ContainerExecutor exec = mock(ContainerExecutor.class); DeletionService delService = mock(DeletionService.class); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); dispatcher.init(conf); dispatcher.start(); try { ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, spyContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); spyService.init(conf); final FsPermission defaultPerm = new FsPermission((short) 0755); // verify directory is not created at initialization for (Path p : localDirs) { p = new Path((new URI(p.toString())).getPath()); Path publicCache = new Path(p, ContainerLocalizer.FILECACHE); verify(spylfs, never()).mkdir(eq(publicCache), eq(defaultPerm), eq(true)); } spyService.start(); final String user = "user0"; final String userFolder = "user0Folder"; // init application final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); when(app.getUser()).thenReturn(user); when(app.getUserFolder()).thenReturn(userFolder); when(app.getAppId()).thenReturn(appId); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); dispatcher.await(); // init container. final Container c = getMockContainer(appId, 42, user, userFolder); // init resources Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); // Queue up public resource localization final LocalResource pubResource1 = getPublicMockedResource(r); final LocalResourceRequest pubReq1 = new LocalResourceRequest(pubResource1); LocalResource pubResource2 = null; do { pubResource2 = getPublicMockedResource(r); } while (pubResource2 == null || pubResource2.equals(pubResource1)); // above call to make sure we don't get identical resources. final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2); Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>(); pubRsrcs.add(pubReq1); pubRsrcs.add(pubReq2); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req.put(LocalResourceVisibility.PUBLIC, pubRsrcs); spyService.handle(new ContainerLocalizationRequestEvent(c, req)); dispatcher.await(); verify(spyService, times(1)).checkAndInitializeLocalDirs(); // verify directory creation for (Path p : localDirs) { p = new Path((new URI(p.toString())).getPath()); Path publicCache = new Path(p, ContainerLocalizer.FILECACHE); verify(spylfs).mkdir(eq(publicCache), eq(defaultPerm), eq(true)); } } finally { dispatcher.stop(); } }
From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService.java
@Test @SuppressWarnings("unchecked") // mocked generics public void testResourceRelease() throws Exception { List<Path> localDirs = new ArrayList<Path>(); String[] sDirs = new String[4]; for (int i = 0; i < 4; ++i) { localDirs.add(lfs.makeQualified(new Path(basedir, i + ""))); sDirs[i] = localDirs.get(i).toString(); }/*from ww w .java 2 s . c o m*/ conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); LocalizerTracker mockLocallilzerTracker = mock(LocalizerTracker.class); DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); //Ignore actual localization EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerBus); ContainerExecutor exec = mock(ContainerExecutor.class); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); DeletionService delService = new DeletionService(exec); delService.init(new Configuration()); delService.start(); ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); doReturn(mockLocallilzerTracker).when(spyService).createLocalizerTracker(isA(Configuration.class)); doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class)); try { spyService.init(conf); spyService.start(); final String user = "user0"; final String userFolder = "user0Folder"; // init application final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); when(app.getUser()).thenReturn(user); when(app.getUserFolder()).thenReturn(userFolder); when(app.getAppId()).thenReturn(appId); spyService.handle( new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); dispatcher.await(); //Get a handle on the trackers after they're setup with INIT_APP_RESOURCES LocalResourcesTracker appTracker = spyService .getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user, appId); LocalResourcesTracker privTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user, appId); LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId); // init container. final Container c = getMockContainer(appId, 42, user, userFolder); // init resources Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); // Send localization requests for one resource of each type. final LocalResource privResource = getPrivateMockedResource(r); final LocalResourceRequest privReq = new LocalResourceRequest(privResource); final LocalResource pubResource = getPublicMockedResource(r); final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource); final LocalResource pubResource2 = getPublicMockedResource(r); final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2); final LocalResource appResource = getAppMockedResource(r); final LocalResourceRequest appReq = new LocalResourceRequest(appResource); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq)); req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq)); req.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq)); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); req2.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq)); req2.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq2)); Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>(); pubRsrcs.add(pubReq); pubRsrcs.add(pubReq2); // Send Request event spyService.handle(new ContainerLocalizationRequestEvent(c, req)); spyService.handle(new ContainerLocalizationRequestEvent(c, req2)); dispatcher.await(); int privRsrcCount = 0; for (LocalizedResource lr : privTracker) { privRsrcCount++; Assert.assertEquals("Incorrect reference count", 2, lr.getRefCount()); Assert.assertEquals(privReq, lr.getRequest()); } Assert.assertEquals(1, privRsrcCount); int pubRsrcCount = 0; for (LocalizedResource lr : pubTracker) { pubRsrcCount++; Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount()); pubRsrcs.remove(lr.getRequest()); } Assert.assertEquals(0, pubRsrcs.size()); Assert.assertEquals(2, pubRsrcCount); int appRsrcCount = 0; for (LocalizedResource lr : appTracker) { appRsrcCount++; Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount()); Assert.assertEquals(appReq, lr.getRequest()); } Assert.assertEquals(1, appRsrcCount); //Send Cleanup Event spyService.handle(new ContainerLocalizationCleanupEvent(c, req)); verify(mockLocallilzerTracker).cleanupPrivLocalizers("container_314159265358979_0003_01_000042"); req2.remove(LocalResourceVisibility.PRIVATE); spyService.handle(new ContainerLocalizationCleanupEvent(c, req2)); dispatcher.await(); pubRsrcs.add(pubReq); pubRsrcs.add(pubReq2); privRsrcCount = 0; for (LocalizedResource lr : privTracker) { privRsrcCount++; Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount()); Assert.assertEquals(privReq, lr.getRequest()); } Assert.assertEquals(1, privRsrcCount); pubRsrcCount = 0; for (LocalizedResource lr : pubTracker) { pubRsrcCount++; Assert.assertEquals("Incorrect reference count", 0, lr.getRefCount()); pubRsrcs.remove(lr.getRequest()); } Assert.assertEquals(0, pubRsrcs.size()); Assert.assertEquals(2, pubRsrcCount); appRsrcCount = 0; for (LocalizedResource lr : appTracker) { appRsrcCount++; } Assert.assertEquals(0, appRsrcCount); } finally { dispatcher.stop(); delService.stop(); } }