List of usage examples for org.apache.hadoop.conf Configuration setFloat
public void setFloat(String name, float value)
name property to a float. From source file:org.apache.tez.runtime.library.conf.TestOrderedPartitionedKVEdgeConfigurer.java
License:Apache License
@Test public void tetCommonConf() { Configuration fromConf = new Configuration(false); fromConf.set("test.conf.key.1", "confkey1"); fromConf.setInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 3); fromConf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.11f); fromConf.setInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB, 123); fromConf.set("io.shouldExist", "io"); Map<String, String> additionalConfs = new HashMap<String, String>(); additionalConfs.put("test.key.2", "key2"); additionalConfs.put(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111"); additionalConfs.put(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f"); additionalConfs.put(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "CustomSorter"); additionalConfs.put("file.shouldExist", "file"); OrderedPartitionedKVEdgeConfigurer.Builder builder = OrderedPartitionedKVEdgeConfigurer .newBuilder("KEY", "VALUE", "PARTITIONER", null).setAdditionalConfiguration("fs.shouldExist", "fs") .setAdditionalConfiguration("test.key.1", "key1") .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "2222") .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, "3333") .setAdditionalConfiguration(additionalConfs).setFromConfiguration(fromConf); OrderedPartitionedKVEdgeConfigurer configuration = builder.build(); byte[] outputBytes = configuration.getOutputPayload(); byte[] inputBytes = configuration.getInputPayload(); OnFileSortedOutputConfiguration rebuiltOutput = new OnFileSortedOutputConfiguration(); rebuiltOutput.fromByteArray(outputBytes); ShuffledMergedInputConfiguration rebuiltInput = new ShuffledMergedInputConfiguration(); rebuiltInput.fromByteArray(inputBytes); Configuration outputConf = rebuiltOutput.conf; Configuration inputConf = rebuiltInput.conf; assertEquals(3, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 0)); assertEquals(1111, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); assertEquals(2222, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT)); assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT)); assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT)); assertEquals(123, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB, 0)); assertEquals("CustomSorter", outputConf.get(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS)); assertEquals(3333, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 0)); assertEquals("io", outputConf.get("io.shouldExist")); assertEquals("file", outputConf.get("file.shouldExist")); assertEquals("fs", outputConf.get("fs.shouldExist")); assertEquals(3, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_SORT_FACTOR, 0)); assertEquals(1111, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); assertEquals(2222, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertEquals(0.11f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.0f), 0.001f);//from ww w . ja v a 2s . com assertEquals(0.22f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); assertEquals(0.33f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.001f); assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_IO_SORT_MB)); assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_INTERNAL_SORTER_CLASS)); assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES)); assertEquals("io", inputConf.get("io.shouldExist")); assertEquals("file", inputConf.get("file.shouldExist")); assertEquals("fs", inputConf.get("fs.shouldExist")); }
From source file:org.apache.tez.runtime.library.conf.TestUnorderedKVEdgeConfig.java
License:Apache License
@Test(timeout = 2000) public void tetCommonConf() { Configuration fromConf = new Configuration(false); fromConf.set("test.conf.key.1", "confkey1"); fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, false); fromConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.11f); fromConf.set("io.shouldExist", "io"); Map<String, String> additionalConfs = new HashMap<String, String>(); additionalConfs.put("test.key.2", "key2"); additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111"); additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f"); additionalConfs.put("file.shouldExist", "file"); UnorderedKVEdgeConfig.Builder builder = UnorderedKVEdgeConfig.newBuilder("KEY", "VALUE") .setAdditionalConfiguration("fs.shouldExist", "fs").setAdditionalConfiguration("test.key.1", "key1") .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "3333") .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") .setAdditionalConfiguration(additionalConfs).setFromConfiguration(fromConf); UnorderedKVEdgeConfig configuration = builder.build(); UnorderedKVOutputConfig rebuiltOutput = new UnorderedKVOutputConfig(); rebuiltOutput.fromUserPayload(configuration.getOutputPayload()); UnorderedKVInputConfig rebuiltInput = new UnorderedKVInputConfig(); rebuiltInput.fromUserPayload(configuration.getInputPayload()); Configuration outputConf = rebuiltOutput.conf; Configuration inputConf = rebuiltInput.conf; assertEquals(false, outputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true)); assertEquals(1111, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); assertEquals(3333, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT)); assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT)); assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT)); assertEquals("io", outputConf.get("io.shouldExist")); assertEquals("file", outputConf.get("file.shouldExist")); assertEquals("fs", outputConf.get("fs.shouldExist")); assertEquals(false, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true)); assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); assertEquals(3333, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertEquals(0.11f,/* w w w .j a va 2s . co m*/ inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.0f), 0.001f); assertEquals(0.22f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); assertEquals(0.33f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.001f); assertEquals("io", inputConf.get("io.shouldExist")); assertEquals("file", inputConf.get("file.shouldExist")); assertEquals("fs", inputConf.get("fs.shouldExist")); }
From source file:org.apache.tez.runtime.library.conf.TestUnorderedPartitionedKVEdgeConfig.java
License:Apache License
@Test(timeout = 2000) public void tetCommonConf() { Configuration fromConf = new Configuration(false); fromConf.set("test.conf.key.1", "confkey1"); fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, false); fromConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.11f); fromConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 123); fromConf.set("io.shouldExist", "io"); Map<String, String> additionalConfs = new HashMap<String, String>(); additionalConfs.put("test.key.2", "key2"); additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111"); additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f"); additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, "2222"); additionalConfs.put("file.shouldExist", "file"); UnorderedPartitionedKVEdgeConfig.Builder builder = UnorderedPartitionedKVEdgeConfig .newBuilder("KEY", "VALUE", "PARTITIONER").setAdditionalConfiguration("fs.shouldExist", "fs") .setAdditionalConfiguration("test.key.1", "key1") .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "3333") .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") .setAdditionalConfiguration(additionalConfs).setFromConfiguration(fromConf); UnorderedPartitionedKVEdgeConfig configuration = builder.build(); UnorderedPartitionedKVOutputConfig rebuiltOutput = new UnorderedPartitionedKVOutputConfig(); rebuiltOutput.fromUserPayload(configuration.getOutputPayload()); UnorderedKVInputConfig rebuiltInput = new UnorderedKVInputConfig(); rebuiltInput.fromUserPayload(configuration.getInputPayload()); Configuration outputConf = rebuiltOutput.conf; Configuration inputConf = rebuiltInput.conf; assertEquals(false, outputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true)); assertEquals(1111, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); assertEquals(3333, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT)); assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT)); assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT)); assertEquals(123,//from w ww . j av a 2 s . com outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 0)); assertEquals(2222, outputConf .getInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, 0)); assertEquals("io", outputConf.get("io.shouldExist")); assertEquals("file", outputConf.get("file.shouldExist")); assertEquals("fs", outputConf.get("fs.shouldExist")); assertEquals(false, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true)); assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); assertEquals(3333, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertEquals(0.11f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.0f), 0.001f); assertEquals(0.22f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); assertEquals(0.33f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.001f); assertNull(inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB)); assertNull(inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES)); assertEquals("io", inputConf.get("io.shouldExist")); assertEquals("file", inputConf.get("file.shouldExist")); assertEquals("fs", inputConf.get("fs.shouldExist")); }
From source file:org.apache.tez.runtime.library.conf.TestUnorderedPartitionedKVEdgeConfigurer.java
License:Apache License
@Test public void tetCommonConf() { Configuration fromConf = new Configuration(false); fromConf.set("test.conf.key.1", "confkey1"); fromConf.setBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, false); fromConf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.11f); fromConf.setInt(TezJobConfig.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 123); fromConf.set("io.shouldExist", "io"); Map<String, String> additionalConfs = new HashMap<String, String>(); additionalConfs.put("test.key.2", "key2"); additionalConfs.put(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111"); additionalConfs.put(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f"); additionalConfs.put(TezJobConfig.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, "2222"); additionalConfs.put("file.shouldExist", "file"); UnorderedPartitionedKVEdgeConfigurer.Builder builder = UnorderedPartitionedKVEdgeConfigurer .newBuilder("KEY", "VALUE", "PARTITIONER", null).setAdditionalConfiguration("fs.shouldExist", "fs") .setAdditionalConfiguration("test.key.1", "key1") .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "3333") .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") .setAdditionalConfiguration(additionalConfs).setFromConfiguration(fromConf); UnorderedPartitionedKVEdgeConfigurer configuration = builder.build(); byte[] outputBytes = configuration.getOutputPayload(); byte[] inputBytes = configuration.getInputPayload(); OnFileUnorderedPartitionedKVOutputConfiguration rebuiltOutput = new OnFileUnorderedPartitionedKVOutputConfiguration(); rebuiltOutput.fromByteArray(outputBytes); ShuffledUnorderedKVInputConfiguration rebuiltInput = new ShuffledUnorderedKVInputConfiguration(); rebuiltInput.fromByteArray(inputBytes); Configuration outputConf = rebuiltOutput.conf; Configuration inputConf = rebuiltInput.conf; assertEquals(false, outputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, true)); assertEquals(1111, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); assertEquals(3333, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT)); assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT)); assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT)); assertEquals(123, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 0)); assertEquals(2222,//w w w. ja va2 s .c o m outputConf.getInt(TezJobConfig.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, 0)); assertEquals("io", outputConf.get("io.shouldExist")); assertEquals("file", outputConf.get("file.shouldExist")); assertEquals("fs", outputConf.get("fs.shouldExist")); assertEquals(false, inputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, true)); assertEquals(1111, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); assertEquals(3333, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertEquals(0.11f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.0f), 0.001f); assertEquals(0.22f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); assertEquals(0.33f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.001f); assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB)); assertNull(inputConf.get(TezJobConfig.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES)); assertEquals("io", inputConf.get("io.shouldExist")); assertEquals("file", inputConf.get("file.shouldExist")); assertEquals("fs", inputConf.get("fs.shouldExist")); }
From source file:org.apache.tez.runtime.library.conf.TestUnorderedUnpartitionedKVEdgeConfigurer.java
License:Apache License
@Test public void tetCommonConf() { Configuration fromConf = new Configuration(false); fromConf.set("test.conf.key.1", "confkey1"); fromConf.setBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, false); fromConf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.11f); fromConf.set("io.shouldExist", "io"); Map<String, String> additionalConfs = new HashMap<String, String>(); additionalConfs.put("test.key.2", "key2"); additionalConfs.put(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111"); additionalConfs.put(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f"); additionalConfs.put("file.shouldExist", "file"); UnorderedUnpartitionedKVEdgeConfigurer.Builder builder = UnorderedUnpartitionedKVEdgeConfigurer .newBuilder("KEY", "VALUE").setAdditionalConfiguration("fs.shouldExist", "fs") .setAdditionalConfiguration("test.key.1", "key1") .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "3333") .setAdditionalConfiguration(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") .setAdditionalConfiguration(additionalConfs).setFromConfiguration(fromConf); UnorderedUnpartitionedKVEdgeConfigurer configuration = builder.build(); byte[] outputBytes = configuration.getOutputPayload(); byte[] inputBytes = configuration.getInputPayload(); OnFileUnorderedKVOutputConfiguration rebuiltOutput = new OnFileUnorderedKVOutputConfiguration(); rebuiltOutput.fromByteArray(outputBytes); ShuffledUnorderedKVInputConfiguration rebuiltInput = new ShuffledUnorderedKVInputConfiguration(); rebuiltInput.fromByteArray(inputBytes); Configuration outputConf = rebuiltOutput.conf; Configuration inputConf = rebuiltInput.conf; assertEquals(false, outputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, true)); assertEquals(1111, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); assertEquals(3333, outputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT)); assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT)); assertNull(outputConf.get(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT)); assertEquals("io", outputConf.get("io.shouldExist")); assertEquals("file", outputConf.get("file.shouldExist")); assertEquals("fs", outputConf.get("fs.shouldExist")); assertEquals(false, inputConf.getBoolean(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD, true)); assertEquals(1111, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); assertEquals(3333, inputConf.getInt(TezJobConfig.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertEquals(0.11f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, 0.0f), 0.001f);//w w w . j a v a 2 s . c om assertEquals(0.22f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); assertEquals(0.33f, inputConf.getFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.001f); assertEquals("io", inputConf.get("io.shouldExist")); assertEquals("file", inputConf.get("file.shouldExist")); assertEquals("fs", inputConf.get("fs.shouldExist")); }
From source file:org.apache.tez.runtime.library.shuffle.common.impl.TestSimpleFetchedInputAllocator.java
License:Apache License
@Test public void testInMemAllocation() throws IOException { String localDirs = "/tmp/" + this.getClass().getName(); Configuration conf = new Configuration(); long jvmMax = Runtime.getRuntime().maxMemory(); LOG.info("jvmMax: " + jvmMax); float bufferPercent = 0.1f; conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, bufferPercent); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 1.0f); conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDirs); long inMemThreshold = (long) (bufferPercent * jvmMax); LOG.info("InMemThreshold: " + inMemThreshold); SimpleFetchedInputAllocator inputManager = new SimpleFetchedInputAllocator(UUID.randomUUID().toString(), conf, Runtime.getRuntime().maxMemory(), inMemThreshold); long requestSize = (long) (0.4f * inMemThreshold); long compressedSize = 1l; LOG.info("RequestSize: " + requestSize); FetchedInput fi1 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(1, 1)); assertEquals(FetchedInput.Type.MEMORY, fi1.getType()); FetchedInput fi2 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(2, 1)); assertEquals(FetchedInput.Type.MEMORY, fi2.getType()); // Over limit by this point. Next reserve should give back a DISK allocation FetchedInput fi3 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(3, 1)); assertEquals(FetchedInput.Type.DISK, fi3.getType()); // Freed one memory allocation. Next should be mem again. fi1.abort();/* www . j a va 2 s.c o m*/ fi1.free(); FetchedInput fi4 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1)); assertEquals(FetchedInput.Type.MEMORY, fi4.getType()); // Freed one disk allocation. Next sould be disk again (no mem freed) fi3.abort(); fi3.free(); FetchedInput fi5 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1)); assertEquals(FetchedInput.Type.DISK, fi5.getType()); }
From source file:org.apache.tez.test.TestFaultTolerance.java
License:Apache License
@Test(timeout = 120000) public void testRandomFailingTasks() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestProcessor.TEZ_FAILING_PROCESSOR_DO_RANDOM_FAIL, true); testConf.setFloat(TestProcessor.TEZ_FAILING_PROCESSOR_RANDOM_FAIL_PROBABILITY, 0.5f); DAG dag = SixLevelsFailingDAG.createDAG("testRandomFailingTasks", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); }
From source file:org.apache.tez.test.TestFaultTolerance.java
License:Apache License
@Test(timeout = 60000) public void testRandomFailingInputs() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestInput.TEZ_FAILING_INPUT_DO_RANDOM_FAIL, true); testConf.setFloat(TestInput.TEZ_FAILING_INPUT_RANDOM_FAIL_PROBABILITY, 0.5f); DAG dag = SixLevelsFailingDAG.createDAG("testRandomFailingInputs", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); }
From source file:org.clueweb.clueweb12.app.DuplicateFiltering.java
License:Apache License
/** * Runs this tool./*from w ww .jav a2s. c om*/ */ @SuppressWarnings({ "static-access", "deprecation" }) public int run(String[] args) throws Exception { Options options = new Options(); options.addOption(OptionBuilder.withArgName("path").hasArg() .withDescription("input path (pfor format expected, add * to retrieve files)") .create(DOCVECTOR_OPTION)); options.addOption( OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(TREC_RESULT_FILE)); options.addOption( OptionBuilder.withArgName("path").hasArg().withDescription("output path").create(OUTPUT_OPTION)); options.addOption( OptionBuilder.withArgName("path").hasArg().withDescription("dictionary").create(DICTIONARY_OPTION)); options.addOption(OptionBuilder.withArgName("int").hasArg().withDescription("topk").create(TOPK)); options.addOption(OptionBuilder.withArgName("float [0-1]").hasArg() .withDescription("cosine similarity threshold").create(SIM_THRESHOLD)); CommandLine cmdline; CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(this.getClass().getName(), options); ToolRunner.printGenericCommandUsage(System.out); System.err.println("Error parsing command line: " + exp.getMessage()); return -1; } if (!cmdline.hasOption(DOCVECTOR_OPTION) || !cmdline.hasOption(OUTPUT_OPTION) || !cmdline.hasOption(DICTIONARY_OPTION) || !cmdline.hasOption(TREC_RESULT_FILE) || !cmdline.hasOption(SIM_THRESHOLD) || !cmdline.hasOption(TOPK)) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(this.getClass().getName(), options); ToolRunner.printGenericCommandUsage(System.out); return -1; } String docvector = cmdline.getOptionValue(DOCVECTOR_OPTION); String trecinput = cmdline.getOptionValue(TREC_RESULT_FILE); String output = cmdline.getOptionValue(OUTPUT_OPTION); String dictionary = cmdline.getOptionValue(DICTIONARY_OPTION); String simThreshold = cmdline.getOptionValue(SIM_THRESHOLD); String topk = cmdline.getOptionValue(TOPK); LOG.info("Tool name: " + DuplicateFiltering.class.getSimpleName()); LOG.info(" - docvector: " + docvector); LOG.info(" - trecinputfile: " + trecinput); LOG.info(" - output: " + output); LOG.info(" - dictionary: " + dictionary); LOG.info(" - cosine similarity threshold: " + SIM_THRESHOLD); LOG.info(" - topk: " + topk); Configuration conf = getConf(); conf.set(DICTIONARY_OPTION, dictionary); conf.setFloat(SIM_THRESHOLD, Float.parseFloat(simThreshold)); conf.set(TREC_RESULT_FILE, trecinput); conf.setInt(TOPK, Integer.parseInt(topk)); conf.set("mapred.task.timeout", "6000000");// default is 600000 FileSystem fs = FileSystem.get(conf); if (fs.exists(new Path(output))) fs.delete(new Path(output)); Job job = new Job(conf, DuplicateFiltering.class.getSimpleName() + ":" + docvector); job.setJarByClass(DuplicateFiltering.class); FileInputFormat.setInputPaths(job, docvector); FileOutputFormat.setOutputPath(job, new Path(output)); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapOutputKeyClass(PairOfIntString.class); job.setMapOutputValueClass(FloatArrayWritable.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(Text.class); job.setMapperClass(MyMapper.class); job.setPartitionerClass(MyPartitioner.class); job.setReducerClass(MyReducer.class); long startTime = System.currentTimeMillis(); job.waitForCompletion(true); LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds"); int numDuplicates = (int) job.getCounters().findCounter(Records.DUPLICATES).getValue(); LOG.info("Number of duplicates: " + numDuplicates); return 0; }
From source file:org.clueweb.clueweb12.app.RMModel.java
License:Apache License
/** * Runs this tool./*from w ww . ja v a 2s .com*/ */ @SuppressWarnings({ "static-access", "deprecation" }) public int run(String[] args) throws Exception { Options options = new Options(); options.addOption(OptionBuilder.withArgName("path").hasArg() .withDescription("input path (pfor format expected, add * to retrieve files)") .create(DOCVECTOR_OPTION)); options.addOption( OptionBuilder.withArgName("path").hasArg().withDescription("output path").create(OUTPUT_OPTION)); options.addOption( OptionBuilder.withArgName("path").hasArg().withDescription("dictionary").create(DICTIONARY_OPTION)); options.addOption(OptionBuilder.withArgName("int").hasArg().withDescription("numFeedbackDocs") .create(NUM_FEEDBACK_DOCS)); options.addOption(OptionBuilder.withArgName("int").hasArg().withDescription("numFeedbackTerms") .create(NUM_FEEDBACK_TERMS)); options.addOption( OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(TREC_RESULT_FILE)); options.addOption( OptionBuilder.withArgName("float").hasArg().withDescription("smoothing").create(SMOOTHING)); CommandLine cmdline; CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(this.getClass().getName(), options); ToolRunner.printGenericCommandUsage(System.out); System.err.println("Error parsing command line: " + exp.getMessage()); return -1; } if (!cmdline.hasOption(DOCVECTOR_OPTION) || !cmdline.hasOption(OUTPUT_OPTION) || !cmdline.hasOption(DICTIONARY_OPTION) || !cmdline.hasOption(TREC_RESULT_FILE) || !cmdline.hasOption(NUM_FEEDBACK_DOCS) || !cmdline.hasOption(SMOOTHING) || !cmdline.hasOption(NUM_FEEDBACK_TERMS)) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(this.getClass().getName(), options); ToolRunner.printGenericCommandUsage(System.out); return -1; } String docvector = cmdline.getOptionValue(DOCVECTOR_OPTION); String output = cmdline.getOptionValue(OUTPUT_OPTION); String dictionary = cmdline.getOptionValue(DICTIONARY_OPTION); String trecinput = cmdline.getOptionValue(TREC_RESULT_FILE); float smoothing = Float.parseFloat(cmdline.getOptionValue(SMOOTHING)); int numDocs = Integer.parseInt(cmdline.getOptionValue(NUM_FEEDBACK_DOCS)); int numTerms = Integer.parseInt(cmdline.getOptionValue(NUM_FEEDBACK_TERMS)); LOG.info("Tool name: " + RMModel.class.getSimpleName()); LOG.info(" - docvector: " + docvector); LOG.info(" - output: " + output); LOG.info(" - dictionary: " + dictionary); LOG.info(" - trecinputfile: " + trecinput); LOG.info(" - numFeedbackDocs: " + numDocs); LOG.info(" - numFeedbackTerms: " + numTerms); LOG.info(" - smoothing: " + smoothing); Configuration conf = getConf(); conf.set(DICTIONARY_OPTION, dictionary); conf.set(TREC_RESULT_FILE, trecinput); conf.setInt(NUM_FEEDBACK_DOCS, numDocs); conf.setInt(NUM_FEEDBACK_TERMS, numTerms); conf.setFloat(SMOOTHING, smoothing); conf.set("mapreduce.map.memory.mb", "10048"); conf.set("mapreduce.map.java.opts", "-Xmx10048m"); conf.set("mapreduce.reduce.memory.mb", "10048"); conf.set("mapreduce.reduce.java.opts", "-Xmx10048m"); conf.set("mapred.task.timeout", "6000000");// default is 600000 FileSystem fs = FileSystem.get(conf); if (fs.exists(new Path(output))) fs.delete(new Path(output)); Job job = new Job(conf, RMModel.class.getSimpleName() + ":" + docvector); job.setJarByClass(RMModel.class); FileInputFormat.setInputPaths(job, docvector); FileOutputFormat.setOutputPath(job, new Path(output)); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(PairOfStringFloat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(FloatWritable.class); job.setMapperClass(MyMapper.class); job.setReducerClass(MyReducer.class); long startTime = System.currentTimeMillis(); job.waitForCompletion(true); LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds"); return 0; }