List of usage examples for org.apache.hadoop.conf Configuration setFloat
public void setFloat(String name, float value)
name property to a float. From source file:org.apache.phoenix.mapreduce.MultiHfileOutputFormat.java
License:Apache License
/** * /*from w ww .j av a 2 s. c om*/ * @param context * @return * @throws IOException */ static <V extends Cell> RecordWriter<TableRowkeyPair, V> createRecordWriter(final TaskAttemptContext context) throws IOException { // Get the path of the temporary output file final Path outputPath = FileOutputFormat.getOutputPath(context); final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath(); final Configuration conf = context.getConfiguration(); final FileSystem fs = outputdir.getFileSystem(conf); final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE); // Invented config. Add to hbase-*.xml if other than default compression. final String defaultCompressionStr = conf.get("hfile.compression", Compression.Algorithm.NONE.getName()); final Algorithm defaultCompression = AbstractHFileWriter.compressionByName(defaultCompressionStr); final boolean compactionExclude = conf.getBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude", false); return new RecordWriter<TableRowkeyPair, V>() { // Map of families to writers and how much has been output on the writer. private final Map<byte[], WriterLength> writers = new TreeMap<byte[], WriterLength>( Bytes.BYTES_COMPARATOR); private byte[] previousRow = HConstants.EMPTY_BYTE_ARRAY; private final byte[] now = Bytes.toBytes(EnvironmentEdgeManager.currentTimeMillis()); private boolean rollRequested = false; @Override public void write(TableRowkeyPair row, V cell) throws IOException { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); // null input == user explicitly wants to flush if (row == null && kv == null) { rollWriters(); return; } // phoenix-2216: start : extract table name from the rowkey String tableName = row.getTableName(); byte[] rowKey = row.getRowkey().get(); long length = kv.getLength(); byte[] family = CellUtil.cloneFamily(kv); byte[] tableAndFamily = join(tableName, Bytes.toString(family)); WriterLength wl = this.writers.get(tableAndFamily); // phoenix-2216: end // If this is a new column family, verify that the directory exists if (wl == null) { // phoenix-2216: start : create a directory for table and family within the output dir Path tableOutputPath = CsvBulkImportUtil.getOutputPath(outputdir, tableName); fs.mkdirs(new Path(tableOutputPath, Bytes.toString(family))); // phoenix-2216: end } // If any of the HFiles for the column families has reached // maxsize, we need to roll all the writers if (wl != null && wl.written + length >= maxsize) { this.rollRequested = true; } // This can only happen once a row is finished though if (rollRequested && Bytes.compareTo(this.previousRow, rowKey) != 0) { rollWriters(); } // create a new WAL writer, if necessary if (wl == null || wl.writer == null) { // phoenix-2216: start : passed even the table name wl = getNewWriter(tableName, family, conf); // phoenix-2216: end } // we now have the proper WAL writer. full steam ahead kv.updateLatestStamp(this.now); wl.writer.append(kv); wl.written += length; // Copy the row so we know when a row transition. this.previousRow = rowKey; } private void rollWriters() throws IOException { for (WriterLength wl : this.writers.values()) { if (wl.writer != null) { LOG.info("Writer=" + wl.writer.getPath() + ((wl.written == 0) ? "" : ", wrote=" + wl.written)); close(wl.writer); } wl.writer = null; wl.written = 0; } this.rollRequested = false; } /* Create a new StoreFile.Writer. * @param family * @return A WriterLength, containing a new StoreFile.Writer. * @throws IOException */ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "BX_UNBOXING_IMMEDIATELY_REBOXED", justification = "Not important") private WriterLength getNewWriter(final String tableName, byte[] family, Configuration conf) throws IOException { WriterLength wl = new WriterLength(); Path tableOutputPath = CsvBulkImportUtil.getOutputPath(outputdir, tableName); Path familydir = new Path(tableOutputPath, Bytes.toString(family)); // phoenix-2216: start : fetching the configuration properties that were set to the table. // create a map from column family to the compression algorithm for the table. final Map<byte[], Algorithm> compressionMap = createFamilyCompressionMap(conf, tableName); final Map<byte[], BloomType> bloomTypeMap = createFamilyBloomTypeMap(conf, tableName); final Map<byte[], Integer> blockSizeMap = createFamilyBlockSizeMap(conf, tableName); // phoenix-2216: end String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_OVERRIDE_CONF_KEY); final Map<byte[], DataBlockEncoding> datablockEncodingMap = createFamilyDataBlockEncodingMap(conf, tableName); final DataBlockEncoding overriddenEncoding; if (dataBlockEncodingStr != null) { overriddenEncoding = DataBlockEncoding.valueOf(dataBlockEncodingStr); } else { overriddenEncoding = null; } Algorithm compression = compressionMap.get(family); compression = compression == null ? defaultCompression : compression; BloomType bloomType = bloomTypeMap.get(family); bloomType = bloomType == null ? BloomType.NONE : bloomType; Integer blockSize = blockSizeMap.get(family); blockSize = blockSize == null ? HConstants.DEFAULT_BLOCKSIZE : blockSize; DataBlockEncoding encoding = overriddenEncoding; encoding = encoding == null ? datablockEncodingMap.get(family) : encoding; encoding = encoding == null ? DataBlockEncoding.NONE : encoding; Configuration tempConf = new Configuration(conf); tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f); HFileContextBuilder contextBuilder = new HFileContextBuilder().withCompression(compression) .withChecksumType(HStore.getChecksumType(conf)) .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(blockSize); contextBuilder.withDataBlockEncoding(encoding); HFileContext hFileContext = contextBuilder.build(); wl.writer = new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), fs) .withOutputDir(familydir).withBloomType(bloomType).withComparator(KeyValue.COMPARATOR) .withFileContext(hFileContext).build(); // join and put it in the writers map . // phoenix-2216: start : holds a map of writers where the // key in the map is a join byte array of table name and family. byte[] tableAndFamily = join(tableName, Bytes.toString(family)); this.writers.put(tableAndFamily, wl); // phoenix-2216: end return wl; } private void close(final StoreFile.Writer w) throws IOException { if (w != null) { w.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(EnvironmentEdgeManager.currentTimeMillis())); w.appendFileInfo(StoreFile.BULKLOAD_TASK_KEY, Bytes.toBytes(context.getTaskAttemptID().toString())); w.appendFileInfo(StoreFile.MAJOR_COMPACTION_KEY, Bytes.toBytes(true)); w.appendFileInfo(StoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY, Bytes.toBytes(compactionExclude)); w.appendTrackedTimestampsToMetadata(); w.close(); } } @Override public void close(TaskAttemptContext c) throws IOException, InterruptedException { for (WriterLength wl : this.writers.values()) { close(wl.writer); } } }; }
From source file:org.apache.tajo.conf.TajoConf.java
License:Apache License
public static void setFloatVar(Configuration conf, ConfVars var, float val) { assert (var.valClass == Float.class); conf.setFloat(var.varname, val); }
From source file:org.apache.tez.dag.app.dag.impl.TestRootInputVertexManager.java
License:Apache License
static RootInputVertexManager createRootInputVertexManager(Configuration conf, VertexManagerPluginContext context, Float min, Float max) { if (min != null) { conf.setFloat(TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION, min); } else {//from ww w .ja va 2s .c om conf.unset(TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION); } if (max != null) { conf.setFloat(TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION, max); } else { conf.unset(TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION); } if (max != null || min != null) { conf.setBoolean(TEZ_ROOT_INPUT_VERTEX_MANAGER_ENABLE_SLOW_START, true); } UserPayload payload; try { payload = TezUtils.createUserPayloadFromConf(conf); } catch (IOException e) { throw new RuntimeException(e); } when(context.getUserPayload()).thenReturn(payload); RootInputVertexManager manager = new RootInputVertexManager(context); manager.initialize(); return manager; }
From source file:org.apache.tez.dag.app.dag.impl.TestVertexScheduler.java
License:Apache License
private ShuffleVertexManager createScheduler(Configuration conf, Vertex vertex, float min, float max) { ShuffleVertexManager scheduler = new ShuffleVertexManager(vertex); conf.setFloat(TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, min); conf.setFloat(TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, max); scheduler.initialize(conf);// ww w. jav a2 s .co m return scheduler; }
From source file:org.apache.tez.dag.library.vertexmanager.TestShuffleVertexManager.java
License:Apache License
private ShuffleVertexManager createManager(Configuration conf, VertexManagerPluginContext context, float min, float max) { conf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, min); conf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, max); UserPayload payload;//from w w w .j a v a 2 s . c o m try { payload = TezUtils.createUserPayloadFromConf(conf); } catch (IOException e) { throw new RuntimeException(e); } when(context.getUserPayload()).thenReturn(payload); ShuffleVertexManager manager = new ShuffleVertexManager(context); manager.initialize(); return manager; }
From source file:org.apache.tez.dag.library.vertexmanager.TestShuffleVertexManagerUtils.java
License:Apache License
static ShuffleVertexManager createShuffleVertexManager(Configuration conf, VertexManagerPluginContext context, Boolean enableAutoParallelism, Long desiredTaskInputSize, Float min, Float max) { if (min != null) { conf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, min); } else {// w ww . j a v a 2s. c o m conf.unset(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION); } if (max != null) { conf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, max); } else { conf.unset(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION); } if (enableAutoParallelism != null) { conf.setBoolean(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL, enableAutoParallelism); } if (desiredTaskInputSize != null) { conf.setLong(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE, desiredTaskInputSize); } UserPayload payload; try { payload = TezUtils.createUserPayloadFromConf(conf); } catch (IOException e) { throw new RuntimeException(e); } when(context.getUserPayload()).thenReturn(payload); ShuffleVertexManager manager = new ShuffleVertexManager(context); manager.initialize(); return manager; }
From source file:org.apache.tez.runtime.library.broadcast.input.TestBroadcastInputManager.java
License:Apache License
@Test public void testInMemAllocation() throws IOException { String localDirs = "/tmp/" + this.getClass().getName(); Configuration conf = new Configuration(); long jvmMax = Runtime.getRuntime().maxMemory(); LOG.info("jvmMax: " + jvmMax); float bufferPercent = 0.1f; conf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_INPUT_BUFFER_PERCENT, bufferPercent); conf.setFloat(TezJobConfig.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 1.0f); conf.setStrings(TezJobConfig.LOCAL_DIRS, localDirs); long inMemThreshold = (long) (bufferPercent * jvmMax); LOG.info("InMemThreshold: " + inMemThreshold); BroadcastInputManager inputManager = new BroadcastInputManager(UUID.randomUUID().toString(), conf); long requestSize = (long) (0.4f * inMemThreshold); long compressedSize = 1l; LOG.info("RequestSize: " + requestSize); FetchedInput fi1 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(1, 1)); assertEquals(FetchedInput.Type.MEMORY, fi1.getType()); FetchedInput fi2 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(2, 1)); assertEquals(FetchedInput.Type.MEMORY, fi2.getType()); // Over limit by this point. Next reserve should give back a DISK allocation FetchedInput fi3 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(3, 1)); assertEquals(FetchedInput.Type.DISK, fi3.getType()); // Freed one memory allocation. Next should be mem again. fi1.abort();/* www.j av a 2 s. com*/ fi1.free(); FetchedInput fi4 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1)); assertEquals(FetchedInput.Type.MEMORY, fi4.getType()); // Freed one disk allocation. Next sould be disk again (no mem freed) fi3.abort(); fi3.free(); FetchedInput fi5 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1)); assertEquals(FetchedInput.Type.DISK, fi5.getType()); }
From source file:org.apache.tez.runtime.library.common.shuffle.impl.TestSimpleFetchedInputAllocator.java
License:Apache License
@Test(timeout = 5000) public void testInMemAllocation() throws IOException { String localDirs = "/tmp/" + this.getClass().getName(); Configuration conf = new Configuration(); long jvmMax = Runtime.getRuntime().maxMemory(); LOG.info("jvmMax: " + jvmMax); float bufferPercent = 0.1f; conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, bufferPercent); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 1.0f); conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDirs); long inMemThreshold = (long) (bufferPercent * jvmMax); LOG.info("InMemThreshold: " + inMemThreshold); SimpleFetchedInputAllocator inputManager = new SimpleFetchedInputAllocator(UUID.randomUUID().toString(), conf, Runtime.getRuntime().maxMemory(), inMemThreshold); long requestSize = (long) (0.4f * inMemThreshold); long compressedSize = 1l; LOG.info("RequestSize: " + requestSize); FetchedInput fi1 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(1, 1)); assertEquals(FetchedInput.Type.MEMORY, fi1.getType()); FetchedInput fi2 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(2, 1)); assertEquals(FetchedInput.Type.MEMORY, fi2.getType()); // Over limit by this point. Next reserve should give back a DISK allocation FetchedInput fi3 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(3, 1)); assertEquals(FetchedInput.Type.DISK, fi3.getType()); // Freed one memory allocation. Next should be mem again. fi1.abort();// www.j a va 2s . co m fi1.free(); FetchedInput fi4 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1)); assertEquals(FetchedInput.Type.MEMORY, fi4.getType()); // Freed one disk allocation. Next sould be disk again (no mem freed) fi3.abort(); fi3.free(); FetchedInput fi5 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1)); assertEquals(FetchedInput.Type.DISK, fi5.getType()); }
From source file:org.apache.tez.runtime.library.common.shuffle.orderedgrouped.TestMergeManager.java
License:Apache License
@Test(timeout = 10000) public void testConfigs() throws IOException { long maxTaskMem = 8192 * 1024 * 1024l; //Test Shuffle fetch buffer and post merge buffer percentage Configuration conf = new TezConfiguration(defaultConf); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.8f); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.5f); Assert.assertTrue(MergeManager.getInitialMemoryRequirement(conf, maxTaskMem) == 6871947776l); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.5f); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.5f); Assert.assertTrue(MergeManager.getInitialMemoryRequirement(conf, maxTaskMem) > Integer.MAX_VALUE); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.9f); Assert.assertTrue(MergeManager.getInitialMemoryRequirement(conf, maxTaskMem) > Integer.MAX_VALUE); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.1f); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.1f); Assert.assertTrue(MergeManager.getInitialMemoryRequirement(conf, maxTaskMem) < Integer.MAX_VALUE); try {/* w w w . j a v a2 s.c o m*/ conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 2.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong buffer percent configuration exception"); } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, -2.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong buffer percent configuration exception"); } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 1.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong post merge buffer percent configuration exception"); } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, -1.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong post merge buffer percent configuration exception"); } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 1.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong shuffle fetch buffer percent configuration exception"); } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, -1.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong shuffle fetch buffer percent configuration exception"); } catch (IllegalArgumentException ie) { } //test post merge mem limit conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.8f); FileSystem localFs = FileSystem.getLocal(conf); LocalDirAllocator localDirAllocator = new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS); InputContext t0inputContext = createMockInputContext(UUID.randomUUID().toString(), maxTaskMem); ExceptionReporter t0exceptionReporter = mock(ExceptionReporter.class); long initialMemoryAvailable = (long) (maxTaskMem * 0.8); MergeManager mergeManager = new MergeManager(conf, localFs, localDirAllocator, t0inputContext, null, null, null, null, t0exceptionReporter, initialMemoryAvailable, null, false, -1); Assert.assertTrue(mergeManager.postMergeMemLimit > Integer.MAX_VALUE); initialMemoryAvailable = 200 * 1024 * 1024l; //initial mem < memlimit mergeManager = new MergeManager(conf, localFs, localDirAllocator, t0inputContext, null, null, null, null, t0exceptionReporter, initialMemoryAvailable, null, false, -1); Assert.assertTrue(mergeManager.postMergeMemLimit == initialMemoryAvailable); }
From source file:org.apache.tez.runtime.library.conf.TestOrderedPartitionedKVEdgeConfig.java
License:Apache License
@Test(timeout = 2000) public void tetCommonConf() { Configuration fromConf = new Configuration(false); fromConf.set("test.conf.key.1", "confkey1"); fromConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 3); fromConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.11f); fromConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 123); fromConf.set("io.shouldExist", "io"); Map<String, String> additionalConfs = new HashMap<String, String>(); additionalConfs.put("test.key.2", "key2"); additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111"); additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, "0.22f"); additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "CustomSorter"); additionalConfs.put("file.shouldExist", "file"); OrderedPartitionedKVEdgeConfig.Builder builder = OrderedPartitionedKVEdgeConfig .newBuilder("KEY", "VALUE", "PARTITIONER").setAdditionalConfiguration("fs.shouldExist", "fs") .setAdditionalConfiguration("test.key.1", "key1") .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, "2222") .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, "0.33f") .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, "3333") .setAdditionalConfiguration(additionalConfs).setFromConfiguration(fromConf); OrderedPartitionedKVEdgeConfig configuration = builder.build(); OrderedPartitionedKVOutputConfig rebuiltOutput = new OrderedPartitionedKVOutputConfig(); rebuiltOutput.fromUserPayload(configuration.getOutputPayload()); OrderedGroupedKVInputConfig rebuiltInput = new OrderedGroupedKVInputConfig(); rebuiltInput.fromUserPayload(configuration.getInputPayload()); Configuration outputConf = rebuiltOutput.conf; Configuration inputConf = rebuiltInput.conf; assertEquals(3, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 0)); assertEquals(1111, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); assertEquals(2222, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT)); assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT)); assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT)); assertEquals(123, outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 0)); assertEquals("CustomSorter", outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS)); assertEquals(3333,//from w w w.j a v a2 s .co m outputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 0)); assertEquals("io", outputConf.get("io.shouldExist")); assertEquals("file", outputConf.get("file.shouldExist")); assertEquals("fs", outputConf.get("fs.shouldExist")); assertEquals(3, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 0)); assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); assertEquals(2222, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); assertEquals(0.11f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.0f), 0.001f); assertEquals(0.22f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f), 0.001f); assertEquals(0.33f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, 0.0f), 0.001f); assertNull(inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB)); assertNull(inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS)); assertNull(inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES)); assertEquals("io", inputConf.get("io.shouldExist")); assertEquals("file", inputConf.get("file.shouldExist")); assertEquals("fs", inputConf.get("fs.shouldExist")); }