Example usage for org.apache.hadoop.conf Configuration getFloat

List of usage examples for org.apache.hadoop.conf Configuration getFloat

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. getFloat

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getFloat.

Prototype

public float getFloat(String name, float defaultValue)

Source Link

Document

Get the value of the name property as a float.

Usage

From source file:org.apache.tajo.conf.TajoConf.java

License:Apache License

public static float getFloatVar(Configuration conf, ConfVars var) {
    assert (var.valClass == Float.class);
    return conf.getFloat(var.varname, var.defaultFloatVal);
}

From source file:org.apache.tajo.conf.TajoConf.java

License:Apache License

public static float getFloatVar(Configuration conf, ConfVars var, float defaultVal) {
    return conf.getFloat(var.varname, defaultVal);
}

From source file:org.apache.tajo.storage.thirdparty.orc.MemoryManager.java

License:Apache License

/**
 * Create the memory manager.//from   www.  j  a va 2  s  . c om
 * @param conf use the configuration to find the maximum size of the memory
 *             pool.
 */
MemoryManager(Configuration conf) {
    HiveConf.ConfVars poolVar = HiveConf.ConfVars.HIVE_ORC_FILE_MEMORY_POOL;
    double maxLoad = conf.getFloat(poolVar.varname, poolVar.defaultFloatVal);
    totalMemoryPool = Math.round(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax() * maxLoad);
    ownerLock.lock();
}

From source file:org.apache.tez.analyzer.plugins.ShuffleTimeAnalyzer.java

License:Apache License

public ShuffleTimeAnalyzer(Configuration config) {
    this.config = config;

    realWorkDoneRatio = config.getFloat(REAL_WORK_DONE_RATIO, REAL_WORK_DONE_RATIO_DEFAULT);
    minShuffleRecords = config.getLong(MIN_SHUFFLE_RECORDS, MIN_SHUFFLE_RECORDS_DEFAULT);
}

From source file:org.apache.tez.analyzer.plugins.SkewAnalyzer.java

License:Apache License

public SkewAnalyzer(Configuration config) {
    this.config = config;
    maxRatio = config.getFloat(ATTEMPT_SHUFFLE_KEY_GROUP_MAX_RATIO,
            ATTEMPT_SHUFFLE_KEY_GROUP_MAX_RATIO_DEFAULT);
    minRatio = config.getFloat(ATTEMPT_SHUFFLE_KEY_GROUP_MIN_RATIO,
            ATTEMPT_SHUFFLE_KEY_GROUP_MIN_RATIO_DEFAULT);
    maxShuffleBytesPerSource = config.getLong(SHUFFLE_BYTES_PER_ATTEMPT_PER_SOURCE,
            SHUFFLE_BYTES_PER_ATTEMPT_PER_SOURCE_DEFAULT);
}

From source file:org.apache.tez.dag.app.dag.impl.ShuffleVertexManager.java

License:Apache License

@Override
public void initialize(Configuration conf) {
    this.slowStartMinSrcCompletionFraction = conf.getFloat(
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION,
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION_DEFAULT);
    this.slowStartMaxSrcCompletionFraction = conf.getFloat(
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION,
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT);

    if (slowStartMinSrcCompletionFraction < 0
            || slowStartMaxSrcCompletionFraction < slowStartMinSrcCompletionFraction) {
        throw new IllegalArgumentException("Invalid values for slowStartMinSrcCompletionFraction"
                + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0 and " + "max cannot be < min.");
    }/*from  w  w  w .  j ava 2 s.  com*/

    enableAutoParallelism = conf.getBoolean(TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL,
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL_DEFAULT);
    desiredTaskInputDataSize = conf.getLong(
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE,
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE_DEFAULT);
    minTaskParallelism = conf.getInt(TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM,
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM_DEFAULT);
    LOG.info("Shuffle Vertex Manager: settings" + " minFrac:" + slowStartMinSrcCompletionFraction + " maxFrac:"
            + slowStartMaxSrcCompletionFraction + " auto:" + enableAutoParallelism + " desiredTaskIput:"
            + desiredTaskInputDataSize + " minTasks:" + minTaskParallelism);
}

From source file:org.apache.tez.dag.app.dag.speculation.legacy.StartEndTimesBase.java

License:Apache License

@Override
public void contextualize(Configuration conf, Vertex vertex) {
    slowTaskRelativeTresholds = conf.getFloat(TezConfiguration.TEZ_AM_LEGACY_SPECULATIVE_SLOWTASK_THRESHOLD,
            1.0f);/*from  www.  j  a  v  a  2  s. c  o  m*/
    this.vertex = vertex;
}

From source file:org.apache.tez.dag.app.speculate.StartEndTimesBase.java

License:Apache License

@Override
public void contextualize(Configuration conf, AppContext context) {
    this.conf = conf;
    this.context = context;

    final DAG dag = context.getDAG();
    for (Entry<TezVertexID, Vertex> entry : dag.getVertices().entrySet()) {
        vertexStatistics.put(entry.getKey(), new DataStatistics());
        slowTaskRelativeTresholds = conf.getFloat(MRJobConfig.SPECULATIVE_SLOWTASK_THRESHOLD, 1.0f);
    }/*w w w .ja v a  2s.c  om*/
}

From source file:org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager.java

License:Apache License

@Override
public void initialize() {
    Configuration conf;
    try {//  ww w .j  ava  2s  .c  o m
        conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
    } catch (IOException e) {
        throw new TezUncheckedException(e);
    }

    this.slowStartMinSrcCompletionFraction = conf.getFloat(
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION,
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION_DEFAULT);
    this.slowStartMaxSrcCompletionFraction = conf.getFloat(
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION,
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT);

    if (slowStartMinSrcCompletionFraction < 0
            || slowStartMaxSrcCompletionFraction < slowStartMinSrcCompletionFraction) {
        throw new IllegalArgumentException("Invalid values for slowStartMinSrcCompletionFraction"
                + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0 and " + "max cannot be < min.");
    }

    enableAutoParallelism = conf.getBoolean(
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL,
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL_DEFAULT);
    desiredTaskInputDataSize = conf.getLong(
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE,
            ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE_DEFAULT);
    minTaskParallelism = Math.max(1,
            conf.getInt(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM,
                    ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM_DEFAULT));
    LOG.info("Shuffle Vertex Manager: settings" + " minFrac:" + slowStartMinSrcCompletionFraction + " maxFrac:"
            + slowStartMaxSrcCompletionFraction + " auto:" + enableAutoParallelism + " desiredTaskIput:"
            + desiredTaskInputDataSize + " minTasks:" + minTaskParallelism);

    Map<String, EdgeProperty> inputs = getContext().getInputVertexEdgeProperties();
    for (Map.Entry<String, EdgeProperty> entry : inputs.entrySet()) {
        srcVertexInfo.put(entry.getKey(), new SourceVertexInfo(entry.getValue()));
        getContext().registerForVertexStateUpdates(entry.getKey(), EnumSet.of(VertexState.CONFIGURED));
        if (entry.getValue().getDataMovementType() == DataMovementType.SCATTER_GATHER) {
            bipartiteSources++;
        }
    }
    if (bipartiteSources == 0) {
        throw new TezUncheckedException("Atleast 1 bipartite source should exist");
    }

    if (enableAutoParallelism) {
        getContext().vertexReconfigurationPlanned();
    }
    // dont track the source tasks here since those tasks may themselves be
    // dynamically changed as the DAG progresses.

}

From source file:org.apache.tez.engine.common.shuffle.impl.MergeManager.java

License:Apache License

public MergeManager(TezTaskAttemptID taskAttemptId, Configuration conf, FileSystem localFS,
        LocalDirAllocator localDirAllocator, TezTaskReporter reporter, Processor combineProcessor,
        TezCounter spilledRecordsCounter, TezCounter reduceCombineInputCounter,
        TezCounter mergedMapOutputsCounter, ExceptionReporter exceptionReporter, Progress mergePhase) {
    this.taskAttemptId = taskAttemptId;
    this.conf = conf;
    this.localDirAllocator = localDirAllocator;
    this.exceptionReporter = exceptionReporter;

    this.reporter = reporter;
    this.combineProcessor = combineProcessor;
    this.reduceCombineInputCounter = reduceCombineInputCounter;
    this.spilledRecordsCounter = spilledRecordsCounter;
    this.mergedMapOutputsCounter = mergedMapOutputsCounter;
    this.mapOutputFile = new TezTaskOutputFiles();
    this.mapOutputFile.setConf(conf);

    this.localFS = localFS;
    this.rfs = ((LocalFileSystem) localFS).getRaw();

    if (ConfigUtils.isIntermediateInputCompressed(conf)) {
        Class<? extends CompressionCodec> codecClass = ConfigUtils.getIntermediateInputCompressorClass(conf,
                DefaultCodec.class);
        codec = ReflectionUtils.newInstance(codecClass, conf);
    } else {/*from ww w .jav a  2  s.  c  o  m*/
        codec = null;
    }

    final float maxInMemCopyUse = conf.getFloat(TezJobConfig.TEZ_ENGINE_SHUFFLE_INPUT_BUFFER_PERCENT,
            TezJobConfig.DEFAULT_TEZ_ENGINE_SHUFFLE_INPUT_BUFFER_PERCENT);
    if (maxInMemCopyUse > 1.0 || maxInMemCopyUse < 0.0) {
        throw new IllegalArgumentException("Invalid value for "
                + TezJobConfig.TEZ_ENGINE_SHUFFLE_INPUT_BUFFER_PERCENT + ": " + maxInMemCopyUse);
    }

    // Allow unit tests to fix Runtime memory
    this.memoryLimit = (long) (conf.getLong(Constants.TEZ_ENGINE_TASK_MEMORY,
            Math.min(Runtime.getRuntime().maxMemory(), Integer.MAX_VALUE)) * maxInMemCopyUse);

    this.ioSortFactor = conf.getInt(TezJobConfig.TEZ_ENGINE_IO_SORT_FACTOR,
            TezJobConfig.DEFAULT_TEZ_ENGINE_IO_SORT_FACTOR);

    final float singleShuffleMemoryLimitPercent = conf.getFloat(
            TezJobConfig.TEZ_ENGINE_SHUFFLE_MEMORY_LIMIT_PERCENT,
            TezJobConfig.DEFAULT_TEZ_ENGINE_SHUFFLE_MEMORY_LIMIT_PERCENT);
    if (singleShuffleMemoryLimitPercent <= 0.0f || singleShuffleMemoryLimitPercent > 1.0f) {
        throw new IllegalArgumentException(
                "Invalid value for " + TezJobConfig.TEZ_ENGINE_SHUFFLE_MEMORY_LIMIT_PERCENT + ": "
                        + singleShuffleMemoryLimitPercent);
    }

    this.maxSingleShuffleLimit = (long) (memoryLimit * singleShuffleMemoryLimitPercent);
    this.memToMemMergeOutputsThreshold = conf.getInt(TezJobConfig.TEZ_ENGINE_SHUFFLE_MEMTOMEM_SEGMENTS,
            ioSortFactor);
    this.mergeThreshold = (long) (this.memoryLimit
            * conf.getFloat(TezJobConfig.TEZ_ENGINE_SHUFFLE_MERGE_PERCENT,
                    TezJobConfig.DEFAULT_TEZ_ENGINE_SHUFFLE_MERGE_PERCENT));
    LOG.info("MergerManager: memoryLimit=" + memoryLimit + ", " + "maxSingleShuffleLimit="
            + maxSingleShuffleLimit + ", " + "mergeThreshold=" + mergeThreshold + ", " + "ioSortFactor="
            + ioSortFactor + ", " + "memToMemMergeOutputsThreshold=" + memToMemMergeOutputsThreshold);

    if (this.maxSingleShuffleLimit >= this.mergeThreshold) {
        throw new RuntimeException("Invlaid configuration: "
                + "maxSingleShuffleLimit should be less than mergeThreshold" + "maxSingleShuffleLimit: "
                + this.maxSingleShuffleLimit + "mergeThreshold: " + this.mergeThreshold);
    }

    boolean allowMemToMemMerge = conf.getBoolean(TezJobConfig.TEZ_ENGINE_SHUFFLE_ENABLE_MEMTOMEM,
            TezJobConfig.DEFAULT_TEZ_ENGINE_SHUFFLE_ENABLE_MEMTOMEM);
    if (allowMemToMemMerge) {
        this.memToMemMerger = new IntermediateMemoryToMemoryMerger(this, memToMemMergeOutputsThreshold);
        this.memToMemMerger.start();
    } else {
        this.memToMemMerger = null;
    }

    this.inMemoryMerger = new InMemoryMerger(this);
    this.inMemoryMerger.start();

    this.onDiskMerger = new OnDiskMerger(this);
    this.onDiskMerger.start();

    this.mergePhase = mergePhase;
}