Example usage for org.apache.hadoop.conf Configuration getLong

List of usage examples for org.apache.hadoop.conf Configuration getLong

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. getLong

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getLong.

Prototype

public long getLong(String name, long defaultValue)

Source Link

Document

Get the value of the name property as a long.

Usage

From source file:org.apache.tephra.txprune.TransactionPruningService.java

License:Apache License

public TransactionPruningService(Configuration conf, TransactionManager txManager) {
    this.conf = conf;
    this.txManager = txManager;
    this.pruneEnabled = conf.getBoolean(TxConstants.TransactionPruning.PRUNE_ENABLE,
            TxConstants.TransactionPruning.DEFAULT_PRUNE_ENABLE);
    this.scheduleInterval = conf.getLong(TxConstants.TransactionPruning.PRUNE_INTERVAL,
            TxConstants.TransactionPruning.DEFAULT_PRUNE_INTERVAL);
}

From source file:org.apache.tez.analyzer.plugins.ShuffleTimeAnalyzer.java

License:Apache License

public ShuffleTimeAnalyzer(Configuration config) {
    this.config = config;

    realWorkDoneRatio = config.getFloat(REAL_WORK_DONE_RATIO, REAL_WORK_DONE_RATIO_DEFAULT);
    minShuffleRecords = config.getLong(MIN_SHUFFLE_RECORDS, MIN_SHUFFLE_RECORDS_DEFAULT);
}

From source file:org.apache.tez.analyzer.plugins.SkewAnalyzer.java

License:Apache License

public SkewAnalyzer(Configuration config) {
    this.config = config;
    maxRatio = config.getFloat(ATTEMPT_SHUFFLE_KEY_GROUP_MAX_RATIO,
            ATTEMPT_SHUFFLE_KEY_GROUP_MAX_RATIO_DEFAULT);
    minRatio = config.getFloat(ATTEMPT_SHUFFLE_KEY_GROUP_MIN_RATIO,
            ATTEMPT_SHUFFLE_KEY_GROUP_MIN_RATIO_DEFAULT);
    maxShuffleBytesPerSource = config.getLong(SHUFFLE_BYTES_PER_ATTEMPT_PER_SOURCE,
            SHUFFLE_BYTES_PER_ATTEMPT_PER_SOURCE_DEFAULT);
}

From source file:org.apache.tez.analyzer.plugins.SlowestVertexAnalyzer.java

License:Apache License

public SlowestVertexAnalyzer(Configuration config) {
    this.config = config;
    this.vertexRuntimeThreshold = Math.max(1, config.getLong(MAX_VERTEX_RUNTIME, MAX_VERTEX_RUNTIME_DEFAULT));

}

From source file:org.apache.tez.analyzer.plugins.SpillAnalyzerImpl.java

License:Apache License

public SpillAnalyzerImpl(Configuration config) {
    this.config = config;
    minOutputBytesPerTask = Math.max(0, config.getLong(OUTPUT_BYTES_THRESHOLD, OUTPUT_BYTES_THRESHOLD_DEFAULT));
    this.csvResult = new CSVResult(headers);
}

From source file:org.apache.tez.common.TestTezUtils.java

License:Apache License

private void checkConf(Configuration conf) {
    Assert.assertEquals(conf.get("test1"), "value1");
    Assert.assertTrue(conf.getBoolean("test2", false));
    Assert.assertEquals(conf.getDouble("test3", 0), 1.2345, 1e-15);
    Assert.assertEquals(conf.getInt("test4", 0), 34567);
    Assert.assertEquals(conf.getLong("test5", 0), 1234567890L);
    String tmp[] = conf.getStrings("test6");
    Assert.assertEquals(tmp.length, 3);/*  w  ww .  java 2  s . c om*/
    Assert.assertEquals(tmp[0], "S1");
    Assert.assertEquals(tmp[1], "S2");
    Assert.assertEquals(tmp[2], "S3");

}

From source file:org.apache.tez.dag.app.dag.impl.ShuffleVertexManager.java

License:Apache License

@Override
public void initialize(Configuration conf) {
    this.slowStartMinSrcCompletionFraction = conf.getFloat(
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION,
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION_DEFAULT);
    this.slowStartMaxSrcCompletionFraction = conf.getFloat(
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION,
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT);

    if (slowStartMinSrcCompletionFraction < 0
            || slowStartMaxSrcCompletionFraction < slowStartMinSrcCompletionFraction) {
        throw new IllegalArgumentException("Invalid values for slowStartMinSrcCompletionFraction"
                + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0 and " + "max cannot be < min.");
    }/*from  w w  w .  j  av  a  2 s .c  om*/

    enableAutoParallelism = conf.getBoolean(TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL,
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL_DEFAULT);
    desiredTaskInputDataSize = conf.getLong(
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE,
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE_DEFAULT);
    minTaskParallelism = conf.getInt(TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM,
            TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM_DEFAULT);
    LOG.info("Shuffle Vertex Manager: settings" + " minFrac:" + slowStartMinSrcCompletionFraction + " maxFrac:"
            + slowStartMaxSrcCompletionFraction + " auto:" + enableAutoParallelism + " desiredTaskIput:"
            + desiredTaskInputDataSize + " minTasks:" + minTaskParallelism);
}

From source file:org.apache.tez.dag.app.rm.DagAwareYarnTaskScheduler.java

License:Apache License

void initialize(AMRMClientAsyncWrapper client) throws Exception {
    super.initialize();
    this.client = client;
    Configuration conf = TezUtils.createConfFromUserPayload(getContext().getInitialUserPayload());
    client.init(conf);//w w  w  .  j av a  2  s . c  o m

    int heartbeatIntervalMax = conf.getInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX,
            TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX_DEFAULT);
    client.setHeartbeatInterval(heartbeatIntervalMax);

    shouldReuseContainers = conf.getBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED,
            TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED_DEFAULT);
    reuseRackLocal = conf.getBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED,
            TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED_DEFAULT);
    reuseNonLocal = conf.getBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED,
            TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED_DEFAULT);
    Preconditions.checkArgument(((!reuseRackLocal && !reuseNonLocal) || (reuseRackLocal)),
            "Re-use Rack-Local cannot be disabled if Re-use Non-Local has been" + " enabled");

    reuseNewContainers = shouldReuseContainers
            && conf.getBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NEW_CONTAINERS_ENABLED,
                    TezConfiguration.TEZ_AM_CONTAINER_REUSE_NEW_CONTAINERS_ENABLED_DEFAULT);

    localitySchedulingDelay = conf.getLong(
            TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS,
            TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS_DEFAULT);
    Preconditions.checkArgument(localitySchedulingDelay >= 0, "Locality Scheduling delay should be >=0");

    idleContainerTimeoutMin = conf.getLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS,
            TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS_DEFAULT);
    Preconditions.checkArgument(idleContainerTimeoutMin >= 0 || idleContainerTimeoutMin == -1,
            "Idle container release min timeout should be either -1 or >=0");

    idleContainerTimeoutMax = conf.getLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS,
            TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS_DEFAULT);
    Preconditions.checkArgument(
            idleContainerTimeoutMax >= 0 && idleContainerTimeoutMax >= idleContainerTimeoutMin,
            "Idle container release max timeout should be >=0 and >= "
                    + TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS);

    sessionNumMinHeldContainers = conf.getInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS,
            TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT);
    Preconditions.checkArgument(sessionNumMinHeldContainers >= 0,
            "Session minimum held containers should be >=0");

    preemptionPercentage = conf.getInt(TezConfiguration.TEZ_AM_PREEMPTION_PERCENTAGE,
            TezConfiguration.TEZ_AM_PREEMPTION_PERCENTAGE_DEFAULT);
    Preconditions.checkArgument(preemptionPercentage >= 0 && preemptionPercentage <= 100,
            "Preemption percentage should be between 0-100");

    numHeartbeatsBetweenPreemptions = conf.getInt(
            TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS,
            TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS_DEFAULT);
    Preconditions.checkArgument(numHeartbeatsBetweenPreemptions >= 1,
            "Heartbeats between preemptions should be >=1");

    preemptionMaxWaitTime = conf.getInt(TezConfiguration.TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS,
            TezConfiguration.TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS_DEFAULT);
    Preconditions.checkArgument(preemptionMaxWaitTime >= 0, "Preemption max wait time must be >=0");

    LOG.info("scheduler initialized with maxRMHeartbeatInterval:" + heartbeatIntervalMax + " reuseEnabled:"
            + shouldReuseContainers + " reuseRack:" + reuseRackLocal + " reuseAny:" + reuseNonLocal
            + " localityDelay:" + localitySchedulingDelay + " preemptPercentage:" + preemptionPercentage
            + " preemptMaxWaitTime:" + preemptionMaxWaitTime + " numHeartbeatsBetweenPreemptions:"
            + numHeartbeatsBetweenPreemptions + " idleContainerMinTimeout:" + idleContainerTimeoutMin
            + " idleContainerMaxTimeout:" + idleContainerTimeoutMax + " sessionMinHeldContainers:"
            + sessionNumMinHeldContainers);
}

From source file:org.apache.tez.dag.app.rm.YarnTaskSchedulerService.java

License:Apache License

@Override
public synchronized void serviceInit(Configuration conf) {

    amRmClient.init(conf);/*from  w  ww .  j a  va2s.c  om*/
    int heartbeatIntervalMax = conf.getInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX,
            TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX_DEFAULT);
    amRmClient.setHeartbeatInterval(heartbeatIntervalMax);

    shouldReuseContainers = conf.getBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED,
            TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED_DEFAULT);
    reuseRackLocal = conf.getBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED,
            TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED_DEFAULT);
    reuseNonLocal = conf.getBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED,
            TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED_DEFAULT);
    Preconditions.checkArgument(((!reuseRackLocal && !reuseNonLocal) || (reuseRackLocal)),
            "Re-use Rack-Local cannot be disabled if Re-use Non-Local has been" + " enabled");

    localitySchedulingDelay = conf.getLong(
            TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS,
            TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS_DEFAULT);
    Preconditions.checkArgument(localitySchedulingDelay >= 0, "Locality Scheduling delay should be >=0");

    idleContainerTimeoutMin = conf.getLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS,
            TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS_DEFAULT);
    Preconditions.checkArgument(idleContainerTimeoutMin >= 0 || idleContainerTimeoutMin == -1,
            "Idle container release min timeout should be either -1 or >=0");

    idleContainerTimeoutMax = conf.getLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS,
            TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS_DEFAULT);
    Preconditions.checkArgument(
            idleContainerTimeoutMax >= 0 && idleContainerTimeoutMax >= idleContainerTimeoutMin,
            "Idle container release max timeout should be >=0 and >= "
                    + TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS);

    sessionNumMinHeldContainers = conf.getInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS,
            TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT);
    Preconditions.checkArgument(sessionNumMinHeldContainers >= 0,
            "Session minimum held containers should be >=0");

    preemptionPercentage = conf.getInt(TezConfiguration.TEZ_AM_PREEMPTION_PERCENTAGE,
            TezConfiguration.TEZ_AM_PREEMPTION_PERCENTAGE_DEFAULT);
    Preconditions.checkArgument(preemptionPercentage >= 0 && preemptionPercentage <= 100,
            "Preemption percentage should be between 0-100");

    numHeartbeatsBetweenPreemptions = conf.getInt(
            TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS,
            TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS_DEFAULT);
    Preconditions.checkArgument(numHeartbeatsBetweenPreemptions >= 1,
            "Heartbeats between preemptions should be >=1");

    delayedContainerManager = new DelayedContainerManager();
    LOG.info("TaskScheduler initialized with configuration: " + "maxRMHeartbeatInterval: "
            + heartbeatIntervalMax + ", containerReuseEnabled: " + shouldReuseContainers + ", reuseRackLocal: "
            + reuseRackLocal + ", reuseNonLocal: " + reuseNonLocal + ", localitySchedulingDelay: "
            + localitySchedulingDelay + ", preemptionPercentage: " + preemptionPercentage
            + ", numHeartbeatsBetweenPreemptions" + numHeartbeatsBetweenPreemptions
            + ", idleContainerMinTimeout=" + idleContainerTimeoutMin + ", idleContainerMaxTimeout="
            + idleContainerTimeoutMax + ", sessionMinHeldContainers=" + sessionNumMinHeldContainers);
}

From source file:org.apache.tez.dag.app.speculate.ExponentiallySmoothedTaskRuntimeEstimator.java

License:Apache License

@Override
public void contextualize(Configuration conf, AppContext context) {
    super.contextualize(conf, context);

    lambda = conf.getLong(MRJobConfig.MR_AM_TASK_ESTIMATOR_SMOOTH_LAMBDA_MS,
            MRJobConfig.DEFAULT_MR_AM_TASK_ESTIMATOR_SMOOTH_LAMBDA_MS);
    smoothedValue = conf.getBoolean(MRJobConfig.MR_AM_TASK_ESTIMATOR_EXPONENTIAL_RATE_ENABLE, true)
            ? SmoothedValue.RATE/*w ww.j a v  a2s .  c  om*/
            : SmoothedValue.TIME_PER_UNIT_PROGRESS;
}