Example usage for org.apache.hadoop.conf Configuration getDouble

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getDouble.

Prototype

public double getDouble(String name, double defaultValue) 

Source Link

Document

Get the value of the name property as a double.

Usage

From source file:org.apache.tez.dag.app.rm.container.AMContainerHelpers.java

License:Apache License

@VisibleForTesting
public static ContainerLaunchContext createContainerLaunchContext(TezDAGID tezDAGID,
        Map<String, LocalResource> commonDAGLRs, Map<ApplicationAccessType, String> acls,
        ContainerId containerId, Map<String, LocalResource> localResources, Map<String, String> vertexEnv,
        String javaOpts, InetSocketAddress taskAttemptListenerAddress, Credentials credentials,
        AppContext appContext, Resource containerResource, Configuration conf) {

    ContainerLaunchContext commonContainerSpec = null;
    synchronized (commonContainerSpecLock) {
        if (!commonContainerSpecs.containsKey(tezDAGID)) {
            commonContainerSpec = createCommonContainerLaunchContext(acls, credentials, commonDAGLRs);
            commonContainerSpecs.put(tezDAGID, commonContainerSpec);
        } else {/*from   w w w  . j  a  v  a 2s .  c  o m*/
            commonContainerSpec = commonContainerSpecs.get(tezDAGID);
        }

        // Ensure that we remove container specs for previous AMs to reduce
        // memory footprint
        if (lastDAGID == null) {
            lastDAGID = tezDAGID;
        } else if (!lastDAGID.equals(tezDAGID)) {
            commonContainerSpecs.remove(lastDAGID);
            lastDAGID = tezDAGID;
        }
    }

    // Fill in the fields needed per-container that are missing in the common
    // spec.
    Map<String, LocalResource> lResources = new TreeMap<String, LocalResource>();
    lResources.putAll(commonContainerSpec.getLocalResources());
    lResources.putAll(localResources);

    // Setup environment by cloning from common env.
    Map<String, String> env = commonContainerSpec.getEnvironment();
    Map<String, String> myEnv = new HashMap<String, String>(env.size());
    myEnv.putAll(env);
    myEnv.putAll(vertexEnv);

    String modifiedJavaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(javaOpts, containerResource,
            conf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
                    TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));
    if (LOG.isDebugEnabled()) {
        if (!modifiedJavaOpts.equals(javaOpts)) {
            LOG.debug("Modified java opts for container" + ", containerId=" + containerId
                    + ", originalJavaOpts=" + javaOpts + ", modifiedJavaOpts=" + modifiedJavaOpts);
        }
    }

    List<String> commands = TezRuntimeChildJVM.getVMCommand(taskAttemptListenerAddress, containerId.toString(),
            appContext.getApplicationID().toString(), appContext.getApplicationAttemptId().getAttemptId(),
            modifiedJavaOpts);

    // Duplicate the ByteBuffers for access by multiple containers.
    Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();
    for (Entry<String, ByteBuffer> entry : commonContainerSpec.getServiceData().entrySet()) {
        myServiceData.put(entry.getKey(), entry.getValue().duplicate());
    }

    // Construct the actual Container
    ContainerLaunchContext container = ContainerLaunchContext.newInstance(lResources, myEnv, commands,
            myServiceData, commonContainerSpec.getTokens().duplicate(), acls);

    return container;
}

---

From source file:org.apache.tez.dag.app.rm.PrimaryTenantYarnTaskSchedulerService.java

License:Apache License

@Override
public synchronized void serviceInit(Configuration conf) {
    this.vcoresPerTask = conf.getInt(TezConfiguration.TEZ_TASK_RESOURCE_CPU_VCORES,
            TezConfiguration.TEZ_TASK_RESOURCE_CPU_VCORES_DEFAULT);

    this.memoryPerTask = conf.getInt(TezConfiguration.TEZ_TASK_RESOURCE_CPU_VCORES,
            TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT);

    this.probabilisticTypeSelection = conf.getBoolean(TezConfiguration.TEZ_PROBABILISTIC_TYPE_SELECTION,
            TezConfiguration.TEZ_PROBABILISTIC_TYPE_SELECTION_DEFAULT);

    this.lowPreferenceWeight = conf.getDouble(TezConfiguration.TEZ_PROBABILISTIC_LOW_PREFERENCE_WEIGHT,
            TezConfiguration.TEZ_PROBABILISTIC_LOW_PREFERENCE_WEIGHT_DEFAULT);

    this.mediumPreferenceWeight = conf.getDouble(TezConfiguration.TEZ_PROBABILISTIC_MEDIUM_PREFERENCE_WEIGHT,
            TezConfiguration.TEZ_PROBABILISTIC_MEDIUM_PREFERENCE_WEIGHT_DEFAULT);

    this.highPreferenceWeight = conf.getDouble(TezConfiguration.TEZ_PROBABILISTIC_HIGH_PREFERENCE_WEIGHT,
            TezConfiguration.TEZ_PROBABILISTIC_HIGH_PREFERENCE_WEIGHT_DEFAULT);

    this.bestFitScheduling = conf.getBoolean(TezConfiguration.TEZ_BEST_FIT_SCHEDULING,
            TezConfiguration.TEZ_BEST_FIT_SCHEDULING_DEFAULT);

    this.thresholdShortAndMedium = conf.getDouble(TezConfiguration.TEZ_THRESHOLD_SHORT_AND_MEDIUM,
            TezConfiguration.TEZ_THRESHOLD_SHORT_AND_MEDIUM_DEFAULT);

    this.thresholdMediumAndLong = conf.getDouble(TezConfiguration.TEZ_THRESHOLD_MEDIUM_AND_LONG,
            TezConfiguration.TEZ_THRESHOLD_MEDIUM_AND_LONG_DEFAULT);

    this.dagExecutionHistoryEnabled = conf.getBoolean(TezConfiguration.TEZ_DAG_EXECUTION_HISTORY_ENABLED,
            TezConfiguration.TEZ_DAG_EXECUTION_HISTORY_ENABLED_DEFAULT);

    this.dagExecutionHistoryPath = conf.get(TezConfiguration.TEZ_DAG_EXECUTION_HISTORY_PATH,
            TezConfiguration.TEZ_DAG_EXECUTION_HISTORY_PATH_DEFAULT);

    // Build the utilization table
    utilizationTable = new UtilizationTable(probabilisticTypeSelection, lowPreferenceWeight,
            mediumPreferenceWeight, highPreferenceWeight, bestFitScheduling, conf);
    // Build the historical execution database
    if (this.dagExecutionHistoryEnabled) {
        File historyProfile = new File(this.dagExecutionHistoryPath);
        if (historyProfile.exists() && !historyProfile.isDirectory()) {
            try {
                FileInputStream fileStream = new FileInputStream(this.dagExecutionHistoryPath);
                InputStreamReader streamReader = new InputStreamReader(fileStream);
                JsonReader jsonReader = new JsonReader(streamReader);
                JsonParser jsonParser = new JsonParser();
                JsonElement jsonElement = jsonParser.parse(jsonReader);

                Set<Entry<String, JsonElement>> entrySet = ((JsonObject) jsonElement).entrySet();
                for (Entry<String, JsonElement> entry : entrySet) {
                    JsonObject map = entry.getValue().getAsJsonObject();
                    this.dagExecutionHistoryDuration.put(entry.getKey(),
                            map.get("total_duration").getAsDouble());
                    this.dagExecutionHistoryTasks.put(entry.getKey(), map.get("concurrent_tasks").getAsInt());
                }//  w ww .j a va2 s  .c o  m
            } catch (FileNotFoundException e) {
                LOG.warn("Can not find the specified history profile: " + historyProfile);
                this.dagExecutionHistoryEnabled = false;
            } catch (IOException e) {
                LOG.warn("Error reading the specified history profile: " + historyProfile);
                this.dagExecutionHistoryEnabled = false;
            }
        } else {
            LOG.warn("Specified DAG execution history profile does not exist: " + historyProfile);
            this.dagExecutionHistoryEnabled = false;
        }
    }

    super.serviceInit(conf);
}

---

• «
• 1
• 2
• »