Example usage for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_NM_PMEM_MB

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_NM_PMEM_MB.

Prototype

int DEFAULT_NM_PMEM_MB

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_NM_PMEM_MB.

Click Source Link

Usage

From source file:com.cloudera.llama.am.mock.MockRMConnector.java

License:Apache License

@Override
public List<NodeInfo> getNodes() throws LlamaException {
    String[] nodeLocations = getConf().getStrings(NODES_KEY, NODES_DEFAULT);
    List<NodeInfo> nodes = new ArrayList<NodeInfo>(nodeLocations.length);
    for (String node : nodeLocations) {
        nodes.add(new NodeInfo(node, (short) YarnConfiguration.DEFAULT_NM_VCORES,
                YarnConfiguration.DEFAULT_NM_PMEM_MB));
    }//  w w w .ja v a  2  s . c  o m
    return nodes;
}

---

From source file:com.cloudera.llama.nm.LlamaNMServer.java

License:Apache License

@Override
protected void startService() {
    int memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB, YarnConfiguration.DEFAULT_NM_PMEM_MB);
    int virtualCores = conf.getInt(YarnConfiguration.NM_VCORES, YarnConfiguration.DEFAULT_NM_VCORES);
    totalCapacity = Resource.newInstance(memoryMb, virtualCores);

    try {/*from   w  ww  . j a va 2  s .  co  m*/
        clientNotificationService = new ClientNotificationService(getServerConf(), null, getMetricRegistry());
        clientNotificationService.start();
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}

---

From source file:de.huberlin.wbi.hiway.am.WorkflowDriver.java

License:Apache License

/**
 * Main run function for the application master. Does more initialization (sic!).
 * Calls the abstract {@link #parseWorkflow()}, then {@link #executeWorkflow()} and finally {@link #finish()}.
 * @return True if there were no errors//from w w w  .j  a  v  a2s.  com
 */
protected boolean run() throws IOException {
    /* log */ Logger.writeToStdout("Starting ApplicationMaster");

    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();

    try (DataOutputBuffer dob = new DataOutputBuffer()) {

        credentials.writeTokenStorageToStream(dob);
        // remove the AM->RM token so that containers cannot access it.
        Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
        while (iter.hasNext()) {
            Token<?> token = iter.next();
            if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
                iter.remove();
            }
        }
        allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

        // Resource Manager communications setup
        RMCallbackHandler allocListener = new RMCallbackHandler(this);
        amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
        amRMClient.init(conf);
        amRMClient.start();

        // Node Managers communications setup
        containerListener = new NMCallbackHandler(this);
        nmClientAsync = new NMClientAsyncImpl(containerListener);
        nmClientAsync.init(conf);
        nmClientAsync.start();

        // get workflow file
        if (hdfs.exists(workflowPath)) {
            Path localPath = new Path(workflowPath.getName());
            hdfs.copyToLocalFile(false, workflowPath, localPath);
            workflowPath = localPath;
            workflowFile = new Data(workflowPath);
            workflowFile.stageOut();
        } else {
            // TODO this doesn't work; the path is triggered when running the application e.g., as hiway workflows/test.dax
            // but stageIn then fails, because in the HDFS, there is only test.dax and not workflows/test.dax
            workflowFile = new Data(workflowPath);
            workflowFile.stageIn();
        }

        // Register self with ResourceManager. This will start heartbeating to the RM.
        /* the hostname of the container running the Hi-WAY ApplicationMaster */
        String appMasterHostname = NetUtils.getHostname();
        /* the port on which the ApplicationMaster listens for status updates from clients */
        int appMasterRpcPort = -1;
        /* the tracking URL to which the ApplicationMaster publishes info for clients to monitor */
        String appMasterTrackingUrl = "";
        RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname,
                appMasterRpcPort, appMasterTrackingUrl);

        // initialize scheduler
        switch (schedulerEnumValue) {
        case roundRobin:
        case heft:
            int workerMemory = conf.getInt(YarnConfiguration.NM_PMEM_MB, YarnConfiguration.DEFAULT_NM_PMEM_MB);
            scheduler = schedulerEnumValue.equals(HiWayConfiguration.HIWAY_SCHEDULERS.roundRobin)
                    ? new RoundRobin(getWorkflowName())
                    : new HEFT(getWorkflowName(), workerMemory / containerMemory);
            break;
        case greedy:
            scheduler = new GreedyQueue(getWorkflowName());
            break;
        case memoryAware:
            scheduler = new MemoryAware(getWorkflowName(), amRMClient);
            break;
        case perfectDaxGQ:
            scheduler = new PerfectDaxGreedyQueue(getWorkflowName());
            break;
        default:
            C3PO c3po = new C3PO(getWorkflowName());
            switch (schedulerEnumValue) {
            case dataAware:
                c3po.setConservatismWeight(0.01d);
                c3po.setnClones(0);
                c3po.setPlacementAwarenessWeight(12d);
                c3po.setOutlookWeight(0.01d);
                break;
            default:
                c3po.setConservatismWeight(3d);
                c3po.setnClones(2);
                c3po.setPlacementAwarenessWeight(1d);
                c3po.setOutlookWeight(2d);
            }
            scheduler = c3po;
        }
        scheduler.init(conf, hdfs, containerMemory, customMemoryMap, containerCores, requestPriority);
        scheduler.initializeProvenanceManager();

        /* log */ logger.writeEntryToLog(new JsonReportEntry(getRunId(), null, null, null, null, null,
                HiwayDBI.KEY_WF_NAME, getWorkflowName()));
        logger.federatedReport = new Data(appId + ".log");

        // parse workflow, obtain ready tasks
        Collection<TaskInstance> readyTasks = parseWorkflow();

        // scheduler updates runtime estimates for all tasks comprising the workflow
        scheduler.updateRuntimeEstimates(getRunId().toString());

        scheduler.addTasks(readyTasks);

        // Dump out information about cluster capability as seen by the resource manager
        maxMem = response.getMaximumResourceCapability().getMemory();
        maxCores = response.getMaximumResourceCapability().getVirtualCores();
        /* log */ Logger.writeToStdout("Max mem capabililty of resources in this cluster " + maxMem);

        // A resource ask cannot exceed the max.
        if (containerMemory > maxMem) {
            /* log */ Logger.writeToStdout("Container memory specified above max threshold of cluster."
                    + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem);
            containerMemory = maxMem;
        }
        if (containerCores > maxCores) {
            /* log */ Logger.writeToStdout("Container vcores specified above max threshold of cluster."
                    + " Using max value." + ", specified=" + containerCores + ", max=" + maxCores);
            containerCores = maxCores;
        }

        // this is the actual work loop:
        // ask for resources until the workflow is done.
        executeWorkflow();

        finish();

    } catch (Exception e) {
        e.printStackTrace(System.out);
        System.exit(-1);
    }
    return success;
}

---