Example usage for org.apache.hadoop.yarn.conf YarnConfiguration NM_VCORES

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration NM_VCORES.

Prototype

String NM_VCORES

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration NM_VCORES.

Click Source Link

Document

Number of Virtual CPU Cores which can be allocated for containers.

Usage

From source file:com.cloudera.llama.am.yarn.TestLlamaAMWithYarn.java

License:Apache License

@Test
public void testResourceRejections() throws Exception {
    try {//from w w w  .j  a  v a2  s  .c o m
        Configuration conf = createMiniYarnConfig(true);
        conf.setInt(YarnConfiguration.NM_VCORES, 1);
        conf.setInt(YarnConfiguration.NM_PMEM_MB, 4096);
        conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 2);
        conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 5020);
        startYarn(conf, 1);
        Configuration llamaConf = getLlamaConfiguration();
        llamaConf.setBoolean(LlamaAM.NORMALIZING_ENABLED_KEY, false);
        LlamaAM llama = LlamaAM.create(llamaConf);
        try {
            llama.start();
            List<NodeInfo> nodes = llama.getNodes();

            //invalid node
            try {
                Resource r = TestUtils.createResource("xyz:-1", Resource.Locality.MUST, 1, 4096);
                llama.reserve(TestUtils.createReservation(UUID.randomUUID(), "u", "root.queue1",
                        Arrays.asList(r), true));
                Assert.fail();
            } catch (LlamaException ex) {
                //NOP
            }

            //over max cpus
            try {
                Resource r = TestUtils.createResource(nodes.get(0).getLocation(), Resource.Locality.MUST, 3,
                        4096);
                llama.reserve(TestUtils.createReservation(UUID.randomUUID(), "u", "root.queue1",
                        Arrays.asList(r), true));
                Assert.fail();
            } catch (LlamaException ex) {
                //NOP
            }

            //over max memory
            try {
                Resource r = TestUtils.createResource(nodes.get(0).getLocation(), Resource.Locality.MUST, 1,
                        4097);
                llama.reserve(TestUtils.createReservation(UUID.randomUUID(), "u", "root.queue1",
                        Arrays.asList(r), true));
                Assert.fail();
            } catch (LlamaException ex) {
                //NOP
            }

            //over node cpus
            try {
                Resource r = TestUtils.createResource(nodes.get(0).getLocation(), Resource.Locality.MUST, 2,
                        4096);
                llama.reserve(TestUtils.createReservation(UUID.randomUUID(), "u", "root.queue1",
                        Arrays.asList(r), true));
                Assert.fail();
            } catch (LlamaException ex) {
                //NOP
            }

            //over node memory
            try {
                Resource r = TestUtils.createResource(nodes.get(0).getLocation(), Resource.Locality.MUST, 1,
                        5021);
                llama.reserve(TestUtils.createReservation(UUID.randomUUID(), "u", "root.queue1",
                        Arrays.asList(r), true));
                Assert.fail();
            } catch (LlamaException ex) {
                //NOP
            }

        } finally {
            llama.stop();
        }
    } finally {
        stopYarn();
    }
}

---

From source file:com.cloudera.llama.nm.LlamaNMServer.java

License:Apache License

@Override
protected void startService() {
    int memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB, YarnConfiguration.DEFAULT_NM_PMEM_MB);
    int virtualCores = conf.getInt(YarnConfiguration.NM_VCORES, YarnConfiguration.DEFAULT_NM_VCORES);
    totalCapacity = Resource.newInstance(memoryMb, virtualCores);

    try {//from  w  w  w .  j  av  a2s.  c  o m
        clientNotificationService = new ClientNotificationService(getServerConf(), null, getMetricRegistry());
        clientNotificationService.start();
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}

---

From source file:org.apache.flink.yarn.AbstractYarnClusterDescriptor.java

License:Apache License

private void isReadyForDeployment() throws YarnDeploymentException {
    if (taskManagerCount <= 0) {
        throw new YarnDeploymentException("Taskmanager count must be positive");
    }//  www . j a  v a 2 s  .c  o  m
    if (this.flinkJarPath == null) {
        throw new YarnDeploymentException("The Flink jar path is null");
    }
    if (this.configurationDirectory == null) {
        throw new YarnDeploymentException("Configuration directory not set");
    }
    if (this.flinkConfigurationPath == null) {
        throw new YarnDeploymentException("Configuration path not set");
    }
    if (this.flinkConfiguration == null) {
        throw new YarnDeploymentException("Flink configuration object has not been set");
    }

    // Check if we don't exceed YARN's maximum virtual cores.
    // The number of cores can be configured in the config.
    // If not configured, it is set to the number of task slots
    int numYarnVcores = conf.getInt(YarnConfiguration.NM_VCORES, YarnConfiguration.DEFAULT_NM_VCORES);
    int configuredVcores = flinkConfiguration.getInteger(ConfigConstants.YARN_VCORES, slots);
    // don't configure more than the maximum configured number of vcores
    if (configuredVcores > numYarnVcores) {
        throw new IllegalConfigurationException(String.format(
                "The number of virtual cores per node were configured with %d"
                        + " but Yarn only has %d virtual cores available. Please note that the number"
                        + " of virtual cores is set to the number of task slots by default unless configured"
                        + " in the Flink config with '%s.'",
                configuredVcores, numYarnVcores, ConfigConstants.YARN_VCORES));
    }

    // check if required Hadoop environment variables are set. If not, warn user
    if (System.getenv("HADOOP_CONF_DIR") == null && System.getenv("YARN_CONF_DIR") == null) {
        LOG.warn("Neither the HADOOP_CONF_DIR nor the YARN_CONF_DIR environment variable is set. "
                + "The Flink YARN Client needs one of these to be set to properly load the Hadoop "
                + "configuration for accessing YARN.");
    }
}

---