Example usage for org.apache.hadoop.yarn.conf YarnConfiguration set

List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration set

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.conf.*
  5. YarnConfiguration
  6. set

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration set.

Prototype

public void set(String name, String value)

Source Link

Document

Set the value of the name property.

Usage

From source file:com.streamsets.datacollector.util.ClusterUtil.java

License:Apache License

public static void setupCluster(String testName, String pipelineJson, YarnConfiguration yarnConfiguration)
        throws Exception {
    System.setProperty("sdc.testing-mode", "true");
    System.setProperty(MiniSDCTestingUtility.PRESERVE_TEST_DIR, "true");
    yarnConfiguration.set("yarn.nodemanager.delete.debug-delay-sec", "600");
    miniSDCTestingUtility = new MiniSDCTestingUtility();
    File dataTestDir = miniSDCTestingUtility.getDataTestDir();

    //copy spark files under the test data directory into a dir called "spark"
    File sparkHome = ClusterUtil.createSparkHome(dataTestDir);

    //start mini yarn cluster
    miniYarnCluster = miniSDCTestingUtility.startMiniYarnCluster(testName, 1, 1, 1, yarnConfiguration);
    Configuration config = miniYarnCluster.getConfig();

    long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10);
    while (config.get(YarnConfiguration.RM_ADDRESS).split(":")[1] == "0") {
        if (System.currentTimeMillis() > deadline) {
            throw new IllegalStateException("Timed out waiting for RM to come up.");
        }/*from  ww  w .j  ava  2s.  c o  m*/
        LOG.debug("RM address still not set in configuration, waiting...");
        TimeUnit.MILLISECONDS.sleep(100);
    }
    LOG.debug("RM at " + config.get(YarnConfiguration.RM_ADDRESS));

    Properties sparkHadoopProps = new Properties();
    for (Map.Entry<String, String> entry : config) {
        sparkHadoopProps.setProperty("spark.hadoop." + entry.getKey(), entry.getValue());
    }

    LOG.debug("Creating spark properties file at " + dataTestDir);
    File propertiesFile = new File(dataTestDir, "spark.properties");
    propertiesFile.createNewFile();
    FileOutputStream sdcOutStream = new FileOutputStream(propertiesFile);
    sparkHadoopProps.store(sdcOutStream, null);
    sdcOutStream.flush();
    sdcOutStream.close();
    // Need to pass this property file to spark-submit for it pick up yarn confs
    System.setProperty(SPARK_PROPERTY_FILE, propertiesFile.getAbsolutePath());

    File sparkBin = new File(sparkHome, "bin");
    for (File file : sparkBin.listFiles()) {
        MiniSDCTestingUtility.setExecutePermission(file.toPath());
    }

    miniSDC = miniSDCTestingUtility.createMiniSDC(MiniSDC.ExecutionMode.CLUSTER);
    miniSDC.startSDC();
    serverURI = miniSDC.getServerURI();
    miniSDC.createPipeline(pipelineJson);
    miniSDC.startPipeline();

    int attempt = 0;
    //Hard wait for 2 minutes
    while (miniSDC.getListOfSlaveSDCURI().size() == 0 && attempt < 24) {
        Thread.sleep(5000);
        attempt++;
        LOG.debug("Attempt no: " + attempt + " to retrieve list of slaves");
    }
    if (miniSDC.getListOfSlaveSDCURI().size() == 0) {
        throw new IllegalStateException("Timed out waiting for slaves to come up.");
    }
}

From source file:gobblin.yarn.GobblinApplicationMasterTest.java

License:Open Source License

@BeforeClass
public void setUp() throws Exception {
    this.testingZKServer = new TestingServer(TEST_ZK_PORT);

    URL url = GobblinApplicationMasterTest.class.getClassLoader()
            .getResource(GobblinApplicationMasterTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);

    Config config = ConfigFactory.parseURL(url).resolve();

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set(YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES, "1");

    String zkConnectionString = config.getString(GobblinYarnConfigurationKeys.ZK_CONNECTION_STRING_KEY);
    YarnHelixUtils.createGobblinYarnHelixCluster(zkConnectionString,
            config.getString(GobblinYarnConfigurationKeys.HELIX_CLUSTER_NAME_KEY));

    this.helixManager = HelixManagerFactory.getZKHelixManager(
            config.getString(GobblinYarnConfigurationKeys.HELIX_CLUSTER_NAME_KEY),
            TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.PARTICIPANT, zkConnectionString);
    this.helixManager.connect();
    this.helixManager.getMessagingService().registerMessageHandlerFactory(
            Message.MessageType.SHUTDOWN.toString(), new TestShutdownMessageHandlerFactory(this));

    this.gobblinApplicationMaster = new GobblinApplicationMaster(TestHelper.TEST_APPLICATION_NAME,
            ConverterUtils.toContainerId(TestHelper.TEST_CONTROLLER_CONTAINER_ID), config, yarnConfiguration);
    this.gobblinApplicationMaster.getEventBus().register(this.gobblinApplicationMaster);
    this.gobblinApplicationMaster.connectHelixManager();
}

From source file:gobblin.yarn.GobblinYarnAppLauncherTest.java

License:Apache License

@BeforeClass
public void setUp() throws Exception {
    // Set java home in environment since it isn't set on some systems
    String javaHome = System.getProperty("java.home");
    setEnv("JAVA_HOME", javaHome);

    final YarnConfiguration clusterConf = new YarnConfiguration();
    clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000");

    MiniYARNCluster miniYARNCluster = this.closer.register(new MiniYARNCluster("TestCluster", 1, 1, 1));
    miniYARNCluster.init(clusterConf);/*from  w ww  .j a  v a2 s .co  m*/
    miniYARNCluster.start();

    // YARN client should not be started before the Resource Manager is up
    AssertWithBackoff.create().logger(LOG).timeoutMs(10000).assertTrue(new Predicate<Void>() {
        @Override
        public boolean apply(Void input) {
            return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
        }
    }, "Waiting for RM");

    this.yarnClient = this.closer.register(YarnClient.createYarnClient());
    this.yarnClient.init(clusterConf);
    this.yarnClient.start();

    // Use a random ZK port
    TestingServer testingZKServer = this.closer.register(new TestingServer(-1));
    LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString());

    // the zk port is dynamically configured
    try (PrintWriter pw = new PrintWriter("dynamic.conf")) {
        File dir = new File("target/dummydir");

        // dummy directory specified in configuration
        dir.mkdir();

        pw.println("gobblin.cluster.zk.connection.string=\"" + testingZKServer.getConnectString() + "\"");
        pw.println("jobconf.fullyQualifiedPath=\"" + dir.getAbsolutePath() + "\"");
    }

    // YARN config is dynamic and needs to be passed to other processes
    try (OutputStream os = new FileOutputStream(new File("yarn-site.xml"))) {
        clusterConf.writeXml(os);
    }

    this.curatorFramework = TestHelper.createZkClient(testingZKServer, this.closer);

    URL url = GobblinYarnAppLauncherTest.class.getClassLoader()
            .getResource(GobblinYarnAppLauncherTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);

    this.config = ConfigFactory.parseURL(url).withValue("gobblin.cluster.zk.connection.string",
            ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())).resolve();

    String zkConnectionString = this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY);
    this.helixManager = HelixManagerFactory.getZKHelixManager(
            this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY),
            TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectionString);

    this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(this.config, clusterConf);
}

From source file:org.apache.ambari.view.slider.SliderAppsViewControllerImpl.java

License:Apache License

/**
 * Dynamically determines Slider client configuration. If unable to determine,
 * <code>null</code> is returned.
 * /*from w ww.ja  v a  2 s . c o m*/
 * @return
 */
private Configuration getSliderClientConfiguration() {
    HdfsConfiguration hdfsConfig = new HdfsConfiguration();
    YarnConfiguration yarnConfig = new YarnConfiguration(hdfsConfig);

    Map<String, String> hadoopConfigs = getHadoopConfigs();
    for (Entry<String, String> entry : hadoopConfigs.entrySet()) {
        String entryValue = entry.getValue();
        if (entryValue == null) {
            entryValue = "";
        }
        yarnConfig.set(entry.getKey(), entryValue);
    }
    yarnConfig.set(PROPERTY_SLIDER_SECURITY_ENABLED, hadoopConfigs.get("security_enabled"));
    if (hadoopConfigs.containsKey(PROPERTY_SLIDER_ZK_QUORUM)) {
        yarnConfig.set(PROPERTY_SLIDER_ZK_QUORUM, hadoopConfigs.get(PROPERTY_SLIDER_ZK_QUORUM));
    }
    return yarnConfig;
}

From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncherTest.java

License:Apache License

@BeforeClass
public void setUp() throws Exception {
    // Set java home in environment since it isn't set on some systems
    String javaHome = System.getProperty("java.home");
    setEnv("JAVA_HOME", javaHome);

    final YarnConfiguration clusterConf = new YarnConfiguration();
    clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000");

    MiniYARNCluster miniYARNCluster = this.closer.register(new MiniYARNCluster("TestCluster", 1, 1, 1));
    miniYARNCluster.init(clusterConf);//from w  ww . ja  v a 2 s  . co  m
    miniYARNCluster.start();

    // YARN client should not be started before the Resource Manager is up
    AssertWithBackoff.create().logger(LOG).timeoutMs(10000).assertTrue(new Predicate<Void>() {
        @Override
        public boolean apply(Void input) {
            return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
        }
    }, "Waiting for RM");

    this.yarnClient = this.closer.register(YarnClient.createYarnClient());
    this.yarnClient.init(clusterConf);
    this.yarnClient.start();

    // Use a random ZK port
    TestingServer testingZKServer = this.closer.register(new TestingServer(-1));
    LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString());

    // the zk port is dynamically configured
    try (PrintWriter pw = new PrintWriter(DYNAMIC_CONF_PATH)) {
        File dir = new File("target/dummydir");

        // dummy directory specified in configuration
        dir.mkdir();

        pw.println("gobblin.cluster.zk.connection.string=\"" + testingZKServer.getConnectString() + "\"");
        pw.println("jobconf.fullyQualifiedPath=\"" + dir.getAbsolutePath() + "\"");
    }

    // YARN config is dynamic and needs to be passed to other processes
    try (OutputStream os = new FileOutputStream(new File(YARN_SITE_XML_PATH))) {
        clusterConf.writeXml(os);
    }

    this.curatorFramework = TestHelper.createZkClient(testingZKServer, this.closer);

    URL url = GobblinYarnAppLauncherTest.class.getClassLoader()
            .getResource(GobblinYarnAppLauncherTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);

    this.config = ConfigFactory.parseURL(url).withValue("gobblin.cluster.zk.connection.string",
            ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())).resolve();

    String zkConnectionString = this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY);
    this.helixManager = HelixManagerFactory.getZKHelixManager(
            this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY),
            TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectionString);

    this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(this.config, clusterConf);
}

From source file:org.apache.samza.autoscaling.utils.YarnUtil.java

License:Apache License

public YarnUtil(String rmAddress, int rmPort) {
    this.httpclient = HttpClientBuilder.create().build();
    this.rmServer = new HttpHost(rmAddress, rmPort, "http");
    log.info("setting rm server to : " + rmServer);
    YarnConfiguration hConfig = new YarnConfiguration();
    hConfig.set(YarnConfiguration.RM_ADDRESS, rmAddress + ":" + YarnConfiguration.DEFAULT_RM_PORT);
    yarnClient = YarnClient.createYarnClient();
    yarnClient.init(hConfig);/*from  w w  w .  j a v  a2 s.co m*/
    yarnClient.start();
}

From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java

License:Apache License

@Test
public void testResourceMapSuccess() {

    Map<String, String> configMap = new HashMap<>();

    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");
    configMap.put("yarn.resources.myResource1.local.name", "readme");
    configMap.put("yarn.resources.myResource1.local.type", "file");
    configMap.put("yarn.resources.myResource1.local.visibility", "public");

    configMap.put("yarn.resources.myResource2.path", "https://host2.com/package");
    configMap.put("yarn.resources.myResource2.local.name", "__package");
    configMap.put("yarn.resources.myResource2.local.type", "archive");
    configMap.put("yarn.resources.myResource2.local.visibility", "private");

    configMap.put("yarn.resources.myResource3.path", "https://host3.com/csr");
    configMap.put("yarn.resources.myResource3.local.name", "csr");
    configMap.put("yarn.resources.myResource3.local.type", "file");
    configMap.put("yarn.resources.myResource3.local.visibility", "application");

    configMap.put("otherconfig", "https://host4.com/not_included");
    configMap.put("yarn.resources.myResource4.local.name", "notExisting");
    configMap.put("yarn.resources.myResource4.local.type", "file");
    configMap.put("yarn.resources.myResource4.local.visibility", "application");

    Config conf = new MapConfig(configMap);

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());
    yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName());

    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf),
            yarnConfiguration);/*  w  w w .  j a v a2 s.  c o  m*/
    Map<String, LocalResource> resourceMap = mapper.getResourceMap();

    assertEquals("resourceMap has 3 resources", 3, resourceMap.size());

    // resource1
    assertEquals("host1.com", resourceMap.get("readme").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("readme").getType());
    assertEquals(LocalResourceVisibility.PUBLIC, resourceMap.get("readme").getVisibility());

    // resource 2
    assertEquals("host2.com", resourceMap.get("__package").getResource().getHost());
    assertEquals(LocalResourceType.ARCHIVE, resourceMap.get("__package").getType());
    assertEquals(LocalResourceVisibility.PRIVATE, resourceMap.get("__package").getVisibility());

    // resource 3
    assertEquals("host3.com", resourceMap.get("csr").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("csr").getType());
    assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("csr").getVisibility());

    // resource 4 should not exist
    assertNull("Resource does not exist with the name myResource4", resourceMap.get("myResource4"));
    assertNull("Resource does not exist with the defined config name notExisting for myResource4 either",
            resourceMap.get("notExisting"));
}

From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java

License:Apache License

@Test
public void testResourceMapWithDefaultValues() {

    Map<String, String> configMap = new HashMap<>();

    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");

    Config conf = new MapConfig(configMap);

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf),
            yarnConfiguration);//from w  ww  . jav  a 2 s  .c o  m
    Map<String, LocalResource> resourceMap = mapper.getResourceMap();

    assertNull("Resource does not exist with a name readme", resourceMap.get("readme"));
    assertNotNull("Resource exists with a name myResource1", resourceMap.get("myResource1"));
    assertEquals("host1.com", resourceMap.get("myResource1").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("myResource1").getType());
    assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("myResource1").getVisibility());
}

From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java

License:Apache License

@Test
public void testResourceMapWithFileStatusFailure() {
    thrown.expect(LocalizerResourceException.class);
    thrown.expectMessage("IO Exception when accessing the resource file status from the filesystem");

    Map<String, String> configMap = new HashMap<>();
    configMap.put("yarn.resources.myResource1.path", "unknown://host1.com/readme");
    configMap.put("yarn.resources.myResource1.local.name", "readme");
    configMap.put("yarn.resources.myResource1.local.type", "file");
    configMap.put("yarn.resources.myResource1.local.visibility", "public");
    Config conf = new MapConfig(configMap);

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());
    yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName());
    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf),
            yarnConfiguration);/*from w  w  w . j a v  a 2 s .com*/
}

From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java

License:Apache License

@Test
public void testResourceMapWithInvalidVisibilityFailure() {
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage(//from  ww w .  ja  va2s .  c o m
            "No enum constant org.apache.hadoop.yarn.api.records.LocalResourceVisibility.INVALIDVISIBILITY");

    Map<String, String> configMap = new HashMap<>();
    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");
    configMap.put("yarn.resources.myResource1.local.name", "readme");
    configMap.put("yarn.resources.myResource1.local.type", "file");
    configMap.put("yarn.resources.myResource1.local.visibility", "invalidVisibility");
    Config conf = new MapConfig(configMap);

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());
    yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName());
    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf),
            yarnConfiguration);
}