Example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM_ADDRESS

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM_ADDRESS.

Prototype

String RM_ADDRESS

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration RM_ADDRESS.

Click Source Link

Document

The address of the applications manager interface in the RM.

Usage

From source file:com.streamsets.datacollector.spark.SparkOnYarnIT.java

License:Apache License

@BeforeClass
public static void setup() throws Exception {
    System.setProperty(MiniSDCTestingUtility.PRESERVE_TEST_DIR, "true");
    miniSDCTestingUtility = new MiniSDCTestingUtility();
    File dataTestDir = miniSDCTestingUtility.getDataTestDir();
    File sparkHome = ClusterUtil.createSparkHome(dataTestDir);

    YarnConfiguration entries = new YarnConfiguration();
    miniYarnCluster = miniSDCTestingUtility.startMiniYarnCluster(TEST_NAME, 1, 1, 1, entries);

    Configuration config = miniYarnCluster.getConfig();
    long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10);
    while (config.get(YarnConfiguration.RM_ADDRESS).split(":")[1] == "0") {
        if (System.currentTimeMillis() > deadline) {
            throw new IllegalStateException("Timed out waiting for RM to come up.");
        }/*from  w  ww .  j a  v a2  s  .  c o  m*/
        LOG.debug("RM address still not set in configuration, waiting...");
        TimeUnit.MILLISECONDS.sleep(100);
    }
    LOG.debug("RM at " + config.get(YarnConfiguration.RM_ADDRESS));

    Properties sparkHadoopProps = new Properties();

    for (Map.Entry<String, String> entry : config) {
        sparkHadoopProps.setProperty("spark.hadoop." + entry.getKey(), entry.getValue());
    }

    LOG.debug("Creating spark properties file at " + dataTestDir);
    File propertiesFile = new File(dataTestDir, "spark.properties");
    propertiesFile.createNewFile();
    FileOutputStream sdcOutStream = new FileOutputStream(propertiesFile);
    sparkHadoopProps.store(sdcOutStream, null);
    sdcOutStream.flush();
    sdcOutStream.close();
    // Need to pass this property file to spark-submit for it pick up yarn confs
    System.setProperty(SPARK_PROPERTY_FILE, propertiesFile.getAbsolutePath());

    URI uri = Resources.getResource("cluster_pipeline.json").toURI();
    pipelineJson = new String(Files.readAllBytes(Paths.get(uri)), StandardCharsets.UTF_8);
    // TODO - Move setup of Kafka in separate class
    setupKafka();

    File sparkBin = new File(sparkHome, "bin");
    for (File file : sparkBin.listFiles()) {
        MiniSDCTestingUtility.setExecutePermission(file.toPath());
    }
}

---

From source file:com.streamsets.datacollector.util.ClusterUtil.java

License:Apache License

public static void setupCluster(String testName, String pipelineJson, YarnConfiguration yarnConfiguration)
        throws Exception {
    System.setProperty("sdc.testing-mode", "true");
    System.setProperty(MiniSDCTestingUtility.PRESERVE_TEST_DIR, "true");
    yarnConfiguration.set("yarn.nodemanager.delete.debug-delay-sec", "600");
    miniSDCTestingUtility = new MiniSDCTestingUtility();
    File dataTestDir = miniSDCTestingUtility.getDataTestDir();

    //copy spark files under the test data directory into a dir called "spark"
    File sparkHome = ClusterUtil.createSparkHome(dataTestDir);

    //start mini yarn cluster
    miniYarnCluster = miniSDCTestingUtility.startMiniYarnCluster(testName, 1, 1, 1, yarnConfiguration);
    Configuration config = miniYarnCluster.getConfig();

    long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10);
    while (config.get(YarnConfiguration.RM_ADDRESS).split(":")[1] == "0") {
        if (System.currentTimeMillis() > deadline) {
            throw new IllegalStateException("Timed out waiting for RM to come up.");
        }//from  w  w  w .j  a  va  2 s .com
        LOG.debug("RM address still not set in configuration, waiting...");
        TimeUnit.MILLISECONDS.sleep(100);
    }
    LOG.debug("RM at " + config.get(YarnConfiguration.RM_ADDRESS));

    Properties sparkHadoopProps = new Properties();
    for (Map.Entry<String, String> entry : config) {
        sparkHadoopProps.setProperty("spark.hadoop." + entry.getKey(), entry.getValue());
    }

    LOG.debug("Creating spark properties file at " + dataTestDir);
    File propertiesFile = new File(dataTestDir, "spark.properties");
    propertiesFile.createNewFile();
    FileOutputStream sdcOutStream = new FileOutputStream(propertiesFile);
    sparkHadoopProps.store(sdcOutStream, null);
    sdcOutStream.flush();
    sdcOutStream.close();
    // Need to pass this property file to spark-submit for it pick up yarn confs
    System.setProperty(SPARK_PROPERTY_FILE, propertiesFile.getAbsolutePath());

    File sparkBin = new File(sparkHome, "bin");
    for (File file : sparkBin.listFiles()) {
        MiniSDCTestingUtility.setExecutePermission(file.toPath());
    }

    miniSDC = miniSDCTestingUtility.createMiniSDC(MiniSDC.ExecutionMode.CLUSTER);
    miniSDC.startSDC();
    serverURI = miniSDC.getServerURI();
    miniSDC.createPipeline(pipelineJson);
    miniSDC.startPipeline();

    int attempt = 0;
    //Hard wait for 2 minutes
    while (miniSDC.getListOfSlaveSDCURI().size() == 0 && attempt < 24) {
        Thread.sleep(5000);
        attempt++;
        LOG.debug("Attempt no: " + attempt + " to retrieve list of slaves");
    }
    if (miniSDC.getListOfSlaveSDCURI().size() == 0) {
        throw new IllegalStateException("Timed out waiting for slaves to come up.");
    }
}

---

From source file:com.twitter.hraven.hadoopJobMonitor.ClusterStatusChecker.java

License:Apache License

/**
 * Get the app list from RM and check status of each
 *//* w w w. j a  v  a2s  . co m*/
@Override
public void run() {
    // 1. get the list of running apps
    LOG.info("Running " + ClusterStatusChecker.class.getName());
    try {
        YarnConfiguration yConf = new YarnConfiguration();
        LOG.info(yConf.get(YarnConfiguration.RM_ADDRESS));
        LOG.info("Getting appList ...");
        // TODO: in future hadoop API we will be able to filter the app list
        EnumSet<YarnApplicationState> states = EnumSet.of(YarnApplicationState.RUNNING);
        List<ApplicationReport> appList = rmDelegate.getApplications(states);
        LOG.info("appList received. size is: " + appList.size());
        for (ApplicationReport appReport : appList)
            checkAppStatus(appReport);
    } catch (YarnRuntimeException e) {
        LOG.error("Error in getting application list from RM", e);
    } catch (YarnException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}

---

From source file:edu.uci.ics.hyracks.yarn.common.protocols.clientrm.YarnClientRMConnection.java

License:Apache License

public YarnClientRMConnection(YarnConfiguration config) {
    this.config = config;
    InetSocketAddress remoteAddress = NetUtils
            .createSocketAddr(config.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));
    Configuration appsManagerServerConf = new Configuration(config);
    appsManagerServerConf.setClass(YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_CLIENT_RESOURCEMANAGER,
            ClientRMSecurityInfo.class, SecurityInfo.class);
    YarnRPC rpc = YarnRPC.create(appsManagerServerConf);
    crmp = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, remoteAddress, appsManagerServerConf));
}

---

From source file:gobblin.yarn.GobblinYarnAppLauncherTest.java

License:Apache License

@BeforeClass
public void setUp() throws Exception {
    // Set java home in environment since it isn't set on some systems
    String javaHome = System.getProperty("java.home");
    setEnv("JAVA_HOME", javaHome);

    final YarnConfiguration clusterConf = new YarnConfiguration();
    clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000");

    MiniYARNCluster miniYARNCluster = this.closer.register(new MiniYARNCluster("TestCluster", 1, 1, 1));
    miniYARNCluster.init(clusterConf);/*  w  w w  . j  a  v a 2 s . c  om*/
    miniYARNCluster.start();

    // YARN client should not be started before the Resource Manager is up
    AssertWithBackoff.create().logger(LOG).timeoutMs(10000).assertTrue(new Predicate<Void>() {
        @Override
        public boolean apply(Void input) {
            return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
        }
    }, "Waiting for RM");

    this.yarnClient = this.closer.register(YarnClient.createYarnClient());
    this.yarnClient.init(clusterConf);
    this.yarnClient.start();

    // Use a random ZK port
    TestingServer testingZKServer = this.closer.register(new TestingServer(-1));
    LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString());

    // the zk port is dynamically configured
    try (PrintWriter pw = new PrintWriter("dynamic.conf")) {
        File dir = new File("target/dummydir");

        // dummy directory specified in configuration
        dir.mkdir();

        pw.println("gobblin.cluster.zk.connection.string=\"" + testingZKServer.getConnectString() + "\"");
        pw.println("jobconf.fullyQualifiedPath=\"" + dir.getAbsolutePath() + "\"");
    }

    // YARN config is dynamic and needs to be passed to other processes
    try (OutputStream os = new FileOutputStream(new File("yarn-site.xml"))) {
        clusterConf.writeXml(os);
    }

    this.curatorFramework = TestHelper.createZkClient(testingZKServer, this.closer);

    URL url = GobblinYarnAppLauncherTest.class.getClassLoader()
            .getResource(GobblinYarnAppLauncherTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);

    this.config = ConfigFactory.parseURL(url).withValue("gobblin.cluster.zk.connection.string",
            ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())).resolve();

    String zkConnectionString = this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY);
    this.helixManager = HelixManagerFactory.getZKHelixManager(
            this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY),
            TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectionString);

    this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(this.config, clusterConf);
}

---

From source file:org.apache.apex.engine.security.TokenRenewer.java

License:Apache License

public TokenRenewer(Context context, boolean renewRMToken, Configuration conf, String destinationFile)
        throws IOException {
    this.renewRMToken = renewRMToken;
    this.destinationFile = destinationFile;
    this.conf = conf;

    if (renewRMToken) {
        tokenLifeTime = (long) (context.getValue(LogicalPlan.TOKEN_REFRESH_ANTICIPATORY_FACTOR)
                * Math.min(context.getValue(LogicalPlan.HDFS_TOKEN_LIFE_TIME),
                        context.getValue(LogicalPlan.RM_TOKEN_LIFE_TIME)));
        tokenRenewalInterval = (long) (context.getValue(LogicalPlan.TOKEN_REFRESH_ANTICIPATORY_FACTOR)
                * Math.min(context.getValue(LogicalPlan.HDFS_TOKEN_RENEWAL_INTERVAL),
                        context.getValue(LogicalPlan.RM_TOKEN_RENEWAL_INTERVAL)));
        rmAddress = conf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS,
                YarnConfiguration.DEFAULT_RM_PORT);
    } else {/*w ww  .  j a v  a 2  s  . c  o  m*/
        tokenLifeTime = (long) (context.getValue(LogicalPlan.TOKEN_REFRESH_ANTICIPATORY_FACTOR)
                * context.getValue(LogicalPlan.HDFS_TOKEN_LIFE_TIME));
        tokenRenewalInterval = (long) (context.getValue(LogicalPlan.TOKEN_REFRESH_ANTICIPATORY_FACTOR)
                * context.getValue(LogicalPlan.HDFS_TOKEN_RENEWAL_INTERVAL));
    }

    principal = context.getValue(LogicalPlan.PRINCIPAL);
    hdfsKeyTabFile = context.getValue(LogicalPlan.KEY_TAB_FILE);

    expiryTime = System.currentTimeMillis() + tokenLifeTime;
    renewTime = expiryTime;

    logger.debug("token life time {} renewal interval {}", tokenLifeTime, tokenRenewalInterval);
    logger.debug("Token expiry time {} renew time {}", expiryTime, renewTime);

    credentials = UserGroupInformation.getCurrentUser().getCredentials();
    // Check credentials are proper at RM
    if (renewRMToken) {
        renewTokens(false, true);
    }
}

---

From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncherTest.java

License:Apache License

@BeforeClass
public void setUp() throws Exception {
    // Set java home in environment since it isn't set on some systems
    String javaHome = System.getProperty("java.home");
    setEnv("JAVA_HOME", javaHome);

    final YarnConfiguration clusterConf = new YarnConfiguration();
    clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000");

    MiniYARNCluster miniYARNCluster = this.closer.register(new MiniYARNCluster("TestCluster", 1, 1, 1));
    miniYARNCluster.init(clusterConf);/*from   w  ww. j  a v a 2s  .c  om*/
    miniYARNCluster.start();

    // YARN client should not be started before the Resource Manager is up
    AssertWithBackoff.create().logger(LOG).timeoutMs(10000).assertTrue(new Predicate<Void>() {
        @Override
        public boolean apply(Void input) {
            return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
        }
    }, "Waiting for RM");

    this.yarnClient = this.closer.register(YarnClient.createYarnClient());
    this.yarnClient.init(clusterConf);
    this.yarnClient.start();

    // Use a random ZK port
    TestingServer testingZKServer = this.closer.register(new TestingServer(-1));
    LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString());

    // the zk port is dynamically configured
    try (PrintWriter pw = new PrintWriter(DYNAMIC_CONF_PATH)) {
        File dir = new File("target/dummydir");

        // dummy directory specified in configuration
        dir.mkdir();

        pw.println("gobblin.cluster.zk.connection.string=\"" + testingZKServer.getConnectString() + "\"");
        pw.println("jobconf.fullyQualifiedPath=\"" + dir.getAbsolutePath() + "\"");
    }

    // YARN config is dynamic and needs to be passed to other processes
    try (OutputStream os = new FileOutputStream(new File(YARN_SITE_XML_PATH))) {
        clusterConf.writeXml(os);
    }

    this.curatorFramework = TestHelper.createZkClient(testingZKServer, this.closer);

    URL url = GobblinYarnAppLauncherTest.class.getClassLoader()
            .getResource(GobblinYarnAppLauncherTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);

    this.config = ConfigFactory.parseURL(url).withValue("gobblin.cluster.zk.connection.string",
            ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())).resolve();

    String zkConnectionString = this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY);
    this.helixManager = HelixManagerFactory.getZKHelixManager(
            this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY),
            TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectionString);

    this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(this.config, clusterConf);
}

---

From source file:org.apache.gobblin.yarn.YarnServiceTest.java

License:Apache License

@BeforeClass
public void setUp() throws Exception {
    // Set java home in environment since it isn't set on some systems
    String javaHome = System.getProperty("java.home");
    setEnv("JAVA_HOME", javaHome);

    this.clusterConf = new YarnConfiguration();
    this.clusterConf.set(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, "100");
    this.clusterConf.set(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, "10000");
    this.clusterConf.set(YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS, "60000");

    this.yarnCluster = this.closer.register(new MiniYARNCluster("YarnServiceTestCluster", 4, 1, 1));
    this.yarnCluster.init(this.clusterConf);
    this.yarnCluster.start();

    // YARN client should not be started before the Resource Manager is up
    AssertWithBackoff.create().logger(LOG).timeoutMs(10000).assertTrue(new Predicate<Void>() {
        @Override/*from   w  w w.  ja v  a  2 s  .  co m*/
        public boolean apply(Void input) {
            return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
        }
    }, "Waiting for RM");

    this.yarnClient = this.closer.register(YarnClient.createYarnClient());
    this.yarnClient.init(this.clusterConf);
    this.yarnClient.start();

    URL url = YarnServiceTest.class.getClassLoader()
            .getResource(YarnServiceTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);

    this.config = ConfigFactory.parseURL(url).resolve();

    // Start a dummy application manager so that the YarnService can use the AM-RM token.
    startApp();

    // create and start the test yarn service
    this.yarnService = new TestYarnService(this.config, "testApp", "appId", this.clusterConf,
            FileSystem.getLocal(new Configuration()), this.eventBus);

    this.yarnService.startUp();
}

---

From source file:org.apache.hama.bsp.YARNBSPJob.java

License:Apache License

public YARNBSPJob(HamaConfiguration conf) throws IOException {
    submitClient = new YARNBSPJobClient(conf);
    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    this.rpc = YarnRPC.create(conf);
    InetSocketAddress rmAddress = NetUtils
            .createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));
    LOG.info("Connecting to ResourceManager at " + rmAddress);

    this.applicationsManager = ((ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class,
            rmAddress, conf));/* w  w w  .  j  av a 2 s  .  co  m*/
}

---

From source file:org.apache.hoya.tools.HoyaUtils.java

License:Apache License

public static InetSocketAddress getRmAddress(Configuration conf) {
    return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS,
            YarnConfiguration.DEFAULT_RM_PORT);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• »