Example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM_ADDRESS

List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration RM_ADDRESS

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.conf.*
  5. YarnConfiguration
  6. RM_ADDRESS

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM_ADDRESS.

Prototype

String RM_ADDRESS

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration RM_ADDRESS.

Click Source Link

Document

The address of the applications manager interface in the RM.

Usage

From source file:org.apache.twill.internal.yarn.YarnUtils.java

License:Apache License

public static InetSocketAddress getRMAddress(Configuration config) {
    return config.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS,
            YarnConfiguration.DEFAULT_RM_PORT);
}

From source file:org.apache.zeppelin.integration.MiniHadoopCluster.java

License:Apache License

@BeforeClass
public void start() throws IOException {
    LOGGER.info("Starting MiniHadoopCluster ...");
    this.hadoopConf = new Configuration();
    new File(configPath).mkdirs();
    // start MiniDFSCluster
    this.dfsCluster = new MiniDFSCluster.Builder(hadoopConf).numDataNodes(2).format(true).waitSafeMode(true)
            .build();//  ww w  . j  ava 2  s  .  c om
    this.dfsCluster.waitActive();
    saveConfig(hadoopConf, configPath + "/core-site.xml");

    // start MiniYarnCluster
    YarnConfiguration baseConfig = new YarnConfiguration(hadoopConf);
    baseConfig.set("yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage", "95");
    this.yarnCluster = new MiniYARNCluster(getClass().getName(), 2, 1, 1);
    yarnCluster.init(baseConfig);

    // Install a shutdown hook for stop the service and kill all running applications.
    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            yarnCluster.stop();
        }
    });

    yarnCluster.start();

    // Workaround for YARN-2642.
    Configuration yarnConfig = yarnCluster.getConfig();
    long start = System.currentTimeMillis();
    while (System.currentTimeMillis() - start < 30 * 1000) {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            throw new IOException(e);
        }
        if (!yarnConfig.get(YarnConfiguration.RM_ADDRESS).split(":")[1].equals("0")) {
            break;
        }
    }
    if (yarnConfig.get(YarnConfiguration.RM_ADDRESS).split(":")[1].equals("0")) {
        throw new IOException("RM not up yes");
    }

    LOGGER.info("RM address in configuration is " + yarnConfig.get(YarnConfiguration.RM_ADDRESS));
    saveConfig(yarnConfig, configPath + "/yarn-site.xml");
}

From source file:org.deeplearning4j.iterativereduce.runtime.yarn.ResourceManagerHandler.java

License:Apache License

public ClientRMProtocol getClientResourceManager() {
    if (clientResourceManager != null)
        return clientResourceManager;

    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    YarnRPC rpc = YarnRPC.create(yarnConf);
    InetSocketAddress rmAddress = NetUtils
            .createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));

    LOG.info("Connecting to the resource manager (client) at " + rmAddress);

    clientResourceManager = (ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, rmAddress, conf);

    return clientResourceManager;
}

From source file:org.elasticsearch.hadoop.yarn.cli.YarnBootstrap.java

License:Apache License

private String buildStatusReport(List<ApplicationReport> esApps) {
    if (esApps.isEmpty()) {
        return String.format("No Elasticsearch YARN clusters found at %s, webapp at %s",
                getConf().get(YarnConfiguration.RM_ADDRESS),
                WebAppUtils.getRMWebAppURLWithoutScheme(getConf()));
    }//from www. ja  va  2  s. c  o  m

    String columnSeparator = "  ";
    StringBuilder sb = new StringBuilder();
    // header
    sb.append("Id                            ");
    sb.append(columnSeparator);
    sb.append("State     ");
    sb.append(columnSeparator);
    sb.append("Status   ");
    sb.append(columnSeparator);
    sb.append("Start Time       ");
    sb.append(columnSeparator);
    sb.append("Finish Time      ");
    sb.append(columnSeparator);
    sb.append("Tracking URL");
    sb.append("\n");

    DateFormat dateFormat = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT);

    for (ApplicationReport appReport : esApps) {
        sb.append(appReport.getApplicationId());
        sb.append(columnSeparator);
        sb.append(box(appReport.getYarnApplicationState().toString(), 10));
        sb.append(columnSeparator);
        sb.append(box(appReport.getFinalApplicationStatus().toString(), 9));
        sb.append(columnSeparator);
        long date = appReport.getStartTime();
        sb.append(date == 0 ? "N/A              " : box(dateFormat.format(new Date(date)), 17));
        sb.append(columnSeparator);
        date = appReport.getFinishTime();
        sb.append(date == 0 ? "N/A              " : box(dateFormat.format(new Date(date)), 17));
        sb.append(columnSeparator);
        sb.append(appReport.getTrackingUrl());
        sb.append(columnSeparator);
        sb.append("\n");
    }

    return sb.toString();
}

From source file:org.huahinframework.manager.rest.service.ApplicationService.java

License:Apache License

/**
 * init// w  w  w .  j a  v a  2 s.  c  o  m
 */
public void init() {
    YarnRPC rpc = YarnRPC.create(getJobConf());
    YarnConfiguration yarnConf = new YarnConfiguration(getJobConf());
    InetSocketAddress rmAddress = NetUtils
            .createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));
    applicationsManager = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, rmAddress, yarnConf));
    recordFactory = RecordFactoryProvider.getRecordFactory(yarnConf);
}

From source file:org.huahinframework.manager.util.JobUtils.java

License:Apache License

/**
 * @param properties//from   w ww .  java  2 s . c o m
 * @return
 */
public static JobConf getJobConf(Properties properties) {
    JobConf jobConf = new JobConf();
    jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    jobConf.set(YarnConfiguration.RM_ADDRESS, properties.getRmAddress());
    jobConf.set(MAPREDUCE_JOBHISTORY_ADDRESS, properties.getMapreduceJobhistoryAddress());
    jobConf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, properties.getFsDefaultFS());
    if (properties.getYarnApplicationClasspath() != null
            && !properties.getYarnApplicationClasspath().isEmpty()) {
        jobConf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, properties.getYarnApplicationClasspath());
    }

    return jobConf;
}

From source file:org.springframework.xd.sqoop.SqoopRunner.java

License:Apache License

protected static Configuration createConfiguration(Map<String, String> configOptions) {

    Configuration configuration = new Configuration();
    setConfigurationProperty(configOptions, configuration, CommonConfigurationKeys.FS_DEFAULT_NAME_KEY);
    setConfigurationProperty(configOptions, configuration, YarnConfiguration.RM_HOSTNAME);
    setConfigurationProperty(configOptions, configuration, YarnConfiguration.RM_ADDRESS);
    setConfigurationProperty(configOptions, configuration, YarnConfiguration.RM_SCHEDULER_ADDRESS);
    setConfigurationProperty(configOptions, configuration, YarnConfiguration.YARN_APPLICATION_CLASSPATH);
    setConfigurationProperty(configOptions, configuration, "mapreduce.framework.name");
    if (StringUtils.hasText(configOptions.get("mapreduce.jobhistory.address"))) {
        setConfigurationProperty(configOptions, configuration, "mapreduce.jobhistory.address");
    }/*from ww  w . ja v a 2  s . co m*/
    if (configOptions.containsKey(SECURITY_AUTH_METHOD)
            && "kerberos".equals(configOptions.get(SECURITY_AUTH_METHOD))) {
        configuration.setBoolean("hadoop.security.authorization", true);
        configuration.set("hadoop.security.authentication", configOptions.get(SECURITY_AUTH_METHOD));
        configuration.set("dfs.namenode.kerberos.principal", configOptions.get(SECURITY_NAMENODE_PRINCIPAL));
        configuration.set("yarn.resourcemanager.principal", configOptions.get(SECURITY_RM_MANAGER_PRINCIPAL));
        if (StringUtils.hasText(configOptions.get(SECURITY_MAPREDUCE_JOBHISTORY_PRINCIPAL))) {
            configuration.set("mapreduce.jobhistory.principal",
                    configOptions.get(SECURITY_MAPREDUCE_JOBHISTORY_PRINCIPAL));
        }
        String userKeytab = configOptions.get(SECURITY_USER_KEYTAB);
        String userPrincipal = configOptions.get(SECURITY_USER_PRINCIPAL);
        UserGroupInformation.setConfiguration(configuration);
        if (StringUtils.hasText(userKeytab)) {
            configuration.set(ConfigurationFactoryBean.USERKEYTAB, userKeytab.trim());
        }
        if (StringUtils.hasText(userPrincipal)) {
            configuration.set(ConfigurationFactoryBean.USERPRINCIPAL, userPrincipal.trim());
        }
        if (StringUtils.hasText(userKeytab) && StringUtils.hasText(userPrincipal)) {
            try {
                SecurityUtil.login(configuration, ConfigurationFactoryBean.USERKEYTAB,
                        ConfigurationFactoryBean.USERPRINCIPAL);
            } catch (Exception e) {
                logger.warn("Cannot login using keytab " + userKeytab + " and principal " + userPrincipal, e);
            }
        }
    }

    for (Entry<String, String> entry : configOptions.entrySet()) {
        String key = entry.getKey();
        if (key.startsWith(SPRING_HADOOP_CONFIG_PREFIX + ".")) {
            String prop = key.substring(SPRING_HADOOP_CONFIG_PREFIX.length() + 1);
            String value = entry.getValue();
            logger.info("Setting configuration property: " + prop + "=" + value);
            configuration.set(prop, value);
        }
    }
    return configuration;
}

From source file:org.springframework.yarn.client.ClientRmTemplate.java

License:Apache License

@Override
protected InetSocketAddress getRpcAddress(Configuration config) {
    return config.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS,
            YarnConfiguration.DEFAULT_RM_PORT);
}

From source file:org.springframework.yarn.client.CommandYarnClient.java

License:Apache License

@Override
public Map<String, String> getEnvironment() {
    // For the environment we set additional env variables
    // from configuration. This is most useful for unit
    // testing where resource manager and hdfs are created 
    // dynamically within mini clusters.
    Map<String, String> env = super.getEnvironment();
    env.put(YarnSystemConstants.RM_ADDRESS, getConfiguration().get(YarnConfiguration.RM_ADDRESS));
    env.put(YarnSystemConstants.FS_ADDRESS,
            getConfiguration().get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY));
    env.put(YarnSystemConstants.SCHEDULER_ADDRESS,
            getConfiguration().get(YarnConfiguration.RM_SCHEDULER_ADDRESS));
    if (log.isDebugEnabled()) {
        log.debug("Setting additional env variables " + YarnSystemConstants.RM_ADDRESS + "="
                + env.get(YarnSystemConstants.RM_ADDRESS) + YarnSystemConstants.FS_ADDRESS + "="
                + env.get(YarnSystemConstants.FS_ADDRESS) + YarnSystemConstants.SCHEDULER_ADDRESS + "="
                + env.get(YarnSystemConstants.SCHEDULER_ADDRESS));
    }// w  w w . j a  va 2 s. co m
    return env;
}

From source file:org.springframework.yarn.support.YarnUtils.java

License:Apache License

/**
 * Gets the address.//from  www .  ja va2  s.c  o m
 *
 * @param conf the Yarn configuration
 * @return the address
 */
public static InetSocketAddress getAddress(Configuration conf) {
    return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS,
            YarnConfiguration.DEFAULT_RM_PORT);
}