List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration RM_ADDRESS
String RM_ADDRESS
To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration RM_ADDRESS.
Click Source Link
From source file:org.apache.hoya.tools.HoyaUtils.java
License:Apache License
public static void setRmAddress(Configuration conf, String rmAddr) { conf.set(YarnConfiguration.RM_ADDRESS, rmAddr); }
From source file:org.apache.hoya.yarn.client.HoyaClient.java
License:Apache License
/** * Verify that the Resource MAnager is configured, if not fail * with a useful error message/* w w w . j a va 2 s. co m*/ * @throws BadCommandArgumentsException the exception raised on an invalid config */ public void verifyManagerSet() throws BadCommandArgumentsException { InetSocketAddress rmAddr = HoyaUtils.getRmAddress(getConfig()); if (!HoyaUtils.isAddressDefined(rmAddr)) { throw new BadCommandArgumentsException("No valid Resource Manager address provided in the argument " + Arguments.ARG_MANAGER + " or the configuration property " + YarnConfiguration.RM_ADDRESS + " value :" + rmAddr); } }
From source file:org.apache.hoya.yarn.params.ClientArgs.java
License:Apache License
@Override public void applyDefinitions(Configuration conf) throws BadCommandArgumentsException { super.applyDefinitions(conf); //RM// ww w.j av a 2 s . c o m if (getManager() != null) { log.debug("Setting RM to {}", getManager()); conf.set(YarnConfiguration.RM_ADDRESS, getManager()); } if (getBasePath() != null) { log.debug("Setting basePath to {}", getBasePath()); conf.set(HoyaXmlConfKeys.KEY_BASE_HOYA_PATH, getBasePath().toString()); } }
From source file:org.apache.samza.autoscaling.utils.YarnUtil.java
License:Apache License
public YarnUtil(String rmAddress, int rmPort) { this.httpclient = HttpClientBuilder.create().build(); this.rmServer = new HttpHost(rmAddress, rmPort, "http"); log.info("setting rm server to : " + rmServer); YarnConfiguration hConfig = new YarnConfiguration(); hConfig.set(YarnConfiguration.RM_ADDRESS, rmAddress + ":" + YarnConfiguration.DEFAULT_RM_PORT); yarnClient = YarnClient.createYarnClient(); yarnClient.init(hConfig);// w w w . j a v a2 s . com yarnClient.start(); }
From source file:org.apache.slider.client.SliderClient.java
License:Apache License
/** * Verify that the Resource Manager is configured (on a non-HA cluster). * with a useful error message// w w w. j a v a 2s.c om * @throws BadCommandArgumentsException the exception raised on an invalid config */ public void verifyBindingsDefined() throws BadCommandArgumentsException { InetSocketAddress rmAddr = SliderUtils.getRmAddress(getConfig()); if (!getConfig().getBoolean(YarnConfiguration.RM_HA_ENABLED, false) && !SliderUtils.isAddressDefined(rmAddr)) { throw new BadCommandArgumentsException("No valid Resource Manager address provided in the argument " + Arguments.ARG_MANAGER + " or the configuration property " + YarnConfiguration.RM_ADDRESS + " value :" + rmAddr); } }
From source file:org.apache.slider.common.params.ClientArgs.java
License:Apache License
@Override public void applyDefinitions(Configuration conf) throws BadCommandArgumentsException { super.applyDefinitions(conf); //RM/*from w ww .j a va2s. c o m*/ if (getManager() != null) { log.debug("Setting RM to {}", getManager()); conf.set(YarnConfiguration.RM_ADDRESS, getManager()); } if (getBasePath() != null) { log.debug("Setting basePath to {}", getBasePath()); conf.set(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH, getBasePath().toString()); } }
From source file:org.apache.tajo.MiniTajoYarnCluster.java
License:Apache License
@Override public void init(Configuration conf) { conf.setSocketAddr(YarnConfiguration.RM_ADDRESS, new InetSocketAddress("127.0.0.1", 0)); conf.setSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, new InetSocketAddress("127.0.0.1", 0)); conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) { conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(), "apps_staging_dir/").getAbsolutePath()); }//www . j a va 2s . c o m conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000"); try { Path stagingPath = FileContext.getFileContext(conf) .makeQualified(new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR))); FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf); if (fc.util().exists(stagingPath)) { LOG.info(stagingPath + " exists! deleting..."); fc.delete(stagingPath, true); } LOG.info("mkdir: " + stagingPath); //mkdir the staging directory so that right permissions are set while running as proxy user fc.mkdir(stagingPath, null, true); //mkdir done directory as well String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf); Path doneDirPath = fc.makeQualified(new Path(doneDir)); fc.mkdir(doneDirPath, null, true); } catch (IOException e) { throw new YarnRuntimeException("Could not create staging directory. ", e); } conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of // which shuffle doesn't happen //configure the shuffle service in NM conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, PullServerAuxService.PULLSERVER_SERVICEID); conf.setClass( String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, PullServerAuxService.PULLSERVER_SERVICEID), PullServerAuxService.class, Service.class); // Non-standard shuffle port conf.setInt(TajoConf.ConfVars.PULLSERVER_PORT.name(), 0); // local directory conf.set(TajoConf.ConfVars.WORKER_TEMPORAL_DIR.name(), "/tmp/tajo-localdir"); conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class, ContainerExecutor.class); // TestMRJobs is for testing non-uberized operation only; see TestUberAM // for corresponding uberized tests. conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); conf.setInt("yarn.nodemanager.delete.debug-delay-sec", 600); super.init(conf); }
From source file:org.apache.tez.common.security.Master.java
License:Apache License
public static InetSocketAddress getMasterAddress(Configuration conf) { return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT); }
From source file:org.apache.tez.mapreduce.client.ResourceMgrDelegate.java
License:Apache License
/** * Delegate responsible for communicating with the Resource Manager's {@link ApplicationClientProtocol}. * @param conf the configuration object. *//*w w w . ja v a 2s .co m*/ public ResourceMgrDelegate(YarnConfiguration conf) { super(); this.conf = conf; client = YarnClient.createYarnClient(); client.init(conf); this.rmAddress = conf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT); client.start(); }
From source file:org.apache.twill.internal.yarn.Hadoop23YarnAppClient.java
License:Apache License
/** * Overrides parent method to adds RM delegation token to the given context. If YARN is running with HA RM, * delegation tokens for each RM service will be added. *//* w ww . j av a 2 s .c o m*/ protected void addRMToken(ContainerLaunchContext context, YarnClient yarnClient, ApplicationId appId) { if (!UserGroupInformation.isSecurityEnabled()) { return; } try { Text renewer = new Text(UserGroupInformation.getCurrentUser().getShortUserName()); org.apache.hadoop.yarn.api.records.Token rmDelegationToken = yarnClient.getRMDelegationToken(renewer); // The following logic is copied from ClientRMProxy.getRMDelegationTokenService, which is not available in // YARN older than 2.4 List<String> services = new ArrayList<>(); if (HAUtil.isHAEnabled(configuration)) { // If HA is enabled, we need to enumerate all RM hosts // and add the corresponding service name to the token service // Copy the yarn conf since we need to modify it to get the RM addresses YarnConfiguration yarnConf = new YarnConfiguration(configuration); for (String rmId : HAUtil.getRMHAIds(configuration)) { yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); InetSocketAddress address = yarnConf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT); services.add(SecurityUtil.buildTokenService(address).toString()); } } else { services.add(SecurityUtil.buildTokenService(YarnUtils.getRMAddress(configuration)).toString()); } Credentials credentials = YarnUtils.decodeCredentials(context.getTokens()); // casting needed for later Hadoop version @SuppressWarnings("RedundantCast") Token<TokenIdentifier> token = ConverterUtils.convertFromYarn(rmDelegationToken, (InetSocketAddress) null); token.setService(new Text(Joiner.on(',').join(services))); credentials.addToken(new Text(token.getService()), token); LOG.debug("Added RM delegation token {} for application {}", token, appId); credentials.addToken(token.getService(), token); context.setTokens(YarnUtils.encodeCredentials(credentials)); } catch (Exception e) { throw Throwables.propagate(e); } }