List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration
public YarnConfiguration(Configuration conf)
From source file:UnmanagedAMLauncher.java
License:Apache License
public boolean init(String[] args) throws ParseException { Options opts = new Options(); opts.addOption("appname", true, "Application Name. Default value - UnmanagedAM"); opts.addOption("priority", true, "Application Priority. Default 0"); opts.addOption("queue", true, "RM Queue in which this application is to be submitted"); opts.addOption("master_memory", true, "Amount of memory in MB to be requested to run the application master"); opts.addOption("cmd", true, "command to start unmanaged AM (required)"); opts.addOption("classpath", true, "additional classpath"); opts.addOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); if (args.length == 0) { printUsage(opts);/*from www.ja va 2 s. c o m*/ throw new IllegalArgumentException("No args specified for client to initialize"); } if (cliParser.hasOption("help")) { printUsage(opts); return false; } appName = cliParser.getOptionValue("appname", "UnmanagedAM"); amPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0")); amQueue = cliParser.getOptionValue("queue", "default"); classpath = cliParser.getOptionValue("classpath", null); amCmd = cliParser.getOptionValue("cmd"); if (amCmd == null) { printUsage(opts); throw new IllegalArgumentException("No cmd specified for application master"); } YarnConfiguration yarnConf = new YarnConfiguration(conf); rmClient = YarnClient.createYarnClient(); rmClient.init(yarnConf); return true; }
From source file:co.cask.cdap.app.runtime.spark.distributed.DistributedSparkProgramRunner.java
License:Apache License
private static YarnConfiguration createConfiguration(YarnConfiguration hConf) { YarnConfiguration configuration = new YarnConfiguration(hConf); configuration.setBoolean(SparkRuntimeContextConfig.HCONF_ATTR_CLUSTER_MODE, true); return configuration; }
From source file:co.cask.cdap.common.guice.ConfigModule.java
License:Apache License
@Override protected void configure() { bind(CConfiguration.class).toInstance(cConf); bind(Configuration.class).toInstance(hConf); bind(SConfiguration.class).toInstance(sConf); bind(YarnConfiguration.class).toInstance(new YarnConfiguration(hConf)); }
From source file:co.cask.cdap.common.guice.TwillModule.java
License:Apache License
/** * Provider method for instantiating {@link YarnTwillRunnerService}. *//*from w w w. ja v a 2s . co m*/ @Singleton @Provides private YarnTwillRunnerService provideYarnTwillRunnerService(CConfiguration configuration, YarnConfiguration yarnConfiguration, LocationFactory locationFactory) { String zkConnectStr = configuration.get(Constants.Zookeeper.QUORUM) + configuration.get(Constants.CFG_TWILL_ZK_NAMESPACE); // Copy the yarn config and set the max heap ratio. YarnConfiguration yarnConfig = new YarnConfiguration(yarnConfiguration); yarnConfig.set(Constants.CFG_TWILL_RESERVED_MEMORY_MB, configuration.get(Constants.CFG_TWILL_RESERVED_MEMORY_MB)); YarnTwillRunnerService runner = new YarnTwillRunnerService(yarnConfig, zkConnectStr, LocationFactories.namespace(locationFactory, "twill")); // Set JVM options based on configuration String jvmOpts = configuration.get(Constants.AppFabric.PROGRAM_JVM_OPTS); runner.setJVMOptions(jvmOpts); return runner; }
From source file:co.cask.cdap.common.security.YarnTokenUtils.java
License:Apache License
/** * Gets a Yarn delegation token and stores it in the given Credentials. * * @return the same Credentials instance as the one given in parameter. *///from ww w . j av a 2s . c om public static Credentials obtainToken(YarnConfiguration configuration, Credentials credentials) { if (!UserGroupInformation.isSecurityEnabled()) { return credentials; } try { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(configuration); yarnClient.start(); try { Text renewer = new Text(UserGroupInformation.getCurrentUser().getShortUserName()); org.apache.hadoop.yarn.api.records.Token rmDelegationToken = yarnClient .getRMDelegationToken(renewer); // TODO: The following logic should be replaced with call to ClientRMProxy.getRMDelegationTokenService after // CDAP-4825 is resolved List<String> services = new ArrayList<>(); if (HAUtil.isHAEnabled(configuration)) { // If HA is enabled, we need to enumerate all RM hosts // and add the corresponding service name to the token service // Copy the yarn conf since we need to modify it to get the RM addresses YarnConfiguration yarnConf = new YarnConfiguration(configuration); for (String rmId : HAUtil.getRMHAIds(configuration)) { yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); InetSocketAddress address = yarnConf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT); services.add(SecurityUtil.buildTokenService(address).toString()); } } else { services.add(SecurityUtil.buildTokenService(YarnUtils.getRMAddress(configuration)).toString()); } Token<TokenIdentifier> token = ConverterUtils.convertFromYarn(rmDelegationToken, (InetSocketAddress) null); token.setService(new Text(Joiner.on(',').join(services))); credentials.addToken(new Text(token.getService()), token); // OK to log, it won't log the credential, only information about the token. LOG.info("Added RM delegation token: {}", token); } finally { yarnClient.stop(); } return credentials; } catch (Exception e) { LOG.error("Failed to get secure token for Yarn.", e); throw Throwables.propagate(e); } }
From source file:co.cask.cdap.explore.security.JobHistoryServerTokenUtils.java
License:Apache License
/** * Gets a JHS delegation token and stores it in the given Credentials. * * @return the same Credentials instance as the one given in parameter. *//*www . ja va2 s . c o m*/ public static Credentials obtainToken(Configuration configuration, Credentials credentials) { if (!UserGroupInformation.isSecurityEnabled()) { return credentials; } String historyServerAddress = configuration.get("mapreduce.jobhistory.address"); HostAndPort hostAndPort = HostAndPort.fromString(historyServerAddress); try { LOG.info("Obtaining delegation token for JHS"); ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate(new YarnConfiguration(configuration)); MRClientCache clientCache = new MRClientCache(configuration, resourceMgrDelegate); MRClientProtocol hsProxy = clientCache.getInitializedHSProxy(); GetDelegationTokenRequest request = new GetDelegationTokenRequestPBImpl(); request.setRenewer(YarnUtils.getYarnTokenRenewer(configuration)); InetSocketAddress address = new InetSocketAddress(hostAndPort.getHostText(), hostAndPort.getPort()); Token<TokenIdentifier> token = ConverterUtils .convertFromYarn(hsProxy.getDelegationToken(request).getDelegationToken(), address); credentials.addToken(new Text(token.getService()), token); return credentials; } catch (Exception e) { LOG.error("Failed to get secure token for JHS at {}.", hostAndPort, e); throw Throwables.propagate(e); } }
From source file:co.cask.cdap.internal.app.runtime.distributed.DistributedWorkflowProgramRunner.java
License:Apache License
private static YarnConfiguration createConfiguration(YarnConfiguration hConf) { YarnConfiguration configuration = new YarnConfiguration(hConf); configuration.setBoolean(HCONF_ATTR_CLUSTER_MODE, true); return configuration; }
From source file:co.cask.cdap.operations.yarn.AbstractYarnStats.java
License:Apache License
protected AbstractYarnStats(Configuration conf) { this.conf = new YarnConfiguration(conf); }
From source file:co.cask.cdap.operations.yarn.YarnInfo.java
License:Apache License
/** * Should only be called when HA is enabled. *//*from w ww.ja v a 2 s . c om*/ private URL getHAWebURL() throws IOException { InetSocketAddress activeRM = null; Collection<String> rmIds = HAUtil.getRMHAIds(conf); if (rmIds.isEmpty()) { throw new IllegalStateException("Resource Manager HA web URL requested in non-HA mode."); } for (String rmId : rmIds) { YarnConfiguration yarnConf = new YarnConfiguration(conf); yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); yarnConf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, conf.get(YarnConfiguration.RM_PRINCIPAL, "")); RMHAServiceTarget rmhaServiceTarget = new RMHAServiceTarget(yarnConf); HAServiceProtocol proxy = rmhaServiceTarget.getProxy(yarnConf, 10000); HAServiceStatus serviceStatus = proxy.getServiceStatus(); if (HAServiceProtocol.HAServiceState.ACTIVE != serviceStatus.getState()) { continue; } activeRM = rmhaServiceTarget.getAddress(); } if (activeRM == null) { throw new IllegalStateException("Could not find an active resource manager"); } return adminToWebappAddress(activeRM); }
From source file:co.cask.tigon.guice.ConfigModule.java
License:Apache License
@Override protected void configure() { bind(CConfiguration.class).toInstance(cConf); bind(Configuration.class).toInstance(hConf); bind(YarnConfiguration.class).toInstance(new YarnConfiguration(hConf)); }