List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration set
public void set(String name, String value)
value of the name property. From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java
License:Apache License
@Test public void testResourceMapWithInvalidTypeFailure() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("No enum constant org.apache.hadoop.yarn.api.records.LocalResourceType.INVALIDTYPE"); Map<String, String> configMap = new HashMap<>(); configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme"); configMap.put("yarn.resources.myResource1.local.name", "readme"); configMap.put("yarn.resources.myResource1.local.type", "invalidType"); configMap.put("yarn.resources.myResource1.local.visibility", "public"); Config conf = new MapConfig(configMap); YarnConfiguration yarnConfiguration = new YarnConfiguration(); yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName()); yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName()); LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf), yarnConfiguration);/*from w w w. j a va 2 s . c om*/ }
From source file:org.apache.samza.validation.YarnJobValidationTool.java
License:Apache License
public static void main(String[] args) throws Exception { CommandLine cmdline = new CommandLine(); OptionParser parser = cmdline.parser(); OptionSpec<String> validatorOpt = parser.accepts("metrics-validator", "The metrics validator class.") .withOptionalArg().ofType(String.class).describedAs("com.foo.bar.ClassName"); OptionSet options = cmdline.parser().parse(args); Config config = cmdline.loadConfig(options); MetricsValidator validator = null;/* ww w .ja v a 2s . com*/ if (options.has(validatorOpt)) { String validatorClass = options.valueOf(validatorOpt); validator = Util.getObj(validatorClass, MetricsValidator.class); } YarnConfiguration hadoopConfig = new YarnConfiguration(); hadoopConfig.set("fs.http.impl", HttpFileSystem.class.getName()); hadoopConfig.set("fs.https.impl", HttpFileSystem.class.getName()); ClientHelper clientHelper = new ClientHelper(hadoopConfig); new YarnJobValidationTool(new JobConfig(config), clientHelper.yarnClient(), validator).run(); }
From source file:org.apache.slider.core.launch.CredentialUtils.java
License:Apache License
/** * Look up and return the resource manager's principal. This method * automatically does the <code>_HOST</code> replacement in the principal and * correctly handles HA resource manager configurations. * * From: YARN-4629// w w w . ja va2 s .c o m * @param conf the {@link Configuration} file from which to read the * principal * @return the resource manager's principal string * @throws IOException thrown if there's an error replacing the host name */ public static String getRMPrincipal(Configuration conf) throws IOException { String principal = conf.get(RM_PRINCIPAL, ""); String hostname; Preconditions.checkState(!principal.isEmpty(), "Not set: " + RM_PRINCIPAL); if (HAUtil.isHAEnabled(conf)) { YarnConfiguration yarnConf = new YarnConfiguration(conf); if (yarnConf.get(RM_HA_ID) == null) { // If RM_HA_ID is not configured, use the first of RM_HA_IDS. // Any valid RM HA ID should work. String[] rmIds = yarnConf.getStrings(RM_HA_IDS); Preconditions.checkState((rmIds != null) && (rmIds.length > 0), "Not set " + RM_HA_IDS); yarnConf.set(RM_HA_ID, rmIds[0]); } hostname = yarnConf.getSocketAddr(RM_ADDRESS, DEFAULT_RM_ADDRESS, DEFAULT_RM_PORT).getHostName(); } else { hostname = conf.getSocketAddr(RM_ADDRESS, DEFAULT_RM_ADDRESS, DEFAULT_RM_PORT).getHostName(); } return SecurityUtil.getServerPrincipal(principal, hostname); }
From source file:org.apache.sqoop.submission.spark.YarnSqoopSparkClient.java
License:Apache License
static YarnConfiguration generateYarnSparkConf(Map<String, String> conf) { YarnConfiguration yarnConf = new YarnConfiguration(); for (Map.Entry<String, String> entry : conf.entrySet()) { yarnConf.set(entry.getKey(), entry.getValue()); }//from w w w . j ava2 s. co m return yarnConf; }
From source file:org.apache.twill.internal.yarn.Hadoop23YarnAppClient.java
License:Apache License
/** * Overrides parent method to adds RM delegation token to the given context. If YARN is running with HA RM, * delegation tokens for each RM service will be added. *//*w ww . j av a2 s .c om*/ protected void addRMToken(ContainerLaunchContext context, YarnClient yarnClient, ApplicationId appId) { if (!UserGroupInformation.isSecurityEnabled()) { return; } try { Text renewer = new Text(UserGroupInformation.getCurrentUser().getShortUserName()); org.apache.hadoop.yarn.api.records.Token rmDelegationToken = yarnClient.getRMDelegationToken(renewer); // The following logic is copied from ClientRMProxy.getRMDelegationTokenService, which is not available in // YARN older than 2.4 List<String> services = new ArrayList<>(); if (HAUtil.isHAEnabled(configuration)) { // If HA is enabled, we need to enumerate all RM hosts // and add the corresponding service name to the token service // Copy the yarn conf since we need to modify it to get the RM addresses YarnConfiguration yarnConf = new YarnConfiguration(configuration); for (String rmId : HAUtil.getRMHAIds(configuration)) { yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); InetSocketAddress address = yarnConf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT); services.add(SecurityUtil.buildTokenService(address).toString()); } } else { services.add(SecurityUtil.buildTokenService(YarnUtils.getRMAddress(configuration)).toString()); } Credentials credentials = YarnUtils.decodeCredentials(context.getTokens()); // casting needed for later Hadoop version @SuppressWarnings("RedundantCast") Token<TokenIdentifier> token = ConverterUtils.convertFromYarn(rmDelegationToken, (InetSocketAddress) null); token.setService(new Text(Joiner.on(',').join(services))); credentials.addToken(new Text(token.getService()), token); LOG.debug("Added RM delegation token {} for application {}", token, appId); credentials.addToken(token.getService(), token); context.setTokens(YarnUtils.encodeCredentials(credentials)); } catch (Exception e) { throw Throwables.propagate(e); } }
From source file:org.apache.zeppelin.integration.MiniHadoopCluster.java
License:Apache License
@BeforeClass public void start() throws IOException { LOGGER.info("Starting MiniHadoopCluster ..."); this.hadoopConf = new Configuration(); new File(configPath).mkdirs(); // start MiniDFSCluster this.dfsCluster = new MiniDFSCluster.Builder(hadoopConf).numDataNodes(2).format(true).waitSafeMode(true) .build();// w ww. ja v a 2s .co m this.dfsCluster.waitActive(); saveConfig(hadoopConf, configPath + "/core-site.xml"); // start MiniYarnCluster YarnConfiguration baseConfig = new YarnConfiguration(hadoopConf); baseConfig.set("yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage", "95"); this.yarnCluster = new MiniYARNCluster(getClass().getName(), 2, 1, 1); yarnCluster.init(baseConfig); // Install a shutdown hook for stop the service and kill all running applications. Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { yarnCluster.stop(); } }); yarnCluster.start(); // Workaround for YARN-2642. Configuration yarnConfig = yarnCluster.getConfig(); long start = System.currentTimeMillis(); while (System.currentTimeMillis() - start < 30 * 1000) { try { Thread.sleep(100); } catch (InterruptedException e) { throw new IOException(e); } if (!yarnConfig.get(YarnConfiguration.RM_ADDRESS).split(":")[1].equals("0")) { break; } } if (yarnConfig.get(YarnConfiguration.RM_ADDRESS).split(":")[1].equals("0")) { throw new IOException("RM not up yes"); } LOGGER.info("RM address in configuration is " + yarnConfig.get(YarnConfiguration.RM_ADDRESS)); saveConfig(yarnConfig, configPath + "/yarn-site.xml"); }
From source file:org.testifyproject.resource.yarn.MiniYarnResource.java
License:Apache License
@Override public YarnConfiguration configure(TestContext testContext, LocalResource localResource, PropertiesReader configReader) { String testName = testContext.getName(); String logDirectory = fileSystemUtil.createPath("target", "yarn", testName); YarnConfiguration configuration = new YarnConfiguration(); configuration.set(YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR, logDirectory); configuration.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64); configuration.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); return configuration; }
From source file:wwh.exec.Main.java
private static YarnConfiguration createClusterConfiguration(String ip) { //create yarn configuration for c0 //A YarnConfiguration object is part of hadoop Yarn and //is allows us to connect to a specific cluster and send application //masters into it. YarnConfiguration configuration = new YarnConfiguration(); //configure the resource manager host name to submit jobs into configuration.set("yarn.resourcemanager.hostname", ip); //configure the hdfs to use configuration.set("fs.defaultFS", "hdfs://" + ip + ":9000"); // configuration.set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, ip + ":2181"); return configuration; }
From source file:yrun.YarnRunner.java
License:Apache License
public static void main(final String[] args) throws IOException, InterruptedException { CommandLine cmd = parse(args, new PrintWriter(System.out)); final String yarnName = cmd.getOptionValue("n"); LOG.info("Yarn name [" + yarnName + "]"); String resourceManagerAddress = cmd.getOptionValue("rma"); LOG.info("Resource Manager Address [" + resourceManagerAddress + "]"); String installPathStr = cmd.getOptionValue("p"); LOG.info("Install Path [" + installPathStr + "]"); final String command = StringUtils.join(" ", cmd.getOptionValues("c")); LOG.info("Command [" + command + "]"); final String queue; if (cmd.hasOption("q")) { queue = cmd.getOptionValue("q"); } else {//from ww w . j ava 2 s . c o m queue = "default"; } LOG.info("Queue [" + queue + "]"); final YarnConfiguration configuration = new YarnConfiguration(); configuration.set(YARN_RESOURCEMANAGER_ADDRESS, resourceManagerAddress); LOG.info("Using resource manager [" + resourceManagerAddress + "]"); final Path installPath = new Path(installPathStr); final List<Path> archivePathList = new ArrayList<Path>(); if (cmd.hasOption("a")) { String[] archivePaths = cmd.getOptionValues("a"); for (String archivePath : archivePaths) { archivePathList.add(new Path(archivePath)); } } final boolean isDaemon = !cmd.hasOption("k"); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); ugi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { YarnRunner yarnRunner = new YarnRunner(configuration, installPath, archivePathList, command, yarnName, queue, isDaemon); yarnRunner.execute(); return null; } }); }