List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration
public YarnConfiguration(Configuration conf)
From source file:co.cask.tigon.guice.TwillModule.java
License:Apache License
/** * Provider method for instantiating {@link org.apache.twill.yarn.YarnTwillRunnerService}. *//*from ww w . j a va 2 s .c o m*/ @Singleton @Provides private YarnTwillRunnerService provideYarnTwillRunnerService(CConfiguration configuration, YarnConfiguration yarnConfiguration, LocationFactory locationFactory) { String zkConnectStr = configuration.get(Constants.Zookeeper.QUORUM) + configuration.get(Constants.CFG_TWILL_ZK_NAMESPACE); // Copy the yarn config and set the max heap ratio. YarnConfiguration yarnConfig = new YarnConfiguration(yarnConfiguration); yarnConfig.set(Constants.CFG_TWILL_RESERVED_MEMORY_MB, configuration.get(Constants.CFG_TWILL_RESERVED_MEMORY_MB)); YarnTwillRunnerService runner = new YarnTwillRunnerService(yarnConfig, zkConnectStr, LocationFactories.namespace(locationFactory, "twill")); // Set JVM options based on configuration runner.setJVMOptions(configuration.get(Constants.Container.PROGRAM_JVM_OPTS)); return runner; }
From source file:com.cloudera.kitten.appmaster.service.ApplicationMasterServiceImpl1.java
License:Open Source License
public ApplicationMasterServiceImpl1(ApplicationMasterParameters parameters, Configuration conf) { this.parameters = Preconditions.checkNotNull(parameters); this.conf = new YarnConfiguration(conf); }
From source file:com.cloudera.kitten.appmaster.service.ResourceManagerConnectionFactory.java
License:Open Source License
@Override public AMRMProtocol connect() { YarnConfiguration yarnConf = new YarnConfiguration(conf); InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS)); LOG.info("Connecting to ResourceManager at " + rmAddress); return ((AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, conf)); }
From source file:com.cloudera.kitten.appmaster.service.WorkflowService.java
License:Open Source License
public WorkflowService(WorkflowParameters parameters, Configuration conf) { this.trackers = new HashMap<String, ContainerTracker>(); this.parameters = Preconditions.checkNotNull(parameters); this.conf = new YarnConfiguration(conf); this.prior = 1; }
From source file:com.cloudera.kitten.client.service.ApplicationsManagerConnectionFactory.java
License:Open Source License
@Override public ClientRMProtocol connect() { YarnConfiguration yarnConf = new YarnConfiguration(conf); InetSocketAddress rmAddress = NetUtils .createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS)); LOG.info("Connecting to ResourceManager at: " + rmAddress); return ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, rmAddress, conf)); }
From source file:com.cloudera.kitten.client.service.YarnClientFactory.java
License:Open Source License
public YarnClientFactory(Configuration conf) { conf = Preconditions.checkNotNull(conf); this.conf = new YarnConfiguration(conf); }
From source file:com.continuuity.weave.internal.yarn.ports.AMRMClientImpl.java
License:Apache License
@Override public synchronized void start() { final YarnConfiguration conf = new YarnConfiguration(getConfig()); final YarnRPC rpc = YarnRPC.create(conf); final InetSocketAddress rmAddress = conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); UserGroupInformation currentUser;//from w ww . ja v a 2 s .c o m try { currentUser = UserGroupInformation.getCurrentUser(); } catch (IOException e) { throw new YarnException(e); } if (UserGroupInformation.isSecurityEnabled()) { String tokenURLEncodedStr = System.getenv().get(ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME); Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>(); try { token.decodeFromUrlString(tokenURLEncodedStr); } catch (IOException e) { throw new YarnException(e); } SecurityUtil.setTokenService(token, rmAddress); if (LOG.isDebugEnabled()) { LOG.debug("AppMasterToken is " + token); } currentUser.addToken(token); } rmClient = currentUser.doAs(new PrivilegedAction<AMRMProtocol>() { @Override public AMRMProtocol run() { return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, conf); } }); LOG.debug("Connecting to ResourceManager at " + rmAddress); super.start(); }
From source file:com.continuuity.weave.yarn.ClusterTestBase.java
License:Open Source License
protected final void doInit() throws IOException { // Starts Zookeeper zkServer = InMemoryZKServer.builder().build(); zkServer.startAndWait();/* www . j a va 2s . com*/ // Start YARN mini cluster YarnConfiguration config = new YarnConfiguration(new Configuration()); // TODO: Hack config.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler" + ".fifo.FifoScheduler"); config.set("yarn.minicluster.fixed.ports", "true"); cluster = new MiniYARNCluster("test-cluster", 1, 1, 1); cluster.init(config); cluster.start(); runnerService = new YarnWeaveRunnerService(config, zkServer.getConnectionStr() + "/weave", new LocalLocationFactory(Files.createTempDir())); runnerService.startAndWait(); }
From source file:com.continuuity.weave.yarn.YarnTestSuite.java
License:Apache License
@BeforeClass public static final void init() throws IOException { // Starts Zookeeper zkServer = InMemoryZKServer.builder().build(); zkServer.startAndWait();/*from w w w .j av a2 s. com*/ // Start YARN mini cluster config = new YarnConfiguration(new Configuration()); if (YarnUtils.isHadoop20()) { config.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler"); } else { config.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"); config.set("yarn.scheduler.capacity.resource-calculator", "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator"); } config.set("yarn.minicluster.fixed.ports", "true"); config.set("yarn.nodemanager.vmem-pmem-ratio", "20.1"); config.set("yarn.nodemanager.vmem-check-enabled", "false"); config.set("yarn.scheduler.minimum-allocation-mb", "128"); config.set("yarn.nodemanager.delete.debug-delay-sec", "3600"); cluster = new MiniYARNCluster("test-cluster", 1, 1, 1); cluster.init(config); cluster.start(); runnerService = createWeaveRunnerService(); runnerService.startAndWait(); }
From source file:com.datatorrent.stram.client.StramClientUtilsTest.java
License:Apache License
@Test public void testRMWebAddress() throws UnknownHostException { Configuration conf = new YarnConfiguration(new Configuration(false)) { @Override// w ww . j a v a 2s . co m public InetSocketAddress getSocketAddr(String name, String defaultAddress, int defaultPort) { String rmId = get(ConfigUtils.RM_HA_ID); if (rmId != null) { name = name + "." + rmId; } return super.getSocketAddr(name, defaultAddress, defaultPort); } }; // basic test conf.setBoolean(CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_KEY, false); conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "192.168.1.1:8032"); conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, "192.168.1.2:8032"); Assert.assertEquals(getHostString("192.168.1.1") + ":8032", StramClientUtils.getSocketConnectString(StramClientUtils.getRMWebAddress(conf, null))); List<InetSocketAddress> addresses = StramClientUtils.getRMAddresses(conf); Assert.assertEquals(1, addresses.size()); Assert.assertEquals(getHostString("192.168.1.1") + ":8032", StramClientUtils.getSocketConnectString(addresses.get(0))); conf.setBoolean(CommonConfigurationKeysPublic.HADOOP_SSL_ENABLED_KEY, true); Assert.assertEquals(getHostString("192.168.1.2") + ":8032", StramClientUtils.getSocketConnectString(StramClientUtils.getRMWebAddress(conf, null))); addresses = StramClientUtils.getRMAddresses(conf); Assert.assertEquals(1, addresses.size()); Assert.assertEquals(getHostString("192.168.1.2") + ":8032", StramClientUtils.getSocketConnectString(addresses.get(0))); // set localhost if host is unknown conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, "someunknownhost.:8032"); Assert.assertEquals(InetAddress.getLocalHost().getCanonicalHostName() + ":8032", StramClientUtils.getSocketConnectString(StramClientUtils.getRMWebAddress(conf, null))); // set localhost conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, "127.0.0.1:8032"); Assert.assertEquals(InetAddress.getLocalHost().getCanonicalHostName() + ":8032", StramClientUtils.getSocketConnectString(StramClientUtils.getRMWebAddress(conf, null))); // test when HA is enabled conf.setBoolean(ConfigUtils.RM_HA_ENABLED, true); conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + ".rm1", "192.168.1.1:8032"); conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + ".rm2", "192.168.1.2:8032"); conf.set("yarn.resourcemanager.ha.rm-ids", "rm1,rm2"); Assert.assertEquals(getHostString("192.168.1.1") + ":8032", StramClientUtils.getSocketConnectString(StramClientUtils.getRMWebAddress(conf, "rm1"))); Assert.assertEquals(getHostString("192.168.1.2") + ":8032", StramClientUtils.getSocketConnectString(StramClientUtils.getRMWebAddress(conf, "rm2"))); addresses = StramClientUtils.getRMAddresses(conf); Assert.assertEquals(2, addresses.size()); Assert.assertEquals(getHostString("192.168.1.1") + ":8032", StramClientUtils.getSocketConnectString(addresses.get(0))); Assert.assertEquals(getHostString("192.168.1.2") + ":8032", StramClientUtils.getSocketConnectString(addresses.get(1))); }