List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration
public YarnConfiguration(Configuration conf)
From source file:org.apache.slider.server.appmaster.SliderAppMaster.java
License:Apache License
/** * pick up the args from the service launcher * @param config configuration/* w w w .j a va2 s. com*/ * @param args argument list */ @Override // RunService public Configuration bindArgs(Configuration config, String... args) throws Exception { YarnConfiguration yarnConfiguration = new YarnConfiguration(super.bindArgs(config, args)); serviceArgs = new SliderAMArgs(args); serviceArgs.parse(); //yarn-ify return SliderUtils.patchConfiguration(yarnConfiguration); }
From source file:org.apache.solr.hadoop.hack.MiniYARNCluster.java
License:Apache License
@Override public void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf)); }
From source file:org.apache.tajo.master.rm.YarnTajoResourceManager.java
License:Apache License
@Override public void init(Configuration conf) { this.conf = conf; connectYarnClient();/* ww w .j a v a 2s .c om*/ final YarnConfiguration yarnConf = new YarnConfiguration(conf); final YarnRPC rpc = YarnRPC.create(conf); final InetSocketAddress rmAddress = conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); UserGroupInformation currentUser; try { currentUser = UserGroupInformation.getCurrentUser(); } catch (IOException e) { throw new YarnRuntimeException(e); } rmClient = currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() { @Override public ApplicationMasterProtocol run() { return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rmAddress, yarnConf); } }); }
From source file:org.apache.tez.auxservices.ShuffleHandler.java
License:Apache License
@Override protected void serviceInit(Configuration conf) throws Exception { manageOsCache = conf.getBoolean(SHUFFLE_MANAGE_OS_CACHE, DEFAULT_SHUFFLE_MANAGE_OS_CACHE); readaheadLength = conf.getInt(SHUFFLE_READAHEAD_BYTES, DEFAULT_SHUFFLE_READAHEAD_BYTES); maxShuffleConnections = conf.getInt(MAX_SHUFFLE_CONNECTIONS, DEFAULT_MAX_SHUFFLE_CONNECTIONS); int maxShuffleThreads = conf.getInt(MAX_SHUFFLE_THREADS, DEFAULT_MAX_SHUFFLE_THREADS); if (maxShuffleThreads == 0) { maxShuffleThreads = 2 * Runtime.getRuntime().availableProcessors(); }/*from w w w . j a v a 2 s .co m*/ shuffleBufferSize = conf.getInt(SHUFFLE_BUFFER_SIZE, DEFAULT_SHUFFLE_BUFFER_SIZE); shuffleTransferToAllowed = conf.getBoolean(SHUFFLE_TRANSFERTO_ALLOWED, (Shell.WINDOWS) ? WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED : DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED); maxSessionOpenFiles = conf.getInt(SHUFFLE_MAX_SESSION_OPEN_FILES, DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES); final String BOSS_THREAD_NAME_PREFIX = "Tez Shuffle Handler Boss #"; NioServerBossPool bossPool = new NioServerBossPool(Executors.newCachedThreadPool(), 1, new ThreadNameDeterminer() { @Override public String determineThreadName(String currentThreadName, String proposedThreadName) throws Exception { return BOSS_THREAD_NAME_PREFIX + currentThreadName.substring(currentThreadName.lastIndexOf('-') + 1); } }); final String WORKER_THREAD_NAME_PREFIX = "Tez Shuffle Handler Worker #"; NioWorkerPool workerPool = new NioWorkerPool(Executors.newCachedThreadPool(), maxShuffleThreads, new ThreadNameDeterminer() { @Override public String determineThreadName(String currentThreadName, String proposedThreadName) throws Exception { return WORKER_THREAD_NAME_PREFIX + currentThreadName.substring(currentThreadName.lastIndexOf('-') + 1); } }); selector = new NioServerSocketChannelFactory(bossPool, workerPool); super.serviceInit(new YarnConfiguration(conf)); }
From source file:org.apache.tez.client.TezClient.java
License:Apache License
/** * Start the client. This establishes a connection to the YARN cluster. * In session mode, this start the App Master thats runs all the DAGs in the * session.//from ww w .jav a 2s .c o m * @throws TezException * @throws IOException */ public synchronized void start() throws TezException, IOException { amConfig.setYarnConfiguration(new YarnConfiguration(amConfig.getTezConfiguration())); frameworkClient = createFrameworkClient(); frameworkClient.init(amConfig.getTezConfiguration(), amConfig.getYarnConfiguration()); frameworkClient.start(); if (this.amConfig.getTezConfiguration().get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "") .equals(atsHistoryLoggingServiceClassName)) { LOG.info("Using " + atsHistoryACLManagerClassName + " to manage Timeline ACLs"); try { historyACLPolicyManager = ReflectionUtils.createClazzInstance(atsHistoryACLManagerClassName); historyACLPolicyManager.setConf(this.amConfig.getYarnConfiguration()); } catch (TezUncheckedException e) { LOG.warn("Could not instantiate object for " + atsHistoryACLManagerClassName + ". ACLs cannot be enforced correctly for history data in Timeline", e); if (!amConfig.getTezConfiguration().getBoolean( TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS_DEFAULT)) { throw e; } historyACLPolicyManager = null; } } if (isSession) { LOG.info("Session mode. Starting session."); TezClientUtils.processTezLocalCredentialsFile(sessionCredentials, amConfig.getTezConfiguration()); Map<String, LocalResource> tezJarResources = getTezJarResources(sessionCredentials); clientTimeout = amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS, TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT); try { if (sessionAppId == null) { sessionAppId = createApplication(); } // Add session token for shuffle TezClientUtils.createSessionToken(sessionAppId.toString(), jobTokenSecretManager, sessionCredentials); ApplicationSubmissionContext appContext = TezClientUtils.createApplicationSubmissionContext( sessionAppId, null, clientName, amConfig, tezJarResources, sessionCredentials, usingTezArchiveDeploy, apiVersionInfo, historyACLPolicyManager); // Set Tez Sessions to not retry on AM crashes if recovery is disabled if (!amConfig.getTezConfiguration().getBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, TezConfiguration.DAG_RECOVERY_ENABLED_DEFAULT)) { appContext.setMaxAppAttempts(1); } frameworkClient.submitApplication(appContext); ApplicationReport appReport = frameworkClient.getApplicationReport(sessionAppId); LOG.info("The url to track the Tez Session: " + appReport.getTrackingUrl()); sessionStarted = true; } catch (YarnException e) { throw new TezException(e); } } }
From source file:org.apache.tez.client.TezSessionConfiguration.java
License:Apache License
public TezSessionConfiguration(AMConfiguration amConfiguration, TezConfiguration tezConfig) { this(amConfiguration, tezConfig, new YarnConfiguration(tezConfig)); }
From source file:org.apache.tez.dag.api.client.DAGClientImpl.java
License:Apache License
public DAGClientImpl(ApplicationId appId, String dagId, TezConfiguration conf, @Nullable FrameworkClient frameworkClient) { this.appId = appId; this.dagId = dagId; this.conf = conf; if (frameworkClient != null && conf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT)) { this.frameworkClient = frameworkClient; } else {//from w ww . j a v a2s . c o m this.frameworkClient = FrameworkClient.createFrameworkClient(conf); this.frameworkClient.init(conf, new YarnConfiguration(conf)); this.frameworkClient.start(); } isATSEnabled = conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "") .equals("org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService"); if (UserGroupInformation.isSecurityEnabled()) { //TODO: enable ATS integration in kerberos secured cluster - see TEZ-1529 isATSEnabled = false; } realClient = new DAGClientRPCImpl(appId, dagId, conf, this.frameworkClient); }
From source file:org.apache.tez.history.ATSImportTool.java
License:Apache License
static boolean hasHttpsPolicy(Configuration conf) { YarnConfiguration yarnConf = new YarnConfiguration(conf); return (HttpConfig.Policy.HTTPS_ONLY == HttpConfig.Policy.fromString( yarnConf.get(YarnConfiguration.YARN_HTTP_POLICY_KEY, YarnConfiguration.YARN_HTTP_POLICY_DEFAULT))); }
From source file:org.apache.tez.mapreduce.client.YARNRunner.java
License:Apache License
/** * Yarn runner incapsulates the client interface of * yarn/*from ww w.j av a 2 s .c o m*/ * @param conf the configuration object for the client */ public YARNRunner(Configuration conf) { this(conf, new ResourceMgrDelegate(new YarnConfiguration(conf))); }
From source file:org.apache.tez.mapreduce.hadoop.TestMRInputHelpers.java
License:Apache License
@BeforeClass public static void setup() throws IOException { try {// w w w . ja v a 2s . c o m conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR); dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).racks(null).build(); remoteFs = dfsCluster.getFileSystem(); } catch (IOException io) { throw new RuntimeException("problem starting mini dfs cluster", io); } Configuration testConf = new YarnConfiguration(dfsCluster.getFileSystem().getConf()); File testConfFile = new File(TEST_ROOT_DIR, "test.xml"); try { testConfFile.createNewFile(); testConf.writeXml(new FileOutputStream(testConfFile)); testConfFile.deleteOnExit(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); throw new RuntimeException(e); } remoteFs.mkdirs(new Path("/tmp/input/")); remoteFs.mkdirs(new Path("/tmp/splitsDirNew/")); remoteFs.mkdirs(new Path("/tmp/splitsDirOld/")); testFilePath = remoteFs.makeQualified(new Path("/tmp/input/test.xml")); remoteFs.copyFromLocalFile(new Path(testConfFile.getAbsolutePath()), testFilePath); FileStatus fsStatus = remoteFs.getFileStatus(testFilePath); Assert.assertTrue(fsStatus.getLen() > 0); oldSplitsDir = remoteFs.makeQualified(new Path("/tmp/splitsDirOld/")); newSplitsDir = remoteFs.makeQualified(new Path("/tmp/splitsDirNew/")); }