Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration.

Prototype

public YarnConfiguration(Configuration conf) 

Source Link

Usage

From source file:org.apache.hama.bsp.YARNBSPJob.java

License:Apache License

public YARNBSPJob(HamaConfiguration conf) throws IOException {
    submitClient = new YARNBSPJobClient(conf);
    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    this.rpc = YarnRPC.create(conf);
    InetSocketAddress rmAddress = NetUtils
            .createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));
    LOG.info("Connecting to ResourceManager at " + rmAddress);

    this.applicationsManager = ((ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class,
            rmAddress, conf));/*from w w w  .  j ava2  s .  c  o  m*/
}

---

From source file:org.apache.hama.bsp.YARNBSPJobClient.java

License:Apache License

public YARNBSPJobClient(HamaConfiguration conf) {
    setConf(conf);
    yarnConf = new YarnConfiguration(conf);
    yarnClient = YarnClient.createYarnClient();
    yarnClient.init(yarnConf);
}

---

From source file:org.apache.hoya.yarn.appmaster.HoyaAppMaster.java

License:Apache License

/**
 * pick up the args from the service launcher
 * @param config/*from w ww. j  a  va2  s.  c om*/
 * @param args argument list
 */
@Override // RunService
public Configuration bindArgs(Configuration config, String... args) throws Exception {
    config = super.bindArgs(config, args);
    serviceArgs = new HoyaAMArgs(args);
    serviceArgs.parse();
    //yarn-ify
    YarnConfiguration yarnConfiguration = new YarnConfiguration(config);
    return HoyaUtils.patchConfiguration(yarnConfiguration);
}

---

From source file:org.apache.hoya.yarn.appmaster.HoyaAppMaster.java

License:Apache License

/**
 * Create and run the cluster.//from ww  w.j a  v a2s  .c o  m
 * @return exit code
 * @throws Throwable on a failure
 */
private int createAndRunCluster(String clustername) throws Throwable {
    HoyaVersionInfo.loadAndPrintVersionInfo(log);

    //load the cluster description from the cd argument
    String hoyaClusterDir = serviceArgs.getHoyaClusterURI();
    URI hoyaClusterURI = new URI(hoyaClusterDir);
    Path clusterDirPath = new Path(hoyaClusterURI);
    HoyaFileSystem fs = getClusterFS();

    // build up information about the running application -this
    // will be passed down to the cluster status
    MapOperations appInformation = new MapOperations();

    AggregateConf instanceDefinition = InstanceIO.loadInstanceDefinitionUnresolved(fs, clusterDirPath);

    log.info("Deploying cluster {}:", instanceDefinition);

    //REVISIT: why is this done?
    appState.updateInstanceDefinition(instanceDefinition);
    File confDir = getLocalConfDir();
    if (!confDir.exists() || !confDir.isDirectory()) {
        log.error("Bad conf dir {}", confDir);
        File parentFile = confDir.getParentFile();
        log.error("Parent dir {}:\n{}", parentFile, HoyaUtils.listDir(parentFile));
        throw new BadCommandArgumentsException("Configuration directory %s doesn't exist", confDir);
    }

    Configuration serviceConf = getConfig();
    // Try to get the proper filtering of static resources through the yarn proxy working
    serviceConf.set("hadoop.http.filter.initializers",
            "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer");

    conf = new YarnConfiguration(serviceConf);
    //get our provider
    MapOperations globalOptions = instanceDefinition.getInternalOperations().getGlobalOptions();
    String providerType = globalOptions.getMandatoryOption(OptionKeys.INTERNAL_PROVIDER_NAME);
    log.info("Cluster provider type is {}", providerType);
    HoyaProviderFactory factory = HoyaProviderFactory.createHoyaProviderFactory(providerType);
    providerService = factory.createServerProvider();
    // init the provider BUT DO NOT START IT YET
    providerService.init(getConfig());
    addService(providerService);

    InetSocketAddress address = HoyaUtils.getRmSchedulerAddress(conf);
    log.info("RM is at {}", address);
    yarnRPC = YarnRPC.create(conf);

    /*
     * Extract the container ID. This is then
     * turned into an (incompete) container
     */
    appMasterContainerID = ConverterUtils.toContainerId(
            HoyaUtils.mandatoryEnvVariable(ApplicationConstants.Environment.CONTAINER_ID.name()));
    appAttemptID = appMasterContainerID.getApplicationAttemptId();

    ApplicationId appid = appAttemptID.getApplicationId();
    log.info("Hoya AM for ID {}", appid.getId());

    appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID, appMasterContainerID.toString());
    appInformation.put(StatusKeys.INFO_AM_APP_ID, appid.toString());
    appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID, appAttemptID.toString());

    UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
    Credentials credentials = currentUser.getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    dob.close();
    // Now remove the AM->RM token so that containers cannot access it.
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        log.info("Token {}", token.getKind());
        if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }
    allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    // set up secret manager
    secretManager = new ClientToAMTokenSecretManager(appAttemptID, null);

    // if not a secure cluster, extract the username -it will be
    // propagated to workers
    if (!UserGroupInformation.isSecurityEnabled()) {
        hoyaUsername = System.getenv(HADOOP_USER_NAME);
        log.info(HADOOP_USER_NAME + "='{}'", hoyaUsername);
    }

    Map<String, String> envVars;

    /**
     * It is critical this section is synchronized, to stop async AM events
     * arriving while registering a restarting AM.
     */
    synchronized (appState) {
        int heartbeatInterval = HEARTBEAT_INTERVAL;

        //add the RM client -this brings the callbacks in
        asyncRMClient = AMRMClientAsync.createAMRMClientAsync(heartbeatInterval, this);
        addService(asyncRMClient);
        //wrap it for the app state model
        rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient);
        //now bring it up
        runChildService(asyncRMClient);

        //nmclient relays callbacks back to this class
        nmClientAsync = new NMClientAsyncImpl("nmclient", this);
        runChildService(nmClientAsync);

        //bring up the Hoya RPC service
        startHoyaRPCServer();

        InetSocketAddress rpcServiceAddr = rpcService.getConnectAddress();
        appMasterHostname = rpcServiceAddr.getHostName();
        appMasterRpcPort = rpcServiceAddr.getPort();
        appMasterTrackingUrl = null;
        log.info("AM Server is listening at {}:{}", appMasterHostname, appMasterRpcPort);
        appInformation.put(StatusKeys.INFO_AM_HOSTNAME, appMasterHostname);
        appInformation.set(StatusKeys.INFO_AM_RPC_PORT, appMasterRpcPort);

        //build the role map
        List<ProviderRole> providerRoles = new ArrayList<ProviderRole>(providerService.getRoles());
        providerRoles.addAll(HoyaAMClientProvider.ROLES);

        // Start up the WebApp and track the URL for it
        webApp = new HoyaAMWebApp();
        WebApps.$for("hoyaam", WebAppApi.class, new WebAppApiImpl(this, appState, providerService), "ws")
                .with(serviceConf).start(webApp);
        appMasterTrackingUrl = "http://" + appMasterHostname + ":" + webApp.port();
        WebAppService<HoyaAMWebApp> webAppService = new WebAppService<HoyaAMWebApp>("hoya", webApp);

        webAppService.init(conf);
        webAppService.start();
        addService(webAppService);

        appInformation.put(StatusKeys.INFO_AM_WEB_URL, appMasterTrackingUrl + "/");
        appInformation.set(StatusKeys.INFO_AM_WEB_PORT, webApp.port());

        // Register self with ResourceManager
        // This will start heartbeating to the RM
        // address = HoyaUtils.getRmSchedulerAddress(asyncRMClient.getConfig());
        log.info("Connecting to RM at {},address tracking URL={}", appMasterRpcPort, appMasterTrackingUrl);
        RegisterApplicationMasterResponse response = asyncRMClient.registerApplicationMaster(appMasterHostname,
                appMasterRpcPort, appMasterTrackingUrl);
        Resource maxResources = response.getMaximumResourceCapability();
        containerMaxMemory = maxResources.getMemory();
        containerMaxCores = maxResources.getVirtualCores();
        appState.setContainerLimits(maxResources.getMemory(), maxResources.getVirtualCores());
        // set the RM-defined maximum cluster values
        appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(containerMaxCores));
        appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(containerMaxMemory));

        boolean securityEnabled = UserGroupInformation.isSecurityEnabled();
        if (securityEnabled) {
            secretManager.setMasterKey(response.getClientToAMTokenMasterKey().array());
            applicationACLs = response.getApplicationACLs();

            //tell the server what the ACLs are 
            rpcService.getServer().refreshServiceAcl(conf, new HoyaAMPolicyProvider());
        }

        // extract container list
        List<Container> liveContainers = AMRestartSupport.retrieveContainersFromPreviousAttempt(response);
        String amRestartSupported = Boolean.toString(liveContainers != null);
        appInformation.put(StatusKeys.INFO_AM_RESTART_SUPPORTED, amRestartSupported);

        //now validate the installation
        Configuration providerConf = providerService.loadProviderConfigurationInformation(confDir);

        providerService.validateApplicationConfiguration(instanceDefinition, confDir, securityEnabled);

        //determine the location for the role history data
        Path historyDir = new Path(clusterDirPath, HISTORY_DIR_NAME);

        //build the instance
        appState.buildInstance(instanceDefinition, providerConf, providerRoles, fs.getFileSystem(), historyDir,
                liveContainers, appInformation);

        // add the AM to the list of nodes in the cluster

        appState.buildAppMasterNode(appMasterContainerID, appMasterHostname, webApp.port(),
                appMasterHostname + ":" + webApp.port());

        // build up environment variables that the AM wants set in every container
        // irrespective of provider and role.
        envVars = new HashMap<String, String>();
        if (hoyaUsername != null) {
            envVars.put(HADOOP_USER_NAME, hoyaUsername);
        }
    }
    String rolesTmpSubdir = appMasterContainerID.toString() + "/roles";

    String amTmpDir = globalOptions.getMandatoryOption(OptionKeys.INTERNAL_AM_TMP_DIR);

    Path tmpDirPath = new Path(amTmpDir);
    Path launcherTmpDirPath = new Path(tmpDirPath, rolesTmpSubdir);
    fs.getFileSystem().mkdirs(launcherTmpDirPath);

    //launcher service
    launchService = new RoleLaunchService(this, providerService, fs, new Path(getGeneratedConfDir()), envVars,
            launcherTmpDirPath);

    runChildService(launchService);

    appState.noteAMLaunched();

    //Give the provider restricted access to the state
    providerService.bind(appState);

    // launch the provider; this is expected to trigger a callback that
    // brings up the service
    launchProviderService(instanceDefinition, confDir);

    try {
        //now block waiting to be told to exit the process
        waitForAMCompletionSignal();
        //shutdown time
    } finally {
        finish();
    }

    return amExitCode;
}

---

From source file:org.apache.hoya.yarn.client.HoyaClient.java

License:Apache License

@Override
public Configuration bindArgs(Configuration config, String... args) throws Exception {
    config = super.bindArgs(config, args);
    log.debug("Binding Arguments");
    serviceArgs = new ClientArgs(args);
    serviceArgs.parse();//from w  w w.  ja v a  2s.c  om
    // yarn-ify
    YarnConfiguration yarnConfiguration = new YarnConfiguration(config);
    return HoyaUtils.patchConfiguration(yarnConfiguration);
}

---

From source file:org.apache.kylin.tool.common.HadoopConfExtractor.java

License:Apache License

public static String extractYarnMasterUrl(Configuration conf) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    final String yarnStatusCheckUrl = config.getYarnStatusCheckUrl();
    Pattern pattern = Pattern.compile("(http(s)?://)([^:]*):([^/])*.*");
    if (yarnStatusCheckUrl != null) {
        Matcher m = pattern.matcher(yarnStatusCheckUrl);
        if (m.matches()) {
            return m.group(1) + m.group(2) + ":" + m.group(3);
        }//ww  w .  jav  a  2 s .  c o  m
    }

    logger.info("kylin.engine.mr.yarn-check-status-url" + " is not set, read from hadoop configuration");

    String webappConfKey, defaultAddr;
    if (YarnConfiguration.useHttps(conf)) {
        webappConfKey = YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS;
        defaultAddr = YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS;
    } else {
        webappConfKey = YarnConfiguration.RM_WEBAPP_ADDRESS;
        defaultAddr = YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS;
    }

    String rmWebHost;
    if (HAUtil.isHAEnabled(conf)) {
        YarnConfiguration yarnConf = new YarnConfiguration(conf);
        String active = RMHAUtils.findActiveRMHAId(yarnConf);
        rmWebHost = HAUtil.getConfValueForRMInstance(HAUtil.addSuffix(webappConfKey, active), defaultAddr,
                yarnConf);
    } else {
        rmWebHost = HAUtil.getConfValueForRMInstance(webappConfKey, defaultAddr, conf);
    }

    if (StringUtils.isEmpty(rmWebHost)) {
        return null;
    }
    if (!rmWebHost.startsWith("http://") && !rmWebHost.startsWith("https://")) {
        rmWebHost = (YarnConfiguration.useHttps(conf) ? "https://" : "http://") + rmWebHost;
    }
    Matcher m = pattern.matcher(rmWebHost);
    Preconditions.checkArgument(m.matches(), "Yarn master URL not found.");
    logger.info("yarn master url: " + rmWebHost);
    return rmWebHost;
}

---

From source file:org.apache.kylin.tool.JobTaskCounterExtractor.java

License:Apache License

private String getRestCheckUrl() {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    final String yarnStatusCheckUrl = config.getYarnStatusCheckUrl();
    Pattern pattern = Pattern.compile("(http://)(.*):.*");
    if (yarnStatusCheckUrl != null) {
        Matcher m = pattern.matcher(yarnStatusCheckUrl);
        m.matches();/*from   w w w.j  a  v  a 2 s .co  m*/
        yarnUrl = m.group(1) + m.group(2) + ":19888";
        return yarnUrl;
    } else {
        logger.info("kylin.job.yarn.app.rest.check.status.url" + " is not set read from hadoop configuration");
    }
    Configuration conf = HadoopUtil.getCurrentConfiguration();
    String rmWebHost = HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf);
    if (HAUtil.isHAEnabled(conf)) {
        YarnConfiguration yarnConf = new YarnConfiguration(conf);
        String active = RMHAUtils.findActiveRMHAId(yarnConf);
        rmWebHost = HAUtil.getConfValueForRMInstance(
                HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, active),
                YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, yarnConf);
    }
    if (StringUtils.isEmpty(rmWebHost)) {
        return null;
    }
    if (rmWebHost.startsWith("http://") || rmWebHost.startsWith("https://")) {
        //do nothing
    } else {
        rmWebHost = "http://" + rmWebHost;
    }
    Matcher m = pattern.matcher(rmWebHost);
    m.matches();
    return m.group(1) + m.group(2) + ":19888";
}

---

From source file:org.apache.kylin.tool.MrJobInfoExtractor.java

License:Apache License

private void extractRestCheckUrl() {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    final String yarnStatusCheckUrl = config.getYarnStatusCheckUrl();
    Pattern pattern = Pattern.compile("(http://)([^:]*):([^/])*.*");
    if (yarnStatusCheckUrl != null) {
        Matcher m = pattern.matcher(yarnStatusCheckUrl);
        if (m.matches()) {
            jobHistoryUrlBase = m.group(1) + m.group(2) + ":19888";
            yarnMasterUrlBase = m.group(1) + m.group(2) + ":" + m.group(3);
        }//from  w ww .  ja v a 2  s  .  c  o  m
    }
    logger.info("kylin.engine.mr.yarn-check-status-url" + " is not set, read from hadoop configuration");

    Configuration conf = HadoopUtil.getCurrentConfiguration();
    String rmWebHost = HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf);
    if (HAUtil.isHAEnabled(conf)) {
        YarnConfiguration yarnConf = new YarnConfiguration(conf);
        String active = RMHAUtils.findActiveRMHAId(yarnConf);
        rmWebHost = HAUtil.getConfValueForRMInstance(
                HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, active),
                YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, yarnConf);
    }
    if (StringUtils.isEmpty(rmWebHost)) {
        return;
    }
    if (!rmWebHost.startsWith("http://") && !rmWebHost.startsWith("https://")) {
        rmWebHost = "http://" + rmWebHost;
    }
    Matcher m = pattern.matcher(rmWebHost);
    Preconditions.checkArgument(m.matches(), "Yarn master URL not found.");
    yarnMasterUrlBase = rmWebHost;
    jobHistoryUrlBase = m.group(1) + HAUtil.getConfValueForRMInstance("mapreduce.jobhistory.webapp.address",
            m.group(2) + ":19888", conf);
}

---

From source file:org.apache.slider.client.SliderClient.java

License:Apache License

/**
 * This is called <i>Before serviceInit is called</i>
 * @param config the initial configuration build up by the
 * service launcher.//from  www  .  jav  a  2s .  c o m
 * @param args argument list list of arguments passed to the command line
 * after any launcher-specific commands have been stripped.
 * @return the post-binding configuration to pass to the <code>init()</code>
 * operation.
 * @throws Exception
 */
@Override
public Configuration bindArgs(Configuration config, String... args) throws Exception {
    config = super.bindArgs(config, args);
    serviceArgs = new ClientArgs(args);
    serviceArgs.parse();
    // yarn-ify
    YarnConfiguration yarnConfiguration = new YarnConfiguration(config);
    return SliderUtils.patchConfiguration(yarnConfiguration);
}

---

From source file:org.apache.slider.core.launch.CredentialUtils.java

License:Apache License

/**
 * Look up and return the resource manager's principal. This method
 * automatically does the <code>_HOST</code> replacement in the principal and
 * correctly handles HA resource manager configurations.
 *
 * From: YARN-4629//from  w  w w  .ja  va 2s .  c om
 * @param conf the {@link Configuration} file from which to read the
 * principal
 * @return the resource manager's principal string
 * @throws IOException thrown if there's an error replacing the host name
 */
public static String getRMPrincipal(Configuration conf) throws IOException {
    String principal = conf.get(RM_PRINCIPAL, "");
    String hostname;
    Preconditions.checkState(!principal.isEmpty(), "Not set: " + RM_PRINCIPAL);

    if (HAUtil.isHAEnabled(conf)) {
        YarnConfiguration yarnConf = new YarnConfiguration(conf);
        if (yarnConf.get(RM_HA_ID) == null) {
            // If RM_HA_ID is not configured, use the first of RM_HA_IDS.
            // Any valid RM HA ID should work.
            String[] rmIds = yarnConf.getStrings(RM_HA_IDS);
            Preconditions.checkState((rmIds != null) && (rmIds.length > 0), "Not set " + RM_HA_IDS);
            yarnConf.set(RM_HA_ID, rmIds[0]);
        }

        hostname = yarnConf.getSocketAddr(RM_ADDRESS, DEFAULT_RM_ADDRESS, DEFAULT_RM_PORT).getHostName();
    } else {
        hostname = conf.getSocketAddr(RM_ADDRESS, DEFAULT_RM_ADDRESS, DEFAULT_RM_PORT).getHostName();
    }
    return SecurityUtil.getServerPrincipal(principal, hostname);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• »