Example usage for org.apache.hadoop.fs FileSystem resolvePath

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem resolvePath.

Prototype

public Path resolvePath(final Path p) throws IOException 

Source Link

Document

Return the fully-qualified path of path, resolving the path through any symlinks or mount point.

Usage

From source file:org.apache.tez.mapreduce.YARNRunner.java

License:Apache License

@Override
public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
        throws IOException, InterruptedException {

    // TEZ-192 - stop using token file
    // Upload only in security mode: TODO
    Path applicationTokensFile = new Path(jobSubmitDir, MRJobConfig.APPLICATION_TOKENS_FILE);
    try {/*from w w w. j a va2 s  .  co  m*/
        ts.writeTokenStorageFile(applicationTokensFile, conf);
    } catch (IOException e) {
        throw new TezUncheckedException(e);
    }

    ApplicationId appId = resMgrDelegate.getApplicationId();

    FileSystem fs = FileSystem.get(conf);
    // Loads the job.xml written by the user.
    JobConf jobConf = new JobConf(new TezConfiguration(conf));

    // Extract individual raw MR configs.
    Configuration[] stageConfs = MultiStageMRConfToTezTranslator.getStageConfs(jobConf);

    // Transform all confs to use Tez keys
    MultiStageMRConfToTezTranslator.translateVertexConfToTez(stageConfs[0], null);
    for (int i = 1; i < stageConfs.length; i++) {
        MultiStageMRConfToTezTranslator.translateVertexConfToTez(stageConfs[i], stageConfs[i - 1]);
    }

    // create inputs to tezClient.submit()

    // FIXME set up job resources
    Map<String, LocalResource> jobLocalResources = createJobLocalResources(stageConfs[0], jobSubmitDir);

    // FIXME createDAG should take the tezConf as a parameter, instead of using
    // MR keys.
    DAG dag = createDAG(fs, jobId, stageConfs, jobSubmitDir, ts, jobLocalResources);

    List<String> vargs = new LinkedList<String>();
    // admin command opts and user command opts
    String mrAppMasterAdminOptions = conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
            MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS);
    warnForJavaLibPath(mrAppMasterAdminOptions, "app master", MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
            MRJobConfig.MR_AM_ADMIN_USER_ENV);
    vargs.add(mrAppMasterAdminOptions);

    // Add AM user command opts
    String mrAppMasterUserOptions = conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
            MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS);
    warnForJavaLibPath(mrAppMasterUserOptions, "app master", MRJobConfig.MR_AM_COMMAND_OPTS,
            MRJobConfig.MR_AM_ENV);
    vargs.add(mrAppMasterUserOptions);

    // Setup the CLASSPATH in environment
    // i.e. add { Hadoop jars, job jar, CWD } to classpath.
    Map<String, String> environment = new HashMap<String, String>();

    // Setup the environment variables for AM
    MRHelpers.updateEnvironmentForMRAM(conf, environment);

    TezConfiguration dagAMConf = getDAGAMConfFromMRConf();

    // Submit to ResourceManager
    try {
        Path appStagingDir = fs.resolvePath(new Path(jobSubmitDir));
        dagClient = tezClient.submitDAGApplication(appId, dag, appStagingDir, ts,
                jobConf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME), vargs, environment,
                jobLocalResources, dagAMConf);

    } catch (TezException e) {
        throw new IOException(e);
    }

    return getJobStatus(jobId);
}

---

• «
• 1
• 2
• »