Example usage for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_QUEUE

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_QUEUE_NAME.

Prototype

String DEFAULT_QUEUE_NAME

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_QUEUE_NAME.

Click Source Link

Document

Default queue name

Usage

From source file:com.cloudera.impala.util.RequestPoolService.java

License:Apache License

/**
 * Resolves the actual pool to use via the allocation placement policy. The policy may
 * change the requested pool./*  w w w  . j av a  2  s.  c om*/
 *
 * @param requestedPool The requested pool. May not be null, an empty string indicates
 * the policy should return the default pool for this user.
 * @param user The user, must not be null or empty.
 * @return the actual pool to use, null if a pool could not be resolved.
 */
@VisibleForTesting
String assignToPool(String requestedPool, String user) throws IOException {
    Preconditions.checkState(running_.get());
    Preconditions.checkNotNull(requestedPool);
    Preconditions.checkArgument(!Strings.isNullOrEmpty(user));
    // Convert the user name to a short name (e.g. 'user1@domain' to 'user1') because
    // assignAppToQueue() will check group membership which should always be done on
    // the short name of the principal.
    String shortName = new User(user).getShortName();
    return allocationConf_.get().getPlacementPolicy().assignAppToQueue(
            requestedPool.isEmpty() ? YarnConfiguration.DEFAULT_QUEUE_NAME : requestedPool, shortName);
}

From source file:com.cloudera.llama.am.LlamaAMServiceImpl.java

License:Apache License

/**
 * Assign reservation to a queue based on the placement policy specified
 * in the alloc conf//  w  w w.  j  a  v  a2 s .  c o  m
 */
// Visible for testing
String assignToQueue(TLlamaAMReservationRequest request) throws LlamaException {
    // Default means no queue requested
    String requestedQueue = (request.isSetQueue()) ? request.getQueue() : YarnConfiguration.DEFAULT_QUEUE_NAME;
    if (requestedQueue == null) {
        requestedQueue = YarnConfiguration.DEFAULT_QUEUE_NAME;
    }
    String user = request.getUser();
    String queue;
    try {
        queue = allocConf.get().getPlacementPolicy().assignAppToQueue(requestedQueue, user);
    } catch (IOException ex) {
        throw new LlamaException(ex, ErrorCode.INTERNAL_ERROR);
    }
    if (queue == null) {
        throw new LlamaException(ErrorCode.RESERVATION_USER_TO_QUEUE_MAPPING_NOT_FOUND, user, requestedQueue);
    }
    LOG.debug("Reservation from user " + user + " with requested queue " + requestedQueue
            + " resolved to queue " + queue);

    return queue;
}

From source file:com.cloudera.llama.am.TestQueueAssignment.java

License:Apache License

@Test
public void testRequestedQueueNotSet() throws Exception {
    TLlamaAMReservationRequest request = mockRequest(null, false, "user");
    Mockito.when(placementPolicy.assignAppToQueue(Mockito.eq(YarnConfiguration.DEFAULT_QUEUE_NAME),
            Mockito.anyString())).thenReturn("resolved");
    String queue = amService.assignToQueue(request);
    amService.checkAccess(request.getUser(), queue, request.getQueue());
    Assert.assertEquals("resolved", queue);
}

From source file:com.cloudera.llama.am.TestQueueAssignment.java

License:Apache License

@Test
public void testRequestedQueueNull() throws Exception {
    TLlamaAMReservationRequest request = mockRequest(null, true, "user");
    Mockito.when(placementPolicy.assignAppToQueue(Mockito.eq(YarnConfiguration.DEFAULT_QUEUE_NAME),
            Mockito.anyString())).thenReturn("resolved");
    String queue = amService.assignToQueue(request);
    amService.checkAccess(request.getUser(), queue, request.getQueue());
    Assert.assertEquals("resolved", queue);
}

From source file:org.apache.hive.jdbc.TestSchedulerQueue.java

License:Apache License

/**
 * Verify that the queue refresh doesn't happen when configured to be off.
 *
 * @throws Exception/* ww  w  .  j a  va 2 s. c om*/
 */
@Test
public void testQueueMappingCheckDisabled() throws Exception {
    miniHS2.setConfProperty(HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname, "false");
    hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user1", "bar");
    verifyProperty(HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname, "false");
    verifyProperty("mapreduce.job.queuename", YarnConfiguration.DEFAULT_QUEUE_NAME);
}

From source file:org.apache.impala.util.RequestPoolService.java

License:Apache License

/**
 * Resolves the actual pool to use via the allocation placement policy. The policy may
 * change the requested pool./* www .jav a 2 s.  c o  m*/
 *
 * @param requestedPool The requested pool. May not be null, an empty string indicates
 * the policy should return the default pool for this user.
 * @param user The user, must not be null or empty.
 * @return the actual pool to use, null if a pool could not be resolved.
 */
@VisibleForTesting
String assignToPool(String requestedPool, String user) throws InternalException, IOException {
    Preconditions.checkState(running_.get());
    Preconditions.checkNotNull(requestedPool);
    Preconditions.checkArgument(!Strings.isNullOrEmpty(user));
    // Convert the user name to a short name (e.g. 'user1@domain' to 'user1') because
    // assignAppToQueue() will check group membership which should always be done on
    // the short name of the principal.
    String shortName = new User(user).getShortName();
    return allocationConf_.get().getPlacementPolicy().assignAppToQueue(
            requestedPool.isEmpty() ? YarnConfiguration.DEFAULT_QUEUE_NAME : requestedPool, shortName);
}

From source file:org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper.java

License:Apache License

public static TezConfiguration getDAGAMConfFromMRConf(Configuration tezConf) {

    // Set Tez parameters based on MR parameters.
    TezConfiguration dagAMConf = new TezConfiguration(tezConf);
    Map<String, String> mrParamToDAGParamMap = DeprecatedKeys.getMRToDAGParamMap();

    for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
        if (dagAMConf.get(entry.getKey()) != null) {
            dagAMConf.set(entry.getValue(), dagAMConf.get(entry.getKey()));
            dagAMConf.unset(entry.getKey());
            if (LOG.isDebugEnabled()) {
                LOG.debug("MR->DAG Translating MR key: " + entry.getKey() + " to Tez key: " + entry.getValue()
                        + " with value " + dagAMConf.get(entry.getValue()));
            }/*from  w ww .ja  v a  2 s  .  com*/
        }
    }

    String env = tezConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV);
    if (tezConf.get(MRJobConfig.MR_AM_ENV) != null) {
        env = (env == null) ? tezConf.get(MRJobConfig.MR_AM_ENV)
                : env + "," + tezConf.get(MRJobConfig.MR_AM_ENV);
    }

    if (env != null) {
        dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_ENV, env);
    }

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
            org.apache.tez.mapreduce.hadoop.MRHelpers.getJavaOptsForMRAM(tezConf));

    String queueName = tezConf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName);

    int amMemMB = tezConf.getInt(MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB);
    int amCores = tezConf.getInt(MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, "" + amMemMB);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, "" + amCores);

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS,
            "" + dagAMConf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));

    if (tezConf.get(MRConfiguration.JOB_CREDENTIALS_BINARY) != null) {
        dagAMConf.setIfUnset(TezConfiguration.TEZ_CREDENTIALS_PATH,
                tezConf.get(MRConfiguration.JOB_CREDENTIALS_BINARY));
    }

    //TODO: Strip out all MR settings

    return dagAMConf;
}

From source file:org.apache.tez.mapreduce.client.YARNRunner.java

License:Apache License

@Override
public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
        throws IOException, InterruptedException {

    // HACK! TEZ-604. Get rid of this once Hive moves all of it's tasks over to Tez native.
    maybeKillSession();/*from   w ww  .j  a va 2 s  .c  o m*/

    ApplicationId appId = resMgrDelegate.getApplicationId();

    FileSystem fs = FileSystem.get(conf);
    // Loads the job.xml written by the user.
    JobConf jobConf = new JobConf(new TezConfiguration(conf));

    // Extract individual raw MR configs.
    Configuration[] stageConfs = MultiStageMRConfToTezTranslator.getStageConfs(jobConf);

    // Transform all confs to use Tez keys
    for (int i = 0; i < stageConfs.length; i++) {
        MRHelpers.translateMRConfToTez(stageConfs[i]);
    }

    // create inputs to tezClient.submit()

    // FIXME set up job resources
    Map<String, LocalResource> jobLocalResources = createJobLocalResources(stageConfs[0], jobSubmitDir);

    // FIXME createDAG should take the tezConf as a parameter, instead of using
    // MR keys.
    DAG dag = createDAG(fs, jobId, stageConfs, jobSubmitDir, ts, jobLocalResources);

    List<String> vargs = new LinkedList<String>();
    // admin command opts and user command opts
    String mrAppMasterAdminOptions = conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
            MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS);
    warnForJavaLibPath(mrAppMasterAdminOptions, "app master", MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
            MRJobConfig.MR_AM_ADMIN_USER_ENV);
    vargs.add(mrAppMasterAdminOptions);

    // Add AM user command opts
    String mrAppMasterUserOptions = conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
            MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS);
    warnForJavaLibPath(mrAppMasterUserOptions, "app master", MRJobConfig.MR_AM_COMMAND_OPTS,
            MRJobConfig.MR_AM_ENV);
    vargs.add(mrAppMasterUserOptions);

    StringBuilder javaOpts = new StringBuilder();
    for (String varg : vargs) {
        javaOpts.append(varg).append(" ");
    }

    // Setup the CLASSPATH in environment
    // i.e. add { Hadoop jars, job jar, CWD } to classpath.
    Map<String, String> environment = new HashMap<String, String>();

    // Setup the environment variables for AM
    MRHelpers.updateEnvBasedOnMRAMEnv(conf, environment);
    StringBuilder envStrBuilder = new StringBuilder();
    boolean first = true;
    for (Entry<String, String> entry : environment.entrySet()) {
        if (!first) {
            envStrBuilder.append(",");
        } else {
            first = false;
        }
        envStrBuilder.append(entry.getKey()).append("=").append(entry.getValue());
    }
    String envStr = envStrBuilder.toString();

    TezConfiguration dagAMConf = getDAGAMConfFromMRConf();
    dagAMConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, javaOpts.toString());
    if (envStr.length() > 0) {
        dagAMConf.set(TezConfiguration.TEZ_AM_LAUNCH_ENV, envStr);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Setting MR AM env to : " + envStr);
        }
    }

    // Submit to ResourceManager
    try {
        dagAMConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, jobSubmitDir);

        // Set Tez parameters based on MR parameters.
        String queueName = jobConf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME);
        dagAMConf.set(TezConfiguration.TEZ_QUEUE_NAME, queueName);

        int amMemMB = jobConf.getInt(MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB);
        int amCores = jobConf.getInt(MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES);
        dagAMConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, amMemMB);
        dagAMConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, amCores);

        dagAMConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS,
                jobConf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));

        tezClient = new MRTezClient("MapReduce", dagAMConf, false, jobLocalResources, ts);
        tezClient.start();
        tezClient.submitDAGApplication(appId, dag);
        tezClient.stop();
    } catch (TezException e) {
        throw new IOException(e);
    }

    return getJobStatus(jobId);
}

From source file:org.apache.tez.mapreduce.YARNRunner.java

License:Apache License

@Override
public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
        throws IOException, InterruptedException {

    // TEZ-192 - stop using token file
    // Upload only in security mode: TODO
    Path applicationTokensFile = new Path(jobSubmitDir, MRJobConfig.APPLICATION_TOKENS_FILE);
    try {//ww w . j  av  a  2 s . c om
        ts.writeTokenStorageFile(applicationTokensFile, conf);
    } catch (IOException e) {
        throw new TezUncheckedException(e);
    }

    ApplicationId appId = resMgrDelegate.getApplicationId();

    FileSystem fs = FileSystem.get(conf);
    // Loads the job.xml written by the user.
    JobConf jobConf = new JobConf(new TezConfiguration(conf));

    // Extract individual raw MR configs.
    Configuration[] stageConfs = MultiStageMRConfToTezTranslator.getStageConfs(jobConf);

    // Transform all confs to use Tez keys
    MultiStageMRConfToTezTranslator.translateVertexConfToTez(stageConfs[0], null);
    for (int i = 1; i < stageConfs.length; i++) {
        MultiStageMRConfToTezTranslator.translateVertexConfToTez(stageConfs[i], stageConfs[i - 1]);
    }

    // create inputs to tezClient.submit()

    // FIXME set up job resources
    Map<String, LocalResource> jobLocalResources = createJobLocalResources(stageConfs[0], jobSubmitDir);

    // FIXME createDAG should take the tezConf as a parameter, instead of using
    // MR keys.
    DAG dag = createDAG(fs, jobId, stageConfs, jobSubmitDir, ts, jobLocalResources);

    List<String> vargs = new LinkedList<String>();
    // admin command opts and user command opts
    String mrAppMasterAdminOptions = conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
            MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS);
    warnForJavaLibPath(mrAppMasterAdminOptions, "app master", MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
            MRJobConfig.MR_AM_ADMIN_USER_ENV);
    vargs.add(mrAppMasterAdminOptions);

    // Add AM user command opts
    String mrAppMasterUserOptions = conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
            MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS);
    warnForJavaLibPath(mrAppMasterUserOptions, "app master", MRJobConfig.MR_AM_COMMAND_OPTS,
            MRJobConfig.MR_AM_ENV);
    vargs.add(mrAppMasterUserOptions);

    // Setup the CLASSPATH in environment
    // i.e. add { Hadoop jars, job jar, CWD } to classpath.
    Map<String, String> environment = new HashMap<String, String>();

    // Setup the environment variables for AM
    MRHelpers.updateEnvironmentForMRAM(conf, environment);

    TezConfiguration dagAMConf = getDAGAMConfFromMRConf();

    // Submit to ResourceManager
    try {
        Path appStagingDir = fs.resolvePath(new Path(jobSubmitDir));
        dagClient = tezClient.submitDAGApplication(appId, dag, appStagingDir, ts,
                jobConf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME), vargs, environment,
                jobLocalResources, dagAMConf);

    } catch (TezException e) {
        throw new IOException(e);
    }

    return getJobStatus(jobId);
}

Example usage for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_QUEUE_NAME

Introduction

Prototype

Document

Usage