Example usage for org.apache.hadoop.conf Configuration get

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration get.

Prototype

public String get(String name)

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:com.cloudera.recordservice.mr.PlanUtil.java

License:Apache License

/**
 * This also handles authentication using credentials. If there is a delegation
 * token in the credentials, that will be used to authenticate the planner
 * connection. Otherwise, if kerberos is enabled, a token will be generated
 * and added to the credentials.//from   w  w w .  j  a v a  2  s. c  o  m
 * TODO: is this behavior sufficient? Do we need to fall back and renew tokens
 * or does the higher level framework (i.e. oozie) do that?
 */
public static SplitsInfo getSplits(Configuration jobConf, Credentials credentials) throws IOException {
    Request request = PlanUtil.getRequest(jobConf);
    RecordServicePlannerClient.Builder builder = getBuilder(jobConf);
    List<NetworkAddress> plannerHostPorts = getPlannerHostPorts(jobConf);
    String kerberosPrincipal = jobConf.get(ConfVars.KERBEROS_PRINCIPAL_CONF.name);
    PlanRequestResult result = null;
    RecordServicePlannerClient planner = PlanUtil.getPlanner(jobConf, builder, plannerHostPorts,
            kerberosPrincipal, credentials);

    try {
        result = planner.planRequest(request);
        if (planner.isKerberosAuthenticated()) {
            // We need to get a delegation token and populate credentials (for the map tasks)
            // TODO: what to set as renewer?
            Token<DelegationTokenIdentifier> delegationToken = TokenUtils
                    .fromTDelegationToken(planner.getDelegationToken(""));
            credentials.addToken(DelegationTokenIdentifier.DELEGATION_KIND, delegationToken);
        }
    } catch (RecordServiceException e) {
        throw new IOException(e);
    } finally {
        if (planner != null)
            planner.close();
    }

    Schema schema = new Schema(result.schema);
    List<InputSplit> splits = new ArrayList<InputSplit>();
    for (Task t : result.tasks) {
        splits.add(new RecordServiceInputSplit(schema, new TaskInfo(t, result.hosts)));
    }
    LOG.debug(String.format("Generated %d splits.", splits.size()));

    // Randomize the order of the splits to mitigate skew.
    Collections.shuffle(splits);
    return new SplitsInfo(splits, schema);
}

From source file:com.cloudera.recordservice.mr.PlanUtil.java

License:Apache License

/**
 * Checks whether planner auto discovery is enabled. This checks the 'conf' to see
 * if the ZooKeeper connection string is defined and is not empty.
 *///  www  .  ja  v a 2 s. com
private static boolean isPlannerDiscoveryEnabled(Configuration conf) {
    String zkConnectString = conf.get(ConfVars.ZOOKEEPER_CONNECTION_STRING_CONF.name);
    return zkConnectString != null && !zkConnectString.isEmpty();
}

From source file:com.cloudera.recordservice.mr.ZooKeeperUtil.java

License:Apache License

/**
 * Returns a list of network addresses for the RecordService planners currently
 * available as maintained by ZooKeeper.
 * @param conf The input client job configuration
 * @return A list of <code>NetworkAddress</code>es for all the planners available
 *//*from  w ww. j ava  2 s . c  o  m*/
public static List<NetworkAddress> getPlanners(Configuration conf) throws IOException {
    String connectionString = conf.get(ConfVars.ZOOKEEPER_CONNECTION_STRING_CONF.name);
    if (connectionString == null || connectionString.trim().isEmpty()) {
        throw new IllegalArgumentException("Zookeeper connect string has to be specified through "
                + ConfVars.ZOOKEEPER_CONNECTION_STRING_CONF.name);
    }
    LOGGER.info("Connecting to zookeeper at: " + connectionString);

    int connectionTimeout = conf.getInt(ConfVars.ZOOKEEPER_CONNECT_TIMEOUTMILLIS_CONF.name,
            CuratorFrameworkFactory.builder().getConnectionTimeoutMs());
    LOGGER.info("Zookeeper connection timeout: " + connectionTimeout);

    String rootNode = conf.get(ConfVars.ZOOKEEPER_ZNODE_CONF.name, RecordServiceConfig.ZOOKEEPER_ZNODE_DEFAULT);
    LOGGER.info("Zookeeper root: " + rootNode);

    CuratorFramework cf = CuratorFrameworkFactory.builder().connectString(connectionString)
            .connectionTimeoutMs(connectionTimeout).aclProvider(new ZooKeeperACLProvider())
            .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build();
    cf.start();

    List<NetworkAddress> result = new ArrayList<NetworkAddress>();
    try {
        for (String path : cf.getChildren().forPath(rootNode + "/planners")) {
            NetworkAddress addr = parsePath(path);
            if (addr != null)
                result.add(parsePath(path));
        }
    } catch (Exception e) {
        cf.close();
        throw new IOException("Could not obtain planner membership" + " from " + connectionString
                + ". Error message: " + e.getMessage(), e);
    }
    cf.close();
    return result;
}

From source file:com.cloudera.recordservice.pig.PigHCatUtil.java

License:Apache License

public static void saveConfigIntoUDFProperties(Properties p, Configuration config, String propName) {
    if (config.get(propName) != null) {
        p.setProperty(propName, config.get(propName));
    }// ww  w  .j  a  v a 2s.co  m
}

From source file:com.cloudera.sa.hbasebulkload.HBASEBulkLoadKeyValueMapper.java

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    Configuration config = context.getConfiguration();
    hbaseTabName = config.get(HBASEBulkLoadConstants.HBASE_TABLE_KEY);
    hbaseColumnFamily = config.get(HBASEBulkLoadConstants.HBASE_COLUMN_FAMILY_KEY);

    hbaseColumnsSeperator = config.get(HBASEBulkLoadConstants.HBASE_COLUMN_SEPERATOR_KEY);
    csvParser = new CSVParser(hbaseColumnsSeperator.charAt(0));
    System.out.println(5);/*from  w w  w  .ja  va 2s  .  co m*/
    hbaseColumns = csvParser.parseLine(config.get(HBASEBulkLoadConstants.HBASE_COLUMNS_KEY));
    noOfColumns = hbaseColumns.length;

}

From source file:com.cloudera.sqoop.mapreduce.DelegatingOutputFormat.java

License:Apache License

@Override
/** {@inheritDoc} */
public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();

    if (null == conf.get(DELEGATE_CLASS_KEY)) {
        throw new IOException("Delegate FieldMapProcessor class is not set.");
    }/*w ww. jav a2  s .  co  m*/
}