Example usage for org.apache.hadoop.conf Configuration getInts

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getInts.

Prototype

public int[] getInts(String name) 

Source Link

Document

Get the value of the name property as a set of comma-delimited int values.

Usage

From source file:org.apache.crunch.kafka.inputformat.KafkaInputFormat.java

License:Apache License

/**
 * Reads the {@code configuration} to determine which topics, partitions, and offsets should be used for reading data.
 *
 * @param configuration the configuration to derive the data to read.
 * @return a map of {@link TopicPartition} to a pair of start and end offsets.
 * @throws IllegalStateException if the {@code configuration} does not have the start and end offsets set properly
 * for a partition.//  w w  w.  j a v  a 2s. c  o  m
 */
public static Map<TopicPartition, Pair<Long, Long>> getOffsets(Configuration configuration) {
    Map<TopicPartition, Pair<Long, Long>> offsets = new HashMap<>();
    //find configuration for all of the topics with defined partitions
    Map<String, String> topicPartitionKeys = configuration.getValByRegex(TOPIC_KEY_REGEX);

    //for each topic start to process it's partitions
    for (String key : topicPartitionKeys.keySet()) {
        String topic = getTopicFromKey(key);
        int[] partitions = configuration.getInts(key);
        //for each partition find and add the start/end offset
        for (int partitionId : partitions) {
            TopicPartition topicPartition = new TopicPartition(topic, partitionId);
            long start = configuration.getLong(generatePartitionStartKey(topic, partitionId), Long.MIN_VALUE);
            long end = configuration.getLong(generatePartitionEndKey(topic, partitionId), Long.MIN_VALUE);

            if (start == Long.MIN_VALUE || end == Long.MIN_VALUE) {
                throw new IllegalStateException("The " + topicPartition + "has an invalid start:" + start
                        + " or end:" + end + " offset configured.");
            }

            offsets.put(topicPartition, Pair.of(start, end));
        }
    }

    return offsets;
}

---

From source file:org.apache.crunch.kafka.record.KafkaInputFormat.java

License:Apache License

/**
 * Reads the {@code configuration} to determine which topics, partitions, and offsets should be used for reading data.
 *
 * @param configuration the configuration to derive the data to read.
 * @return a map of {@link TopicPartition} to a pair of start and end offsets.
 * @throws IllegalStateException if the {@code configuration} does not have the start and end offsets set properly
 *                               for a partition.
 *//*from w  w w.  ja  v  a2  s  . c  om*/
public static Map<TopicPartition, Pair<Long, Long>> getOffsets(Configuration configuration) {
    Map<TopicPartition, Pair<Long, Long>> offsets = new HashMap<>();
    //find configuration for all of the topics with defined partitions
    Map<String, String> topicPartitionKeys = configuration.getValByRegex(TOPIC_KEY_REGEX);

    //for each topic start to process it's partitions
    for (String key : topicPartitionKeys.keySet()) {
        String topic = getTopicFromKey(key);
        int[] partitions = configuration.getInts(key);
        //for each partition find and add the start/end offset
        for (int partitionId : partitions) {
            TopicPartition topicPartition = new TopicPartition(topic, partitionId);
            long start = configuration.getLong(generatePartitionStartKey(topic, partitionId), Long.MIN_VALUE);
            long end = configuration.getLong(generatePartitionEndKey(topic, partitionId), Long.MIN_VALUE);

            if (start == Long.MIN_VALUE || end == Long.MIN_VALUE) {
                throw new IllegalStateException("The " + topicPartition + " has an invalid start:" + start
                        + " or end:" + end + " offset configured.");
            }

            offsets.put(topicPartition, Pair.of(start, end));
        }
    }

    return offsets;
}

---

From source file:org.apache.nutch.hostdb.UpdateHostDbReducer.java

License:Apache License

/**
  * Configures the thread pool and prestarts all resolver threads.
  *///from   www  .  ja  va  2 s.c  om
@Override
public void setup(Reducer<Text, NutchWritable, Text, HostDatum>.Context context) {
    Configuration conf = context.getConfiguration();
    purgeFailedHostsThreshold = conf.getInt(UpdateHostDb.HOSTDB_PURGE_FAILED_HOSTS_THRESHOLD, -1);
    numResolverThreads = conf.getInt(UpdateHostDb.HOSTDB_NUM_RESOLVER_THREADS, 10);
    recheckInterval = conf.getInt(UpdateHostDb.HOSTDB_RECHECK_INTERVAL, 86400) * 1000;
    checkFailed = conf.getBoolean(UpdateHostDb.HOSTDB_CHECK_FAILED, false);
    checkNew = conf.getBoolean(UpdateHostDb.HOSTDB_CHECK_NEW, false);
    checkKnown = conf.getBoolean(UpdateHostDb.HOSTDB_CHECK_KNOWN, false);
    force = conf.getBoolean(UpdateHostDb.HOSTDB_FORCE_CHECK, false);
    numericFields = conf.getStrings(UpdateHostDb.HOSTDB_NUMERIC_FIELDS);
    stringFields = conf.getStrings(UpdateHostDb.HOSTDB_STRING_FIELDS);
    percentiles = conf.getInts(UpdateHostDb.HOSTDB_PERCENTILES);

    // What fields do we need to collect metadata from
    if (numericFields != null) {
        numericFieldWritables = new Text[numericFields.length];
        for (int i = 0; i < numericFields.length; i++) {
            numericFieldWritables[i] = new Text(numericFields[i]);
        }
    }

    if (stringFields != null) {
        stringFieldWritables = new Text[stringFields.length];
        for (int i = 0; i < stringFields.length; i++) {
            stringFieldWritables[i] = new Text(stringFields[i]);
        }
    }

    // Initialize the thread pool with our queue
    executor = new ThreadPoolExecutor(numResolverThreads, numResolverThreads, 5, TimeUnit.SECONDS, queue);

    // Run all threads in the pool
    executor.prestartAllCoreThreads();
}

---