Example usage for org.apache.hadoop.conf Configuration getBoolean

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getBoolean.

Prototype

public boolean getBoolean(String name, boolean defaultValue) 

Source Link

Document

Get the value of the name property as a boolean.

Usage

From source file:edu.isi.mavuno.score.GetTopResults.java

License:Apache License

public int run() throws ClassNotFoundException, InterruptedException, IOException {
    Configuration conf = getConf();

    String inputPath = MavunoUtils.getRequiredParam("Mavuno.GetTopResults.InputPath", conf);
    String outputPath = MavunoUtils.getRequiredParam("Mavuno.GetTopResults.OutputPath", conf);
    int numResults = Integer.parseInt(MavunoUtils.getRequiredParam("Mavuno.GetTopResults.NumResults", conf));
    boolean sequenceFileOutputFormat = conf.getBoolean("Mavuno.GetTopResults.SequenceFileOutputFormat", false);

    sLogger.info("Tool name: GetTopResults");
    sLogger.info(" - Input path: " + inputPath);
    sLogger.info(" - Number of results: " + numResults);
    sLogger.info(" - Output path: " + outputPath);

    Job job = new Job(conf);
    job.setJobName("GetTopResults");

    FileInputFormat.addInputPath(job, new Path(inputPath));
    FileOutputFormat.setOutputPath(job, new Path(outputPath));

    job.setInputFormatClass(TextInputFormat.class);
    job.setSortComparatorClass(ContextPatternWritable.Comparator.class);
    job.setPartitionerClass(ContextPatternWritable.IdPartitioner.class);

    if (sequenceFileOutputFormat) {
        job.setOutputFormatClass(SequenceFileOutputFormat.class);
    } else {/*from   w ww  .  j  av  a 2 s .c  om*/
        job.setOutputFormatClass(TextOutputFormat.class);
    }

    job.setMapOutputKeyClass(ContextPatternWritable.class);
    job.setMapOutputValueClass(DoubleWritable.class);

    job.setOutputKeyClass(ContextPatternWritable.class);
    job.setOutputValueClass(DoubleWritable.class);

    job.setMapperClass(MyMapper.class);
    job.setReducerClass(MyReducer.class);

    job.waitForCompletion(true);

    return 0;
}

---

From source file:edu.iu.daal_subgraph.SCDaalCollectiveMapper.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {

    LOG.info("start setup");

    Configuration configuration = context.getConfiguration();
    numMappers = configuration.getInt(SCConstants.NUM_MAPPERS, 10);
    templateFile = configuration.get(SCConstants.TEMPLATE_PATH);
    useLocalMultiThread = configuration.getBoolean(SCConstants.USE_LOCAL_MULTITHREAD, true);
    rotation_pipeline = configuration.getBoolean(SCConstants.ROTATION_PIPELINE, true);

    LOG.info("init templateFile");
    LOG.info(templateFile);//from   www. j  a va2  s  .com

    numThreads = configuration.getInt(SCConstants.THREAD_NUM, 10);

    //always use the maximum hardware threads to load in data and convert data 
    harpThreads = Runtime.getRuntime().availableProcessors();
    LOG.info("Num Threads " + numThreads);
    LOG.info("Num harp load data threads " + harpThreads);

    numCores = configuration.getInt(SCConstants.CORE_NUM, 24);
    affinity = configuration.get(SCConstants.THD_AFFINITY);
    omp_opt = configuration.get(SCConstants.OMPSCHEDULE);
    tpc = configuration.getInt(SCConstants.TPC, 2);

    send_array_limit = (configuration.getInt(SCConstants.SENDLIMIT, 250)) * 1024L * 1024L;
    nbr_split_len = configuration.getInt(SCConstants.NBRTASKLEN, 0);

    numIteration = configuration.getInt(SCConstants.NUM_ITERATION, 10);
    LOG.info("Harp-DAAL Subgraph Counting Iteration: " + numIteration);

}

---

From source file:edu.iu.examples.BaseExampleMapper.java

License:Apache License

/**
 * Mapper configuration./*  w  w w.j  a v a2s  .  co m*/
 */
@Override
protected void setup(Context context) {
    Configuration configuration = context.getConfiguration();
    cmd = configuration.get(Constants.ARGS_OPERATION, "allreduce");
    numMappers = configuration.getInt(Constants.ARGS_MAPPERS, 1);
    numPartitions = configuration.getInt(Constants.ARGS_PARTITIONS, 1);
    elements = configuration.getInt(Constants.ARGS_ELEMENTS, 1);
    numIterations = configuration.getInt(Constants.ARGS_ITERATIONS, 1);
    dataType = configuration.get(Constants.ARGS_DATA_TYPE, "double");
    verify = configuration.getBoolean(Constants.ARGS_VERIFY, false);
    LOG.info("Example operation " + cmd);
    LOG.info("Num Mappers " + numMappers);
    LOG.info("Num Partitions " + numPartitions);
    LOG.info("Bytes per Partition " + elements);
    LOG.info("Num Iterations " + numIterations);
    LOG.info("Data type " + dataType);
    LOG.info("Verify " + verify);
}

---

From source file:edu.iu.lda.LDAMPCollectiveMapper.java

License:Apache License

/**
 * Mapper configuration./*  w  w w .  ja  v a 2  s  . c o m*/
 */
@Override
protected void setup(Context context) {
    LOG.info(
            "start setup: " + new SimpleDateFormat("yyyyMMdd_HHmmss").format(Calendar.getInstance().getTime()));
    long startTime = System.currentTimeMillis();
    Configuration configuration = context.getConfiguration();
    numTopics = configuration.getInt(Constants.NUM_TOPICS, 100);
    alpha = configuration.getDouble(Constants.ALPHA, 0.1);
    beta = configuration.getDouble(Constants.BETA, 0.001);
    numIterations = configuration.getInt(Constants.NUM_ITERATIONS, 100);
    numThreads = configuration.getInt(Constants.NUM_THREADS, 16);
    scheduleRatio = configuration.getDouble(Constants.SCHEDULE_RATIO, 2.0);
    minBound = configuration.getInt(Constants.MIN_BOUND, Constants.TRAIN_MIN_THRESHOLD);
    maxBound = configuration.getInt(Constants.MAX_BOUND, Constants.TRAIN_MAX_THRESHOLD);
    if (minBound <= 0 || minBound > 100) {
        minBound = Constants.TRAIN_MIN_THRESHOLD;
    }
    if (maxBound <= 0 || maxBound > 100) {
        maxBound = Constants.TRAIN_MAX_THRESHOLD;
    }
    if (maxBound < minBound) {
        maxBound = minBound;
    }
    if (maxBound == 100) {
        minBound = 100;
        enableTuning = false;
    } else {
        enableTuning = true;
    }
    time = enableTuning ? 1000L : 1000000000L;
    hasOverTrained = false;
    lastUnderTrainIte = 0;
    breakPeriod = 0;
    modelDirPath = configuration.get(Constants.MODEL_DIR, "");
    printModel = configuration.getBoolean(Constants.PRINT_MODEL, false);
    printInterval = 10;
    freeInterval = 10;
    numModelSlices = 2;
    computeTime = 0L;
    waitTime = 0L;
    long endTime = System.currentTimeMillis();
    LOG.info("config (ms): " + (endTime - startTime));
    LOG.info("Num Topics " + numTopics);
    LOG.info("Alpha " + alpha);
    LOG.info("Beta " + beta);
    LOG.info("Num Iterations " + numIterations);
    LOG.info("numThreads\\scheduleRaito " + numThreads + "\\" + scheduleRatio);
    LOG.info("enableTuning\\Time\\Bounds " + enableTuning + "\\" + time + "\\" + minBound + "\\" + maxBound);
    LOG.info("Model Dir Path " + modelDirPath);
    LOG.info("Print Model " + printModel);
    LOG.info("Model Slices " + numModelSlices);
    LOG.info("Container Memory " + configuration.get("mapreduce.map.collective.memory.mb"));
    LOG.info("Java Memory " + configuration.get("mapreduce.map.collective.java.opts"));
}

---

From source file:edu.iu.subgraph.SCCollectiveMapper.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {

    LOG.info("start setup");

    Configuration configuration = context.getConfiguration();
    numMappers = configuration.getInt(SCConstants.NUM_MAPPERS, 10);
    templateFile = configuration.get(SCConstants.TEMPLATE_PATH);
    useLocalMultiThread = configuration.getBoolean(SCConstants.USE_LOCAL_MULTITHREAD, true);
    rotation_pipeline = configuration.getBoolean(SCConstants.ROTATION_PIPELINE, true);

    LOG.info("init templateFile");
    LOG.info(templateFile);/*from w  ww  .j a  v  a2 s  .  c  o  m*/

    numThreads = configuration.getInt(SCConstants.THREAD_NUM, 10);
    numCores = configuration.getInt(SCConstants.CORE_NUM, 24);
    affinity = configuration.get(SCConstants.THD_AFFINITY);
    tpc = configuration.getInt(SCConstants.TPC, 2);

    //always use the maximum hardware threads to load in data and convert data 
    harpThreads = Runtime.getRuntime().availableProcessors();
    LOG.info("Num Threads " + numThreads);
    LOG.info("Num harp load data threads " + harpThreads);

    send_array_limit = (configuration.getInt(SCConstants.SENDLIMIT, 250)) * 1024L * 1024L;

    numIteration = configuration.getInt(SCConstants.NUM_ITERATION, 10);
    LOG.info("Subgraph Counting Iteration: " + numIteration);

    numModelSlices = 2;
}

---

From source file:edu.rosehulman.CollocMapper.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    Configuration conf = context.getConfiguration();
    this.maxShingleSize = conf.getInt(MAX_SHINGLE_SIZE, DEFAULT_MAX_SHINGLE_SIZE);

    this.emitUnigrams = conf.getBoolean(CollocDriver.EMIT_UNIGRAMS, CollocDriver.DEFAULT_EMIT_UNIGRAMS);

    if (log.isInfoEnabled()) {
        log.info("Max Ngram size is {}", this.maxShingleSize);
        log.info("Emit Unitgrams is {}", emitUnigrams);
    }//ww w  . j ava 2 s  .c  o  m
}

---

From source file:edu.rosehulman.TFPartialVectorReducer.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    Configuration conf = context.getConfiguration();

    dimension = conf.getInt(PartialVectorMerger.DIMENSION, Integer.MAX_VALUE);
    sequentialAccess = conf.getBoolean(PartialVectorMerger.SEQUENTIAL_ACCESS, false);
    namedVector = conf.getBoolean(PartialVectorMerger.NAMED_VECTOR, false);
    maxNGramSize = conf.getInt(DictionaryVectorizer.MAX_NGRAMS, maxNGramSize);

    //MAHOUT-1247
    Path dictionaryFile = HadoopUtil.getSingleCachedFile(conf);
    // key is word value is id
    for (Pair<Writable, IntWritable> record : new SequenceFileIterable<Writable, IntWritable>(dictionaryFile,
            true, conf)) {/*from  w w  w  . j  a  va  2  s  .  co  m*/
        dictionary.put(record.getFirst().toString(), record.getSecond().get());
    }
}

---

From source file:edu.stolaf.cs.wmrserver.JobServiceHandler.java

License:Apache License

public JobServiceHandler(Configuration conf) throws IOException {
    _homeDir = getHome(conf);//  www .jav a 2s. c  om
    _tempDir = getTempDir(conf);
    _langSupportDir = new File(conf.get("wmr.lang.support.dir", "lang-support"));
    _enforceInputContainment = conf.getBoolean("wmr.input.containment.enforce", false);
    _disallowLocalInput = conf.getBoolean("wmr.input.disallow.local", true);
    _outputPageSize = getOutputPageSize(conf);
    _quotaEnabled = conf.getBoolean("wmr.quota.enable", true) && conf.getBoolean("wmr.quota.user.enable", true);
    _quotaAttempts = conf.getInt("wmr.quota.user.attempts", 20);
    _quotaDuration = conf.getInt("wmr.quota.user.duration", 10);

    // Resolve relative lang support dir
    if (!_langSupportDir.isAbsolute())
        _langSupportDir = new File(System.getProperty("wmr.home.dir"), _langSupportDir.toString());

    // Load language configuration
    File wmrConfFile = new File(_langSupportDir, LANG_CONF_FILE);
    if (!wmrConfFile.exists())
        throw new IOException("Language configuration could not be found: " + wmrConfFile.toString());
    try {
        _languageConf = new HierarchicalINIConfiguration(wmrConfFile);
    } catch (ConfigurationException ex) {
        throw new IOException("The language configuration could not be loaded.", ex);
    }

    _hadoopEngine = new HadoopEngine(conf);
    _testJobEngine = new TestJobEngine(conf);
}

---

From source file:edu.uci.ics.pregelix.api.util.BspUtils.java

License:Apache License

/**
 * Get the job configuration parameter whether the vertex states will increase dynamically
 * /* ww  w.  j  a  v  a 2 s. c  om*/
 * @param conf
 *            the job configuration
 * @return the boolean setting of the parameter, by default it is false
 */
public static boolean getDynamicVertexValueSize(Configuration conf) {
    return conf.getBoolean(PregelixJob.INCREASE_STATE_LENGTH, false);
}

---

From source file:edu.umd.cloud9.webgraph.driver.wt10g.GenericExtractLinks.java

License:Apache License

@Override
public int runTool() throws Exception {

    Configuration conf = getConf();
    Job job = new Job(conf);

    int numReducers = conf.getInt("Cloud9.Reducers", 200);

    String inputPath = conf.get("Cloud9.InputPath");
    String outputPath = conf.get("Cloud9.OutputPath");

    String mappingFile = conf.get("Cloud9.DocnoMappingFile");

    FileSystem fs = FileSystem.get(conf);
    if (!fs.exists(new Path(mappingFile))) {
        throw new RuntimeException("Error: Docno mapping data file " + mappingFile + " doesn't exist!");
    }//  w w w .j a v a  2 s.c o  m

    DistributedCache.addCacheFile(new Path(mappingFile).toUri(), job.getConfiguration());

    job.setJobName("ExtractLinks");
    conf.set("mapred.child.java.opts", "-Xmx2048m");
    conf.setInt("mapred.task.timeout", 60000000);

    job.setNumReduceTasks(numReducers);

    job.setMapperClass(GenericExtractLinks.Map.class);
    job.setCombinerClass(GenericExtractLinks.Reduce.class);
    job.setReducerClass(GenericExtractLinks.Reduce.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(ArrayListWritable.class);

    configer.applyJobConfig(job);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);

    SequenceFileOutputFormat.setCompressOutput(job, true);
    SequenceFileOutputFormat.setOutputCompressionType(job, SequenceFile.CompressionType.BLOCK);

    recursivelyAddInputPaths(job, inputPath);

    FileOutputFormat.setOutputPath(job, new Path(outputPath));

    LOG.info("ExtractLinks");
    LOG.info(" - input path: " + inputPath);
    LOG.info(" - output path: " + outputPath);
    LOG.info(" - mapping file: " + mappingFile);
    LOG.info(" - include internal links? " + conf.getBoolean("Cloud9.IncludeInternalLinks", false));

    job.waitForCompletion(true);
    return 0;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »