Example usage for org.apache.hadoop.conf Configuration getFloat

List of usage examples for org.apache.hadoop.conf Configuration getFloat

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. getFloat

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getFloat.

Prototype

public float getFloat(String name, float defaultValue)

Source Link

Document

Get the value of the name property as a float.

Usage

From source file:org.apache.nutch.searcher.more.TypeQueryFilter.java

License:Apache License

public void setConf(Configuration conf) {
    this.conf = conf;
    setBoost(conf.getFloat("query.type.boost", 0.0f));
}

From source file:org.apache.nutch.searcher.site.SiteQueryFilter.java

License:Apache License

public void setConf(Configuration conf) {
    this.conf = conf;
    setBoost(conf.getFloat("query.site.boost", 0.0f));
}

From source file:org.apache.nutchbase.crawl.AdaptiveFetchScheduleHbase.java

License:Apache License

public void setConf(Configuration conf) {
    super.setConf(conf);
    if (conf == null)
        return;//from  w w w .  j a va2 s .  co m
    INC_RATE = conf.getFloat("db.fetch.schedule.adaptive.inc_rate", 0.2f);
    DEC_RATE = conf.getFloat("db.fetch.schedule.adaptive.dec_rate", 0.2f);
    MIN_INTERVAL = conf.getInt("db.fetch.schedule.adaptive.min_interval", 60);
    MAX_INTERVAL = conf.getInt("db.fetch.schedule.adaptive.max_interval", FetchSchedule.SECONDS_PER_DAY * 365); // 1 year
    SYNC_DELTA = conf.getBoolean("db.fetch.schedule.adaptive.sync_delta", true);
    SYNC_DELTA_RATE = conf.getFloat("db.fetch.schedule.adaptive.sync_delta_rate", 0.2f);
}

From source file:org.apache.nutchbase.protocol.http.api.HttpBase.java

License:Apache License

public void setConf(Configuration conf) {
    this.conf = conf;
    this.proxyHost = conf.get("http.proxy.host");
    this.proxyPort = conf.getInt("http.proxy.port", 8080);
    this.useProxy = (proxyHost != null && proxyHost.length() > 0);
    this.timeout = conf.getInt("http.timeout", 10000);
    this.maxContent = conf.getInt("http.content.limit", 64 * 1024);
    this.maxDelays = conf.getInt("http.max.delays", 3);
    this.maxThreadsPerHost = conf.getInt("fetcher.threads.per.host", 1);
    this.userAgent = getAgentString(conf.get("http.agent.name"), conf.get("http.agent.version"),
            conf.get("http.agent.description"), conf.get("http.agent.url"), conf.get("http.agent.email"));
    this.serverDelay = (long) (conf.getFloat("fetcher.server.delay", 1.0f) * 1000);
    this.maxCrawlDelay = (long) (conf.getInt("fetcher.max.crawl.delay", -1) * 1000);
    // backward-compatible default setting
    this.byIP = conf.getBoolean("fetcher.threads.per.host.by.ip", true);
    this.mimeTypes = new MimeUtil(conf);
    this.useHttp11 = conf.getBoolean("http.useHttp11", false);
    this.robots.setConf(conf);
    this.checkBlocking = conf.getBoolean(Protocol.CHECK_BLOCKING, true);
    this.checkRobots = conf.getBoolean(Protocol.CHECK_ROBOTS, true);
    logConf();// ww  w  . j  av  a2  s .  co  m
}

From source file:org.apache.oozie.service.ConfigurationService.java

License:Apache License

public static float getFloat(String name) {
    Configuration conf = Services.get().getConf();
    return conf.getFloat(name, ConfigUtils.FLOAT_DEFAULT);
}

From source file:org.apache.parquet.hadoop.ParquetOutputFormat.java

License:Apache License

public RecordWriter<Void, T> getRecordWriter(Configuration conf, Path file, CompressionCodecName codec)
        throws IOException, InterruptedException {
    final WriteSupport<T> writeSupport = getWriteSupport(conf);

    CodecFactory codecFactory = new CodecFactory(conf);
    long blockSize = getLongBlockSize(conf);
    if (INFO)//from   ww w .j av  a2 s.  c  o m
        LOG.info("Parquet block size to " + blockSize);
    int pageSize = getPageSize(conf);
    if (INFO)
        LOG.info("Parquet page size to " + pageSize);
    int dictionaryPageSize = getDictionaryPageSize(conf);
    if (INFO)
        LOG.info("Parquet dictionary page size to " + dictionaryPageSize);
    boolean enableDictionary = getEnableDictionary(conf);
    if (INFO)
        LOG.info("Dictionary is " + (enableDictionary ? "on" : "off"));
    boolean validating = getValidation(conf);
    if (INFO)
        LOG.info("Validation is " + (validating ? "on" : "off"));
    WriterVersion writerVersion = getWriterVersion(conf);
    if (INFO)
        LOG.info("Writer version is: " + writerVersion);
    int maxPaddingSize = getMaxPaddingSize(conf);
    if (INFO)
        LOG.info("Maximum row group padding size is " + maxPaddingSize + " bytes");

    WriteContext init = writeSupport.init(conf);
    ParquetFileWriter w = new ParquetFileWriter(conf, init.getSchema(), file, Mode.CREATE, blockSize,
            maxPaddingSize);
    w.start();

    float maxLoad = conf.getFloat(ParquetOutputFormat.MEMORY_POOL_RATIO,
            MemoryManager.DEFAULT_MEMORY_POOL_RATIO);
    long minAllocation = conf.getLong(ParquetOutputFormat.MIN_MEMORY_ALLOCATION,
            MemoryManager.DEFAULT_MIN_MEMORY_ALLOCATION);
    if (memoryManager == null) {
        memoryManager = new MemoryManager(maxLoad, minAllocation);
    } else if (memoryManager.getMemoryPoolRatio() != maxLoad) {
        LOG.warn("The configuration " + MEMORY_POOL_RATIO + " has been set. It should not "
                + "be reset by the new value: " + maxLoad);
    }

    return new ParquetRecordWriter<T>(w, writeSupport, init.getSchema(), init.getExtraMetaData(), blockSize,
            pageSize, codecFactory.getCompressor(codec, pageSize), dictionaryPageSize, enableDictionary,
            validating, writerVersion, memoryManager);
}

From source file:org.apache.parquet.hadoop.UnmaterializableRecordCounter.java

License:Apache License

public UnmaterializableRecordCounter(Configuration conf, long totalNumRecords) {
    this(conf.getFloat(BAD_RECORD_THRESHOLD_CONF_KEY, DEFAULT_THRESHOLD), totalNumRecords);
}

From source file:org.apache.pig.CounterBasedErrorHandler.java

License:Apache License

public CounterBasedErrorHandler() {
    Configuration conf = UDFContext.getUDFContext().getJobConf();
    this.minErrors = conf.getLong(PigConfiguration.PIG_ERROR_HANDLING_MIN_ERROR_RECORDS, 0);
    this.errorThreshold = conf.getFloat(PigConfiguration.PIG_ERROR_HANDLING_THRESHOLD_PERCENT, 0.0f);
}

From source file:org.apache.pig.impl.builtin.PoissonSampleLoader.java

License:Apache License

@SuppressWarnings("rawtypes")
@Override//from   w  ww .  jav  a 2s  .co  m
public void prepareToRead(RecordReader reader, PigSplit split) throws IOException {
    super.prepareToRead(reader, split);
    numRowsSampled = 0;
    avgTupleMemSz = 0;
    rowNum = 0;
    skipInterval = -1;
    memToSkipPerSample = 0;
    numRowSplTupleReturned = false;
    newSample = null;

    Configuration conf = split.getConf();
    sampleRate = conf.getInt(PigConfiguration.SAMPLE_RATE, DEFAULT_SAMPLE_RATE);
    heapPerc = conf.getFloat(PigConfiguration.PERC_MEM_AVAIL, PartitionSkewedKeys.DEFAULT_PERCENT_MEMUSAGE);
}

From source file:org.apache.pig.impl.util.SpillableMemoryManager.java

License:Apache License

public void configure(Configuration conf) {

    spillFileSizeThreshold = conf.getLong("pig.spill.size.threshold", spillFileSizeThreshold);
    gcActivationSize = conf.getLong("pig.spill.gc.activation.size", gcActivationSize);
    float memoryThresholdFraction = conf.getFloat(PigConfiguration.PIG_SPILL_MEMORY_USAGE_THRESHOLD_FRACTION,
            MEMORY_THRESHOLD_FRACTION_DEFAULT);
    float collectionThresholdFraction = conf.getFloat(PigConfiguration.PIG_SPILL_COLLECTION_THRESHOLD_FRACTION,
            COLLECTION_THRESHOLD_FRACTION_DEFAULT);
    long unusedMemoryThreshold = conf.getLong(PigConfiguration.PIG_SPILL_UNUSED_MEMORY_THRESHOLD_SIZE,
            UNUSED_MEMORY_THRESHOLD_DEFAULT);
    configureMemoryThresholds(memoryThresholdFraction, collectionThresholdFraction, unusedMemoryThreshold);
}