Example usage for org.apache.hadoop.conf Configuration getFloat

List of usage examples for org.apache.hadoop.conf Configuration getFloat

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. getFloat

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getFloat.

Prototype

public float getFloat(String name, float defaultValue)

Source Link

Document

Get the value of the name property as a float.

Usage

From source file:org.apache.ambari.servicemonitor.jobs.ProbableFileOperation.java

License:Apache License

/**
 * Create an operation/*from   ww w .  ja va  2 s  . co  m*/
 *
 * @param name prefix for the
 * @param conf configuration to save
 */
public ProbableFileOperation(String name, Configuration conf) {
    super(conf);
    this.name = name;
    path = new Path(conf.get(name + PATH, "/"));
    probability = conf.getFloat(name + PROBABILITY, 0.0f);
    rng = new Random(conf.getInt(name + SEED, 0));
    delay = conf.getInt(name + SLEEPTIME, 0);
    cachedFS = new OnDemandFS(getConf());
}

From source file:org.apache.giraph.examples.RandomWalkWorkerContext.java

License:Apache License

/**
 * Set static variables from Configuration
 *
 * @param configuration the conf//from ww  w. ja va  2 s. c  o m
 */
private void setStaticVars(Configuration configuration) {
    MAX_SUPERSTEPS = configuration.getInt(RandomWalkComputation.MAX_SUPERSTEPS, DEFAULT_MAX_SUPERSTEPS);
    TELEPORTATION_PROBABILITY = configuration.getFloat(RandomWalkComputation.TELEPORTATION_PROBABILITY,
            DEFAULT_TELEPORTATION_PROBABILITY);
    SOURCES = initializeSources(configuration);
}

From source file:org.apache.lens.cube.parse.StorageTableResolver.java

License:Apache License

StorageTableResolver(Configuration conf) {
    this.conf = conf;
    this.supportedStorages = getSupportedStorages(conf);
    this.allStoragesSupported = (supportedStorages == null);
    this.failOnPartialData = conf.getBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false);
    String str = conf.get(CubeQueryConfUtil.VALID_STORAGE_DIM_TABLES);
    validDimTables = StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ","));
    this.processTimePartCol = conf.get(CubeQueryConfUtil.PROCESS_TIME_PART_COL);
    String maxIntervalStr = conf.get(CubeQueryConfUtil.QUERY_MAX_INTERVAL);
    if (maxIntervalStr != null) {
        this.maxInterval = UpdatePeriod.valueOf(maxIntervalStr);
    } else {/*from w  w w .  j a  v a2  s  .co m*/
        this.maxInterval = null;
    }
    rangeWriter = ReflectionUtils.newInstance(conf.getClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS,
            CubeQueryConfUtil.DEFAULT_TIME_RANGE_WRITER, TimeRangeWriter.class), this.conf);
    String formatStr = conf.get(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT);
    if (formatStr != null) {
        partWhereClauseFormat = new SimpleDateFormat(formatStr);
    }
    this.phase = PHASE.first();
    completenessThreshold = conf.getFloat(CubeQueryConfUtil.COMPLETENESS_THRESHOLD,
            CubeQueryConfUtil.DEFAULT_COMPLETENESS_THRESHOLD);
    completenessPartCol = conf.get(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL);
}

From source file:org.apache.mahout.classifier.naivebayes.BayesUtils.java

License:Apache License

public static NaiveBayesModel readModelFromDir(Path base, Configuration conf) {

    float alphaI = conf.getFloat(ThetaMapper.ALPHA_I, 1.0f);
    boolean isComplementary = conf.getBoolean(NaiveBayesModel.COMPLEMENTARY_MODEL, true);

    // read feature sums and label sums
    Vector scoresPerLabel = null;
    Vector scoresPerFeature = null;
    for (Pair<Text, VectorWritable> record : new SequenceFileDirIterable<Text, VectorWritable>(
            new Path(base, TrainNaiveBayesJob.WEIGHTS), PathType.LIST, PathFilters.partFilter(), conf)) {
        String key = record.getFirst().toString();
        VectorWritable value = record.getSecond();
        if (key.equals(TrainNaiveBayesJob.WEIGHTS_PER_FEATURE)) {
            scoresPerFeature = value.get();
        } else if (key.equals(TrainNaiveBayesJob.WEIGHTS_PER_LABEL)) {
            scoresPerLabel = value.get();
        }//  www.  j a  v a  2 s.c om
    }

    Preconditions.checkNotNull(scoresPerFeature);
    Preconditions.checkNotNull(scoresPerLabel);

    Matrix scoresPerLabelAndFeature = new SparseMatrix(scoresPerLabel.size(), scoresPerFeature.size());
    for (Pair<IntWritable, VectorWritable> entry : new SequenceFileDirIterable<IntWritable, VectorWritable>(
            new Path(base, TrainNaiveBayesJob.SUMMED_OBSERVATIONS), PathType.LIST, PathFilters.partFilter(),
            conf)) {
        scoresPerLabelAndFeature.assignRow(entry.getFirst().get(), entry.getSecond().get());
    }

    // perLabelThetaNormalizer is only used by the complementary model, we do not instantiate it for the standard model
    Vector perLabelThetaNormalizer = null;
    if (isComplementary) {
        perLabelThetaNormalizer = scoresPerLabel.like();
        for (Pair<Text, VectorWritable> entry : new SequenceFileDirIterable<Text, VectorWritable>(
                new Path(base, TrainNaiveBayesJob.THETAS), PathType.LIST, PathFilters.partFilter(), conf)) {
            if (entry.getFirst().toString().equals(TrainNaiveBayesJob.LABEL_THETA_NORMALIZER)) {
                perLabelThetaNormalizer = entry.getSecond().get();
            }
        }
        Preconditions.checkNotNull(perLabelThetaNormalizer);
    }

    return new NaiveBayesModel(scoresPerLabelAndFeature, scoresPerFeature, scoresPerLabel,
            perLabelThetaNormalizer, alphaI, isComplementary);
}

From source file:org.apache.mahout.classifier.naivebayes.trainer.NaiveBayesThetaComplementaryMapper.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    Configuration conf = context.getConfiguration();
    URI[] localFiles = DistributedCache.getCacheFiles(conf);
    if (localFiles == null || localFiles.length < 2) {
        throw new IllegalArgumentException("missing paths from the DistributedCache");
    }// www  .j a  va2 s .  c  o  m
    alphaI = conf.getFloat(NaiveBayesTrainer.ALPHA_I, 1.0f);
    Path weightFile = new Path(localFiles[0].getPath());
    for (Pair<Text, VectorWritable> record : new SequenceFileIterable<Text, VectorWritable>(weightFile, true,
            conf)) {
        Text key = record.getFirst();
        VectorWritable value = record.getSecond();
        if (key.toString().equals(BayesConstants.FEATURE_SUM)) {
            featureSum = value.get();
        } else if (key.toString().equals(BayesConstants.LABEL_SUM)) {
            labelSum = value.get();
        }
    }
    perLabelThetaNormalizer = labelSum.like();
    totalSum = labelSum.zSum();
    vocabCount = featureSum.getNumNondefaultElements();

    Path labelMapFile = new Path(localFiles[1].getPath());
    // key is word value is id
    for (Pair<Writable, IntWritable> record : new SequenceFileIterable<Writable, IntWritable>(labelMapFile,
            true, conf)) {
        labelMap.put(record.getFirst().toString(), record.getSecond().get());
    }
}

From source file:org.apache.mahout.classifier.naivebayes.trainer.NaiveBayesThetaMapper.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    Configuration conf = context.getConfiguration();
    URI[] localFiles = DistributedCache.getCacheFiles(conf);
    if (localFiles == null || localFiles.length < 2) {
        throw new IllegalArgumentException("missing paths from the DistributedCache");
    }/*from   w ww .jav  a 2s.  c o m*/
    alphaI = conf.getFloat(NaiveBayesTrainer.ALPHA_I, 1.0f);
    Path weightFile = new Path(localFiles[0].getPath());

    for (Pair<Text, VectorWritable> record : new SequenceFileIterable<Text, VectorWritable>(weightFile, true,
            conf)) {
        Text key = record.getFirst();
        VectorWritable value = record.getSecond();
        if (key.toString().equals(BayesConstants.FEATURE_SUM)) {
            featureSum = value.get();
        } else if (key.toString().equals(BayesConstants.LABEL_SUM)) {
            labelSum = value.get();
        }
    }
    perLabelThetaNormalizer = labelSum.like();
    vocabCount = featureSum.getNumNondefaultElements();

    Path labelMapFile = new Path(localFiles[1].getPath());

    // key is word value is id
    for (Pair<Writable, IntWritable> record : new SequenceFileIterable<Writable, IntWritable>(labelMapFile,
            true, conf)) {
        labelMap.put(record.getFirst().toString(), record.getSecond().get());
    }
}

From source file:org.apache.mahout.classifier.naivebayes.training.ThetaMapper.java

License:Apache License

@Override
protected void setup(Context ctx) throws IOException, InterruptedException {
    super.setup(ctx);
    Configuration conf = ctx.getConfiguration();

    float alphaI = conf.getFloat(ALPHA_I, 1.0f);
    Map<String, Vector> scores = BayesUtils.readScoresFromCache(conf);

    trainer = new ComplementaryThetaTrainer(scores.get(TrainNaiveBayesJob.WEIGHTS_PER_FEATURE),
            scores.get(TrainNaiveBayesJob.WEIGHTS_PER_LABEL), alphaI);
}

From source file:org.apache.mahout.classifier.naivebayes.training.TrainUtils.java

License:Apache License

static NaiveBayesModel readModelFromTempDir(Path base, Configuration conf) {

    float alphaI = conf.getFloat(ThetaMapper.ALPHA_I, 1.0f);

    // read feature sums and label sums
    Vector scoresPerLabel = null;
    Vector scoresPerFeature = null;
    for (Pair<Text, VectorWritable> record : new SequenceFileDirIterable<Text, VectorWritable>(
            new Path(base, TrainNaiveBayesJob.WEIGHTS), PathType.LIST, PathFilters.partFilter(), conf)) {
        String key = record.getFirst().toString();
        VectorWritable value = record.getSecond();
        if (key.equals(TrainNaiveBayesJob.WEIGHTS_PER_FEATURE)) {
            scoresPerFeature = value.get();
        } else if (key.equals(TrainNaiveBayesJob.WEIGHTS_PER_LABEL)) {
            scoresPerLabel = value.get();
        }/*from  www  . j  a v a  2  s  .  c  o  m*/
    }

    Preconditions.checkNotNull(scoresPerFeature);
    Preconditions.checkNotNull(scoresPerLabel);

    Matrix scoresPerLabelAndFeature = new SparseMatrix(
            new int[] { scoresPerLabel.size(), scoresPerFeature.size() });
    for (Pair<IntWritable, VectorWritable> entry : new SequenceFileDirIterable<IntWritable, VectorWritable>(
            new Path(base, TrainNaiveBayesJob.SUMMED_OBSERVATIONS), PathType.LIST, PathFilters.partFilter(),
            conf)) {
        scoresPerLabelAndFeature.assignRow(entry.getFirst().get(), entry.getSecond().get());
    }

    Vector perlabelThetaNormalizer = null;
    for (Pair<Text, VectorWritable> entry : new SequenceFileDirIterable<Text, VectorWritable>(
            new Path(base, TrainNaiveBayesJob.THETAS), PathType.LIST, PathFilters.partFilter(), conf)) {
        if (entry.getFirst().toString().equals(TrainNaiveBayesJob.LABEL_THETA_NORMALIZER)) {
            perlabelThetaNormalizer = entry.getSecond().get();
        }
    }

    Preconditions.checkNotNull(perlabelThetaNormalizer);

    return new NaiveBayesModel(scoresPerLabelAndFeature, scoresPerFeature, scoresPerLabel,
            perlabelThetaNormalizer, alphaI);
}

From source file:org.apache.mahout.clustering.classify.ClusterClassificationMapper.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);

    Configuration conf = context.getConfiguration();
    String clustersIn = conf.get(ClusterClassificationConfigKeys.CLUSTERS_IN);
    threshold = conf.getFloat(ClusterClassificationConfigKeys.OUTLIER_REMOVAL_THRESHOLD, 0.0f);
    emitMostLikely = conf.getBoolean(ClusterClassificationConfigKeys.EMIT_MOST_LIKELY, false);

    clusterModels = Lists.newArrayList();

    if (clustersIn != null && !clustersIn.isEmpty()) {
        Path clustersInPath = new Path(clustersIn);
        clusterModels = populateClusterModels(clustersInPath, conf);
        ClusteringPolicy policy = ClusterClassifier.readPolicy(finalClustersPath(clustersInPath));
        clusterClassifier = new ClusterClassifier(clusterModels, policy);
    }/*from  www  . ja  v  a  2s. c  o  m*/
    clusterId = new IntWritable();
}

From source file:org.apache.mahout.clustering.lda.cvb.CachingCVB0Mapper.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    log.info("Retrieving configuration");
    Configuration conf = context.getConfiguration();
    float eta = conf.getFloat(CVB0Driver.TERM_TOPIC_SMOOTHING, Float.NaN);
    float alpha = conf.getFloat(CVB0Driver.DOC_TOPIC_SMOOTHING, Float.NaN);
    long seed = conf.getLong(CVB0Driver.RANDOM_SEED, 1234L);
    numTopics = conf.getInt(CVB0Driver.NUM_TOPICS, -1);
    int numTerms = conf.getInt(CVB0Driver.NUM_TERMS, -1);
    int numUpdateThreads = conf.getInt(CVB0Driver.NUM_UPDATE_THREADS, 1);
    int numTrainThreads = conf.getInt(CVB0Driver.NUM_TRAIN_THREADS, 4);
    maxIters = conf.getInt(CVB0Driver.MAX_ITERATIONS_PER_DOC, 10);
    float modelWeight = conf.getFloat(CVB0Driver.MODEL_WEIGHT, 1.0f);

    log.info("Initializing read model");
    Path[] modelPaths = CVB0Driver.getModelPaths(conf);
    if (modelPaths != null && modelPaths.length > 0) {
        readModel = new TopicModel(conf, eta, alpha, null, numUpdateThreads, modelWeight, modelPaths);
    } else {/*from ww w.jav a  2 s  . c o  m*/
        log.info("No model files found");
        readModel = new TopicModel(numTopics, numTerms, eta, alpha, RandomUtils.getRandom(seed), null,
                numTrainThreads, modelWeight);
    }

    log.info("Initializing write model");
    writeModel = modelWeight == 1 ? new TopicModel(numTopics, numTerms, eta, alpha, null, numUpdateThreads)
            : readModel;

    log.info("Initializing model trainer");
    modelTrainer = new ModelTrainer(readModel, writeModel, numTrainThreads, numTopics, numTerms);
    modelTrainer.start();
}