Example usage for org.apache.hadoop.conf Configuration getBoolean

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getBoolean.

Prototype

public boolean getBoolean(String name, boolean defaultValue) 

Source Link

Document

Get the value of the name property as a boolean.

Usage

From source file:com.twitter.hraven.mapreduce.JobFileTableMapper.java

License:Apache License

@Override
protected void setup(Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Put>.Context context)
        throws java.io.IOException, InterruptedException {
    Configuration myConf = context.getConfiguration();
    jobHistoryByIdService = new JobHistoryByIdService(myConf);
    appVersionService = new AppVersionService(myConf);
    rawService = new JobHistoryRawService(myConf);
    // set aggregation to false by default
    aggregationFlag = myConf.getBoolean(AggregationConstants.AGGREGATION_FLAG_NAME, false);
    if (!aggregationFlag) {
        LOG.info("Aggregation is turned off ");
    }/*  w  w w .jav a  2 s. c  om*/
    reAggregationFlag = myConf.getBoolean(AggregationConstants.RE_AGGREGATION_FLAG_NAME, false);
    if (reAggregationFlag) {
        LOG.info("Re-aggregation is turned ON, will be aggregating stats again "
                + " for jobs even if already aggregated status is true in raw table ");
    }
    appSummaryService = new AppSummaryService(myConf);

    keyCount = 0;
}

---

From source file:com.voole.hobbit2.camus.hive.order.mixed.jobcontrol.ControlledJob.java

License:Apache License

/**
 * Submit this job to mapred. The state becomes RUNNING if submission 
 * is successful, FAILED otherwise.  /*ww  w. j av  a  2 s. c  o  m*/
 */
protected synchronized void submit() {
    try {
        Configuration conf = job.getConfiguration();
        if (conf.getBoolean(CREATE_DIR, false)) {
            FileSystem fs = FileSystem.get(conf);
            Path inputPaths[] = FileInputFormat.getInputPaths(job);
            for (int i = 0; i < inputPaths.length; i++) {
                if (!fs.exists(inputPaths[i])) {
                    try {
                        fs.mkdirs(inputPaths[i]);
                    } catch (IOException e) {

                    }
                }
            }
        }
        job.submit();
        this.state = State.RUNNING;
    } catch (Exception ioe) {
        LOG.info(getJobName() + " got an error while submitting ", ioe);
        this.state = State.FAILED;
        this.message = StringUtils.stringifyException(ioe);
    }
}

---

From source file:com.willetinc.hadoop.mapreduce.dynamodb.DynamoDBQueryInputFormat.java

License:Apache License

public static boolean getInterpolateAcrossRangeKeyValues(Configuration conf) {
    return conf.getBoolean(DynamoDBConfiguration.RANGE_KEY_INTERPOLATE_PROPERTY, false);
}

---

From source file:com.willetinc.hadoop.mapreduce.dynamodb.DynamoDBQueryInputFormatTest.java

License:Apache License

@Test
public void testGetInterpolateAcrossRangeKeyValues() {
    Configuration conf = createMock(Configuration.class);
    expect(conf.getBoolean(DynamoDBConfiguration.RANGE_KEY_INTERPOLATE_PROPERTY, false)).andReturn(true);

    replay(conf);/*  ww  w.  j a  va  2 s  .co m*/
    boolean result = DynamoDBQueryInputFormat.getInterpolateAcrossRangeKeyValues(conf);
    assertEquals(true, result);
    verify(conf);
}

---

From source file:com.yahoo.glimmer.indexing.preprocessor.TuplesToResourcesMapper.java

License:Open Source License

protected void setup(Mapper<LongWritable, Text, Text, Object>.Context context)
        throws java.io.IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    boolean includeContexts = conf.getBoolean(INCLUDE_CONTEXTS_KEY, true);
    setIncludeContexts(includeContexts);

    TupleFilter filter = TupleFilterSerializer.deserialize(conf);
    if (filter != null) {
        LOG.info("Using TupleFilter:\n" + filter.toString());
        setFilter(filter);//from  ww  w.ja v a  2  s  .  c om
    } else {
        LOG.info("No TupleFilter given. Processing all tuples.");
    }

    extraResources = conf.getStrings(EXTRA_RESOURCES);
}

---

From source file:com.yahoo.glimmer.indexing.RDFDocumentFactory.java

License:Open Source License

public static boolean getWithContexts(Configuration conf) {
    return conf.getBoolean(CONF_WITH_CONTEXTS_KEY, true);
}

---

From source file:com.yahoo.ycsb.bulk.hbase.BulkDataGeneratorJob.java

License:Apache License

int createSplitsFile(Configuration conf, String splitsFile) throws IOException, InvalidInputException {
    int splitCount = conf.getInt(ARG_KEY_SPLIT_COUNT, 0);

    if (splitCount <= 0) {
        throw new InvalidInputException(
                "Invalid or unspecified split count:" + splitCount + "\nSpecify it in: " + ARG_KEY_SPLIT_COUNT);
    }//from w  ww . java2 s. c o  m

    String rowPrefix = conf.get(ARG_KEY_ROW_PREFIX, "row");
    String rowFormat = DataGenerator.getKeyFormat(rowPrefix);
    boolean hashKeys = conf.getBoolean(ARG_KEY_HASH_KEYS, false);
    long start = conf.getInt(ARG_KEY_RANGE_START, 0);
    long end = conf.getInt(ARG_KEY_RANGE_END, 0);

    FileSystem fs = FileSystem.get(conf);
    Path splitsPath = new Path(splitsFile);
    Path plainPath = new Path(splitsFile + "-debug");
    PrintStream out = new PrintStream(new BufferedOutputStream(fs.create(splitsPath)));
    PrintStream plain = new PrintStream(new BufferedOutputStream(fs.create(plainPath)));

    if (hashKeys) {
        start = conf.getInt(ARG_KEY_HASHED_RANGE_START, 0);
        end = conf.getInt(ARG_KEY_HASHED_RANGE_END, Integer.MAX_VALUE);
    }

    long rangeSize = Math.max(1, (end - start + 1) / (splitCount + 1));
    long rangeStart = start + rangeSize;

    System.err.println("Generating splits file: " + splitsFile + "\nrangeStart:" + rangeStart + "\nrangeSize: "
            + rangeSize + "\nsplitCount: " + splitCount + "\nrangeEnd: " + end);

    int i = 0;
    try {
        while (rangeStart < end && splitCount > 0) {
            out.println(new String(Base64.encodeBase64(String.format(rowFormat, rangeStart).getBytes())));
            plain.println(String.format(rowFormat, rangeStart));
            rangeStart += rangeSize;
            splitCount--;
            i++;
        }
    } finally {
        out.close();
        plain.close();
    }
    System.err.println("Splits created: " + i);
    return i;
}

---

From source file:com.yahoo.ycsb.bulk.hbase.BulkDataGeneratorJob.java

License:Apache License

/**
 * The splits file should be generated according to these parameters:
 * - OK: flags specifying whether or not the splits file should be
 *   generated//from w w  w. j ava  2  s .com
 * - OK: #of splits / partitions for the inserts, this should be
 *   explicitly specified.
 * - OK: the name of the generated file (to store the split info).
 */
public String getSplitsFile(Job job, String workdir) throws IOException, InvalidInputException {

    Configuration conf = job.getConfiguration();

    /* Read property for splits file */
    String splitsFile = conf.get(ARG_KEY_SPLITFILE, workdir + "/autogen-splits.txt");
    boolean genSplits = conf.getBoolean(ARG_KEY_GENTABLESPLITS, false);

    if (genSplits) {
        // generate splits
        this.createSplitsFile(conf, splitsFile);
    }

    return splitsFile;
}

---

From source file:com.yahoo.ycsb.bulk.hbase.BulkDataGeneratorJob.java

License:Apache License

/**
 * Parameters for bulk loader specified through the config file:
 *
 * - prefix for the row keys/*w w w.jav  a  2 s.  c o m*/
 * - range start
 * - range end (inclusive)
 * - num splits (or number of partitions).
 * - user
 * - password
 * - table
 *
 * For the accepted default options
 * @see org.apache.hadoop.util.Tool#run(java.lang.String[])
 */
public int run(String[] args) throws Exception {
    Configuration conf = this.getConf();

    Util.printArgs("run", args, System.err);
    printKeyValues(conf, ARG_KEYS, System.err);

    if (args.length > 1 || (args.length == 1 && "-help".compareToIgnoreCase(args[0]) == 0)) {
        System.err.println("Usage: " + this.getClass().getName()
                + "input_path [generic options] [input_paths...] ouptut_path");
        GenericOptionsParser.printGenericCommandUsage(System.err);
        return 1;
    }

    // Time run
    long startTime = System.currentTimeMillis();
    String workdir;

    if (args.length == 1) {
        /* override workdir in the config if it is specified in the
         * command line
         */
        conf.set(ARG_KEY_OUTDIR, args[0]);
        workdir = args[0];
    }

    workdir = conf.get(ARG_KEY_OUTDIR);

    if (workdir == null) {
        System.err.println("No output directory specified");
        return 1;
    }

    /* Initialize job, check parameters and decide which mapper to use */
    Job job = new Job(conf, conf.get(ARG_KEY_JOBNAME, "YCSB KV data generator"));

    /* these settings are the same (i.e., fixed) independent of the
     * parameters */
    job.setJarByClass(this.getClass());
    // job.setInputFormatClass(TextInputFormat.class);
    job.setInputFormatClass(NLineInputFormat.class);

    /* these settings should depend on the type of output file */
    job.setOutputFormatClass(HFileOutputFormat.class);
    /* not sure the next two are needed */
    job.setOutputKeyClass(ImmutableBytesWritable.class);
    job.setOutputValueClass(KeyValue.class);

    this.createInputFile(job, workdir);

    HFileOutputFormat.setOutputPath(job, new Path(workdir + "/files"));

    /* depending on whether the keys need to be sorted and hashed, then
     * decide which mapper and reducer to use 
     */
    boolean hashKeys = conf.getBoolean(ARG_KEY_HASH_KEYS, false);
    boolean sortKeys = conf.getBoolean(ARG_KEY_SORTKEYS, true);

    /* get splits file name: side-effect -> this may generate a splits file  */
    String splitsfile = this.getSplitsFile(job, workdir);

    if (sortKeys && hashKeys) { /* do a full map reduce job */
        job.setMapperClass(RowGeneratorMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setPartitionerClass(RangePartitioner.class);

        if (splitsfile == null) {
            /* Auto generate the splits file either from:
             * - the input key ranges
             * - from the current table splits
             */
            throw new InvalidInputException("No splits specified");
        }

        /* Set splits file */
        RangePartitioner.setSplitFile(job, splitsfile);

        /* Add reducer (based on mapper code) */
        job.setReducerClass(RowGeneratorReduce.class);

        /* the number of reducers is dependent on the number of
         * partitions
         */
        int numReduce = conf.getInt(ARG_KEY_NUMREDUCE, 1);
        job.setNumReduceTasks(numReduce);
    } else { /* perform a map only job */
        job.setMapperClass(RowGeneratorMapOnly.class);
        /* map output key and value types are the same as
         * for the job
         */
        job.setMapOutputKeyClass(job.getOutputKeyClass());
        job.setMapOutputValueClass(job.getOutputValueClass());
        job.setNumReduceTasks(0);
    }

    job.waitForCompletion(true);

    //        JobClient.runJob(conf);
    SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss.SSS z");
    SimpleDateFormat ddf = new SimpleDateFormat("HH:mm:ss.SSS");
    ddf.setTimeZone(TimeZone.getTimeZone("UTC"));
    long endTime = System.currentTimeMillis();
    System.out.println("Start time (ms): " + df.format(new Date(startTime)) + " -- " + startTime);
    System.out.println("End time (ms): " + df.format(new Date(endTime)) + " -- " + endTime);
    System.out
            .println("Elapsed time (ms): " + ddf.format(endTime - startTime) + " -- " + (endTime - startTime));
    return 0;
}

---

From source file:com.yourcompany.hadoop.mapreduce.KoreanWordcountMapper.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    Configuration configuration = context.getConfiguration();
    exactMatch = configuration.getBoolean("exactMatch", false);
    bigrammable = configuration.getBoolean("bigrammable", false);
    hasOrigin = configuration.getBoolean("hasOrigin", false);
    originCNoun = configuration.getBoolean("originCNoun", false);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »