Example usage for org.apache.hadoop.conf Configuration setBoolean

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setBoolean.

Prototype

public void setBoolean(String name, boolean value) 

Source Link

Document

Set the value of the name property to a boolean.

Usage

From source file:org.apache.jena.hadoop.rdf.io.input.AbstractNodeTupleInputFormatTests.java

License:Apache License

/**
 * Tests behaviour when ignoring bad tuples is disabled
 * /*from   ww  w .j  a  v  a2s.c om*/
 * @throws InterruptedException
 * @throws IOException
 */
@Test(expected = IOException.class)
public final void fail_on_bad_input_02() throws IOException, InterruptedException {
    Configuration config = this.prepareConfiguration();
    config.setBoolean(RdfIOConstants.INPUT_IGNORE_BAD_TUPLES, false);
    Assert.assertFalse(config.getBoolean(RdfIOConstants.INPUT_IGNORE_BAD_TUPLES, true));
    testSingleInput(config, mixed, 1, MIXED_SIZE / 2);
}

---

From source file:org.apache.jena.hadoop.rdf.io.input.AbstractNodeTupleInputFormatTests.java

License:Apache License

/**
 * Tests for input splitting//from w  w w.  j  a va 2s  .  co  m
 * 
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public final void split_input_01() throws IOException, InterruptedException {
    Assume.assumeTrue(this.canSplitInputs());

    Configuration config = this.prepareConfiguration();
    config.setBoolean(RdfIOConstants.INPUT_IGNORE_BAD_TUPLES, false);
    Assert.assertEquals(Integer.MAX_VALUE, config.getInt(HadoopIOConstants.MAX_LINE_LENGTH, Integer.MAX_VALUE));
    this.testSplitInputs(config, new File[] { small }, 100, SMALL_SIZE);
}

---

From source file:org.apache.jena.hadoop.rdf.io.input.AbstractNodeTupleInputFormatTests.java

License:Apache License

/**
 * Tests for input splitting//w w w  .  j ava  2  s . c  o  m
 * 
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public final void split_input_02() throws IOException, InterruptedException {
    Assume.assumeTrue(this.canSplitInputs());

    Configuration config = this.prepareConfiguration();
    config.setBoolean(RdfIOConstants.INPUT_IGNORE_BAD_TUPLES, false);
    config.setLong(NLineInputFormat.LINES_PER_MAP, 10);
    Assert.assertEquals(Integer.MAX_VALUE, config.getInt(HadoopIOConstants.MAX_LINE_LENGTH, Integer.MAX_VALUE));
    this.testSplitInputs(config, new File[] { small }, 10, SMALL_SIZE);
}

---

From source file:org.apache.jena.hadoop.rdf.io.input.AbstractNodeTupleInputFormatTests.java

License:Apache License

/**
 * Tests for input splitting//from  w  w w.j  a  va 2s  .  c  o  m
 * 
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public final void split_input_03() throws IOException, InterruptedException {
    Assume.assumeTrue(this.canSplitInputs());

    Configuration config = this.prepareConfiguration();
    config.setBoolean(RdfIOConstants.INPUT_IGNORE_BAD_TUPLES, false);
    config.setLong(NLineInputFormat.LINES_PER_MAP, 100);
    Assert.assertEquals(Integer.MAX_VALUE, config.getInt(HadoopIOConstants.MAX_LINE_LENGTH, Integer.MAX_VALUE));
    this.testSplitInputs(config, new File[] { large }, 100, LARGE_SIZE);
}

---

From source file:org.apache.jena.hadoop.rdf.io.input.turtle.TurtleInputTest.java

License:Apache License

@Test
public void turtle_with_prefixes_01() throws IOException, InterruptedException {
    // Try to reproduce JENA-1075

    // Create test data
    File f = folder.newFile("prefixes.ttl");

    try (FileWriter writer = new FileWriter(f)) {
        //@formatter:off
        writer.write(StrUtils.strjoinNL("@prefix : <http://test/ns#> .", ":s :p :o ."));
        //@formatter:on
        writer.close();//from w  w  w.j  a va 2  s  . com
    }

    Configuration config = this.prepareConfiguration();
    config.setBoolean(RdfIOConstants.INPUT_IGNORE_BAD_TUPLES, false);
    this.testSingleInput(config, f, 1, 1);

    // Clean up
    if (f.exists())
        f.delete();
}

---

From source file:org.apache.jena.tdbloader4.FirstDriver.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.printf("Usage: %s [generic options] <input> <output>\n", getClass().getName());
        ToolRunner.printGenericCommandUsage(System.err);
        return -1;
    }//from ww  w  .  ja v  a  2  s  .  c o  m

    Configuration configuration = getConf();
    boolean useCompression = configuration.getBoolean(Constants.OPTION_USE_COMPRESSION,
            Constants.OPTION_USE_COMPRESSION_DEFAULT);

    if (useCompression) {
        configuration.setBoolean("mapred.compress.map.output", true);
        configuration.set("mapred.output.compression.type", "BLOCK");
        configuration.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec");
    }

    Job job = new Job(configuration);
    job.setJobName(Constants.NAME_FIRST);
    job.setJarByClass(getClass());

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setInputFormatClass(NQuadsInputFormat.class);
    job.setMapperClass(FirstMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(NullWritable.class);

    job.setReducerClass(FirstReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    Utils.setReducers(job, configuration, log);

    job.setOutputFormatClass(TextOutputFormat.class);

    if (log.isDebugEnabled())
        Utils.log(job, log);

    return job.waitForCompletion(true) ? 0 : 1;
}

---

From source file:org.apache.jena.tdbloader4.InferDriver.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 3) {
        System.err.printf("Usage: %s [generic options] <vocabulary> <input> <output>\n", getClass().getName());
        ToolRunner.printGenericCommandUsage(System.err);
        return -1;
    }/* w w  w .jav  a  2s.c  om*/

    Configuration configuration = getConf();
    boolean useCompression = configuration.getBoolean(Constants.OPTION_USE_COMPRESSION,
            Constants.OPTION_USE_COMPRESSION_DEFAULT);

    if (useCompression) {
        configuration.setBoolean("mapred.compress.map.output", true);
        configuration.set("mapred.output.compression.type", "BLOCK");
        configuration.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec");
    }

    boolean overrideOutput = configuration.getBoolean(Constants.OPTION_OVERRIDE_OUTPUT,
            Constants.OPTION_OVERRIDE_OUTPUT_DEFAULT);
    FileSystem fs = FileSystem.get(new Path(args[2]).toUri(), configuration);
    if (overrideOutput) {
        fs.delete(new Path(args[2]), true);
    }

    // All the mappers need to have the vocabulary/ontology available, typically they are very small
    Path vocabulary = new Path(args[0]);
    DistributedCache.addCacheFile(vocabulary.toUri(), configuration);

    Job job = new Job(configuration);
    job.setJobName(Constants.NAME_INFER);
    job.setJarByClass(getClass());

    FileInputFormat.addInputPath(job, new Path(args[1]));
    FileOutputFormat.setOutputPath(job, new Path(args[2]));

    job.setInputFormatClass(NQuadsInputFormat.class);
    job.setMapperClass(InferMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(NullWritable.class);

    job.setNumReduceTasks(0); // map only job

    job.setOutputFormatClass(TextOutputFormat.class);

    if (log.isDebugEnabled())
        Utils.log(job, log);

    return job.waitForCompletion(true) ? 0 : 1;
}

---

From source file:org.apache.jena.tdbloader4.SecondDriver.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.printf("Usage: %s [generic options] <input> <output>\n", getClass().getName());
        ToolRunner.printGenericCommandUsage(System.err);
        return -1;
    }/*from   w w w  . ja  v  a  2s.c  o  m*/

    Configuration configuration = getConf();
    boolean useCompression = configuration.getBoolean(Constants.OPTION_USE_COMPRESSION,
            Constants.OPTION_USE_COMPRESSION_DEFAULT);

    if (useCompression) {
        configuration.setBoolean("mapred.compress.map.output", true);
        configuration.set("mapred.output.compression.type", "BLOCK");
        configuration.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec");
    }

    Job job = new Job(configuration);
    job.setJobName(Constants.NAME_SECOND);
    job.setJarByClass(getClass());

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setInputFormatClass(NQuadsInputFormat.class);
    job.setMapperClass(SecondMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);

    job.setReducerClass(SecondReducer.class);
    job.setOutputKeyClass(LongWritable.class);
    job.setOutputValueClass(Text.class);

    Utils.setReducers(job, configuration, log);

    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    if (useCompression) {
        SequenceFileOutputFormat.setCompressOutput(job, true);
        SequenceFileOutputFormat.setOutputCompressorClass(job, GzipCodec.class);
        SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK);
    }

    if (log.isDebugEnabled())
        Utils.log(job, log);

    return job.waitForCompletion(true) ? 0 : 1;
}

---

From source file:org.apache.jena.tdbloader4.StatsDriver.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.printf("Usage: %s [generic options] <input> <output>\n", getClass().getName());
        ToolRunner.printGenericCommandUsage(System.err);
        return -1;
    }/*from ww  w. j a  v a  2 s .  c om*/

    Configuration configuration = getConf();
    boolean useCompression = configuration.getBoolean(Constants.OPTION_USE_COMPRESSION,
            Constants.OPTION_USE_COMPRESSION_DEFAULT);

    if (useCompression) {
        configuration.setBoolean("mapred.compress.map.output", true);
        configuration.set("mapred.output.compression.type", "BLOCK");
        configuration.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec");
    }

    boolean overrideOutput = configuration.getBoolean(Constants.OPTION_OVERRIDE_OUTPUT,
            Constants.OPTION_OVERRIDE_OUTPUT_DEFAULT);
    FileSystem fs = FileSystem.get(new Path(args[1]).toUri(), configuration);
    if (overrideOutput) {
        fs.delete(new Path(args[1]), true);
    }

    Job job = new Job(configuration);
    job.setJobName(Constants.NAME_STATS);
    job.setJarByClass(getClass());

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setInputFormatClass(NQuadsInputFormat.class);

    job.setMapperClass(StatsMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);
    job.setCombinerClass(StatsReducer.class);

    job.setReducerClass(StatsReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    // we use the combiner, 1 reducer here is not a problem
    configuration.set(Constants.OPTION_NUM_REDUCERS, "1");
    Utils.setReducers(job, configuration, log);

    job.setOutputFormatClass(TextOutputFormat.class);

    if (log.isDebugEnabled())
        Utils.log(job, log);

    return job.waitForCompletion(true) ? 0 : 1;
}

---

From source file:org.apache.jena.tdbloader4.ThirdDriver.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.printf("Usage: %s [generic options] <input> <output>\n", getClass().getName());
        ToolRunner.printGenericCommandUsage(System.err);
        return -1;
    }//from   w  ww  .  j  a  v  a2  s  .co m

    log.debug("input: {}, output: {}", args[0], args[1]);

    Configuration configuration = getConf();
    boolean useCompression = configuration.getBoolean(Constants.OPTION_USE_COMPRESSION,
            Constants.OPTION_USE_COMPRESSION_DEFAULT);
    log.debug("Compression is {}", useCompression ? "enabled" : "disabled");

    if (useCompression) {
        configuration.setBoolean("mapred.compress.map.output", true);
        configuration.set("mapred.output.compression.type", "BLOCK");
        configuration.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec");
    }

    Job job = new Job(configuration);
    job.setJobName(Constants.NAME_THIRD);
    job.setJarByClass(getClass());

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileInputFormat.setInputPathFilter(job, ExcludeNodeTableFilter.class);
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setInputFormatClass(SequenceFileInputFormat.class);

    job.setMapperClass(ThirdMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);

    job.setReducerClass(ThirdReducer.class);
    job.setOutputKeyClass(LongQuadWritable.class);
    job.setOutputValueClass(NullWritable.class);

    Utils.setReducers(job, configuration, log);

    job.setOutputFormatClass(SequenceFileOutputFormat.class);

    if (useCompression) {
        SequenceFileOutputFormat.setCompressOutput(job, true);
        SequenceFileOutputFormat.setOutputCompressorClass(job, GzipCodec.class);
        SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK);
    }

    if (log.isDebugEnabled())
        Utils.log(job, log);

    return job.waitForCompletion(true) ? 0 : 1;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »