Example usage for org.apache.hadoop.conf Configuration getBoolean

List of usage examples for org.apache.hadoop.conf Configuration getBoolean

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. getBoolean

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getBoolean.

Prototype

public boolean getBoolean(String name, boolean defaultValue)

Source Link

Document

Get the value of the name property as a boolean.

Usage

From source file:org.apache.impala.service.JniFrontend.java

License:Apache License

/**
 * Returns an error message if short circuit reads are enabled but misconfigured.
 * Otherwise, returns an empty string,//from   w w w.  j ava 2 s.  c om
 */
private String checkShortCircuitRead(Configuration conf) {
    if (!conf.getBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY,
            DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_DEFAULT)) {
        LOG.info("Short-circuit reads are not enabled.");
        return "";
    }

    StringBuilder output = new StringBuilder();
    String errorMessage = "Invalid short-circuit reads configuration:\n";
    String prefix = "  - ";
    StringBuilder errorCause = new StringBuilder();

    // dfs.domain.socket.path must be set properly
    String domainSocketPath = conf.getTrimmed(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
            DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_DEFAULT);
    if (domainSocketPath.isEmpty()) {
        errorCause.append(prefix);
        errorCause.append(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY);
        errorCause.append(" is not configured.\n");
    } else {
        // The socket path parent directory must be readable and executable.
        File socketFile = new File(domainSocketPath);
        File socketDir = socketFile.getParentFile();
        if (socketDir == null || !socketDir.canRead() || !socketDir.canExecute()) {
            errorCause.append(prefix);
            errorCause.append("Impala cannot read or execute the parent directory of ");
            errorCause.append(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY);
            errorCause.append("\n");
        }
    }

    // dfs.client.use.legacy.blockreader.local must be set to false
    if (conf.getBoolean(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL,
            DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL_DEFAULT)) {
        errorCause.append(prefix);
        errorCause.append(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL);
        errorCause.append(" should not be enabled.\n");
    }

    if (errorCause.length() > 0) {
        output.append(errorMessage);
        output.append(errorCause);
    }

    return output.toString();
}

From source file:org.apache.impala.yarn.server.resourcemanager.scheduler.fair.QueuePlacementPolicy.java

License:Apache License

/**
 * Build a simple queue placement policy from the allow-undeclared-pools and
 * user-as-default-queue configuration options.
 *//* ww w. ja  v  a 2 s  .  com*/
public static QueuePlacementPolicy fromConfiguration(Configuration conf,
        Map<FSQueueType, Set<String>> configuredQueues) {
    boolean create = conf.getBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS,
            FairSchedulerConfiguration.DEFAULT_ALLOW_UNDECLARED_POOLS);
    boolean userAsDefaultQueue = conf.getBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE,
            FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE);
    List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>();
    rules.add(new QueuePlacementRule.Specified().initialize(create, null));
    if (userAsDefaultQueue) {
        rules.add(new QueuePlacementRule.User().initialize(create, null));
    }
    if (!userAsDefaultQueue || !create) {
        rules.add(new QueuePlacementRule.Default().initialize(true, null));
    }
    try {
        return new QueuePlacementPolicy(rules, configuredQueues, conf);
    } catch (AllocationConfigurationException ex) {
        throw new RuntimeException("Should never hit exception when loading" + "placement policy from conf",
                ex);
    }
}

From source file:org.apache.jena.grande.giraph.FoafShortestPathsVertex.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    log.debug("run({})", Utils.toString(args));
    Preconditions.checkArgument(args.length == 4,
            "run: Must have 4 arguments <input path> <output path> <source vertex uri> <# of workers>");

    Configuration configuration = getConf();
    boolean overrideOutput = configuration.getBoolean(Constants.OPTION_OVERWRITE_OUTPUT,
            Constants.OPTION_OVERWRITE_OUTPUT_DEFAULT);
    FileSystem fs = FileSystem.get(new Path(args[1]).toUri(), configuration);
    if (overrideOutput) {
        fs.delete(new Path(args[1]), true);
    }/*from  w ww.  ja  v  a 2  s .c om*/

    GiraphConfiguration giraphConfiguration = new GiraphConfiguration(getConf());
    giraphConfiguration.setVertexClass(getClass());
    giraphConfiguration.setVertexInputFormatClass(TurtleVertexInputFormat.class);
    giraphConfiguration.setVertexOutputFormatClass(TurtleVertexOutputFormat.class);
    giraphConfiguration.set(SOURCE_URI, args[2]);
    giraphConfiguration.setWorkerConfiguration(Integer.parseInt(args[3]), Integer.parseInt(args[3]), 100.0f);

    GiraphJob job = new GiraphJob(getConf(), getClass().getName());
    FileInputFormat.addInputPath(job.getInternalJob(), new Path(args[0]));
    FileOutputFormat.setOutputPath(job.getInternalJob(), new Path(args[1]));
    return job.run(true) ? 0 : -1;
}

From source file:org.apache.jena.grande.giraph.sssps.SingleSourceShortestPaths.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    log.debug("run({})", Utils.toString(args));
    Preconditions.checkArgument(args.length == 4,
            "run: Must have 4 arguments <input path> <output path> <source vertex> <# of workers>");

    Configuration configuration = getConf();
    boolean overrideOutput = configuration.getBoolean(Constants.OPTION_OVERWRITE_OUTPUT,
            Constants.OPTION_OVERWRITE_OUTPUT_DEFAULT);
    FileSystem fs = FileSystem.get(new Path(args[1]).toUri(), configuration);
    if (overrideOutput) {
        fs.delete(new Path(args[1]), true);
    }/*  w w w.j a v a 2s .co  m*/

    GiraphConfiguration giraphConfiguration = new GiraphConfiguration(getConf());
    giraphConfiguration.setVertexClass(getClass());
    giraphConfiguration.setVertexInputFormatClass(IntIntNullIntTextInputFormat.class);
    giraphConfiguration.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
    giraphConfiguration.set(SOURCE_VERTEX, args[2]);
    giraphConfiguration.setWorkerConfiguration(Integer.parseInt(args[3]), Integer.parseInt(args[3]), 100.0f);

    GiraphJob job = new GiraphJob(giraphConfiguration, getClass().getName());
    FileInputFormat.addInputPath(job.getInternalJob(), new Path(args[0]));
    FileOutputFormat.setOutputPath(job.getInternalJob(), new Path(args[1]));
    return job.run(true) ? 0 : -1;
}

From source file:org.apache.jena.grande.mapreduce.Rdf2AdjacencyListDriver.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        System.err.printf("Usage: %s [generic options] <input> <output>\n", getClass().getName());
        ToolRunner.printGenericCommandUsage(System.err);
        return -1;
    }//from www  .j  a v a 2  s . c  o m

    Configuration configuration = getConf();
    boolean useCompression = configuration.getBoolean(Constants.OPTION_USE_COMPRESSION,
            Constants.OPTION_USE_COMPRESSION_DEFAULT);

    if (useCompression) {
        configuration.setBoolean("mapred.compress.map.output", true);
        configuration.set("mapred.output.compression.type", "BLOCK");
        configuration.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec");
    }

    boolean overrideOutput = configuration.getBoolean(Constants.OPTION_OVERWRITE_OUTPUT,
            Constants.OPTION_OVERWRITE_OUTPUT_DEFAULT);
    FileSystem fs = FileSystem.get(new Path(args[1]).toUri(), configuration);
    if (overrideOutput) {
        fs.delete(new Path(args[1]), true);
    }

    Job job = new Job(configuration);
    job.setJobName(Constants.RDF_2_ADJACENCY_LIST);
    job.setJarByClass(getClass());

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setInputFormatClass(NQuadsInputFormat.class);

    job.setMapperClass(Rdf2AdjacencyListMapper.class);
    job.setMapOutputKeyClass(NodeWritable.class);
    job.setMapOutputValueClass(QuadWritable.class);

    job.setReducerClass(Rdf2AdjacencyListReducer.class);
    job.setOutputKeyClass(NullWritable.class);
    job.setOutputValueClass(Text.class);

    Utils.setReducers(job, configuration, log);

    job.setOutputFormatClass(TextOutputFormat.class);

    if (log.isDebugEnabled())
        Utils.log(job, log);

    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:org.apache.jena.hadoop.rdf.io.input.AbstractNodeTupleInputFormatTests.java

License:Apache License

/**
 * Tests behaviour when ignoring bad tuples is disabled
 * //w  ww  . j a va2  s.  co m
 * @throws InterruptedException
 * @throws IOException
 */
@Test(expected = IOException.class)
public final void fail_on_bad_input_01() throws IOException, InterruptedException {
    Configuration config = this.prepareConfiguration();
    config.setBoolean(RdfIOConstants.INPUT_IGNORE_BAD_TUPLES, false);
    Assert.assertFalse(config.getBoolean(RdfIOConstants.INPUT_IGNORE_BAD_TUPLES, true));
    testSingleInput(config, bad, 1, 0);
}

From source file:org.apache.jena.hadoop.rdf.io.input.AbstractNodeTupleInputFormatTests.java

License:Apache License

/**
 * Tests behaviour when ignoring bad tuples is disabled
 * /*from   w w w.j av  a2s . c o  m*/
 * @throws InterruptedException
 * @throws IOException
 */
@Test(expected = IOException.class)
public final void fail_on_bad_input_02() throws IOException, InterruptedException {
    Configuration config = this.prepareConfiguration();
    config.setBoolean(RdfIOConstants.INPUT_IGNORE_BAD_TUPLES, false);
    Assert.assertFalse(config.getBoolean(RdfIOConstants.INPUT_IGNORE_BAD_TUPLES, true));
    testSingleInput(config, mixed, 1, MIXED_SIZE / 2);
}

From source file:org.apache.jena.hadoop.rdf.io.input.readers.AbstractBlockBasedNodeTupleReader.java

License:Apache License

@Override
public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    LOG.debug("initialize({}, {})", genericSplit, context);

    // Assuming file split
    if (!(genericSplit instanceof FileSplit))
        throw new IOException("This record reader only supports FileSplit inputs");
    FileSplit split = (FileSplit) genericSplit;

    // Configuration
    Configuration config = context.getConfiguration();
    this.ignoreBadTuples = config.getBoolean(RdfIOConstants.INPUT_IGNORE_BAD_TUPLES, true);
    if (this.ignoreBadTuples)
        LOG.warn(//from  w  ww .  j a v  a 2 s .c  o m
                "Configured to ignore bad tuples, parsing errors will be logged and further parsing aborted but no user visible errors will be thrown.  Consider setting {} to false to disable this behaviour",
                RdfIOConstants.INPUT_IGNORE_BAD_TUPLES);

    // Figure out what portion of the file to read
    start = split.getStart();
    long end = start + split.getLength();
    final Path file = split.getPath();
    long totalLength = file.getFileSystem(context.getConfiguration()).getFileStatus(file).getLen();
    boolean readToEnd = end == totalLength;
    CompressionCodecFactory factory = new CompressionCodecFactory(config);
    this.compressionCodecs = factory.getCodec(file);

    LOG.info(String.format("Got split with start %d and length %d for file with total length of %d",
            new Object[] { start, split.getLength(), totalLength }));

    // Open the file and prepare the input stream
    FileSystem fs = file.getFileSystem(config);
    FSDataInputStream fileIn = fs.open(file);
    this.length = split.getLength();
    if (start > 0)
        fileIn.seek(start);

    if (this.compressionCodecs != null) {
        // Compressed input
        // For compressed input NLineInputFormat will have failed to find
        // any line breaks and will give us a split from 0 -> (length - 1)
        // Add 1 and re-verify readToEnd so we can abort correctly if ever
        // given a partial split of a compressed file
        end++;
        readToEnd = end == totalLength;
        if (start > 0 || !readToEnd)
            throw new IOException(
                    "This record reader can only be used with compressed input where the split is a whole file");
        input = new TrackedInputStream(this.compressionCodecs.createInputStream(fileIn));
    } else {
        // Uncompressed input

        if (readToEnd) {
            input = new TrackedInputStream(fileIn);
        } else {
            // Need to limit the portion of the file we are reading
            input = new BlockInputStream(fileIn, split.getLength());
        }
    }

    // Set up background thread for parser
    iter = this.getPipedIterator();
    this.stream = this.getPipedStream(iter, this.input);
    RDFParserBuilder builder = RdfIOUtils.createRDFParserBuilder(context, file);
    Runnable parserRunnable = this.createRunnable(this, this.input, stream, this.getRdfLanguage(), builder);

    this.parserThread = new Thread(parserRunnable);
    this.parserThread.setDaemon(true);
    this.parserThread.start();
}

From source file:org.apache.jena.hadoop.rdf.io.input.readers.AbstractLineBasedNodeTupleReader.java

License:Apache License

@Override
public final void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    LOG.debug("initialize({}, {})", genericSplit, context);

    // Assuming file split
    if (!(genericSplit instanceof FileSplit))
        throw new IOException("This record reader only supports FileSplit inputs");
    FileSplit split = (FileSplit) genericSplit;

    // Intermediate : RDFParser but need to make a Iterator<Quad/Triple>
    LabelToNode labelToNode = RdfIOUtils.createLabelToNode(context, split.getPath());
    maker = new ParserProfileStd(RiotLib.factoryRDF(labelToNode), ErrorHandlerFactory.errorHandlerStd,
            IRIResolver.create(), PrefixMapFactory.createForInput(), null, true, false);

    Configuration config = context.getConfiguration();
    this.ignoreBadTuples = config.getBoolean(RdfIOConstants.INPUT_IGNORE_BAD_TUPLES, true);
    if (this.ignoreBadTuples)
        LOG.warn(// www .j av a  2s  . c  om
                "Configured to ignore bad tuples, parsing errors will be logged and the bad line skipped but no errors will be thrownConsider setting {} to false to disable this behaviour",
                RdfIOConstants.INPUT_IGNORE_BAD_TUPLES);

    // Figure out what portion of the file to read
    this.maxLineLength = config.getInt(HadoopIOConstants.MAX_LINE_LENGTH, Integer.MAX_VALUE);
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();
    long totalLength = file.getFileSystem(context.getConfiguration()).getFileStatus(file).getLen();
    compressionCodecs = new CompressionCodecFactory(config);
    final CompressionCodec codec = compressionCodecs.getCodec(file);

    LOG.info(String.format("Got split with start %d and length %d for file with total length of %d",
            new Object[] { start, split.getLength(), totalLength }));

    // Open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(config);
    FSDataInputStream fileIn = fs.open(file);
    boolean skipFirstLine = false;
    if (codec != null) {
        // Compressed input
        // For compressed input NLineInputFormat will have failed to find
        // any line breaks and will give us a split from 0 -> (length - 1)
        // Add 1 and verify we got complete split
        if (totalLength > split.getLength() + 1)
            throw new IOException(
                    "This record reader can only be used with compressed input where the split covers the whole file");
        in = new LineReader(codec.createInputStream(fileIn), config);
        estLength = end;
        end = Long.MAX_VALUE;
    } else {
        // Uncompressed input
        if (start != 0) {
            skipFirstLine = true;
            --start;
            fileIn.seek(start);
        }
        in = new LineReader(fileIn, config);
    }
    // Skip first line and re-establish "start".
    // This is to do with how line reader reads lines and how
    // NLineInputFormat will provide the split information to use
    if (skipFirstLine) {
        start += in.readLine(new Text(), 0, (int) Math.min(Integer.MAX_VALUE, end - start));
    }
    this.pos = start;
}

From source file:org.apache.jena.hadoop.rdf.io.input.readers.AbstractWholeFileNodeTupleReader.java

License:Apache License

@Override
public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    LOG.debug("initialize({}, {})", genericSplit, context);

    // Assuming file split
    if (!(genericSplit instanceof FileSplit))
        throw new IOException("This record reader only supports FileSplit inputs");
    FileSplit split = (FileSplit) genericSplit;

    // Configuration
    Configuration config = context.getConfiguration();
    this.ignoreBadTuples = config.getBoolean(RdfIOConstants.INPUT_IGNORE_BAD_TUPLES, true);
    if (this.ignoreBadTuples)
        LOG.warn(//from ww w . jav  a2  s .com
                "Configured to ignore bad tuples, parsing errors will be logged and further parsing aborted but no user visible errors will be thrown.  Consider setting {} to false to disable this behaviour",
                RdfIOConstants.INPUT_IGNORE_BAD_TUPLES);

    // Figure out what portion of the file to read
    if (split.getStart() > 0)
        throw new IOException("This record reader requires a file split which covers the entire file");
    final Path file = split.getPath();
    long totalLength = file.getFileSystem(context.getConfiguration()).getFileStatus(file).getLen();
    CompressionCodecFactory factory = new CompressionCodecFactory(config);
    this.compressionCodecs = factory.getCodec(file);

    LOG.info(String.format("Got split with start %d and length %d for file with total length of %d",
            new Object[] { split.getStart(), split.getLength(), totalLength }));

    if (totalLength > split.getLength())
        throw new IOException("This record reader requires a file split which covers the entire file");

    // Open the file and prepare the input stream
    FileSystem fs = file.getFileSystem(config);
    FSDataInputStream fileIn = fs.open(file);
    this.length = split.getLength();
    if (this.compressionCodecs != null) {
        // Compressed input
        input = new TrackedInputStream(this.compressionCodecs.createInputStream(fileIn));
    } else {
        // Uncompressed input
        input = new TrackedInputStream(fileIn);
    }

    // Set up background thread for parser
    iter = this.getPipedIterator();
    this.stream = this.getPipedStream(iter, this.input);
    RDFParserBuilder builder = RdfIOUtils.createRDFParserBuilder(context, file);
    Runnable parserRunnable = this.createRunnable(this, this.input, stream, this.getRdfLanguage(), builder);
    this.parserThread = new Thread(parserRunnable);
    this.parserThread.setDaemon(true);
    this.parserThread.start();
}