Example usage for org.apache.hadoop.conf Configuration setBoolean

List of usage examples for org.apache.hadoop.conf Configuration setBoolean

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. setBoolean

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setBoolean.

Prototype

public void setBoolean(String name, boolean value)

Source Link

Document

Set the value of the name property to a boolean.

Usage

From source file:org.apache.cassandra.hadoop.cql3.CqlBulkOutputFormat.java

License:Apache License

public static void setDeleteSourceOnSuccess(Configuration conf, boolean deleteSrc) {
    conf.setBoolean(DELETE_SOURCE, deleteSrc);
}

From source file:org.apache.crunch.impl.mr.plan.DotfileUtil.java

License:Apache License

/**
 * Enable the creation of debugging dotfiles (which explain various stages in the job planning process).
 *
 * @param conf pipeline configuration//  w w  w. ja  va2  s  .  c o m
 */
public static void enableDebugDotfiles(Configuration conf) {
    conf.setBoolean(PlanningParameters.DEBUG_DOTFILES_ENABLED, true);
}

From source file:org.apache.crunch.impl.mr.plan.DotfileUtil.java

License:Apache License

/**
 * Disable the creation of debugging dotfiles.
 *
 * @param conf pipeline configuration// w ww  .  jav a  2s .com
 */
public static void disableDebugDotfiles(Configuration conf) {
    conf.setBoolean(PlanningParameters.DEBUG_DOTFILES_ENABLED, false);
}

From source file:org.apache.crunch.MaterializeIT.java

License:Apache License

@Test
public void testMaterializeNoFailure() throws IOException {
    Configuration conf = tmpDir.getDefaultConfiguration();
    conf.setBoolean("crunch.empty.materialize.on.failure", true);
    runMaterializeWithFailure(new MRPipeline(MaterializeIT.class, conf));
}

From source file:org.apache.crunch.MultipleOutputIT.java

License:Apache License

@Test
public void testCountersDisabled() throws IOException {
    Configuration configuration = tmpDir.getDefaultConfiguration();
    configuration.setBoolean(CrunchOutputs.CRUNCH_DISABLE_OUTPUT_COUNTERS, true);

    PipelineResult result = run(new MRPipeline(MultipleOutputIT.class, configuration),
            WritableTypeFamily.getInstance());

    assertEquals(1, result.getStageResults().size());
    StageResult stageResult = result.getStageResults().get(0);

    assertFalse(stageResult.getCounterNames().containsKey(CrunchOutputs.CRUNCH_OUTPUTS));
}

From source file:org.apache.crunch.types.avro.AvroGroupedTableType.java

License:Apache License

@Override
public void configureShuffle(Job job, GroupingOptions options) {
    AvroTableType<K, V> att = (AvroTableType<K, V>) tableType;
    String schemaJson = att.getSchema().toString();
    Configuration conf = job.getConfiguration();

    if (att.hasReflect()) {
        if (att.hasSpecific()) {
            Avros.checkCombiningSpecificAndReflectionSchemas();
        }/*  ww w  . j a  v  a 2  s  .  c  o  m*/
        conf.setBoolean(AvroJob.MAP_OUTPUT_IS_REFLECT, true);
    }
    conf.set(AvroJob.MAP_OUTPUT_SCHEMA, schemaJson);
    job.setSortComparatorClass(AvroKeyComparator.class);
    job.setMapOutputKeyClass(AvroKey.class);
    job.setMapOutputValueClass(AvroValue.class);
    if (options != null) {
        options.configure(job);
    }

    Avros.configureReflectDataFactory(conf);

    Collection<String> serializations = job.getConfiguration().getStringCollection("io.serializations");
    if (!serializations.contains(SafeAvroSerialization.class.getName())) {
        serializations.add(SafeAvroSerialization.class.getName());
        job.getConfiguration().setStrings("io.serializations", serializations.toArray(new String[0]));
    }
}

From source file:org.apache.drill.exec.store.parquet.ParquetReaderConfig.java

License:Apache License

public Configuration addCountersToConf(Configuration conf) {
    Configuration newConfig = new Configuration(conf);
    newConfig.setBoolean(ENABLE_BYTES_READ_COUNTER, enableBytesReadCounter);
    newConfig.setBoolean(ENABLE_BYTES_TOTAL_COUNTER, enableBytesTotalCounter);
    newConfig.setBoolean(ENABLE_TIME_READ_COUNTER, enableTimeReadCounter);
    return newConfig;
}

From source file:org.apache.drill.exec.store.parquet.ParquetScanBatchCreator.java

License:Apache License

@Override
public ScanBatch getBatch(FragmentContext context, ParquetRowGroupScan rowGroupScan, List<RecordBatch> children)
        throws ExecutionSetupException {
    Preconditions.checkArgument(children.isEmpty());
    OperatorContext oContext = context.newOperatorContext(rowGroupScan);

    final ImplicitColumnExplorer columnExplorer = new ImplicitColumnExplorer(context,
            rowGroupScan.getColumns());/*from   w w  w  .j  av a2  s. c o  m*/

    if (!columnExplorer.isStarQuery()) {
        rowGroupScan = new ParquetRowGroupScan(rowGroupScan.getUserName(), rowGroupScan.getStorageEngine(),
                rowGroupScan.getRowGroupReadEntries(), columnExplorer.getTableColumns(),
                rowGroupScan.getSelectionRoot());
        rowGroupScan.setOperatorId(rowGroupScan.getOperatorId());
    }

    DrillFileSystem fs;
    try {
        fs = oContext.newFileSystem(rowGroupScan.getStorageEngine().getFsConf());
    } catch (IOException e) {
        throw new ExecutionSetupException(String.format("Failed to create DrillFileSystem: %s", e.getMessage()),
                e);
    }
    Configuration conf = new Configuration(fs.getConf());
    conf.setBoolean(ENABLE_BYTES_READ_COUNTER, false);
    conf.setBoolean(ENABLE_BYTES_TOTAL_COUNTER, false);
    conf.setBoolean(ENABLE_TIME_READ_COUNTER, false);

    // keep footers in a map to avoid re-reading them
    Map<String, ParquetMetadata> footers = Maps.newHashMap();
    List<RecordReader> readers = Lists.newArrayList();
    List<Map<String, String>> implicitColumns = Lists.newArrayList();
    Map<String, String> mapWithMaxColumns = Maps.newLinkedHashMap();
    for (RowGroupReadEntry e : rowGroupScan.getRowGroupReadEntries()) {
        /*
        Here we could store a map from file names to footers, to prevent re-reading the footer for each row group in a file
        TODO - to prevent reading the footer again in the parquet record reader (it is read earlier in the ParquetStorageEngine)
        we should add more information to the RowGroupInfo that will be populated upon the first read to
        provide the reader with all of th file meta-data it needs
        These fields will be added to the constructor below
        */
        try {
            Stopwatch timer = Stopwatch.createUnstarted();
            if (!footers.containsKey(e.getPath())) {
                timer.start();
                ParquetMetadata footer = ParquetFileReader.readFooter(conf, new Path(e.getPath()));
                long timeToRead = timer.elapsed(TimeUnit.MICROSECONDS);
                logger.trace("ParquetTrace,Read Footer,{},{},{},{},{},{},{}", "", e.getPath(), "", 0, 0, 0,
                        timeToRead);
                footers.put(e.getPath(), footer);
            }
            if (!context.getOptions().getOption(ExecConstants.PARQUET_NEW_RECORD_READER).bool_val
                    && !isComplex(footers.get(e.getPath()))) {
                readers.add(new ParquetRecordReader(context, e.getPath(), e.getRowGroupIndex(), fs,
                        CodecFactory.createDirectCodecFactory(fs.getConf(),
                                new ParquetDirectByteBufferAllocator(oContext.getAllocator()), 0),
                        footers.get(e.getPath()), rowGroupScan.getColumns()));
            } else {
                ParquetMetadata footer = footers.get(e.getPath());
                readers.add(new DrillParquetReader(context, footer, e, columnExplorer.getTableColumns(), fs));
            }

            Map<String, String> implicitValues = columnExplorer.populateImplicitColumns(e,
                    rowGroupScan.getSelectionRoot());
            implicitColumns.add(implicitValues);
            if (implicitValues.size() > mapWithMaxColumns.size()) {
                mapWithMaxColumns = implicitValues;
            }

        } catch (IOException e1) {
            throw new ExecutionSetupException(e1);
        }
    }

    // all readers should have the same number of implicit columns, add missing ones with value null
    Map<String, String> diff = Maps.transformValues(mapWithMaxColumns, Functions.constant((String) null));
    for (Map<String, String> map : implicitColumns) {
        map.putAll(Maps.difference(map, diff).entriesOnlyOnRight());
    }

    return new ScanBatch(rowGroupScan, context, oContext, readers.iterator(), implicitColumns);
}

From source file:org.apache.drill.exec.store.parquet.TestParquetReaderConfig.java

License:Apache License

@Test
public void testAddConfigToConf() {
    Configuration conf = new Configuration();
    conf.setBoolean(ParquetReaderConfig.ENABLE_BYTES_READ_COUNTER, true);
    conf.setBoolean(ParquetReaderConfig.ENABLE_BYTES_TOTAL_COUNTER, true);
    conf.setBoolean(ParquetReaderConfig.ENABLE_TIME_READ_COUNTER, true);

    ParquetReaderConfig readerConfig = ParquetReaderConfig.builder().withConf(conf).build();
    Configuration newConf = readerConfig.addCountersToConf(new Configuration());
    checkConfigValue(newConf, ParquetReaderConfig.ENABLE_BYTES_READ_COUNTER, "true");
    checkConfigValue(newConf, ParquetReaderConfig.ENABLE_BYTES_TOTAL_COUNTER, "true");
    checkConfigValue(newConf, ParquetReaderConfig.ENABLE_TIME_READ_COUNTER, "true");

    conf = new Configuration();
    conf.setBoolean(ParquetReaderConfig.ENABLE_BYTES_READ_COUNTER, false);
    conf.setBoolean(ParquetReaderConfig.ENABLE_BYTES_TOTAL_COUNTER, false);
    conf.setBoolean(ParquetReaderConfig.ENABLE_TIME_READ_COUNTER, false);

    readerConfig = ParquetReaderConfig.builder().withConf(conf).build();
    newConf = readerConfig.addCountersToConf(new Configuration());
    checkConfigValue(newConf, ParquetReaderConfig.ENABLE_BYTES_READ_COUNTER, "false");
    checkConfigValue(newConf, ParquetReaderConfig.ENABLE_BYTES_TOTAL_COUNTER, "false");
    checkConfigValue(newConf, ParquetReaderConfig.ENABLE_TIME_READ_COUNTER, "false");
}

From source file:org.apache.eagle.app.utils.HadoopSecurityUtil.java

License:Apache License

public static void login(Configuration kConfig) throws IOException {
    String keytab = kConfig.get(EAGLE_KEYTAB_FILE_KEY);
    String principal = kConfig.get(EAGLE_PRINCIPAL_KEY);
    if (keytab == null || principal == null || keytab.isEmpty() || principal.isEmpty()) {
        return;/*from ww  w .  java 2 s. c  om*/
    }

    kConfig.setBoolean("hadoop.security.authorization", true);
    kConfig.set("hadoop.security.authentication", "kerberos");
    UserGroupInformation.setConfiguration(kConfig);
    UserGroupInformation.loginUserFromKeytab(kConfig.get(EAGLE_PRINCIPAL_KEY),
            kConfig.get(EAGLE_KEYTAB_FILE_KEY));
}