Example usage for org.apache.hadoop.conf Configuration getBoolean

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getBoolean.

Prototype

public boolean getBoolean(String name, boolean defaultValue) 

Source Link

Document

Get the value of the name property as a boolean.

Usage

From source file:com.sensei.indexing.hadoop.reduce.ShardWriter.java

License:Apache License

private void setParameters(Configuration conf) {
    int maxFieldLength = conf.getInt(SenseiJobConfig.MAX_FIELD_LENGTH, -1);
    if (maxFieldLength > 0) {
        writer.setMaxFieldLength(maxFieldLength);
    }/*from w w w .j  a v a  2s . c  o m*/
    writer.setUseCompoundFile(conf.getBoolean(SenseiJobConfig.USE_COMPOUND_FILE, false));
    maxNumSegments = conf.getInt(SenseiJobConfig.MAX_NUM_SEGMENTS, -1);

    if (maxFieldLength > 0) {
        logger.info(SenseiJobConfig.MAX_FIELD_LENGTH + " = " + writer.getMaxFieldLength());
    }
    logger.info(SenseiJobConfig.USE_COMPOUND_FILE + " = " + writer.getUseCompoundFile());
    logger.info(SenseiJobConfig.MAX_NUM_SEGMENTS + " = " + maxNumSegments);
}

---

From source file:com.splicemachine.fs.s3.PrestoS3FileSystem.java

License:Apache License

@Override
public void initialize(URI uri, Configuration conf) throws IOException {
    requireNonNull(uri, "uri is null");
    requireNonNull(conf, "conf is null");
    super.initialize(uri, conf);
    setConf(conf);/*from w w  w .  j a v a  2  s  . com*/

    this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
    this.workingDirectory = new Path(PATH_SEPARATOR).makeQualified(this.uri, new Path(PATH_SEPARATOR));

    HiveS3Config defaults = new HiveS3Config();
    this.stagingDirectory = new File(
            conf.get(S3_STAGING_DIRECTORY, defaults.getS3StagingDirectory().toString()));
    this.maxAttempts = conf.getInt(S3_MAX_CLIENT_RETRIES, defaults.getS3MaxClientRetries()) + 1;
    this.maxBackoffTime = Duration
            .valueOf(conf.get(S3_MAX_BACKOFF_TIME, defaults.getS3MaxBackoffTime().toString()));
    this.maxRetryTime = Duration.valueOf(conf.get(S3_MAX_RETRY_TIME, defaults.getS3MaxRetryTime().toString()));
    int maxErrorRetries = conf.getInt(S3_MAX_ERROR_RETRIES, defaults.getS3MaxErrorRetries());
    boolean sslEnabled = conf.getBoolean(S3_SSL_ENABLED, defaults.isS3SslEnabled());
    Duration connectTimeout = Duration
            .valueOf(conf.get(S3_CONNECT_TIMEOUT, defaults.getS3ConnectTimeout().toString()));
    Duration socketTimeout = Duration
            .valueOf(conf.get(S3_SOCKET_TIMEOUT, defaults.getS3SocketTimeout().toString()));
    int maxConnections = conf.getInt(S3_MAX_CONNECTIONS, defaults.getS3MaxConnections());
    long minFileSize = conf.getLong(S3_MULTIPART_MIN_FILE_SIZE, defaults.getS3MultipartMinFileSize().toBytes());
    long minPartSize = conf.getLong(S3_MULTIPART_MIN_PART_SIZE, defaults.getS3MultipartMinPartSize().toBytes());
    this.useInstanceCredentials = conf.getBoolean(S3_USE_INSTANCE_CREDENTIALS,
            defaults.isS3UseInstanceCredentials());
    this.pinS3ClientToCurrentRegion = conf.getBoolean(S3_PIN_CLIENT_TO_CURRENT_REGION,
            defaults.isPinS3ClientToCurrentRegion());
    this.sseEnabled = conf.getBoolean(S3_SSE_ENABLED, defaults.isS3SseEnabled());
    this.sseType = PrestoS3SseType.valueOf(conf.get(S3_SSE_TYPE, defaults.getS3SseType().name()));
    this.sseKmsKeyId = conf.get(S3_SSE_KMS_KEY_ID, defaults.getS3SseKmsKeyId());
    String userAgentPrefix = conf.get(S3_USER_AGENT_PREFIX, defaults.getS3UserAgentPrefix());

    ClientConfiguration configuration = new ClientConfiguration().withMaxErrorRetry(maxErrorRetries)
            .withProtocol(sslEnabled ? Protocol.HTTPS : Protocol.HTTP)
            .withConnectionTimeout(toIntExact(connectTimeout.toMillis()))
            .withSocketTimeout(toIntExact(socketTimeout.toMillis())).withMaxConnections(maxConnections)
            .withUserAgentPrefix(userAgentPrefix).withUserAgentSuffix(S3_USER_AGENT_SUFFIX);

    this.s3 = createAmazonS3Client(uri, conf, configuration);

    transferConfig.setMultipartUploadThreshold(minFileSize);
    transferConfig.setMinimumUploadPartSize(minPartSize);
}

---

From source file:com.splicemachine.orc.OrcConf.java

License:Open Source License

public static boolean getBoolVar(Configuration conf, OrcConf.ConfVars var) {
    return conf.getBoolean(var.varname, var.defaultBoolVal);
}

---

From source file:com.splout.db.engine.MySQLOutputFormat.java

License:Apache License

/**
 * Loads global variable configuration//www.jav a 2  s  .  co  m
 */
protected void loadGlobalConf() {
    Configuration conf = getConf();
    if (conf.get(GLOBAL_AUTO_TRIM_STRING) != null) {
        globalAutoTrim = conf.getBoolean(GLOBAL_AUTO_TRIM_STRING, true);
    }
    if (conf.get(GLOBAL_STRING_FIELD_SIZE) != null) {
        globalStringFieldSize = conf.getInt(GLOBAL_STRING_FIELD_SIZE, 255);
    }
}

---

From source file:com.spotify.hdfs2cass.cassandra.utils.CassandraPartitioner.java

License:Open Source License

@Override
public void setConf(Configuration conf) {
    this.conf = conf;

    final String partitionerParam = conf.get(CassandraParams.SCRUB_CASSANDRACLUSTER_PARTITIONER_CONFIG);
    logger.info(CassandraParams.SCRUB_CASSANDRACLUSTER_PARTITIONER_CONFIG + ": " + partitionerParam);
    if (partitionerParam == null) {
        throw new RuntimeException("Didn't get any cassandra partitioner information");
    }/*from   w  w w .j  av  a  2s  .  co m*/

    try {
        partitioner = (IPartitioner) Class.forName(partitionerParam).newInstance();
    } catch (Exception ex) {
        throw new RuntimeException("Invalid partitioner class name: " + partitionerParam);
    }

    final String rangePerReducerStr = conf.get(CassandraParams.SCRUB_CASSANDRACLUSTER_RANGE_PER_REDUCER_CONFIG);
    if (rangePerReducerStr == null) {
        throw new RuntimeException("Didn't get cassandra range per reducer");
    }

    rangePerReducer = new BigInteger(rangePerReducerStr);

    final String reducersStr = conf.get(CassandraParams.SCRUB_CASSANDRACLUSTER_REDUCERS_CONFIG);
    if (reducersStr == null) {
        throw new RuntimeException("Failed to get list of reducers");
    }

    final String[] parts = StringUtils.splitByWholeSeparatorPreserveAllTokens(reducersStr, ",");
    if ((parts == null) || (parts.length == 0)) {
        throw new RuntimeException("Didn't get any valid list of reducers");
    }

    reducers = new ArrayList<>(parts.length);
    for (String part : parts) {
        reducers.add(Integer.parseInt(part));
    }

    distributeRandomly = conf.getBoolean(CassandraParams.SCRUB_CASSANDRACLUSTER_DISTRIBUTE_RANDOMLY_CONFIG,
            false);
    if (distributeRandomly) {
        random = new Random();
    }

    logger.info("CP: range per reducer: {}, reducers: {}, distribute randomly: {}",
            new Object[] { rangePerReducerStr, Arrays.toString(reducers.toArray()), distributeRandomly });
}

---

From source file:com.ssamples.hbase.stochasticbalancer.StochasticLoadBalancer.java

License:Apache License

@Override
public synchronized void setConf(Configuration conf) {
    super.setConf(conf);
    maxSteps = conf.getInt(MAX_STEPS_KEY, maxSteps);
    stepsPerRegion = conf.getInt(STEPS_PER_REGION_KEY, stepsPerRegion);
    maxRunningTime = conf.getLong(MAX_RUNNING_TIME_KEY, maxRunningTime);
    runMaxSteps = conf.getBoolean(RUN_MAX_STEPS_KEY, runMaxSteps);

    numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, numRegionLoadsToRemember);
    isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable);
    minCostNeedBalance = conf.getFloat(MIN_COST_NEED_BALANCE_KEY, minCostNeedBalance);
    if (localityCandidateGenerator == null) {
        localityCandidateGenerator = new LocalityBasedCandidateGenerator(services);
    }// www  . ja v a 2s.c  o m
    localityCost = new ServerLocalityCostFunction(conf, services);
    rackLocalityCost = new RackLocalityCostFunction(conf, services);

    if (this.candidateGenerators == null) {
        candidateGenerators = Lists.newArrayList();
        candidateGenerators.add(new RandomCandidateGenerator());
        candidateGenerators.add(new LoadCandidateGenerator());
        candidateGenerators.add(localityCandidateGenerator);
        candidateGenerators.add(new RegionReplicaRackCandidateGenerator());
    }
    regionLoadFunctions = new CostFromRegionLoadFunction[] { new ReadRequestCostFunction(conf),
            new CPRequestCostFunction(conf), new WriteRequestCostFunction(conf),
            new MemStoreSizeCostFunction(conf), new StoreFileCostFunction(conf) };
    regionReplicaHostCostFunction = new RegionReplicaHostCostFunction(conf);
    regionReplicaRackCostFunction = new RegionReplicaRackCostFunction(conf);
    costFunctions = new CostFunction[] { new RegionCountSkewCostFunction(conf),
            new PrimaryRegionCountSkewCostFunction(conf), new MoveCostFunction(conf), localityCost,
            rackLocalityCost, new TableSkewCostFunction(conf), regionReplicaHostCostFunction,
            regionReplicaRackCostFunction, regionLoadFunctions[0], regionLoadFunctions[1],
            regionLoadFunctions[2], regionLoadFunctions[3], regionLoadFunctions[4] };
    curFunctionCosts = new Double[costFunctions.length];
    tempFunctionCosts = new Double[costFunctions.length];
    LOG.info("Loaded config; maxSteps=" + maxSteps + ", stepsPerRegion=" + stepsPerRegion + ", maxRunningTime="
            + maxRunningTime + ", isByTable=" + isByTable + ", etc.");
}

---

From source file:com.ssamples.hbase.stochasticbalancer.StochasticLoadBalancerNew.java

License:Apache License

@Override
public synchronized void setConf(Configuration conf) {
    super.setConf(conf);
    maxSteps = conf.getInt(MAX_STEPS_KEY, maxSteps);
    stepsPerRegion = conf.getInt(STEPS_PER_REGION_KEY, stepsPerRegion);
    maxRunningTime = conf.getLong(MAX_RUNNING_TIME_KEY, maxRunningTime);
    runMaxSteps = conf.getBoolean(RUN_MAX_STEPS_KEY, runMaxSteps);

    numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, numRegionLoadsToRemember);
    isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable);
    minCostNeedBalance = conf.getFloat(MIN_COST_NEED_BALANCE_KEY, minCostNeedBalance);
    if (localityCandidateGenerator == null) {
        localityCandidateGenerator = new LocalityBasedCandidateGenerator(services);
    }//  w ww.  j ava2  s  .c o  m
    localityCost = new ServerLocalityCostFunction(conf, services);
    rackLocalityCost = new RackLocalityCostFunction(conf, services);

    if (this.candidateGenerators == null) {
        candidateGenerators = Lists.newArrayList();
        candidateGenerators.add(new RandomCandidateGenerator());
        candidateGenerators.add(new LoadCandidateGenerator());
        candidateGenerators.add(localityCandidateGenerator);
        candidateGenerators.add(new RegionReplicaRackCandidateGenerator());
    }
    regionLoadFunctions = new CostFromRegionLoadFunction[] { new ReadRequestCostFunction(conf),
            new CPRequestCostFunction(conf), new WriteRequestCostFunction(conf),
            new MemStoreSizeCostFunction(conf), new StoreFileCostFunction(conf) };
    regionReplicaHostCostFunction = new RegionReplicaHostCostFunction(conf);
    regionReplicaRackCostFunction = new RegionReplicaRackCostFunction(conf);
    costFunctions = new CostFunction[] {
            //new RegionCountSkewCostFunction(conf),
            new PrimaryRegionCountSkewCostFunction(conf), new MoveCostFunction(conf), localityCost,
            rackLocalityCost,
            //new TableSkewCostFunction(conf),
            new TableSkewCostFunctionNew(conf),
            //new TableRegionSkewCostFunction(conf),
            new ServerResourceCostFunction(conf), regionReplicaHostCostFunction, regionReplicaRackCostFunction,
            regionLoadFunctions[0], regionLoadFunctions[1], regionLoadFunctions[2], regionLoadFunctions[3],
            regionLoadFunctions[4] };
    curFunctionCosts = new Double[costFunctions.length];
    tempFunctionCosts = new Double[costFunctions.length];
    LOG.info("Loaded config; maxSteps=" + maxSteps + ", stepsPerRegion=" + stepsPerRegion + ", maxRunningTime="
            + maxRunningTime + ", isByTable=" + isByTable + ", etc.");
}

---

From source file:com.streamsets.pipeline.stage.destination.mapreduce.jobtype.avroconvert.AvroConversionBaseMapper.java

License:Apache License

@Override
protected void map(String input, String output, Context context) throws IOException, InterruptedException {
    FileSystem fs = FileSystem.get(context.getConfiguration());
    Configuration conf = context.getConfiguration();

    LOG.info("Converting input file: {}", input);
    LOG.info("Output directory: {}", output);
    Path inputPath = new Path(input);
    Path outputDir = new Path(output);
    fs.mkdirs(outputDir);/*  w ww . ja  va2  s  . c o  m*/

    Path tempFile = new Path(outputDir, getTempFilePrefix() + inputPath.getName());
    if (fs.exists(tempFile)) {
        if (conf.getBoolean(AvroConversionCommonConstants.OVERWRITE_TMP_FILE, false)) {
            fs.delete(tempFile, true);
        } else {
            throw new IOException("Temporary file " + tempFile + " already exists.");
        }
    }
    LOG.info("Using temp file: {}", tempFile);

    // Output file is the same as input except of dropping .avro extension if it exists and appending .parquet or .orc
    String outputFileName = inputPath.getName().replaceAll("\\.avro$", "") + getOutputFileSuffix();
    Path finalFile = new Path(outputDir, outputFileName);
    LOG.info("Final path will be: {}", finalFile);

    // Avro reader
    SeekableInput seekableInput = new FsInput(inputPath, conf);
    DatumReader<GenericRecord> reader = new GenericDatumReader<>();
    FileReader<GenericRecord> fileReader = DataFileReader.openReader(seekableInput, reader);
    Schema avroSchema = fileReader.getSchema();

    initializeWriter(tempFile, avroSchema, conf, context);

    LOG.info("Started reading input file");
    long recordCount = 0;
    try {
        while (fileReader.hasNext()) {
            GenericRecord record = fileReader.next();
            handleAvroRecord(record);

            context.getCounter(Counters.PROCESSED_RECORDS).increment(1);
            recordCount++;
        }
    } catch (Exception e) {
        // Various random stuff can happen while converting, so we wrap the underlying exception with more details
        String message = String.format("Exception at offset %d (record %d): %s", fileReader.tell(), recordCount,
                e.toString());
        throw new IOException(message, e);
    }
    LOG.info("Done reading input file");
    closeWriter();

    LOG.info("Moving temporary file {} to final destination {}", tempFile, finalFile);
    fs.rename(tempFile, finalFile);

    if (!context.getConfiguration().getBoolean(AvroConversionCommonConstants.KEEP_INPUT_FILE, false)) {
        LOG.info("Removing input file", inputPath);
        fs.delete(inputPath, true);
    }

    LOG.info("Done converting input file into output directory {}", output);
}

---

From source file:com.streamsets.pipeline.stage.destination.SimpleTestInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext jobContext) throws IOException, InterruptedException {
    Configuration conf = jobContext.getConfiguration();

    if (conf.getBoolean(THROW_EXCEPTION, false)) {
        throw new IOException("Throwing exception as instructed, failure in bootstraping MR job.");
    }//  www  .  j av  a  2s.co m

    String fileLocation = conf.get(FILE_LOCATION);
    if (fileLocation != null) {
        FileUtils.writeStringToFile(new File(fileLocation), conf.get(FILE_VALUE));
    }

    return Collections.emptyList();
}

---

From source file:com.talis.hadoop.rdf.collation.QuadsCollater.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    Configuration configuration = getConf();

    boolean useCompression = configuration.getBoolean(Constants.OPTION_USE_COMPRESSION,
            Constants.OPTION_USE_COMPRESSION_DEFAULT);
    if (useCompression) {
        configuration.setBoolean("mapred.compress.map.output", true);
        configuration.set("mapred.output.compression.type", "BLOCK");
        configuration.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec");
    }//from w ww. j a  va 2 s  .c  o m

    boolean overrideOutput = configuration.getBoolean(Constants.OPTION_OVERRIDE_OUTPUT,
            Constants.OPTION_OVERRIDE_OUTPUT_DEFAULT);
    FileSystem fs = FileSystem.get(new Path(args[1]).toUri(), configuration);
    if (overrideOutput) {
        fs.delete(new Path(args[1]), true);
    }

    Job job = new Job(configuration);
    job.setJobName(JOB_NAME);
    job.setJarByClass(getClass());

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    FileOutputFormat.setCompressOutput(job, true);

    job.setInputFormatClass(NQuadsInputFormat.class);
    job.setMapperClass(CollationMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(QuadWritable.class);

    job.setReducerClass(CollationReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(QuadArrayWritable.class);

    job.setOutputFormatClass(SequenceFileOutputFormat.class);

    if (LOG.isDebugEnabled())
        Utils.log(job, LOG);

    return job.waitForCompletion(true) ? 0 : 1;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »