Example usage for org.apache.hadoop.conf Configuration getBoolean

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getBoolean.

Prototype

public boolean getBoolean(String name, boolean defaultValue) 

Source Link

Document

Get the value of the name property as a boolean.

Usage

From source file:com.datatorrent.contrib.machinedata.Application.java

License:Open Source License

/**
 * Create the DAG//from  www.j  a va  2s . c  o  m
 */
@Override
public void populateDAG(DAG dag, Configuration conf) {

    int partitions = conf.getInt(Application.class.getName() + ".partitions", 1);
    int unifier_count = conf.getInt(Application.class.getName() + ".unifier_count", 2);

    dag.setAttribute(DAG.APPLICATION_NAME, "MachineDataApplication");
    dag.setAttribute(DAG.DEBUG, false);
    dag.setAttribute(DAG.STREAMING_WINDOW_SIZE_MILLIS, streamingWindowSizeMilliSeconds);

    InputReceiver randomGen = getRandomInformationTupleGenerator("InputReceiver", dag);
    setDefaultOutputPortQueueCapacity(dag, randomGen.outputInline);
    dag.setAttribute(randomGen, OperatorContext.INITIAL_PARTITION_COUNT, partitions);

    DimensionGenerator dimensionGenerator = dag.addOperator("GenerateDimensions", DimensionGenerator.class);
    dag.setAttribute(dimensionGenerator, Context.OperatorContext.APPLICATION_WINDOW_COUNT,
            appWindowCountMinute);
    setDefaultOutputPortQueueCapacity(dag, dimensionGenerator.outputInline);
    setDefaultOutputPortQueueCapacity(dag, dimensionGenerator.output);
    setDefaultInputPortQueueCapacity(dag, dimensionGenerator.inputPort);
    dag.addStream("generate_dimensions", randomGen.outputInline, dimensionGenerator.inputPort)
            .setLocality(Locality.CONTAINER_LOCAL);
    dag.setInputPortAttribute(dimensionGenerator.inputPort, PortContext.PARTITION_PARALLEL, true);

    if (conf.getBoolean("machinedata.calculate.average", true)) {
        MachineInfoAveragingPrerequisitesOperator prereqAverageOper = addAverageCalculation(dag, conf);
        dag.addStream("prereq_calculation", dimensionGenerator.outputInline, prereqAverageOper.inputPort)
                .setLocality(Locality.THREAD_LOCAL);
        dag.setOutputPortAttribute(prereqAverageOper.outputPort, PortContext.UNIFIER_LIMIT, unifier_count);
    }

    /*
    CalculatorOperator calculatorOperator = addCalculator(dag, conf);
    dag.addStream("dimension_generator_to_calculator", dimensionGenerator.output, calculatorOperator.dataPort);
            
    if (conf.getBoolean("machinedata.calculate.percentile", false)) {
      calculatorOperator.setComputePercentile(true);
    }
            
    if (conf.getBoolean("machinedata.calculate.sd", false)) {
      calculatorOperator.setComputeSD(true);
    }
            
            
    if (conf.getBoolean("machinedata.calculate.max", false)) {
      calculatorOperator.setComputeMax(true);
    }
    */

}

---

From source file:com.datatorrent.demos.ads.Application.java

License:Open Source License

private void configure(DAG dag, Configuration conf) {

    if (StreamingApplication.Environment.CLUSTER == conf.getEnum(StreamingApplication.ENVIRONMENT,
            StreamingApplication.Environment.LOCAL)) {
        setLocalMode();/*from ww w . jav a2  s.  c o m*/
        // settings only affect distributed mode
        AttributeMap attributes = dag.getAttributes();
        if (attributes.get(DAGContext.CONTAINER_MEMORY_MB) == null) {
            attributes.put(DAGContext.CONTAINER_MEMORY_MB, 2048);
        }
        if (attributes.get(DAGContext.MASTER_MEMORY_MB) == null) {
            attributes.put(DAGContext.MASTER_MEMORY_MB, 1024);
        }
        if (attributes.get(DAGContext.CONTAINERS_MAX_COUNT) == null) {
            attributes.put(DAGContext.CONTAINERS_MAX_COUNT, 1);
        }
    } else if (StreamingApplication.Environment.LOCAL == conf.getEnum(StreamingApplication.ENVIRONMENT,
            StreamingApplication.Environment.CLUSTER)) {
        setLocalMode();
    }

    this.generatorVTuplesBlast = conf.getInt(P_generatorVTuplesBlast, this.generatorVTuplesBlast);
    this.generatorMaxWindowsCount = conf.getInt(P_generatorMaxWindowsCount, this.generatorMaxWindowsCount);
    this.locality = conf.getBoolean(P_allInline, false) ? Locality.CONTAINER_LOCAL : null;
    this.numGenerators = conf.getInt(P_numGenerators, this.numGenerators);

}

---

From source file:com.datatorrent.demos.ads.Application.java

License:Open Source License

@Override
public void populateDAG(DAG dag, Configuration conf) {
    configure(dag, conf);/*from   w w  w.j a  va 2  s  .c o m*/
    //dag.setAttribute(DAG.APPLICATION_NAME, "AdsApplication");
    dag.setAttribute(DAG.STREAMING_WINDOW_SIZE_MILLIS, WINDOW_SIZE_MILLIS); // set the streaming window size to 1 millisec

    //dag.getAttributes().attr(DAG.CONTAINERS_MAX_COUNT).setIfAbsent(9);
    EventGenerator viewGen = getPageViewGenOperator("viewGen", dag);
    dag.getMeta(viewGen).getAttributes().put(OperatorContext.INITIAL_PARTITION_COUNT, numGenerators);
    dag.setOutputPortAttribute(viewGen.hash_data, PortContext.QUEUE_CAPACITY, 32 * 1024);

    EventClassifier adviews = getAdViewsStampOperator("adviews", dag);
    dag.setOutputPortAttribute(adviews.data, PortContext.QUEUE_CAPACITY, 32 * 1024);
    dag.setInputPortAttribute(adviews.event, PortContext.QUEUE_CAPACITY, 32 * 1024);

    FilteredEventClassifier<Double> insertclicks = getInsertClicksOperator("insertclicks", dag);
    dag.setInputPortAttribute(insertclicks.data, PortContext.QUEUE_CAPACITY, 32 * 1024);

    SumCountMap<String, Double> viewAggregate = getSumOperator("viewAggr", dag);
    dag.setAttribute(viewAggregate, OperatorContext.APPLICATION_WINDOW_COUNT, applicationWindow);
    dag.setInputPortAttribute(viewAggregate.data, PortContext.QUEUE_CAPACITY, 32 * 1024);

    SumCountMap<String, Double> clickAggregate = getSumOperator("clickAggr", dag);
    dag.setAttribute(clickAggregate, OperatorContext.APPLICATION_WINDOW_COUNT, applicationWindow);

    dag.setInputPortAttribute(adviews.event, PortContext.PARTITION_PARALLEL, true);
    dag.addStream("views", viewGen.hash_data, adviews.event).setLocality(Locality.CONTAINER_LOCAL);
    dag.setInputPortAttribute(insertclicks.data, PortContext.PARTITION_PARALLEL, true);
    dag.setInputPortAttribute(viewAggregate.data, PortContext.PARTITION_PARALLEL, true);
    DAG.StreamMeta viewsAggStream = dag
            .addStream("viewsaggregate", adviews.data, insertclicks.data, viewAggregate.data)
            .setLocality(Locality.CONTAINER_LOCAL);

    if (conf.getBoolean(P_enableHdfs, false)) {
        HdfsHashMapOutputOperator viewsToHdfs = dag.addOperator("viewsToHdfs", new HdfsHashMapOutputOperator());
        viewsToHdfs.setAppend(false);
        viewsToHdfs.setCloseCurrentFile(true);
        viewsToHdfs.setFilePath("file:///tmp/adsdemo/views-%(operatorId)-part%(partIndex)");
        dag.setInputPortAttribute(viewsToHdfs.input, PortContext.PARTITION_PARALLEL, true);
        viewsAggStream.addSink(viewsToHdfs.input);
    }

    dag.setInputPortAttribute(clickAggregate.data, PortContext.PARTITION_PARALLEL, true);
    dag.addStream("clicksaggregate", insertclicks.filter, clickAggregate.data)
            .setLocality(Locality.CONTAINER_LOCAL);

    QuotientMap<String, Integer> ctr = getQuotientOperator("ctr", dag);
    SumCountMap<String, Double> cost = getSumOperator("cost", dag);
    SumCountMap<String, Double> revenue = getSumOperator("rev", dag);
    MarginMap<String, Double> margin = getMarginOperator("margin", dag);
    StreamMerger<HashMap<String, Integer>> merge = getStreamMerger("countmerge", dag);
    ThroughputCounter<String, Integer> tuple_counter = getThroughputCounter("tuple_counter", dag);

    dag.addStream("adviewsdata", viewAggregate.sum, cost.data);
    dag.addStream("clicksdata", clickAggregate.sum, revenue.data);
    dag.addStream("viewtuplecount", viewAggregate.count, ctr.denominator, merge.data1).setLocality(locality);
    dag.addStream("clicktuplecount", clickAggregate.count, ctr.numerator, merge.data2).setLocality(locality);
    dag.addStream("total count", merge.out, tuple_counter.data).setLocality(locality);

    InputPort<Object> revconsole = getConsolePort(dag, "revConsole", false);
    InputPort<Object> costconsole = getConsolePort(dag, "costConsole", false);
    InputPort<Object> marginconsole = getConsolePort(dag, "marginConsole", false);
    InputPort<Object> ctrconsole = getConsolePort(dag, "ctrConsole", false);
    InputPort<Object> viewcountconsole = getConsolePort(dag, "viewCountConsole", false);

    dag.addStream("revenuedata", revenue.sum, margin.denominator, revconsole).setLocality(locality);
    dag.addStream("costdata", cost.sum, margin.numerator, costconsole).setLocality(locality);
    dag.addStream("margindata", margin.margin, marginconsole).setLocality(locality);
    dag.addStream("ctrdata", ctr.quotient, ctrconsole).setLocality(locality);
    dag.addStream("tuplecount", tuple_counter.count, viewcountconsole).setLocality(locality);

}

---

From source file:com.datatorrent.demos.dimensions.ads.ApplicationWithHDHT.java

License:Open Source License

@Override
public void populateDAG(DAG dag, Configuration conf) {
    InputItemGenerator input = dag.addOperator("InputGenerator", InputItemGenerator.class);
    DimensionsComputation<AdInfo, AdInfo.AdInfoAggregateEvent> dimensions = dag.addOperator(
            "DimensionsComputation", new DimensionsComputation<AdInfo, AdInfo.AdInfoAggregateEvent>());
    dag.getMeta(dimensions).getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 4);
    String[] dimensionSpecs = new String[] { "time=" + TimeUnit.MINUTES, "time=" + TimeUnit.MINUTES + ":adUnit",
            "time=" + TimeUnit.MINUTES + ":advertiserId", "time=" + TimeUnit.MINUTES + ":publisherId",
            "time=" + TimeUnit.MINUTES + ":advertiserId:adUnit",
            "time=" + TimeUnit.MINUTES + ":publisherId:adUnit",
            "time=" + TimeUnit.MINUTES + ":publisherId:advertiserId",
            "time=" + TimeUnit.MINUTES + ":publisherId:advertiserId:adUnit" };

    AdInfoAggregator[] aggregators = new AdInfoAggregator[dimensionSpecs.length];
    for (int i = dimensionSpecs.length; i-- > 0;) {
        AdInfoAggregator aggregator = new AdInfoAggregator();
        aggregator.init(dimensionSpecs[i]);
        aggregators[i] = aggregator;//  w  w w  .  j  a v a  2s  .  c o m
    }
    dimensions.setAggregators(aggregators);

    AdsDimensionStoreOperator store = dag.addOperator("Store", AdsDimensionStoreOperator.class);
    TFileImpl hdsFile = new TFileImpl.DefaultTFileImpl();
    store.setFileStore(hdsFile);
    store.setAggregator(new AdInfoAggregator());
    dag.setAttribute(store, Context.OperatorContext.COUNTERS_AGGREGATOR,
            new BasicCounters.LongAggregator<MutableLong>());

    Operator.OutputPort<String> queryPort;
    Operator.InputPort<Object> queryResultPort;
    if (conf.getBoolean(PROP_USE_WEBSOCKETS, false)) {
        String gatewayAddress = dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS);
        URI uri = URI.create("ws://" + gatewayAddress + "/pubsub");
        //LOG.info("WebSocket with gateway at: {}", gatewayAddress);
        PubSubWebSocketInputOperator<String> wsIn = dag.addOperator("Query",
                new PubSubWebSocketInputOperator<String>());
        wsIn.setUri(uri);
        queryPort = wsIn.outputPort;
        PubSubWebSocketOutputOperator<Object> wsOut = dag.addOperator("QueryResult",
                new PubSubWebSocketOutputOperator<Object>());
        wsOut.setUri(uri);
        queryResultPort = wsOut.input;
    } else {
        KafkaSinglePortStringInputOperator queries = dag.addOperator("Query",
                new KafkaSinglePortStringInputOperator());
        queries.setConsumer(new SimpleKafkaConsumer());
        queryPort = queries.outputPort;
        KafkaSinglePortOutputOperator<Object, Object> queryResult = dag.addOperator("QueryResult",
                new KafkaSinglePortOutputOperator<Object, Object>());
        queryResult.getConfigProperties().put("serializer.class", KafkaJsonEncoder.class.getName());
        queryResultPort = queryResult.inputPort;
    }

    dag.addStream("InputStream", input.outputPort, dimensions.data).setLocality(Locality.CONTAINER_LOCAL);
    dag.addStream("DimensionalData", dimensions.output, store.input);
    dag.addStream("Query", queryPort, store.query);
    dag.addStream("QueryResult", store.queryResult, queryResultPort);
}

---

From source file:com.datatorrent.stram.StreamingContainerParent.java

License:Apache License

protected void startRpcServer() {
    Configuration conf = getConfig();
    LOG.info("Config: " + conf);
    LOG.info("Listener thread count " + listenerThreadCount);
    try {//w w  w .j a  va 2s  .  c om
        server = new RPC.Builder(conf).setProtocol(StreamingContainerUmbilicalProtocol.class).setInstance(this)
                .setBindAddress("0.0.0.0").setPort(0).setNumHandlers(listenerThreadCount)
                .setSecretManager(tokenSecretManager).setVerbose(false).build();

        // Enable service authorization?
        if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
            //refreshServiceAcls(conf, new MRAMPolicyProvider());
            server.refreshServiceAcl(conf, new PolicyProvider() {

                @Override
                public Service[] getServices() {
                    return (new Service[] { new Service(StreamingContainerUmbilicalProtocol.class.getName(),
                            StreamingContainerUmbilicalProtocol.class) });
                }

            });
        }

        server.start();
        this.address = NetUtils.getConnectAddress(server);
        LOG.info("Container callback server listening at " + this.address);
    } catch (IOException e) {
        throw new YarnRuntimeException(e);
    }
}

---

From source file:com.elex.dmp.vectorizer.TFPartialVectorReducer.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    Configuration conf = context.getConfiguration();
    URI[] localFiles = DistributedCache.getCacheFiles(conf);
    Preconditions.checkArgument(localFiles != null && localFiles.length >= 1,
            "missing paths from the DistributedCache");

    dimension = conf.getInt(PartialVectorMerger.DIMENSION, Integer.MAX_VALUE);
    sequentialAccess = conf.getBoolean(PartialVectorMerger.SEQUENTIAL_ACCESS, false);
    namedVector = conf.getBoolean(PartialVectorMerger.NAMED_VECTOR, false);
    maxNGramSize = conf.getInt(DictionaryVectorizer.MAX_NGRAMS, maxNGramSize);

    Path dictionaryFile = new Path(localFiles[0].getPath());
    // key is word value is id
    for (Pair<Writable, IntWritable> record : new SequenceFileIterable<Writable, IntWritable>(dictionaryFile,
            true, conf)) {//from w  ww.jav a 2  s .c  o  m
        dictionary.put(record.getFirst().toString(), record.getSecond().get());
    }
}

---

From source file:com.emc.greenplum.gpdb.hdfsconnector.ConnectorUtil.java

License:Open Source License

/**
 * Helper routine to login to secure Hadoop. If it's not configured to use
 * security (in the core-site.xml) then return
 *
 * Create a LoginContext using config in $GPHOME/lib/hadoop/jaas.conf and search for a valid TGT
 * which matches HADOOP_SECURITY_USERNAME.
 * Check if the TGT needs to be renewed or recreated and use installed kinit command to handle the
 * credential cache//from   w w  w .  ja  v  a2 s.c o  m
 *
 * @param conf the configuration
 */
protected static void loginSecureHadoop(Configuration conf) throws IOException, InterruptedException {
    // if security config does not exist then assume no security
    if (conf.get(HADOOP_SECURITY_USERNAME) == null || conf.get(HADOOP_SECURITY_USER_KEYTAB_FILE) == null) {
        return;
    }

    String principal = SecurityUtil.getServerPrincipal(conf.get(HADOOP_SECURITY_USERNAME),
            InetAddress.getLocalHost().getCanonicalHostName());
    String jaasConf = System.getenv("GPHOME") + "/lib/hadoop/jaas.conf";
    System.setProperty("java.security.auth.login.config", jaasConf);
    Boolean kinitDisabled = conf.getBoolean(HADOOP_DISABLE_KINIT, false);

    /*
       Attempt to find the TGT from the users ticket cache and check if its a valid TGT
       If the TGT needs to be renewed or recreated then we use kinit binary command so the cache can be persisted
       allowing future queries to reuse cached TGT's
            
       If user disables kinit then we fail back SecurityUtil.login which will always perform a AS_REQ
       followed by a TGS_REQ to the KDC and set the global login context.  the problem with this method is if you have 300
       or more GPDB segments then every gphdfs query will issue 300 AS_REQ to the KDC and may result in intermittent failures
       or longer running queries if the KDC can not keep up with the demand
    */
    try {
        LoginContext login = new LoginContext("gphdfs");
        login.login();
        Subject subject = login.getSubject();
        Set<KerberosTicket> tickets = subject.getPrivateCredentials(KerberosTicket.class);

        // find the TGT that matches the configured principal
        for (KerberosTicket ticket : tickets) {
            if (ticket.getClient().toString().equals(principal)) {
                long start = ticket.getStartTime().getTime();
                long end = ticket.getEndTime().getTime();
                long current = System.currentTimeMillis();
                Long rtime = start + (long) ((end - start) * .8); // compute start time of ticket plus 80% to find the refresh window

                if (current <= rtime && ticket.isCurrent()) { // Ticket is current so no further action required
                    return;
                } else if (current >= rtime && ticket.isRenewable() && !kinitDisabled) { // Ticket needs to be renewed and is renewable
                    String[] kinitRefresh = { "kinit", "-R" };
                    Process kinitRenew = Runtime.getRuntime().exec(kinitRefresh);
                    int rt = kinitRenew.waitFor();
                    if (rt == 0) {
                        return;
                    }

                }
                break;
            }
        }
    } catch (LoginException | InterruptedException e) {
        if (kinitDisabled) {
            SecurityUtil.login(conf, HADOOP_SECURITY_USER_KEYTAB_FILE, HADOOP_SECURITY_USERNAME);
            return;
        }
        /* if kinit is not disabled then do nothing because we will request a new TGT and update the ticket cache
        * regardless if login or kinit refresh failed initially
        */
    }

    // fail back to securityutil if kinit is disabled
    if (kinitDisabled) { // login from keytab
        SecurityUtil.login(conf, HADOOP_SECURITY_USER_KEYTAB_FILE, HADOOP_SECURITY_USERNAME);
        return;
    }

    // if we made it here then there is not a current TGT found in cache that matches the principal and we need to request a new TGT
    String[] kinitCmd = { "kinit", "-kt", conf.get(HADOOP_SECURITY_USER_KEYTAB_FILE), principal };
    try {
        Process kinit = Runtime.getRuntime().exec(kinitCmd);
        int rt = kinit.waitFor();
        if (rt != 0) {
            BufferedReader errOut = new BufferedReader(new InputStreamReader(kinit.getErrorStream()));
            String line;
            String errOutput = "";
            while ((line = errOut.readLine()) != null) {
                errOutput += line;
            }
            throw new IOException(String.format(
                    "Failed to Acquire TGT using command \"kinit -kt\" with configured keytab and principal settings:\n%s",
                    errOutput));
        }
    } catch (InterruptedException e) {
        throw new InterruptedException(String.format(
                "command \"kinit -kt\" with configured keytab and principal settings:\n%s", e.getMessage()));
    }
}

---

From source file:com.facebook.hive.orc.OrcConf.java

License:Open Source License

public static boolean getBoolVar(Configuration conf, ConfVars var) {
    return conf.getBoolean(var.varname, var.defaultBoolVal);
}

---

From source file:com.facebook.hiveio.common.HadoopUtils.java

License:Apache License

/**
 * Check if output committer needs success marker
 * @param conf Configuration to use/*from  ww w .j av  a2  s . co m*/
 * @return true if success marker required
 */
public static boolean needSuccessMarker(Configuration conf) {
    return conf.getBoolean("mapreduce.fileoutputcommitter.marksuccessfuljobs", false);
}

---

From source file:com.facebook.hiveio.conf.BooleanConfOption.java

License:Apache License

/**
 * Lookup value in Configuration//  w  w w. j av  a2s  . c om
 * @param conf Configuration
 * @return value for key in conf, or defaultValue if not present
 */
public boolean get(Configuration conf) {
    return conf.getBoolean(getKey(), defaultValue);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »