Example usage for org.apache.hadoop.conf Configuration getInt

List of usage examples for org.apache.hadoop.conf Configuration getInt

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. getInt

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getInt.

Prototype

public int getInt(String name, int defaultValue)

Source Link

Document

Get the value of the name property as an int.

Usage

From source file:com.datatorrent.contrib.frauddetect.Application.java

License:Open Source License

public AverageAlertingOperator getAverageAlertingOperator(String name, DAG dag, Configuration conf) {
    AverageAlertingOperator oper = dag.addOperator(name, AverageAlertingOperator.class);
    oper.setThreshold(conf.getInt(AVG_THRESHOLD_PROPERTY, 1200));
    dag.setAttribute(oper, OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCount);
    return oper;// w  w w  .j  a  v a  2s . c o m
}

From source file:com.datatorrent.contrib.frauddetect.Application.java

License:Open Source License

public CreditCardAmountSamplerOperator getTransactionAmountSamplerOperator(String name, DAG dag,
        Configuration conf) {
    CreditCardAmountSamplerOperator oper = dag.addOperator(name, CreditCardAmountSamplerOperator.class);
    oper.setThreshold(conf.getInt(CC_THRESHOLD_PROPERTY, 420));
    dag.setAttribute(oper, OperatorContext.APPLICATION_WINDOW_COUNT, amountSamplerWindowCount);
    return oper;/*w  w  w. ja  v a2  s  . c  o m*/
}

From source file:com.datatorrent.contrib.machinedata.Application.java

License:Open Source License

/**
 * This method returns new RedisOutputOperator Operator
 * @param name the name of the operator in DAG
 * @param dag the DAG instance/* ww  w .  j  a  v a  2 s  . c  o  m*/
 * @param conf the configuration object
 * @param database the database instance id
 * @return RedisOutputOperator
 */
private RedisKeyValPairOutputOperator<MachineKey, Map<String, String>> getRedisOutputOperator(String name,
        DAG dag, Configuration conf, int database) {
    RedisKeyValPairOutputOperator<MachineKey, Map<String, String>> oper = dag.addOperator(name,
            new RedisKeyValPairOutputOperator<MachineKey, Map<String, String>>());
    String host = conf.get("machinedata.redis.host", "localhost");
    int port = conf.getInt("machinedata.redis.port", 6379);
    RedisStore store = oper.getStore();
    store.setHost(host);
    store.setPort(port);
    store.setDbIndex(database);
    dag.setAttribute(oper, Context.OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCountMinute);
    return oper;
}

From source file:com.datatorrent.contrib.machinedata.Application.java

License:Open Source License

/**
 * This method returns new SmtpOutputOperator Operator
 * @param name the name of the operator in DAG
 * @param dag the DAG instance/*from   w w w .j a v a  2  s .c  o  m*/
 * @param conf the configuration object
 * @return SmtpOutputOperator
 */
private SmtpOutputOperator getSmtpOutputOperator(String name, DAG dag, Configuration conf) {
    SmtpOutputOperator mailOper = new SmtpOutputOperator();

    String from = conf.get("machinedata.smtp.from", "admin@datatorrent.com");
    String recipient = conf.get("machinedata.smtp.recipient", "atul@datatorrent.com");
    String subject = conf.get("machinedata.smtp.subject", "Alert!!!");
    String content = conf.get("machinedata.smtp.content", "{}");
    String host = conf.get("machinedata.smtp.host", "localhost");
    int port = conf.getInt("machinedata.smtp.port", 25);
    boolean useSsl = conf.getBoolean("machinedata.smtp.ssl", false);

    mailOper.setFrom(from);
    mailOper.addRecipient(SmtpOutputOperator.RecipientType.TO, recipient);
    mailOper.setSubject(subject);
    mailOper.setContent(content);
    mailOper.setSmtpHost(host);
    mailOper.setSmtpPort(port);
    // mailOper.setSmtpUserName(userName);
    // mailOper.setSmtpPassword(password);
    mailOper.setUseSsl(useSsl);

    dag.addOperator(name, mailOper);

    return mailOper;

}

From source file:com.datatorrent.contrib.machinedata.Application.java

License:Open Source License

/**
 * This function sets up the DAG for calculating the average
 * @param dag the DAG instance/*from w w w.  jav a  2  s.c o  m*/
 * @param conf the configuration instance
 * @return MachineInfoAveragingPrerequisitesOperator
 */
private MachineInfoAveragingPrerequisitesOperator addAverageCalculation(DAG dag, Configuration conf) {
    MachineInfoAveragingPrerequisitesOperator prereqAverageOper = getMachineInfoAveragingPrerequisitesOperator(
            "PrereqAverage", dag);
    dag.setInputPortAttribute(prereqAverageOper.inputPort, PortContext.PARTITION_PARALLEL, true);
    dag.setAttribute(prereqAverageOper, Context.OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCountMinute);

    setDefaultInputPortQueueCapacity(dag, prereqAverageOper.inputPort);
    setDefaultOutputPortQueueCapacity(dag, prereqAverageOper.outputPort);

    int partitions = conf.getInt(Application.class.getName() + ".averagePartitions", 2);
    MachineInfoAveragingOperator averageOperator = getMachineInfoAveragingOperator("Average", dag);
    dag.setAttribute(averageOperator, OperatorContext.INITIAL_PARTITION_COUNT, partitions);
    setDefaultInputPortQueueCapacity(dag, averageOperator.inputPort);
    setDefaultOutputPortQueueCapacity(dag, averageOperator.outputPort);

    RedisKeyValPairOutputOperator<MachineKey, Map<String, String>> redisAvgOperator = getRedisOutputOperator(
            "RedisAverageOutput", dag, conf, conf.getInt("machinedata.redis.db", 5));
    setDefaultInputPortQueueCapacity(dag, redisAvgOperator.input);
    dag.setInputPortAttribute(redisAvgOperator.input, PortContext.PARTITION_PARALLEL, true);
    dag.addStream("avg_output", averageOperator.outputPort, redisAvgOperator.input);

    SmtpOutputOperator smtpOutputOperator = getSmtpOutputOperator("SmtpAvgOperator", dag, conf);

    dag.addStream("inter_avg", prereqAverageOper.outputPort, averageOperator.inputPort);
    dag.addStream("avg_alert_mail", averageOperator.smtpAlert, smtpOutputOperator.input);

    return prereqAverageOper;

}

From source file:com.datatorrent.contrib.machinedata.Application.java

License:Open Source License

/**
 * This function sets up the DAG for calculating the percentile/sd/max/min
 * @param dag the DAG instance// w  ww . j a va  2s  .c o m
 * @param conf the configuration instance
 * @return CalculatorOperator
 */
private CalculatorOperator addCalculator(DAG dag, Configuration conf) {
    CalculatorOperator oper = dag.addOperator("Calculator", CalculatorOperator.class);
    dag.setAttribute(oper, Context.OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCountMinute);
    int partitions = conf.getInt(Application.class.getName() + ".calculatorPartitions", 5);
    dag.setAttribute(oper, OperatorContext.INITIAL_PARTITION_COUNT, partitions);

    setDefaultInputPortQueueCapacity(dag, oper.dataPort);

    // Percentile
    setDefaultOutputPortQueueCapacity(dag, oper.percentileOutputPort);

    ConsoleOutputOperator console = dag.addOperator("console_percentile", ConsoleOutputOperator.class);
    setDefaultInputPortQueueCapacity(dag, console.input);
    console.silent = true;
    dag.addStream("percentile_output", oper.percentileOutputPort, console.input);
    // TODO: Change back to Redis
    //    RedisOutputOperator redisPercentileOutput = getRedisOutputOperator("RedisPercentileOutput", dag, conf, conf.getInt("machinedata.percentile.redis.db", 34));
    //    setDefaultInputPortQueueCapacity(dag, redisPercentileOutput.inputInd);
    //    dag.addStream("percentile_output", oper.percentileOutputPort, redisPercentileOutput.inputInd);

    // SD
    setDefaultOutputPortQueueCapacity(dag, oper.sdOutputPort);
    ConsoleOutputOperator console1 = dag.addOperator("console_sd", ConsoleOutputOperator.class);
    setDefaultInputPortQueueCapacity(dag, console1.input);
    console.silent = true;
    dag.addStream("sd_output", oper.sdOutputPort, console1.input);

    // TODO: Change back to redis
    //    RedisOutputOperator redisSDOperator = getRedisOutputOperator("RedisSDOutput", dag, conf, conf.getInt("machinedata.sd.redis.db", 33));
    //    setDefaultInputPortQueueCapacity(dag, redisSDOperator.inputInd);
    //    dag.addStream("sd_output", oper.sdOutputPort, redisSDOperator.inputInd);

    // Max
    setDefaultOutputPortQueueCapacity(dag, oper.maxOutputPort);

    ConsoleOutputOperator console2 = dag.addOperator("console_max", ConsoleOutputOperator.class);
    setDefaultInputPortQueueCapacity(dag, console2.input);
    console.silent = true;
    dag.addStream("max_output", oper.maxOutputPort, console2.input);

    /*TODO: change back to Redis
    RedisOutputOperator redisMaxOutput = getRedisOutputOperator("RedisMaxOutput", dag, conf, conf.getInt("machinedata.max.redis.db", 32));
    setDefaultInputPortQueueCapacity(dag, redisMaxOutput.inputInd);
    dag.addStream("max_output", oper.maxOutputPort, redisMaxOutput.inputInd);
    */

    SmtpOutputOperator smtpOutputOperator = getSmtpOutputOperator("SmtpCalcOperator", dag, conf);
    dag.addStream("calc_alert_mail", oper.smtpAlert, smtpOutputOperator.input);

    return oper;
}

From source file:com.datatorrent.contrib.machinedata.Application.java

License:Open Source License

/**
 * Create the DAG// w w w  . ja  va  2 s  .c  o  m
 */
@Override
public void populateDAG(DAG dag, Configuration conf) {

    int partitions = conf.getInt(Application.class.getName() + ".partitions", 1);
    int unifier_count = conf.getInt(Application.class.getName() + ".unifier_count", 2);

    dag.setAttribute(DAG.APPLICATION_NAME, "MachineDataApplication");
    dag.setAttribute(DAG.DEBUG, false);
    dag.setAttribute(DAG.STREAMING_WINDOW_SIZE_MILLIS, streamingWindowSizeMilliSeconds);

    InputReceiver randomGen = getRandomInformationTupleGenerator("InputReceiver", dag);
    setDefaultOutputPortQueueCapacity(dag, randomGen.outputInline);
    dag.setAttribute(randomGen, OperatorContext.INITIAL_PARTITION_COUNT, partitions);

    DimensionGenerator dimensionGenerator = dag.addOperator("GenerateDimensions", DimensionGenerator.class);
    dag.setAttribute(dimensionGenerator, Context.OperatorContext.APPLICATION_WINDOW_COUNT,
            appWindowCountMinute);
    setDefaultOutputPortQueueCapacity(dag, dimensionGenerator.outputInline);
    setDefaultOutputPortQueueCapacity(dag, dimensionGenerator.output);
    setDefaultInputPortQueueCapacity(dag, dimensionGenerator.inputPort);
    dag.addStream("generate_dimensions", randomGen.outputInline, dimensionGenerator.inputPort)
            .setLocality(Locality.CONTAINER_LOCAL);
    dag.setInputPortAttribute(dimensionGenerator.inputPort, PortContext.PARTITION_PARALLEL, true);

    if (conf.getBoolean("machinedata.calculate.average", true)) {
        MachineInfoAveragingPrerequisitesOperator prereqAverageOper = addAverageCalculation(dag, conf);
        dag.addStream("prereq_calculation", dimensionGenerator.outputInline, prereqAverageOper.inputPort)
                .setLocality(Locality.THREAD_LOCAL);
        dag.setOutputPortAttribute(prereqAverageOper.outputPort, PortContext.UNIFIER_LIMIT, unifier_count);
    }

    /*
    CalculatorOperator calculatorOperator = addCalculator(dag, conf);
    dag.addStream("dimension_generator_to_calculator", dimensionGenerator.output, calculatorOperator.dataPort);
            
    if (conf.getBoolean("machinedata.calculate.percentile", false)) {
      calculatorOperator.setComputePercentile(true);
    }
            
    if (conf.getBoolean("machinedata.calculate.sd", false)) {
      calculatorOperator.setComputeSD(true);
    }
            
            
    if (conf.getBoolean("machinedata.calculate.max", false)) {
      calculatorOperator.setComputeMax(true);
    }
    */

}

From source file:com.datatorrent.demos.ads.Application.java

License:Open Source License

private void configure(DAG dag, Configuration conf) {

    if (StreamingApplication.Environment.CLUSTER == conf.getEnum(StreamingApplication.ENVIRONMENT,
            StreamingApplication.Environment.LOCAL)) {
        setLocalMode();//from ww  w  . j a  va 2s . c om
        // settings only affect distributed mode
        AttributeMap attributes = dag.getAttributes();
        if (attributes.get(DAGContext.CONTAINER_MEMORY_MB) == null) {
            attributes.put(DAGContext.CONTAINER_MEMORY_MB, 2048);
        }
        if (attributes.get(DAGContext.MASTER_MEMORY_MB) == null) {
            attributes.put(DAGContext.MASTER_MEMORY_MB, 1024);
        }
        if (attributes.get(DAGContext.CONTAINERS_MAX_COUNT) == null) {
            attributes.put(DAGContext.CONTAINERS_MAX_COUNT, 1);
        }
    } else if (StreamingApplication.Environment.LOCAL == conf.getEnum(StreamingApplication.ENVIRONMENT,
            StreamingApplication.Environment.CLUSTER)) {
        setLocalMode();
    }

    this.generatorVTuplesBlast = conf.getInt(P_generatorVTuplesBlast, this.generatorVTuplesBlast);
    this.generatorMaxWindowsCount = conf.getInt(P_generatorMaxWindowsCount, this.generatorMaxWindowsCount);
    this.locality = conf.getBoolean(P_allInline, false) ? Locality.CONTAINER_LOCAL : null;
    this.numGenerators = conf.getInt(P_numGenerators, this.numGenerators);

}

From source file:com.datatorrent.demos.machinedata.Application.java

License:Open Source License

/**
 * This function sets up the DAG for calculating the average
 *
 * @param dag  the DAG instance// ww  w. j a v a 2s.  c  o m
 * @param conf the configuration instance
 * @return MachineInfoAveragingPrerequisitesOperator
 */
private MachineInfoAveragingPrerequisitesOperator addAverageCalculation(DAG dag, Configuration conf) {
    MachineInfoAveragingPrerequisitesOperator prereqAverageOper = getMachineInfoAveragingPrerequisitesOperator(
            "Aggregator", dag);
    dag.setInputPortAttribute(prereqAverageOper.inputPort, PortContext.PARTITION_PARALLEL, true);
    dag.setAttribute(prereqAverageOper, Context.OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCountMinute);

    setDefaultInputPortQueueCapacity(dag, prereqAverageOper.inputPort);
    setDefaultOutputPortQueueCapacity(dag, prereqAverageOper.outputPort);

    int partitions = conf.getInt(Application.class.getName() + ".averagePartitions", 2);
    MachineInfoAveragingOperator averageOperator = getMachineInfoAveragingOperator("AverageCalculator", dag);
    dag.setAttribute(averageOperator, OperatorContext.INITIAL_PARTITION_COUNT, partitions);
    setDefaultInputPortQueueCapacity(dag, averageOperator.inputPort);
    setDefaultOutputPortQueueCapacity(dag, averageOperator.outputPort);

    RedisKeyValPairOutputOperator<MachineKey, Map<String, String>> redisAvgOperator = getRedisOutputOperator(
            "Persister", dag, conf, conf.getInt("machinedata.redis.db", 5));
    setDefaultInputPortQueueCapacity(dag, redisAvgOperator.input);
    dag.setInputPortAttribute(redisAvgOperator.input, PortContext.PARTITION_PARALLEL, true);
    dag.addStream("Average", averageOperator.outputPort, redisAvgOperator.input);

    SmtpOutputOperator smtpOutputOperator = getSmtpOutputOperator("Alerter", dag, conf);

    dag.addStream("Aggregates", prereqAverageOper.outputPort, averageOperator.inputPort);
    dag.addStream("Alerts", averageOperator.smtpAlert, smtpOutputOperator.input);

    return prereqAverageOper;

}

From source file:com.datatorrent.demos.machinedata.Application.java

License:Open Source License

/**
 * Create the DAG/*from w  ww.  ja va  2 s .  c o  m*/
 */
@Override
public void populateDAG(DAG dag, Configuration conf) {

    int partitions = conf.getInt(Application.class.getName() + ".partitions", 1);
    int unifier_count = conf.getInt(Application.class.getName() + ".unifier_count", 2);

    //dag.setAttribute(DAG.APPLICATION_NAME, "MachineDataApplication");
    dag.setAttribute(DAG.DEBUG, false);
    dag.setAttribute(DAG.STREAMING_WINDOW_SIZE_MILLIS, streamingWindowSizeMilliSeconds);

    InputReceiver randomGen = getRandomInformationTupleGenerator("Receiver", dag);
    setDefaultOutputPortQueueCapacity(dag, randomGen.outputInline);
    dag.setAttribute(randomGen, OperatorContext.INITIAL_PARTITION_COUNT, partitions);

    DimensionGenerator dimensionGenerator = dag.addOperator("DimensionsGenerator", DimensionGenerator.class);
    dag.setAttribute(dimensionGenerator, Context.OperatorContext.APPLICATION_WINDOW_COUNT,
            appWindowCountMinute);
    setDefaultOutputPortQueueCapacity(dag, dimensionGenerator.outputInline);
    setDefaultOutputPortQueueCapacity(dag, dimensionGenerator.output);
    setDefaultInputPortQueueCapacity(dag, dimensionGenerator.inputPort);
    dag.addStream("Events", randomGen.outputInline, dimensionGenerator.inputPort)
            .setLocality(Locality.CONTAINER_LOCAL);
    dag.setInputPortAttribute(dimensionGenerator.inputPort, PortContext.PARTITION_PARALLEL, true);

    MachineInfoAveragingPrerequisitesOperator prereqAverageOper = addAverageCalculation(dag, conf);
    dag.addStream("DimensionalData", dimensionGenerator.outputInline, prereqAverageOper.inputPort)
            .setLocality(Locality.THREAD_LOCAL);
    dag.setOutputPortAttribute(prereqAverageOper.outputPort, PortContext.UNIFIER_LIMIT, unifier_count);

}