Example usage for org.apache.hadoop.conf Configuration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf) 

Source Link

Document

Add a configuration resource.

Usage

From source file:com.datatorrent.benchmark.testbench.EventClassifierNumberToHashDoubleAppTest.java

License:Open Source License

@Test
public void testEventClassifierNumberToHashDoubleApp() throws Exception {
    Logger logger = LoggerFactory.getLogger(EventClassifierNumberToHashDoubleAppTest.class);
    Configuration conf = new Configuration();
    LocalMode lm = LocalMode.newInstance();
    InputStream is = getClass().getResourceAsStream("/dt-site-testbench.xml");
    conf.addResource(is);
    conf.get("dt.application.EventClassifierNumberToHashDoubleApp.operator.eventClassify.key_keys");
    conf.get("dt.application.EventClassifierNumberToHashDoubleApp.operator.eventClassify.s_start");
    conf.get("dt.application.EventClassifierNumberToHashDoubleApp.operator.eventClassify.s_end");
    try {/*from  ww w  .ja  v a  2 s . com*/
        lm.prepareDAG(new EventClassifierNumberToHashDoubleApp(), conf);
        LocalMode.Controller lc = lm.getController();
        lc.run(20000);
    } catch (Exception ex) {
        logger.info(ex.getMessage());
    }
    is.close();
}

---

From source file:com.datatorrent.benchmark.testbench.EventGeneratorAppTest.java

License:Open Source License

@Test
public void testEventGeneratorApp() throws Exception {
    Logger logger = LoggerFactory.getLogger(EventGeneratorAppTest.class);
    Configuration conf = new Configuration();

    LocalMode lm = LocalMode.newInstance();

    InputStream is = getClass().getResourceAsStream("/dt-site-testbench.xml");
    conf.addResource(is);

    conf.get("dt.application.EventGeneratorApp.operator.eventGenerator.keysHelper");
    conf.get("dt.application.EventGeneratorApp.operator.eventGenerator.weightsHelper");
    conf.get("dt.application.EventGeneratorApp.operator.eventGenerator.valuesHelper");
    try {/*from ww  w  . j  a  v a  2 s.co  m*/
        lm.prepareDAG(new EventGeneratorApp(), conf);
        LocalMode.Controller lc = lm.getController();
        lc.run(20000);
    } catch (Exception ex) {
        logger.info(ex.getMessage());
    }
    is.close();
}

---

From source file:com.datatorrent.benchmark.testbench.EventIncrementerAppTest.java

License:Open Source License

@Test
public void testEventIncrementerApp() throws Exception {
    Logger logger = LoggerFactory.getLogger(EventIncrementerAppTest.class);
    Configuration conf = new Configuration();
    LocalMode lm = LocalMode.newInstance();
    InputStream is = getClass().getResourceAsStream("/dt-site-testbench.xml");
    conf.addResource(is);
    conf.get("dt.application.EventIncrementerApp.operator.hmapOper.seed");
    conf.get("dt.application.EventIncrementerApp.operator.hmapOper.keys");
    conf.get("dt.application.EventIncrementerApp.operator.hmapOper.numKeys");
    try {//w w w .  j a  v  a  2s .c  om
        lm.prepareDAG(new EventIncrementerApp(), conf);
        LocalMode.Controller lc = lm.getController();
        lc.run(20000);
    } catch (Exception ex) {
        logger.info(ex.getMessage());
    }
    is.close();
}

---

From source file:com.datatorrent.benchmark.testbench.FilterClassifierAppTest.java

License:Open Source License

@Test
public void testFilterClassifierApp() throws Exception {
    Logger logger = LoggerFactory.getLogger(FilterClassifierAppTest.class);
    Configuration conf = new Configuration();
    LocalMode lm = LocalMode.newInstance();
    InputStream is = getClass().getResourceAsStream("/dt-site-testbench.xml");
    conf.addResource(is);
    conf.get("dt.application.FilterClassifier.operator.hmapOper.keys");
    conf.get("dt.application.FilterClassifier.operator.hmapOper.numKeys");
    try {//from   w  w  w .  jav a 2  s .  c  om
        lm.prepareDAG(new FilterClassifierApp(), conf);
        LocalMode.Controller lc = lm.getController();
        lc.run(20000);
    } catch (Exception ex) {
        logger.info(ex.getMessage());
    }
    is.close();
}

---

From source file:com.datatorrent.benchmark.testbench.FilteredEventClassifierAppTest.java

License:Open Source License

@Test
public void testFilterClassifierApp() throws Exception {
    Logger logger = LoggerFactory.getLogger(FilteredEventClassifierAppTest.class);
    Configuration conf = new Configuration();
    LocalMode lm = LocalMode.newInstance();
    InputStream is = getClass().getResourceAsStream("/dt-site-testbench.xml");
    conf.addResource(is);
    conf.get("dt.application.FilteredEventClassifierApp.operator.hmapOper.keys");
    conf.get("dt.application.FilteredEventClassifierApp.operator.hmapOper.numKeys");
    try {//w  w  w .  j ava2 s  .com
        lm.prepareDAG(new FilteredEventClassifierApp(), conf);
        LocalMode.Controller lc = lm.getController();
        lc.run(20000);
    } catch (Exception ex) {
        logger.info(ex.getMessage());
    }
    is.close();
}

---

From source file:com.datatorrent.benchmark.testbench.ThroughputCounterAppTest.java

License:Open Source License

@Test
public void testThroughputCounterApp() throws Exception {
    Logger logger = LoggerFactory.getLogger(ThroughputCounterAppTest.class);
    Configuration conf = new Configuration();
    LocalMode lm = LocalMode.newInstance();
    InputStream is = getClass().getResourceAsStream("/dt-site-testbench.xml");
    conf.addResource(is);
    conf.get("dt.application.ThroughputCounterApp.operator.hmapOper.keys");
    conf.get("dt.application.ThroughputCounterApp.operator.hmapOper.numKeys");
    try {//www  . ja  v a  2  s. co  m
        lm.prepareDAG(new ThroughputCounterApp(), conf);
        LocalMode.Controller lc = lm.getController();
        lc.run(20000);
    } catch (Exception ex) {
        logger.info(ex.getMessage());
    }
    is.close();
}

---

From source file:com.datatorrent.demos.dimensions.generic.GenericAppTest.java

License:Open Source License

@Test
public void testApplication() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource("META-INF/properties.xml");
    conf.set("dt.operator.DimensionsComputation.attr.APPLICATION_WINDOW_COUNT", "1");
    conf.set("dt.operator.QueryResult.prop.configProperties(metadata.broker.list)", "localhost:9092");
    conf.set("dt.operator.DimensionsStore.fileStore.basePath", "target/HDSApplicationTestStore");
    conf.set("dt.operator.Query.brokerSet", "localhost:9092");
    conf.set("dt.operator.Query.topic", kafkaQueryTopic);
    conf.set("dt.operator.QueryResult.topic", kafkaQueryResultTopic);
    conf.set("dt.operator.DimensionsComputation.attr.APPLICATION_WINDOW_COUNT", "2");
    conf.set("dt.operator.InputGenerator.numPublishers", "2");
    conf.set("dt.loggers.level", "server.*:INFO");

    GenericDimensionsApplication app = new GenericDimensionsApplication();
    lma.prepareDAG(app, conf);//w w w . ja  va  2 s .c  o  m
    LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    lc.runAsync();

    //Write messages to kafkaQueryTopic
    KafkaTestProducer kafkaQuery = new KafkaTestProducer(kafkaQueryTopic);

    // Query should be able to support keys of String or Number type
    String testQuery = "{\n" + " \"id\": \"query1\",\n" + " \"keys\": {\"publisherId\": \"1\"},\n"
            + " \"kafka\": {\"queryTopic\":\"GenericDimensionsQuery\",\"resultTopic\":\"GenericDimensionsQueryResult\"}"
            + "}";

    List<String> testQueryMessages = new ArrayList<String>();
    testQueryMessages.add(testQuery);
    kafkaQuery.setMessages(testQueryMessages);
    kafkaQuery.run();

    // Setup a message listener to receive the query results
    CountDownLatch latch = new CountDownLatch(100);
    KafkaTestConsumer queryResultsListener = new KafkaTestConsumer(kafkaQueryResultTopic);
    queryResultsListener.setLatch(latch);
    new Thread(queryResultsListener).start();

    // Wait to receive messages
    latch.await(15, TimeUnit.SECONDS);
    lc.shutdown();

    // Evaluate results
    String lastMessage;
    LOG.info("Sent " + kafkaQuery.getSendCount() + " messages to " + kafkaQueryTopic);
    LOG.info("Received " + queryResultsListener.holdingBuffer.size() + " messages from Kafka on "
            + kafkaQueryResultTopic + " topic");
    Assert.assertTrue("Minimum messages received from Kafka " + queryResultsListener.holdingBuffer,
            queryResultsListener.holdingBuffer.size() >= 1);

    while (!queryResultsListener.holdingBuffer.isEmpty()) {
        lastMessage = queryResultsListener.getMessage(queryResultsListener.holdingBuffer.poll());
        Assert.assertNotNull("Did not receive message from Kafka", lastMessage);
        LOG.info("received:\n{}", lastMessage);
    }
}

---

From source file:com.datatorrent.demos.ml.classifier.nb.ApplicationTest.java

@Test
public void testApplication() throws IOException, Exception {
    try {//from  w  w w  . j ava2  s.  c  om
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        lma.prepareDAG(new NaiveBayesDemo(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.run(10000); // runs for 10 seconds and quits
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

---

From source file:com.datatorrent.demos.mobile.ApplicationTest.java

License:Open Source License

/**
 * Test of getApplication method, of class Application.
 *//*from   w w  w.  j av  a  2 s  .  co  m*/
@Test
public void testGetApplication() throws Exception {
    Configuration conf = new Configuration(false);
    conf.addResource("dt-site-mobile.xml");
    Server server = new Server(0);
    SamplePubSubWebSocketServlet servlet = new SamplePubSubWebSocketServlet();
    ServletHolder sh = new ServletHolder(servlet);
    ServletContextHandler contextHandler = new ServletContextHandler(server, "/",
            ServletContextHandler.SESSIONS);
    contextHandler.addServlet(sh, "/pubsub");
    contextHandler.addServlet(sh, "/*");
    server.start();
    Connector connector[] = server.getConnectors();
    conf.set("dt.attr.GATEWAY_CONNECT_ADDRESS", "localhost:" + connector[0].getLocalPort());
    URI uri = URI.create("ws://localhost:" + connector[0].getLocalPort() + "/pubsub");

    PubSubWebSocketOutputOperator<Object> outputOperator = new PubSubWebSocketOutputOperator<Object>();
    outputOperator.setName("testOutputOperator");
    outputOperator.setUri(uri);
    outputOperator.setTopic(conf.get("dt.application.MobileDemo.operator.QueryLocation.topic"));

    PubSubWebSocketInputOperator<Map<String, String>> inputOperator = new PubSubWebSocketInputOperator<Map<String, String>>();
    inputOperator.setName("testInputOperator");
    inputOperator.setUri(uri);
    inputOperator.setTopic(conf.get("dt.application.MobileDemo.operator.LocationResults.topic"));

    CollectorTestSink<Object> sink = new CollectorTestSink<Object>();
    inputOperator.outputPort.setSink(sink);

    Map<String, String> data = new HashMap<String, String>();
    data.put("command", "add");
    data.put("phone", "5559990");

    Application app = new Application();
    LocalMode lma = LocalMode.newInstance();
    lma.prepareDAG(app, conf);
    LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    lc.runAsync();
    Thread.sleep(5000);
    inputOperator.setup(null);
    outputOperator.setup(null);
    inputOperator.activate(null);
    outputOperator.beginWindow(0);
    outputOperator.input.process(data);
    outputOperator.endWindow();
    inputOperator.beginWindow(0);
    int timeoutMillis = 5000;
    while (sink.collectedTuples.size() < 5 && timeoutMillis > 0) {
        inputOperator.emitTuples();
        timeoutMillis -= 20;
        Thread.sleep(20);
    }
    inputOperator.endWindow();
    lc.shutdown();
    inputOperator.teardown();
    outputOperator.teardown();
    server.stop();
    Assert.assertTrue("size of output is 5 ", sink.collectedTuples.size() == 5);
    for (Object obj : sink.collectedTuples) {
        Assert.assertEquals("Expected phone number", "5559990", ((Map<String, String>) obj).get("phone"));
    }
}

---

From source file:com.datatorrent.demos.mrmonitor.MrMonitoringApplicationTest.java

License:Open Source License

@Test
public void testApplication() throws Exception {
    Configuration conf = new Configuration(false);
    conf.addResource("dt-site-monitoring.xml");
    Server server = new Server(0);
    SamplePubSubWebSocketServlet servlet = new SamplePubSubWebSocketServlet();
    ServletHolder sh = new ServletHolder(servlet);
    ServletContextHandler contextHandler = new ServletContextHandler(server, "/",
            ServletContextHandler.SESSIONS);
    contextHandler.addServlet(sh, "/pubsub");
    contextHandler.addServlet(sh, "/*");
    server.start();/*w  w w .j av a 2  s . co m*/
    Connector connector[] = server.getConnectors();
    conf.set("dt.attr.GATEWAY_CONNECT_ADDRESS", "localhost:" + connector[0].getLocalPort());

    MRMonitoringApplication application = new MRMonitoringApplication();
    LocalMode lma = LocalMode.newInstance();
    lma.prepareDAG(application, conf);
    LocalMode.Controller lc = lma.getController();
    lc.run(10000);
    server.stop();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »