Example usage for org.apache.hadoop.conf Configuration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf) 

Source Link

Document

Add a configuration resource.

Usage

From source file:org.apache.apex.examples.JdbcToJdbc.ApplicationTest.java

License:Apache License

@Test
@Ignore//from   w w  w.  ja  v a 2  s. c o m
public void testApplication() throws IOException, Exception {
    try {
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-JdbcToJdbcApp.xml"));
        lma.prepareDAG(new JdbcToJdbcApp(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.run(50000); // runs for 10 seconds and quits
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

---

From source file:org.apache.apex.examples.JdbcToJdbc.JdbcOperatorTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    try {//from w  w  w.  java  2 s.c o  m
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-JdbcToJdbcApp.xml"));
        lma.prepareDAG(new JdbcToJdbcApp(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.runAsync();

        // wait for records to be added to table
        Thread.sleep(5000);

        Assert.assertEquals("Events in store", 10, getNumOfEventsInStore());
        dropTable();

    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

---

From source file:org.apache.apex.examples.kafka.exactlyonceoutput.ApplicationTest.java

License:Apache License

private Configuration getConfig() {
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-KafkaExactlyOnceOutput.xml"));
    conf.set("dt.operator.passthrough.prop.directoryPath", directory);
    conf.set("dt.operator.validationToFile.prop.filePath", directory);
    conf.set("dt.operator.kafkaTopicExactly.prop.clusters", broker);
    conf.set("dt.operator.kafkaTopicAtLeast.prop.clusters", broker);
    conf.set("dt.operator.kafkaOutputOperator.prop.properties(bootstrap.servers)", broker);
    conf.set("dt.operator.kafkaExactlyOnceOutputOperator.prop.properties(bootstrap.servers)", broker);
    tuplesUntilKill = conf.get("dt.operator.passthrough.prop.tuplesUntilKill");
    return conf;//from  w  w  w.  jav a  2  s  . c o m
}

---

From source file:org.apache.apex.examples.kafka.hdfs2kafka.ApplicationTest.java

License:Apache License

private Configuration getConfig() {
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-hdfs2kafka.xml"));
    conf.set("dt.operator.lines.prop.directory", directory);
    conf.set("dt.operator.kafkaOutput.prop.producerProperties",
            "serializer.class=kafka.serializer.StringEncoder,producer.type=async,metadata.broker.list="
                    + BROKER);/*w w w. j av a 2  s  . co  m*/

    return conf;
}

---

From source file:org.apache.apex.examples.mobile.ApplicationTest.java

License:Apache License

/**
 * Test of getApplication method, of class Application.
 *//*from w w w. j a v  a 2  s.c om*/
@Test
public void testGetApplication() throws Exception {
    Configuration conf = new Configuration(false);
    conf.addResource("dt-site-mobile.xml");
    Server server = new Server(0);
    Servlet servlet = new SamplePubSubWebSocketServlet();
    ServletHolder sh = new ServletHolder(servlet);
    ServletContextHandler contextHandler = new ServletContextHandler(server, "/",
            ServletContextHandler.SESSIONS);
    contextHandler.addServlet(sh, "/pubsub");
    contextHandler.addServlet(sh, "/*");
    server.start();
    Connector[] connector = server.getConnectors();
    conf.set("dt.attr.GATEWAY_CONNECT_ADDRESS", "localhost:" + connector[0].getLocalPort());
    URI uri = PubSubHelper.getURI("localhost:" + connector[0].getLocalPort());

    PubSubWebSocketOutputOperator<Object> outputOperator = new PubSubWebSocketOutputOperator<Object>();
    outputOperator.setUri(uri);
    outputOperator.setTopic(conf.get("dt.application.MobileExample.operator.QueryLocation.topic"));

    PubSubWebSocketInputOperator<Map<String, String>> inputOperator = new PubSubWebSocketInputOperator<Map<String, String>>();
    inputOperator.setUri(uri);
    inputOperator.setTopic(conf.get("dt.application.MobileExample.operator.LocationResults.topic"));

    CollectorTestSink<Object> sink = new CollectorTestSink<Object>();
    inputOperator.outputPort.setSink(sink);

    Map<String, String> data = new HashMap<String, String>();
    data.put("command", "add");
    data.put("phone", "5559990");

    Application app = new Application();
    LocalMode lma = LocalMode.newInstance();
    lma.prepareDAG(app, conf);
    LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    lc.runAsync();
    Thread.sleep(5000);
    inputOperator.setup(null);
    outputOperator.setup(null);
    inputOperator.activate(null);
    outputOperator.beginWindow(0);
    outputOperator.input.process(data);
    outputOperator.endWindow();
    inputOperator.beginWindow(0);
    int timeoutMillis = 5000;
    while (sink.collectedTuples.size() < 5 && timeoutMillis > 0) {
        inputOperator.emitTuples();
        timeoutMillis -= 20;
        Thread.sleep(20);
    }
    inputOperator.endWindow();
    lc.shutdown();
    inputOperator.teardown();
    outputOperator.teardown();
    server.stop();
    Assert.assertTrue("size of output is 5 ", sink.collectedTuples.size() == 5);
    for (Object obj : sink.collectedTuples) {
        Assert.assertEquals("Expected phone number", "5559990", ((Map<String, String>) obj).get("phone"));
    }
}

---

From source file:org.apache.apex.examples.mrmonitor.MrMonitoringApplicationTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    Configuration conf = new Configuration(false);
    conf.addResource("dt-site-monitoring.xml");
    Server server = new Server(0);
    Servlet servlet = new SamplePubSubWebSocketServlet();
    ServletHolder sh = new ServletHolder(servlet);
    ServletContextHandler contextHandler = new ServletContextHandler(server, "/",
            ServletContextHandler.SESSIONS);
    contextHandler.addServlet(sh, "/pubsub");
    contextHandler.addServlet(sh, "/*");
    server.start();/*from  w  w  w. ja  va 2  s  .c o m*/
    Connector[] connector = server.getConnectors();
    conf.set("dt.attr.GATEWAY_CONNECT_ADDRESS", "localhost:" + connector[0].getLocalPort());

    MRMonitoringApplication application = new MRMonitoringApplication();
    LocalMode lma = LocalMode.newInstance();
    lma.prepareDAG(application, conf);
    LocalMode.Controller lc = lma.getController();
    lc.run(10000);
    server.stop();
}

---

From source file:org.apache.apex.examples.mroperator.ReduceOperator.java

License:Apache License

@Override
public void setup(OperatorContext context) {
    reporter = new ReporterImpl(ReporterImpl.ReporterType.Reducer, new Counters());
    if (context != null) {
        operatorId = context.getId();/*from w  w  w .ja  v  a  2  s.c o  m*/
    }
    cacheObject = new HashMap<K1, List<V1>>();
    outputCollector = new OutputCollectorImpl<K2, V2>();
    if (reduceClass != null) {
        try {
            reduceObj = reduceClass.newInstance();
        } catch (Exception e) {
            logger.info("can't instantiate object {}", e.getMessage());
            throw new RuntimeException(e);
        }
        Configuration conf = new Configuration();
        InputStream stream = null;
        if (configFile != null && configFile.length() > 0) {
            logger.info("system /{}", configFile);
            stream = ClassLoader.getSystemResourceAsStream("/" + configFile);
            if (stream == null) {
                logger.info("system {}", configFile);
                stream = ClassLoader.getSystemResourceAsStream(configFile);
            }
        }
        if (stream != null) {
            logger.info("found our stream... so adding it");
            conf.addResource(stream);
        }
        reduceObj.configure(new JobConf(conf));
    }

}

---

From source file:org.apache.apex.examples.parser.jsonparser.ApplicationTest.java

License:Apache License

@Test
public void testApplication() throws IOException, Exception {

    try {//from  www.  j av  a  2s  . c o m
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        lma.prepareDAG(new Application(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.run(10 * 1000); // runs for 30 seconds and quits
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

---

From source file:org.apache.apex.examples.parser.regexparser.RegexParserApplicationTest.java

License:Apache License

@Test
public void testApplication() throws IOException, Exception {
    try {/*from  ww w  .j a  va2  s  .com*/
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/properties-regexParserApplication.xml"));
        conf.setLong("dt.application.RegexParser.operator.logGenerator.prop.tupleRate", 10);
        final String dataFolderPath = conf.get("dt.application.RegexParser.operator.*.prop.filePath");
        final String dataFileName = conf
                .get("dt.application.RegexParser.operator.regexWriter.prop.outputFileName");

        FileUtils.deleteDirectory(new File(dataFolderPath));
        lma.prepareDAG(new RegexParserApplication(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.setHeartbeatMonitoringEnabled(false);
        ((StramLocalCluster) lc).setExitCondition(new Callable<Boolean>() {
            @Override
            public Boolean call() throws Exception {
                if (new File(dataFolderPath).exists()) {
                    Collection<File> files = FileUtils.listFiles(new File(dataFolderPath),
                            new WildcardFileFilter(dataFileName + "*"), null);
                    if (files.size() >= 1) {
                        File parsedFile = files.iterator().next();
                        String fileData = FileUtils.readFileToString(parsedFile);
                        String[] regexData = fileData.split("\n");
                        return regexData.length == 10;
                    }
                }
                return false;
            }
        });

        lc.run(30 * 1000); // runs for 30 seconds and quitxs

        Collection<File> files = FileUtils.listFiles(new File(dataFolderPath),
                new WildcardFileFilter(dataFileName + "*"), null);
        File parsedFile = files.iterator().next();
        String fileData = FileUtils.readFileToString(parsedFile);
        String[] logData = fileData.split("\n");
        for (String logLine : logData) {
            Assert.assertTrue(logLine.contains("id=" + 101));
            Assert.assertTrue(logLine.contains("signInId=" + "'11111@psop.com'"));
            Assert.assertTrue(logLine.contains("serviceId=" + "'IP1234-NPB12345_00'"));
            Assert.assertTrue(logLine.contains("accountId=" + "'11111'"));
            Assert.assertTrue(logLine.contains("platform=" + "'pik'"));
        }
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

---

From source file:org.apache.apex.examples.partition.ApplicationTest.java

License:Apache License

@Test
public void testApplication() throws IOException, Exception {
    try {/*  ww w  .j  a  v  a  2s. c o m*/
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        lma.prepareDAG(new Application(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.run(5000); // runs for 5 seconds and quits
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »