Example usage for org.apache.hadoop.conf Configuration addResource

List of usage examples for org.apache.hadoop.conf Configuration addResource

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf)

Source Link

Document

Add a configuration resource.

Usage

From source file:org.apache.apex.examples.enricher.ApplicationTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(Launcher.LaunchMode.EMBEDDED);
    Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
    launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
    EnricherAppWithJSONFile enricherAppWithJSONFile = new EnricherAppWithJSONFile();
    enricherAppWithJSONFile.outputFn = outputFn;
    Launcher.AppHandle appHandle = launcher.launchApp(enricherAppWithJSONFile, conf, launchAttributes);
    int sleepTimeCounterForLoopExit = 0;
    int sleepTimePerIteration = 500;
    // wait until expected result count or timeout
    while (results.size() < enricherAppWithJSONFile.getDataGenerator().getLimit()) {
        sleepTimeCounterForLoopExit += sleepTimePerIteration;
        if (sleepTimeCounterForLoopExit > 30000) {
            break;
        }//  w w  w . j  a  v  a  2  s .  c o m
        Thread.sleep(sleepTimePerIteration);
    }
    appHandle.shutdown(ShutdownMode.KILL);
    assertTrue(results.size() >= enricherAppWithJSONFile.getDataGenerator().getLimit());
    assertEnrichedOutput();
}

From source file:org.apache.apex.examples.exactlyonce.ExactlyOnceFileOutputAppTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    File targetDir = new File(TARGET_DIR);
    FileUtils.deleteDirectory(targetDir);
    FileUtils.forceMkdir(targetDir);//w w  w . ja  v  a2s . co m

    KafkaUnit ku = kafkaUnitRule.getKafkaUnit();
    String topicName = "testTopic";
    // topic creation is async and the producer may also auto-create it
    ku.createTopic(topicName, 1);

    // produce test data
    String[] words = "count count the words from kafka and store them in a file".split("\\s+");
    for (String word : words) {
        ku.sendMessages(new KeyedMessage<String, String>(topicName, word));
    }

    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    conf.set("apex.operator.kafkaInput.prop.topics", topicName);
    conf.set("apex.operator.kafkaInput.prop.clusters", "localhost:" + brokerPort);
    conf.set("apex.operator.kafkaInput.prop.maxTuplesPerWindow", "2"); // consume one word per window
    conf.set("apex.operator.kafkaInput.prop.initialOffset", "EARLIEST");
    conf.set("apex.operator.fileWriter.prop.filePath", TARGET_DIR);

    EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
    Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
    launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true); // terminate after results are available
    AppHandle appHandle = launcher.launchApp(new ExactlyOnceFileOutputApp(), conf, launchAttributes);

    long timeout = System.currentTimeMillis() + 60000; // 60s timeout

    File outputFile = new File(TARGET_DIR, ExactlyOnceFileOutputApp.FileWriter.FILE_NAME_PREFIX);
    while (!outputFile.exists() && timeout > System.currentTimeMillis()) {
        Thread.sleep(1000);
        LOG.debug("Waiting for {}", outputFile);
    }

    Assert.assertTrue("output file exists " + ExactlyOnceFileOutputApp.FileWriter.FILE_NAME_PREFIX,
            outputFile.exists() && outputFile.isFile());

    String result = FileUtils.readFileToString(outputFile);
    Assert.assertTrue(result.contains("count=2"));

    appHandle.shutdown(ShutdownMode.KILL);
}

From source file:org.apache.apex.examples.exactlyonce.ExactlyOnceJdbcOutputTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    KafkaUnit ku = kafkaUnitRule.getKafkaUnit();
    String topicName = "testTopic";
    // topic creation is async and the producer may also auto-create it
    ku.createTopic(topicName, 1);//  w  w  w .j a  v  a  2s .c  om

    // produce test data
    String[] words = "count the words from kafka and store them in the db".split("\\s+");
    for (String word : words) {
        ku.sendMessages(new KeyedMessage<String, String>(topicName, word));
    }

    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    conf.set("apex.operator.kafkaInput.prop.topics", topicName);
    conf.set("apex.operator.kafkaInput.prop.clusters", "localhost:" + brokerPort);
    conf.set("apex.operator.kafkaInput.prop.maxTuplesPerWindow", "1"); // consume one word per window
    conf.set("apex.operator.kafkaInput.prop.initialOffset", "EARLIEST");
    conf.set("apex.operator.store.prop.store.databaseDriver", DB_DRIVER);
    conf.set("apex.operator.store.prop.store.databaseUrl", DB_URL);

    EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
    Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
    launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true); // terminate after results are available
    AppHandle appHandle = launcher.launchApp(new ExactlyOnceJdbcOutputApp(), conf, launchAttributes);
    HashSet<String> wordsSet = Sets.newHashSet(words);
    Connection con = DriverManager.getConnection(DB_URL);
    Statement stmt = con.createStatement();
    int rowCount = 0;
    long timeout = System.currentTimeMillis() + 30000; // 30s timeout
    while (rowCount < wordsSet.size() && timeout > System.currentTimeMillis()) {
        Thread.sleep(500);
        String countQuery = "SELECT count(*) from " + TABLE_NAME;
        ResultSet resultSet = stmt.executeQuery(countQuery);
        resultSet.next();
        rowCount = resultSet.getInt(1);
        resultSet.close();
        LOG.info("current row count in {} is {}", TABLE_NAME, rowCount);
    }
    Assert.assertEquals("number of words", wordsSet.size(), rowCount);
    appHandle.shutdown(ShutdownMode.KILL);
}

From source file:org.apache.apex.examples.FileToJdbcApp.ApplicationTest.java

License:Apache License

@Test
public void testCsvParserApp() throws IOException, Exception {
    try {//from   w  w w.  j ava2 s . com
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(new File("src/test/resources/test-FileToJdbcApp.xml").toURI().toURL());

        lma.prepareDAG(new FileToJdbcCsvParser(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.runAsync(); // test will terminate after results are available

        // wait for records to be added to table
        Thread.sleep(5000);

        Assert.assertEquals("Events in store", 10, getNumOfEventsInStore());
        cleanTable();

    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

From source file:org.apache.apex.examples.FileToJdbcApp.ApplicationTest.java

License:Apache License

@Test
public void testCustomParserApp() throws IOException, Exception {
    try {/*from   ww  w.  ja  va  2 s .  c  om*/
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(new File("src/test/resources/test-FileToJdbcApp.xml").toURI().toURL());

        lma.prepareDAG(new FileToJdbcCustomParser(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.runAsync(); // test will terminate after results are available

        // wait for records to be added to table
        Thread.sleep(5000);

        Assert.assertEquals("Events in store", 10, getNumOfEventsInStore());
        cleanTable();

    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

From source file:org.apache.apex.examples.filter.ApplicationTest.java

License:Apache License

@Test
public void testApplication() throws IOException, Exception {
    try {//www .  ja v  a  2  s . c om
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        conf.set("dt.application.FilterExample.operator.selectedOutput.prop.filePath", outputDir);
        conf.set("dt.application.FilterExample.operator.rejectedOutput.prop.filePath", outputDir);
        final File selectedfile = FileUtils.getFile(outputDir, "selected.txt_8.0");
        final File rejectedfile = FileUtils.getFile(outputDir, "rejected.txt_6.0");

        lma.prepareDAG(new Application(), conf);
        LocalMode.Controller lc = lma.getController();

        ((StramLocalCluster) lc).setExitCondition(new Callable<Boolean>() {
            @Override
            public Boolean call() throws Exception {
                if (selectedfile.exists() && rejectedfile.exists()) {
                    return true;
                }
                return false;
            }
        });

    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

From source file:org.apache.apex.examples.frauddetect.FrauddetectApplicationTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    try {//ww  w .  j  a  v  a  2  s . co m
        Application application = new Application();
        Configuration conf = new Configuration(false);
        conf.addResource("dt-site-frauddetect.xml");
        LocalMode lma = LocalMode.newInstance();
        lma.prepareDAG(application, conf);
        lma.getController().run(120000);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

From source file:org.apache.apex.examples.innerjoin.InnerJoinApplicationTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    lma.prepareDAG(new InnerJoinApplication(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.runAsync();/*from  w  w  w.j  ava2  s  . com*/
    Thread.sleep(10 * 1000);
    lc.shutdown();
}

From source file:org.apache.apex.examples.JdbcIngest.ApplicationTest.java

License:Apache License

@Test
@Ignore// ww w.j  a  v a 2s.  com
public void testApplication() throws IOException, Exception {
    try {
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-SimpleJdbcToHDFSApp.xml"));
        lma.prepareDAG(new JdbcHDFSApp(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.run(10000); // runs for 10 seconds and quits
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

From source file:org.apache.apex.examples.JdbcIngest.JdbcInputAppTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    try {/*from  w w  w .  ja  va  2 s .c  o  m*/
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-SimpleJdbcToHDFSApp.xml"));
        lma.prepareDAG(new JdbcHDFSApp(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.runAsync();

        // wait for output files to roll
        Thread.sleep(5000);

        String[] extensions = { "dat.0", "tmp" };
        Collection<File> list = FileUtils.listFiles(new File(FILE_NAME), extensions, false);
        Assert.assertEquals("Records in file", 10, FileUtils.readLines(list.iterator().next()).size());

    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}