Example usage for org.apache.hadoop.conf Configuration addResource

List of usage examples for org.apache.hadoop.conf Configuration addResource

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf)

Source Link

Document

Add a configuration resource.

Usage

From source file:org.apache.apex.examples.recordReader.ApplicationTest.java

License:Apache License

@Test
public void testApplication() throws IOException, Exception {
    try {//  ww  w.  j  av  a2  s  .c  o  m
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        conf.set("dt.application.RecordReaderExample.operator.fileOutput.prop.filePath", outputDir);
        File outputfile = FileUtils.getFile(outputDir, "output.txt_5.0");

        lma.prepareDAG(new Application(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.runAsync();

        // wait for tuples to show up
        while (!outputfile.exists()) {
            Thread.sleep(1000);
        }

        lc.shutdown();
        Assert.assertTrue(FileUtils.contentEquals(
                FileUtils.getFile(
                        conf.get("dt.application.RecordReaderExample.operator.recordReader.prop.files")),
                outputfile));

    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

From source file:org.apache.apex.examples.transform.ApplicationTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    lma.prepareDAG(new Application(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.runAsync();//from   w  w  w.j  a  v a  2 s .c om
    Thread.sleep(10 * 1000);
    lc.shutdown();
}

From source file:org.apache.apex.examples.transform.SimpleTransformApplicationTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(Launcher.LaunchMode.EMBEDDED);
    Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
    launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
    SimpleTransformApplication simpleTransformApplication = new SimpleTransformApplication();
    simpleTransformApplication.outputFn = outputFn;
    Launcher.AppHandle appHandle = launcher.launchApp(simpleTransformApplication, conf, launchAttributes);
    int sleepTimeCounterForLoopExit = 0;
    int sleepTimePerIteration = 500;
    // wait until expected result count or timeout
    while (results.size() < simpleTransformApplication.pojoDataGenerator.getMaxTuples()) {
        sleepTimeCounterForLoopExit += sleepTimePerIteration;
        if (sleepTimeCounterForLoopExit > 30000) {
            break;
        }/*  w  w  w .  ja v a2 s .  com*/
        Thread.sleep(sleepTimePerIteration);
    }
    appHandle.shutdown(Launcher.ShutdownMode.KILL);
    assertEquals(results.size(), simpleTransformApplication.pojoDataGenerator.getMaxTuples());
    assertTransformationsGenerated(simpleTransformApplication);
}

From source file:org.apache.apex.examples.wordcount.ApplicationTest.java

License:Apache License

@Test
public void testSomeMethod() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource("dt-site-wordcount.xml");
    lma.prepareDAG(new Application(), conf);
    LocalMode.Controller lc = lma.getController();
    long start = System.currentTimeMillis();
    lc.run(300000);/*from  ww  w .  j  a  va  2s  .  c o  m*/
    long end = System.currentTimeMillis();
    long time = end - start;
    LOG.debug("Test used " + time + " ms");
}

From source file:org.apache.apex.malhar.python.PythonExecutorApplicationTest.java

License:Apache License

@JepPythonTestContext(jepPythonBasedTest = true)
@Test/*  www .  j a  va 2 s .  c om*/
public void testApplication() throws Exception {
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(Launcher.LaunchMode.EMBEDDED);
    Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
    launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
    PythonExecutorApplication pythonExecutorApplication = new PythonExecutorApplication();
    pythonExecutorApplication.outputFn = outputFn;
    Launcher.AppHandle appHandle = launcher.launchApp(pythonExecutorApplication, conf, launchAttributes);
    int sleepTimeCounterForLoopExit = 0;
    int sleepTimePerIteration = 1000;
    // wait until expected result count or timeout
    while (results.size() < pythonExecutorApplication.pojoDataGenerator.getMaxTuples()) {
        sleepTimeCounterForLoopExit += sleepTimePerIteration;
        if (sleepTimeCounterForLoopExit > 30000) {
            break;
        }
        LOG.info("Test sleeping until the application time out is reached");
        Thread.sleep(sleepTimePerIteration);
    }
    appHandle.shutdown(Launcher.ShutdownMode.KILL);
    assertEquals(pythonExecutorApplication.pojoDataGenerator.getMaxTuples(), results.size());
}

From source file:org.apache.apex.malhar.sql.sample.FusionStyleSQLApplicationTest.java

License:Apache License

@Test
public void test() throws Exception {
    try {/*from   w  ww .j  a v a 2  s .c  o m*/
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        conf.addResource(
                this.getClass().getResourceAsStream("/META-INF/properties-FusionStyleSQLApplication.xml"));

        conf.set("dt.operator.KafkaInput.prop.topics", testTopicData);
        conf.set("dt.operator.KafkaInput.prop.clusters", kafka.getBroker());
        conf.set("folderPath", outputFolder);
        conf.set("fileName", "out.tmp");

        FusionStyleSQLApplication app = new FusionStyleSQLApplication();

        lma.prepareDAG(app, conf);

        LocalMode.Controller lc = lma.getController();

        lc.runAsync();
        kafka.publish(testTopicData,
                Arrays.asList("15/02/2016 10:15:00 +0000,1,paint1,11", "15/02/2016 10:16:00 +0000,2,paint2,12",
                        "15/02/2016 10:17:00 +0000,3,paint3,13", "15/02/2016 10:18:00 +0000,4,paint4,14",
                        "15/02/2016 10:19:00 +0000,5,paint5,15", "15/02/2016 10:10:00 +0000,6,abcde6,16"));

        Assert.assertTrue(PureStyleSQLApplicationTest.waitTillFileIsPopulated(outputFolder, 40000));
        lc.shutdown();

        File file = new File(outputFolder);
        File file1 = new File(outputFolder + file.list()[0]);
        List<String> strings = FileUtils.readLines(file1);

        String[] actualLines = strings.toArray(new String[strings.size()]);
        String[] expectedLines = new String[] { "15/02/2016 10:18:00 +0000,15/02/2016 12:00:00 +0000,OILPAINT4",
                "", "15/02/2016 10:19:00 +0000,15/02/2016 12:00:00 +0000,OILPAINT5", "" };
        Assert.assertEquals(expectedLines.length, actualLines.length);
        for (int i = 0; i < actualLines.length; i++) {
            Assert.assertEquals(expectedLines[i], actualLines[i]);
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
}

From source file:org.apache.apex.malhar.sql.sample.PureStyleSQLApplicationTest.java

License:Apache License

@Test
public void test() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-PureStyleSQLApplication.xml"));

    conf.set("broker", kafka.getBroker());
    conf.set("topic", testTopicData);
    conf.set("outputFolder", outputFolder);
    conf.set("destFileName", "out.tmp");

    PureStyleSQLApplication app = new PureStyleSQLApplication();

    lma.prepareDAG(app, conf);// www  . ja  v  a  2 s. c o  m

    LocalMode.Controller lc = lma.getController();

    lc.runAsync();
    kafka.publish(testTopicData,
            Arrays.asList("15/02/2016 10:15:00 +0000,1,paint1,11", "15/02/2016 10:16:00 +0000,2,paint2,12",
                    "15/02/2016 10:17:00 +0000,3,paint3,13", "15/02/2016 10:18:00 +0000,4,paint4,14",
                    "15/02/2016 10:19:00 +0000,5,paint5,15", "15/02/2016 10:10:00 +0000,6,abcde6,16"));

    Assert.assertTrue(waitTillFileIsPopulated(outputFolder, 40000));
    lc.shutdown();

    File file = new File(outputFolder);
    File file1 = new File(outputFolder + file.list()[0]);
    List<String> strings = FileUtils.readLines(file1);

    String[] actualLines = strings.toArray(new String[strings.size()]);

    String[] expectedLines = new String[] { "15/02/2016 10:18:00 +0000,15/02/2016 12:00:00 +0000,OILPAINT4", "",
            "15/02/2016 10:19:00 +0000,15/02/2016 12:00:00 +0000,OILPAINT5", "" };

    Assert.assertEquals(expectedLines.length, actualLines.length);
    for (int i = 0; i < expectedLines.length; i++) {
        Assert.assertEquals(expectedLines[i], actualLines[i]);
    }
}

From source file:org.apache.apex.malhar.sql.sample.SQLApplicationWithAPITest.java

License:Apache License

@Test
public void test() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties-SQLApplicationWithAPI.xml"));

    SQLApplicationWithAPI app = new SQLApplicationWithAPI();

    lma.prepareDAG(app, conf);/* w  ww.  jav  a2  s .c  o m*/

    LocalMode.Controller lc = lma.getController();

    PrintStream originalSysout = System.out;
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    System.setOut(new PrintStream(baos));

    lc.runAsync();
    SQLApplicationWithModelFileTest.waitTillStdoutIsPopulated(baos, 30000);
    lc.shutdown();

    System.setOut(originalSysout);

    String[] sout = baos.toString().split(System.lineSeparator());
    Collection<String> filter = Collections2.filter(Arrays.asList(sout),
            Predicates.containsPattern("Delta Record:"));

    String[] actualLines = filter.toArray(new String[filter.size()]);
    Assert.assertTrue(actualLines[0].contains("RowTime=Mon Feb 15 10:15:00 GMT 2016, Product=paint1"));
    Assert.assertTrue(actualLines[1].contains("RowTime=Mon Feb 15 10:16:00 GMT 2016, Product=paint2"));
    Assert.assertTrue(actualLines[2].contains("RowTime=Mon Feb 15 10:17:00 GMT 2016, Product=paint3"));
    Assert.assertTrue(actualLines[3].contains("RowTime=Mon Feb 15 10:18:00 GMT 2016, Product=paint4"));
    Assert.assertTrue(actualLines[4].contains("RowTime=Mon Feb 15 10:19:00 GMT 2016, Product=paint5"));
    Assert.assertTrue(actualLines[5].contains("RowTime=Mon Feb 15 10:10:00 GMT 2016, Product=abcde6"));
}

From source file:org.apache.apex.malhar.sql.sample.SQLApplicationWithModelFileTest.java

License:Apache License

@Test
public void test() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
    conf.addResource(/*from   w  w  w .  ja va  2 s  .  c  o m*/
            this.getClass().getResourceAsStream("/META-INF/properties-SQLApplicationWithModelFile.xml"));

    SQLApplicationWithModelFile app = new SQLApplicationWithModelFile();

    lma.prepareDAG(app, conf);

    LocalMode.Controller lc = lma.getController();

    PrintStream originalSysout = System.out;
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    System.setOut(new PrintStream(baos));

    lc.runAsync();
    waitTillStdoutIsPopulated(baos, 30000);
    lc.shutdown();

    System.setOut(originalSysout);

    String[] sout = baos.toString().split(System.lineSeparator());
    Collection<String> filter = Collections2.filter(Arrays.asList(sout),
            Predicates.containsPattern("Delta Record:"));

    String[] actualLines = filter.toArray(new String[filter.size()]);
    Assert.assertTrue(actualLines[0].contains("RowTime=Mon Feb 15 10:15:00 GMT 2016, Product=paint1"));
    Assert.assertTrue(actualLines[1].contains("RowTime=Mon Feb 15 10:16:00 GMT 2016, Product=paint2"));
    Assert.assertTrue(actualLines[2].contains("RowTime=Mon Feb 15 10:17:00 GMT 2016, Product=paint3"));
    Assert.assertTrue(actualLines[3].contains("RowTime=Mon Feb 15 10:18:00 GMT 2016, Product=paint4"));
    Assert.assertTrue(actualLines[4].contains("RowTime=Mon Feb 15 10:19:00 GMT 2016, Product=paint5"));
    Assert.assertTrue(actualLines[5].contains("RowTime=Mon Feb 15 10:10:00 GMT 2016, Product=abcde6"));
}

From source file:org.apache.atlas.security.SecureClientUtils.java

License:Apache License

public static URLConnectionClientHandler getClientConnectionHandler(DefaultClientConfig config,
        org.apache.commons.configuration.Configuration clientConfig, String doAsUser,
        final UserGroupInformation ugi) {
    config.getProperties().put(URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND,
            true);/*from  w  ww.ja  v a  2 s  . co m*/
    Configuration conf = new Configuration();
    conf.addResource(conf.get(SSLFactory.SSL_CLIENT_CONF_KEY, SecurityProperties.SSL_CLIENT_PROPERTIES));
    UserGroupInformation.setConfiguration(conf);
    final ConnectionConfigurator connConfigurator = newConnConfigurator(conf);
    String authType = "simple";
    if (clientConfig != null) {
        authType = clientConfig.getString("atlas.http.authentication.type", "simple");
    }
    Authenticator authenticator = new PseudoDelegationTokenAuthenticator();
    if (!authType.equals("simple")) {
        authenticator = new KerberosDelegationTokenAuthenticator();
    }
    authenticator.setConnectionConfigurator(connConfigurator);
    final DelegationTokenAuthenticator finalAuthenticator = (DelegationTokenAuthenticator) authenticator;
    final DelegationTokenAuthenticatedURL.Token token = new DelegationTokenAuthenticatedURL.Token();
    HttpURLConnectionFactory httpURLConnectionFactory = null;
    try {
        UserGroupInformation ugiToUse = ugi != null ? ugi : UserGroupInformation.getCurrentUser();
        final UserGroupInformation actualUgi = (ugiToUse
                .getAuthenticationMethod() == UserGroupInformation.AuthenticationMethod.PROXY)
                        ? ugiToUse.getRealUser()
                        : ugiToUse;
        LOG.info("Real User: {}, is from ticket cache? {}", actualUgi, actualUgi.isLoginTicketBased());
        if (StringUtils.isEmpty(doAsUser)) {
            doAsUser = actualUgi.getShortUserName();
        }
        LOG.info("doAsUser: {}", doAsUser);
        final String finalDoAsUser = doAsUser;
        httpURLConnectionFactory = new HttpURLConnectionFactory() {
            @Override
            public HttpURLConnection getHttpURLConnection(final URL url) throws IOException {
                try {
                    return actualUgi.doAs(new PrivilegedExceptionAction<HttpURLConnection>() {
                        @Override
                        public HttpURLConnection run() throws Exception {
                            try {
                                return new DelegationTokenAuthenticatedURL(finalAuthenticator, connConfigurator)
                                        .openConnection(url, token, finalDoAsUser);
                            } catch (Exception e) {
                                throw new IOException(e);
                            }
                        }
                    });
                } catch (Exception e) {
                    if (e instanceof IOException) {
                        throw (IOException) e;
                    } else {
                        throw new IOException(e);
                    }
                }
            }
        };
    } catch (IOException e) {
        LOG.warn("Error obtaining user", e);
    }

    return new URLConnectionClientHandler(httpURLConnectionFactory);
}