Example usage for org.apache.hadoop.conf Configuration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf) 

Source Link

Document

Add a configuration resource.

Usage

From source file:com.datatorrent.stram.plan.logical.LogicalPlanConfigurationTest.java

License:Apache License

@Test
public void testAppNameAttribute() {
    StreamingApplication app = new AnnotatedApplication();
    Configuration conf = new Configuration(false);
    conf.addResource(StramClientUtils.DT_SITE_XML_FILE);

    LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);

    Properties properties = new Properties();
    properties.put(StreamingApplication.DT_PREFIX + "application.TestAliasApp.class", app.getClass().getName());

    builder.addFromProperties(properties, null);

    LogicalPlan dag = new LogicalPlan();
    String appPath = app.getClass().getName().replace(".", "/") + ".class";
    dag.setAttribute(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME, "testApp");
    builder.prepareDAG(dag, app, appPath);

    Assert.assertEquals("Application name", "testApp",
            dag.getAttributes().get(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME));
}

---

From source file:com.datatorrent.stram.plan.logical.LogicalPlanConfigurationTest.java

License:Apache License

@Test
public void testAppAlias() {
    StreamingApplication app = new AnnotatedApplication();
    Configuration conf = new Configuration(false);
    conf.addResource(StramClientUtils.DT_SITE_XML_FILE);

    LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);

    Properties properties = new Properties();
    properties.put(StreamingApplication.DT_PREFIX + "application.TestAliasApp.class", app.getClass().getName());

    builder.addFromProperties(properties, null);

    LogicalPlan dag = new LogicalPlan();
    String appPath = app.getClass().getName().replace(".", "/") + ".class";
    builder.prepareDAG(dag, app, appPath);

    Assert.assertEquals("Application name", "TestAliasApp",
            dag.getAttributes().get(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME));
}

---

From source file:com.datatorrent.stram.plan.logical.LogicalPlanConfigurationTest.java

License:Apache License

@Test
public void testAppAnnotationAlias() {
    StreamingApplication app = new AnnotatedApplication();
    Configuration conf = new Configuration(false);
    conf.addResource(StramClientUtils.DT_SITE_XML_FILE);

    LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);

    LogicalPlan dag = new LogicalPlan();
    String appPath = app.getClass().getName().replace(".", "/") + ".class";
    builder.prepareDAG(dag, app, appPath);

    Assert.assertEquals("Application name", "AnnotatedAlias",
            dag.getAttributes().get(com.datatorrent.api.Context.DAGContext.APPLICATION_NAME));
}

---

From source file:com.datatorrent.stram.plan.LogicalPlanConfigurationTest.java

License:Apache License

/**
 * Test read from dt-site.xml in Hadoop configuration format.
 *//*from ww  w. j  a  v a  2 s  .  com*/
@Test
public void testLoadFromConfigXml() {
    Configuration conf = new Configuration(false);
    conf.addResource(StramClientUtils.DT_SITE_XML_FILE);
    //Configuration.dumpConfiguration(conf, new PrintWriter(System.out));

    LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);

    LogicalPlan dag = new LogicalPlan();
    builder.populateDAG(dag);
    dag.validate();

    assertEquals("number of operator confs", 6, dag.getAllOperators().size());

    OperatorMeta operator1 = assertNode(dag, "operator1");
    OperatorMeta operator2 = assertNode(dag, "operator2");
    OperatorMeta operator3 = assertNode(dag, "operator3");
    OperatorMeta operator4 = assertNode(dag, "operator4");

    assertNotNull("operatorConf for root", operator1);
    assertEquals("operatorId set", "operator1", operator1.getName());

    // verify operator instantiation
    assertEquals(operator1.getOperator().getClass(), TestGeneratorInputOperator.class);
    TestGeneratorInputOperator GenericTestNode = (TestGeneratorInputOperator) operator1.getOperator();
    assertEquals("myStringPropertyValue", GenericTestNode.getMyStringProperty());

    // check links
    assertEquals("operator1 inputs", 0, operator1.getInputStreams().size());
    assertEquals("operator1 outputs", 1, operator1.getOutputStreams().size());
    StreamMeta n1n2 = operator2.getInputStreams()
            .get(operator2.getMeta(((GenericTestOperator) operator2.getOperator()).inport1));
    assertNotNull("n1n2", n1n2);

    // output/input stream object same
    assertEquals("rootNode out is operator2 in", n1n2, operator1.getOutputStreams()
            .get(operator1.getMeta(((TestGeneratorInputOperator) operator1.getOperator()).outport)));
    assertEquals("n1n2 source", operator1, n1n2.getSource().getOperatorMeta());
    Assert.assertEquals("n1n2 targets", 1, n1n2.getSinks().size());
    Assert.assertEquals("n1n2 target", operator2, n1n2.getSinks().get(0).getOperatorWrapper());

    assertEquals("stream name", "n1n2", n1n2.getName());
    Assert.assertEquals("n1n2 not inline (default)", null, n1n2.getLocality());

    // operator 2 streams to operator 3 and operator 4
    assertEquals("operator 2 number of outputs", 1, operator2.getOutputStreams().size());
    StreamMeta fromNode2 = operator2.getOutputStreams().values().iterator().next();

    Set<OperatorMeta> targetNodes = new HashSet<OperatorMeta>();
    for (LogicalPlan.InputPortMeta ip : fromNode2.getSinks()) {
        targetNodes.add(ip.getOperatorWrapper());
    }
    Assert.assertEquals("outputs " + fromNode2, Sets.newHashSet(operator3, operator4), targetNodes);

    OperatorMeta operator6 = assertNode(dag, "operator6");

    List<OperatorMeta> rootNodes = dag.getRootOperators();
    assertEquals("number root operators", 2, rootNodes.size());
    assertTrue("root operator2", rootNodes.contains(operator1));
    assertTrue("root operator6", rootNodes.contains(operator6));

    for (OperatorMeta n : rootNodes) {
        printTopology(n, dag, 0);
    }

}

---

From source file:com.datatorrent.stram.util.SecurityUtilsTest.java

License:Apache License

private Configuration setupConfiguration(AuthScheme authScheme) {
    Configuration conf = new Configuration();
    if (authScheme != null) {
        conf.set(SecurityUtils.HADOOP_HTTP_AUTH_PROP, authScheme.getName());
        conf.addResource("security/dt-site-" + authScheme.getName() + ".xml");
    }/*www. j a  v a  2s  .co m*/
    return conf;
}

---

From source file:com.datos.vfs.provider.hdfs.HdfsFileSystem.java

License:Apache License

/**
 * Resolve FileName into FileObject./*w  w  w . j a  v  a  2 s  .  c o  m*/
 * @param name The name of a file on the HdfsFileSystem.
 * @return resolved FileObject.
 * @throws FileSystemException if an error occurred.
 */
@Override
public FileObject resolveFile(final FileName name) throws FileSystemException {
    synchronized (this) {
        if (this.fs == null) {
            final String hdfsUri = name.getRootURI();
            final HdfsFileSystemConfigBuilder builder = HdfsFileSystemConfigBuilder.getInstance();
            final FileSystemOptions options = getFileSystemOptions();
            final String[] configNames = builder.getConfigNames(options);
            final Path[] configPaths = builder.getConfigPaths(options);
            final URL[] configURLs = builder.getConfigURLs(options);
            final InputStream configStream = builder.getConfigInputStream(options);
            final Configuration configConfiguration = builder.getConfigConfiguration(options);

            final Configuration conf = new Configuration(true);
            conf.set(FileSystem.FS_DEFAULT_NAME_KEY, hdfsUri);

            // Load any alternate configuration parameters that may have been specified
            // no matter where they might come from
            if (configNames != null) {
                for (String configName : configNames) {
                    log.debug("Adding HDFS configuration resource: " + configName);
                    conf.addResource(configName);
                }
            }
            if (configPaths != null) {
                for (Path path : configPaths) {
                    log.debug("Adding HDFS configuration path: " + path);
                    conf.addResource(path);
                }
            }
            if (configURLs != null) {
                for (URL url : configURLs) {
                    log.debug("Adding HDFS configuration URL: " + url);
                    conf.addResource(url);
                }
            }
            if (configStream != null) {
                log.debug("Adding HDFS configuration stream");
                conf.addResource(configStream);
            }
            if (configConfiguration != null) {
                log.debug("Adding HDFS configuration object");
                conf.addResource(configConfiguration);
            }

            try {
                fs = FileSystem.get(conf);
            } catch (final IOException e) {
                log.error("Error connecting to filesystem " + hdfsUri, e);
                throw new FileSystemException("Error connecting to filesystem " + hdfsUri, e);
            }
        }
    }

    final boolean useCache = null != getContext().getFileSystemManager().getFilesCache();
    FileObject file;
    if (useCache) {
        file = this.getFileFromCache(name);
    } else {
        file = null;
    }
    if (null == file) {
        String path = null;
        try {
            path = URLDecoder.decode(name.getPath(), "UTF-8");
        } catch (final UnsupportedEncodingException e) {
            path = name.getPath();
        }
        final Path filePath = new Path(path);
        file = new HdfsFileObject((AbstractFileName) name, this, fs, filePath);
        if (useCache) {
            this.putFileToCache(file);
        }
    }
    /**
     * resync the file information if requested
     */
    if (getFileSystemManager().getCacheStrategy().equals(CacheStrategy.ON_RESOLVE)) {
        file.refresh();
    }
    return file;
}

---

From source file:com.davidgildeh.hadoop.utils.FileUtils.java

License:Apache License

/**
 * Opens the HDFS FileSystem so file operations can be run. Configuration is
 * loaded and will automatically load Hadoop environment settings
 * //from   w ww  .j ava  2  s .  c o m
 * @return  The HDFS FileSystem, null if failure 
 * @throws IOException
 */
private static FileSystem getFileSystem(Path filePath) throws IOException {

    // Check if we have local Configuration for HDFS Set, if not it will default to local file system
    Configuration conf = new Configuration();
    if (System.getenv("HADOOP_HOME") != null) {
        LOG.info("Loading Hadoop Configuration Files under " + System.getenv("HADOOP_HOME"));
        Path coreSitePath = new Path(System.getenv("HADOOP_HOME"), "conf/core-site.xml");
        conf.addResource(coreSitePath);
        Path hdfsSitePath = new Path(System.getenv("HADOOP_HOME"), "conf/hdfs-site.xml");
        conf.addResource(hdfsSitePath);
    } else {
        LOG.info("HADOOP_HOME Not Set. Using Local File System.");
    }

    return filePath.getFileSystem(conf);
}

---

From source file:com.digitalpebble.behemoth.BehemothConfiguration.java

License:Apache License

/** Add the standard Behemoth resources to {@link Configuration}. */
private static Configuration addBehemothResources(Configuration conf) {
    conf.addResource("behemoth-default.xml");
    conf.addResource("behemoth-site.xml");
    return conf;/*from   www.  j a  va 2s  . c  om*/
}

---

From source file:com.dt.weather.app.ApplicationTest.java

@Test
public void testApplication() throws IOException, Exception {
    try {//from   w w w.ja va 2 s .  c om
        LocalMode lma = LocalMode.newInstance();
        // Configuration conf = new Configuration(false);
        Configuration conf = new Application().readPropertiesFile(
                "/Users/dev/checkout/personalGit/devel/DTStreaming/WeatherStreamingApp/src/test/resources/localmode.properties");
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        lma.prepareDAG(new Application(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.run(500000); // runs for 60 seconds and quits
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

---

From source file:com.dt.weather.app.WeatherAppTest.java

@Test
public void testApplication() throws IOException, Exception {
    try {//from   ww  w.j  ava  2 s .  c  om
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new WeatherApp().readPropertiesFile(
                "/Users/dev/checkout/personalGit/devel/DTStreaming/WeatherStreamingApp/src/test/resources/localmode.properties");
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        lma.prepareDAG(new WeatherApp(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.run(500000); // runs for 60 seconds and quits
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »