Example usage for org.apache.hadoop.conf Configuration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf) 

Source Link

Document

Add a configuration resource.

Usage

From source file:com.datatorrent.lib.db.jdbc.JdbcPojoOperatorApplicationTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    try {//from w ww .jav  a  2 s  . c o m
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/JdbcProperties.xml"));
        lma.prepareDAG(new JdbcPojoOperatorApplication(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.setHeartbeatMonitoringEnabled(false);
        ((StramLocalCluster) lc).setExitCondition(new Callable<Boolean>() {
            @Override
            public Boolean call() throws Exception {
                return getNumOfRowsinTable(TABLE_POJO_NAME) == 10;
            }
        });
        lc.run(10000);// runs for 10 seconds and quits
        Assert.assertEquals("rows in db", 10, getNumOfRowsinTable(TABLE_POJO_NAME));
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

---

From source file:com.datatorrent.maprapp.ApplicationTest.java

@Test
public void testApplication() throws IOException, Exception {
    try {//  w w  w.jav a  2  s . c o m
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        lma.prepareDAG(new Application(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.runAsync();
        while (!check()) {
            System.out.println("Sleeping...");
            Thread.sleep(1000);
        }
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

---

From source file:com.datatorrent.stram.cli.ApexCli.java

License:Apache License

DTConfiguration getLaunchAppPackageProperties(AppPackage ap, ConfigPackage cp,
        LaunchCommandLineInfo commandLineInfo, String appName) throws Exception {
    DTConfiguration launchProperties = new DTConfiguration();

    List<AppInfo> applications = getAppsFromPackageAndConfig(ap, cp, commandLineInfo.useConfigApps);

    AppInfo selectedApp = null;//from   w w  w.  j a v a 2 s.c  om
    for (AppInfo app : applications) {
        if (app.name.equals(appName)) {
            selectedApp = app;
            break;
        }
    }
    Map<String, String> defaultProperties = selectedApp == null ? ap.getDefaultProperties()
            : selectedApp.defaultProperties;
    Set<String> requiredProperties = new TreeSet<>(
            selectedApp == null ? ap.getRequiredProperties() : selectedApp.requiredProperties);

    for (Map.Entry<String, String> entry : defaultProperties.entrySet()) {
        launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
        requiredProperties.remove(entry.getKey());
    }

    // settings specified in the user's environment take precedence over defaults in package.
    // since both are merged into a single -conf option below, apply them on top of the defaults here.
    File confFile = new File(StramClientUtils.getUserDTDirectory(), StramClientUtils.DT_SITE_XML_FILE);
    if (confFile.exists()) {
        Configuration userConf = new Configuration(false);
        userConf.addResource(new Path(confFile.toURI()));
        Iterator<Entry<String, String>> it = userConf.iterator();
        while (it.hasNext()) {
            Entry<String, String> entry = it.next();
            // filter relevant entries
            if (entry.getKey().startsWith(StreamingApplication.DT_PREFIX)) {
                launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
                requiredProperties.remove(entry.getKey());
            }
        }
    }

    if (commandLineInfo.apConfigFile != null) {
        DTConfiguration givenConfig = new DTConfiguration();
        givenConfig.loadFile(new File(ap.tempDirectory() + "/conf/" + commandLineInfo.apConfigFile));
        for (Map.Entry<String, String> entry : givenConfig) {
            launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
            requiredProperties.remove(entry.getKey());
        }
    }
    if (cp != null) {
        Map<String, String> properties = cp.getProperties(appName);
        for (Map.Entry<String, String> entry : properties.entrySet()) {
            launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
            requiredProperties.remove(entry.getKey());
        }
    } else if (commandLineInfo.configFile != null) {
        DTConfiguration givenConfig = new DTConfiguration();
        givenConfig.loadFile(new File(commandLineInfo.configFile));
        for (Map.Entry<String, String> entry : givenConfig) {
            launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
            requiredProperties.remove(entry.getKey());
        }
    }
    if (commandLineInfo.overrideProperties != null) {
        for (Map.Entry<String, String> entry : commandLineInfo.overrideProperties.entrySet()) {
            launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
            requiredProperties.remove(entry.getKey());
        }
    }

    // now look at whether it is in default configuration
    for (Map.Entry<String, String> entry : conf) {
        if (StringUtils.isNotBlank(entry.getValue())) {
            requiredProperties.remove(entry.getKey());
        }
    }
    if (!requiredProperties.isEmpty()) {
        throw new CliException("Required properties not set: " + StringUtils.join(requiredProperties, ", "));
    }

    //StramClientUtils.evalProperties(launchProperties);
    return launchProperties;
}

---

From source file:com.datatorrent.stram.cli.DTCli.java

License:Apache License

DTConfiguration getLaunchAppPackageProperties(AppPackage ap, ConfigPackage cp,
        LaunchCommandLineInfo commandLineInfo, String appName) throws Exception {
    DTConfiguration launchProperties = new DTConfiguration();
    List<AppInfo> applications = ap.getApplications();
    AppInfo selectedApp = null;/*from www .j  a v a 2s  . com*/
    for (AppInfo app : applications) {
        if (app.name.equals(appName)) {
            selectedApp = app;
            break;
        }
    }
    Map<String, String> defaultProperties = selectedApp == null ? ap.getDefaultProperties()
            : selectedApp.defaultProperties;
    Set<String> requiredProperties = new TreeSet<String>(
            selectedApp == null ? ap.getRequiredProperties() : selectedApp.requiredProperties);

    for (Map.Entry<String, String> entry : defaultProperties.entrySet()) {
        launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
        requiredProperties.remove(entry.getKey());
    }

    // settings specified in the user's environment take precedence over defaults in package.
    // since both are merged into a single -conf option below, apply them on top of the defaults here.
    File confFile = new File(StramClientUtils.getUserDTDirectory(), StramClientUtils.DT_SITE_XML_FILE);
    if (confFile.exists()) {
        Configuration userConf = new Configuration(false);
        userConf.addResource(new Path(confFile.toURI()));
        Iterator<Entry<String, String>> it = userConf.iterator();
        while (it.hasNext()) {
            Entry<String, String> entry = it.next();
            // filter relevant entries
            if (entry.getKey().startsWith(StreamingApplication.DT_PREFIX)) {
                launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
                requiredProperties.remove(entry.getKey());
            }
        }
    }

    if (commandLineInfo.apConfigFile != null) {
        DTConfiguration givenConfig = new DTConfiguration();
        givenConfig.loadFile(new File(ap.tempDirectory() + "/conf/" + commandLineInfo.apConfigFile));
        for (Map.Entry<String, String> entry : givenConfig) {
            launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
            requiredProperties.remove(entry.getKey());
        }
    }
    if (cp != null) {
        Map<String, String> properties = cp.getProperties(appName);
        for (Map.Entry<String, String> entry : properties.entrySet()) {
            launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
            requiredProperties.remove(entry.getKey());
        }
    } else if (commandLineInfo.configFile != null) {
        DTConfiguration givenConfig = new DTConfiguration();
        givenConfig.loadFile(new File(commandLineInfo.configFile));
        for (Map.Entry<String, String> entry : givenConfig) {
            launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
            requiredProperties.remove(entry.getKey());
        }
    }
    if (commandLineInfo.overrideProperties != null) {
        for (Map.Entry<String, String> entry : commandLineInfo.overrideProperties.entrySet()) {
            launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
            requiredProperties.remove(entry.getKey());
        }
    }

    // now look at whether it is in default configuration
    for (Map.Entry<String, String> entry : conf) {
        if (StringUtils.isNotBlank(entry.getValue())) {
            requiredProperties.remove(entry.getKey());
        }
    }
    if (!requiredProperties.isEmpty()) {
        throw new CliException("Required properties not set: " + StringUtils.join(requiredProperties, ", "));
    }

    //StramClientUtils.evalProperties(launchProperties);
    return launchProperties;
}

---

From source file:com.datatorrent.stram.client.StramAppLauncher.java

License:Apache License

public static Configuration getOverriddenConfig(Configuration conf, String overrideConfFileName,
        Map<String, String> overrideProperties) throws IOException {
    if (overrideConfFileName != null) {
        File overrideConfFile = new File(overrideConfFileName);
        if (overrideConfFile.exists()) {
            LOG.info("Loading settings: " + overrideConfFile.toURI());
            conf.addResource(new Path(overrideConfFile.toURI()));
        } else {//  w  w w .j  a va 2 s .c  o  m
            throw new IOException("Problem opening file " + overrideConfFile);
        }
    }
    if (overrideProperties != null) {
        for (Map.Entry<String, String> entry : overrideProperties.entrySet()) {
            conf.set(entry.getKey(), entry.getValue());
        }
    }
    return conf;
}

---

From source file:com.datatorrent.stram.client.StramClientUtils.java

License:Apache License

public static Configuration addDTDefaultResources(Configuration conf) {
    conf.addResource(DT_DEFAULT_XML_FILE);
    return conf;
}

---

From source file:com.datatorrent.stram.client.StramClientUtils.java

License:Apache License

public static void addDTLocalResources(Configuration conf) {
    conf.addResource(DT_DEFAULT_XML_FILE);
    if (!isDevelopmentMode()) {
        addDTSiteResources(conf, new File(StramClientUtils.getConfigDir(), StramClientUtils.DT_SITE_XML_FILE));
    }/*from   ww w.ja v a2 s . c om*/
    addDTSiteResources(conf,
            new File(StramClientUtils.getUserDTDirectory(), StramClientUtils.DT_SITE_XML_FILE));
}

---

From source file:com.datatorrent.stram.client.StramClientUtils.java

License:Apache License

private static Configuration addDTSiteResources(Configuration conf, File confFile) {
    if (confFile.exists()) {
        LOG.info("Loading settings: " + confFile.toURI());
        conf.addResource(new Path(confFile.toURI()));
    } else {/*from   w ww . j a  va 2 s  . c  o m*/
        LOG.info("Configuration file {} is not found. Skipping...", confFile.toURI());
    }
    return conf;
}

---

From source file:com.datatorrent.stram.plan.logical.LogicalPlanConfigurationTest.java

License:Apache License

/**
 * Test read from dt-site.xml in Hadoop configuration format.
 *//*from www.j  a  v a  2 s  .  c  om*/
@Test
public void testLoadFromConfigXml() {
    Configuration conf = new Configuration(false);
    conf.addResource(StramClientUtils.DT_SITE_XML_FILE);

    LogicalPlanConfiguration builder = new LogicalPlanConfiguration(conf);

    LogicalPlan dag = new LogicalPlan();
    builder.populateDAG(dag);
    dag.validate();

    assertEquals("number of operator confs", 6, dag.getAllOperators().size());

    OperatorMeta operator1 = assertNode(dag, "operator1");
    OperatorMeta operator2 = assertNode(dag, "operator2");
    OperatorMeta operator3 = assertNode(dag, "operator3");
    OperatorMeta operator4 = assertNode(dag, "operator4");

    assertNotNull("operatorConf for root", operator1);
    assertEquals("operatorId set", "operator1", operator1.getName());

    // verify operator instantiation
    assertEquals(operator1.getOperator().getClass(), TestGeneratorInputOperator.class);
    TestGeneratorInputOperator GenericTestNode = (TestGeneratorInputOperator) operator1.getOperator();
    assertEquals("myStringPropertyValue", GenericTestNode.getMyStringProperty());

    // check links
    assertEquals("operator1 inputs", 0, operator1.getInputStreams().size());
    assertEquals("operator1 outputs", 1, operator1.getOutputStreams().size());
    StreamMeta n1n2 = operator2.getInputStreams()
            .get(operator2.getMeta(((GenericTestOperator) operator2.getOperator()).inport1));
    assertNotNull("n1n2", n1n2);

    // output/input stream object same
    assertEquals("rootNode out is operator2 in", n1n2, operator1.getOutputStreams()
            .get(operator1.getMeta(((TestGeneratorInputOperator) operator1.getOperator()).outport)));
    assertEquals("n1n2 source", operator1, n1n2.getSource().getOperatorMeta());
    Assert.assertEquals("n1n2 targets", 1, n1n2.getSinks().size());
    Assert.assertEquals("n1n2 target", operator2, n1n2.getSinks().get(0).getOperatorWrapper());

    assertEquals("stream name", "n1n2", n1n2.getName());
    Assert.assertEquals("n1n2 not inline (default)", null, n1n2.getLocality());

    // operator 2 streams to operator 3 and operator 4
    assertEquals("operator 2 number of outputs", 1, operator2.getOutputStreams().size());
    StreamMeta fromNode2 = operator2.getOutputStreams().values().iterator().next();

    Set<OperatorMeta> targetNodes = Sets.newHashSet();
    for (LogicalPlan.InputPortMeta ip : fromNode2.getSinks()) {
        targetNodes.add(ip.getOperatorWrapper());
    }
    Assert.assertEquals("outputs " + fromNode2, Sets.newHashSet(operator3, operator4), targetNodes);

    OperatorMeta operator6 = assertNode(dag, "operator6");

    List<OperatorMeta> rootNodes = dag.getRootOperators();
    assertEquals("number root operators", 2, rootNodes.size());
    assertTrue("root operator2", rootNodes.contains(operator1));
    assertTrue("root operator6", rootNodes.contains(operator6));

    for (OperatorMeta n : rootNodes) {
        printTopology(n, dag, 0);
    }

}

---

From source file:com.datatorrent.stram.plan.logical.LogicalPlanConfigurationTest.java

License:Apache License

@Test
public void testPrepareDAG() {
    final MutableBoolean appInitialized = new MutableBoolean(false);
    StreamingApplication app = new StreamingApplication() {
        @Override//w  w w  . j a va  2  s .c o m
        public void populateDAG(DAG dag, Configuration conf) {
            Assert.assertEquals("", "hostname:9090", dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS));
            dag.setAttribute(DAG.GATEWAY_CONNECT_ADDRESS, "hostname:9091");
            appInitialized.setValue(true);
        }
    };
    Configuration conf = new Configuration(false);
    conf.addResource(StramClientUtils.DT_SITE_XML_FILE);
    LogicalPlanConfiguration pb = new LogicalPlanConfiguration(conf);

    LogicalPlan dag = new LogicalPlan();
    pb.prepareDAG(dag, app, "testconfig");

    Assert.assertTrue("populateDAG called", appInitialized.booleanValue());
    Assert.assertEquals("populateDAG overrides attribute", "hostname:9091",
            dag.getValue(DAG.GATEWAY_CONNECT_ADDRESS));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »