List of usage examples for org.apache.hadoop.conf Configuration size
public int size()
From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java
License:Apache License
public void testParseJobXmlAndConfiguration() throws Exception { String str = "<java>" + "<job-xml>job1.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" + "<configuration>" + "<property><name>p1</name><value>v1a</value></property>" + "<property><name>p2</name><value>v2</value></property>" + "</configuration>" + "</java>"; Element xml = XmlUtils.parseXml(str); Path appPath = new Path(getFsTestCaseDir(), "app"); getFileSystem().mkdirs(appPath);//from w ww .j av a 2s.c o m XConfiguration jConf = new XConfiguration(); jConf.set("p1", "v1b"); jConf.set("p3", "v3a"); OutputStream os = getFileSystem().create(new Path(appPath, "job1.xml")); jConf.writeXml(os); os.close(); jConf = new XConfiguration(); jConf.set("p4", "v4"); jConf.set("p3", "v3b"); os = getFileSystem().create(new Path(appPath, "job2.xml")); jConf.writeXml(os); os.close(); Configuration conf = new XConfiguration(); assertEquals(0, conf.size()); JavaActionExecutor.parseJobXmlAndConfiguration(createContext("<java/>", null), xml, appPath, conf); assertEquals(4, conf.size()); assertEquals("v1a", conf.get("p1")); assertEquals("v2", conf.get("p2")); assertEquals("v3b", conf.get("p3")); assertEquals("v4", conf.get("p4")); }
From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java
License:Apache License
public void testParseJobXmlAndConfigurationWithELExpression() throws Exception { String str = "<java>" + "<job-xml>job1.xml</job-xml>" + "<job-xml>job2.xml</job-xml>" + "<configuration>" + "<property><name>p1</name><value>v1a</value></property>" + "<property><name>p2</name><value>v2</value></property>" + "</configuration>" + "</java>"; Element xml = XmlUtils.parseXml(str); Path appPath = new Path(getFsTestCaseDir(), "app"); getFileSystem().mkdirs(appPath);/* w w w.j a v a 2 s .c o m*/ XConfiguration jConf = new XConfiguration(); jConf.set("p3", "${v3}"); jConf.set("p4", "${v4}"); jConf.set("user", "${wf:user()}"); OutputStream os = getFileSystem().create(new Path(appPath, "job1.xml")); jConf.writeXml(os); os.close(); jConf = new XConfiguration(); jConf.set("p5", "v5"); jConf.set("p6", "v6"); os = getFileSystem().create(new Path(appPath, "job2.xml")); jConf.writeXml(os); os.close(); Configuration conf = new XConfiguration(); assertEquals(0, conf.size()); JavaActionExecutor .parseJobXmlAndConfiguration(createContext( "<configuration>" + "<property><name>v3</name><value>v3a</value></property>" + "<property><name>v4</name><value>v4a</value></property>" + "</configuration>", null), xml, appPath, conf); assertEquals(7, conf.size()); assertEquals("v1a", conf.get("p1")); assertEquals("v2", conf.get("p2")); assertEquals("v3a", conf.get("p3")); assertEquals("v4a", conf.get("p4")); assertEquals("v5", conf.get("p5")); assertEquals("v6", conf.get("p6")); assertEquals("test", conf.get("user")); }
From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java
License:Apache License
public void testJobXmlAndNonDefaultNamenode() throws Exception { // By default the job.xml file is taken from the workflow application // namenode, regadless the namenode specified for the action. To specify // a job.xml on another namenode use a fully qualified file path. Path appPath = new Path(getFsTestCaseDir(), "app"); getFileSystem().mkdirs(appPath);/*w w w . j ava 2s .c om*/ Path jobXmlAbsolutePath = new Path(getFsTestCaseDir().toUri().getPath(), "jobxmlpath/job.xml"); assertTrue(jobXmlAbsolutePath.isAbsolute() && jobXmlAbsolutePath.toUri().getAuthority() == null); Path jobXmlAbsolutePath2 = new Path(getFsTestCaseDir().toUri().getPath(), "jobxmlpath/job3.xml"); assertTrue(jobXmlAbsolutePath2.isAbsolute() && jobXmlAbsolutePath2.toUri().getAuthority() == null); Path jobXmlQualifiedPath = new Path(getFs2TestCaseDir(), "jobxmlpath/job4.xml"); assertTrue(jobXmlQualifiedPath.toUri().getAuthority() != null); // Use non-default name node (second filesystem) and three job-xml configurations: // 1. Absolute (but not fully qualified) path located in the first filesystem // 2. Without path (fist filesystem) // 3. Absolute (but not fully qualified) path located in the both filesystems // (first should be used) // 4. Fully qualified path located in the second filesystem String str = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNode2Uri() + "</name-node>" + "<job-xml>" + jobXmlAbsolutePath.toString() + "</job-xml>" + "<job-xml>job2.xml</job-xml>" + "<job-xml>" + jobXmlAbsolutePath2.toString() + "</job-xml>" + "<job-xml>" + jobXmlQualifiedPath.toString() + "</job-xml>" + "<configuration>" + "<property><name>p1</name><value>v1a</value></property>" + "<property><name>p2</name><value>v2</value></property>" + "</configuration>" + "</java>"; Element xml = XmlUtils.parseXml(str); XConfiguration jConf = new XConfiguration(); jConf.set("p1", "v1b"); jConf.set("p3", "v3a"); OutputStream os = getFileSystem().create(jobXmlAbsolutePath); jConf.writeXml(os); os.close(); jConf = new XConfiguration(); jConf.set("p4", "v4"); jConf.set("p3", "v3b"); os = getFileSystem().create(new Path(appPath, "job2.xml")); jConf.writeXml(os); os.close(); // This configuration is expected to be used jConf = new XConfiguration(); jConf.set("p5", "v5a"); jConf.set("p6", "v6a"); os = getFileSystem().create(jobXmlAbsolutePath2); jConf.writeXml(os); os.close(); // This configuration is expected to be ignored jConf = new XConfiguration(); jConf.set("p5", "v5b"); jConf.set("p6", "v6b"); os = getFileSystem2().create(new Path(jobXmlAbsolutePath2.toUri().getPath())); jConf.writeXml(os); os.close(); jConf = new XConfiguration(); jConf.set("p7", "v7a"); jConf.set("p8", "v8a"); os = getFileSystem2().create(jobXmlQualifiedPath); jConf.writeXml(os); os.close(); Context context = createContext("<java/>", null); Configuration conf = new JavaActionExecutor().createBaseHadoopConf(context, xml); int confSize0 = conf.size(); JavaActionExecutor.parseJobXmlAndConfiguration(context, xml, appPath, conf); assertEquals(confSize0 + 8, conf.size()); assertEquals("v1a", conf.get("p1")); assertEquals("v2", conf.get("p2")); assertEquals("v3b", conf.get("p3")); assertEquals("v4", conf.get("p4")); assertEquals("v5a", conf.get("p5")); assertEquals("v6a", conf.get("p6")); assertEquals("v7a", conf.get("p7")); assertEquals("v8a", conf.get("p8")); }
From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java
License:Apache License
public void testDefaultConfigurationInLauncher() throws Exception { JavaActionExecutor ae = new JavaActionExecutor(); Element actionXmlWithConfiguration = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>" + "<property><name>oozie.launcher.a</name><value>AA</value></property>" + "<property><name>b</name><value>BB</value></property>" + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>"); Element actionXmlWithoutConfiguration = XmlUtils .parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<main-class>MAIN-CLASS</main-class>" + "</java>"); Configuration conf = new Configuration(false); Assert.assertEquals(0, conf.size()); conf.set("mapred.job.tracker", getJobTrackerUri()); ae.setupLauncherConf(conf, actionXmlWithConfiguration, null, null); assertEquals(getJobTrackerUri(), conf.get("mapred.job.tracker")); assertEquals("AA", conf.get("oozie.launcher.a")); assertEquals("AA", conf.get("a")); assertEquals("action.barbar", conf.get("oozie.launcher.action.foofoo")); assertEquals("action.barbar", conf.get("action.foofoo")); assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); if (conf.size() == 7) { assertEquals(getJobTrackerUri(), conf.get("mapreduce.jobtracker.address")); } else {/*from w w w . j ava 2 s . c om*/ assertEquals(6, conf.size()); } conf = new Configuration(false); Assert.assertEquals(0, conf.size()); conf.set("mapred.job.tracker", getJobTrackerUri()); ae.setupLauncherConf(conf, actionXmlWithoutConfiguration, null, null); assertEquals(getJobTrackerUri(), conf.get("mapred.job.tracker")); assertEquals("action.barbar", conf.get("oozie.launcher.action.foofoo")); assertEquals("action.barbar", conf.get("action.foofoo")); assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); if (conf.size() == 5) { assertEquals(getJobTrackerUri(), conf.get("mapreduce.jobtracker.address")); } else { assertEquals(4, conf.size()); } }
From source file:org.apache.oozie.command.coord.CoordUpdateXCommand.java
License:Apache License
public CoordUpdateXCommand(boolean dryrun, Configuration conf, String jobId) { super(dryrun, conf); this.jobId = jobId; isConfChange = conf.size() == 0 ? false : true; }
From source file:org.apache.oozie.command.coord.CoordUpdateXCommand.java
License:Apache License
public CoordUpdateXCommand(boolean dryrun, Configuration conf, String jobId, boolean showDiff) { super(dryrun, conf); this.jobId = jobId; this.showDiff = showDiff; isConfChange = conf.size() == 0 ? false : true; }
From source file:org.apache.oozie.command.wf.SubmitMRCommand.java
License:Open Source License
private Element generateMRSection(Configuration conf, Namespace ns) { Element mapreduce = new Element("map-reduce", ns); Element jt = new Element("job-tracker", ns); jt.addContent(conf.get(XOozieClient.JT)); mapreduce.addContent(jt);//from w w w .j a v a 2s. co m Element nn = new Element("name-node", ns); nn.addContent(conf.get(XOozieClient.NN)); mapreduce.addContent(nn); if (conf.size() > MANDATORY_OOZIE_CONFS.size()) { // excluding JT, NN, // LIBPATH // configuration section Element configuration = generateConfigurationSection(conf, ns); if (configuration != null) { mapreduce.addContent(configuration); } // file section addFileSection(mapreduce, conf, ns); // archive section addArchiveSection(mapreduce, conf, ns); } return mapreduce; }
From source file:org.apache.oozie.command.wf.SubmitMRXCommand.java
License:Apache License
@Override protected Element generateSection(Configuration conf, Namespace ns) { Element mapreduce = new Element("map-reduce", ns); Element jt = new Element("job-tracker", ns); String newJTVal = conf.get(DEPRECATE_MAP.get(XOozieClient.JT)); jt.addContent(newJTVal != null ? newJTVal : (conf.get(XOozieClient.JT))); mapreduce.addContent(jt);//from w w w. ja v a 2 s .co m Element nn = new Element("name-node", ns); String newNNVal = conf.get(DEPRECATE_MAP.get(XOozieClient.NN)); nn.addContent(newNNVal != null ? newNNVal : (conf.get(XOozieClient.NN))); mapreduce.addContent(nn); if (conf.size() > MANDATORY_OOZIE_CONFS.size()) { // excluding JT, NN, // LIBPATH // configuration section Element configuration = generateConfigurationSection(conf, ns); if (configuration != null) { mapreduce.addContent(configuration); } // file section addFileSection(mapreduce, conf, ns); // archive section addArchiveSection(mapreduce, conf, ns); } return mapreduce; }
From source file:org.apache.oozie.test.ZKXTestCase.java
License:Apache License
protected void setUp(Configuration conf) throws Exception { super.setUp(); Services services = new Services(); if (conf != null && conf.size() > 0) { for (Iterator<Entry<String, String>> itr = (Iterator<Entry<String, String>>) conf.iterator(); itr .hasNext();) {//from w w w.java2 s . c om Entry<String, String> entry = itr.next(); services.getConf().set(entry.getKey(), entry.getValue()); } } services.init(); setUpZK(); }
From source file:org.apache.oozie.util.Instrumentation.java
License:Apache License
/** * Return the current system configuration as a Map<String,String>. * * @return the current system configuration as a Map<String,String>. *//*from w w w . j a va 2 s. c om*/ public Map<String, String> getConfiguration() { final Configuration maskedConf = Services.get().get(ConfigurationService.class).getMaskedConfiguration(); return new Map<String, String>() { public int size() { return maskedConf.size(); } public boolean isEmpty() { return maskedConf.size() == 0; } public boolean containsKey(Object o) { return maskedConf.get((String) o) != null; } public boolean containsValue(Object o) { throw new UnsupportedOperationException(); } public String get(Object o) { return maskedConf.get((String) o); } public String put(String s, String s1) { throw new UnsupportedOperationException(); } public String remove(Object o) { throw new UnsupportedOperationException(); } public void putAll(Map<? extends String, ? extends String> map) { throw new UnsupportedOperationException(); } public void clear() { throw new UnsupportedOperationException(); } public Set<String> keySet() { Set<String> set = new LinkedHashSet<String>(); for (Entry<String, String> entry : maskedConf) { set.add(entry.getKey()); } return set; } public Collection<String> values() { Set<String> set = new LinkedHashSet<String>(); for (Entry<String, String> entry : maskedConf) { set.add(entry.getValue()); } return set; } public Set<Entry<String, String>> entrySet() { Set<Entry<String, String>> set = new LinkedHashSet<Entry<String, String>>(); for (Entry<String, String> entry : maskedConf) { set.add(entry); } return set; } }; }