Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.oozie.service; import java.io.File; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.InputStream; import java.io.StringWriter; import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.Properties; import org.apache.commons.logging.LogFactory; import org.apache.oozie.client.rest.RestConstants; import org.apache.oozie.test.EmbeddedServletContainer; import org.apache.oozie.test.ZKXTestCase; import org.apache.oozie.util.DateUtils; import org.apache.oozie.util.XLogErrorStreamer; import org.apache.oozie.util.XLogFilter; import org.apache.oozie.util.XLogStreamer; import org.apache.oozie.util.ZKUtils; public class TestZKXLogStreamingService extends ZKXTestCase { @Override protected void setUp() throws Exception { super.setUp(); } @Override protected void tearDown() throws Exception { super.tearDown(); } public void testRegisterUnregister() throws Exception { assertEquals(0, ZKUtils.getUsers().size()); ZKXLogStreamingService zkxlss = new ZKXLogStreamingService(); try { zkxlss.init(Services.get()); assertEquals(1, ZKUtils.getUsers().size()); assertEquals(zkxlss, ZKUtils.getUsers().iterator().next()); zkxlss.destroy(); assertEquals(0, ZKUtils.getUsers().size()); } finally { zkxlss.destroy(); } } public void testDisableLogOverWS() throws Exception { Properties props = new Properties(); // Test missing logfile props.setProperty("log4j.appender.oozie.File", ""); File propsFile = new File(getTestCaseConfDir(), "test-disable-log-over-ws-log4j.properties"); FileOutputStream fos = new FileOutputStream(propsFile); props.store(fos, ""); setSystemProperty(XLogService.LOG4J_FILE, propsFile.getName()); assertTrue(doStreamDisabledCheck()); // Test non-absolute path for logfile props.setProperty("log4j.appender.oozie.File", "oozie.log"); fos = new FileOutputStream(propsFile); props.store(fos, ""); assertTrue(doStreamDisabledCheck()); // Test missing appender class props.setProperty("log4j.appender.oozie.File", "${oozie.log.dir}/oozie.log"); props.setProperty("log4j.appender.oozie", ""); fos = new FileOutputStream(propsFile); props.store(fos, ""); assertTrue(doStreamDisabledCheck()); // Test appender class not DailyRollingFileAppender or RollingFileAppender props.setProperty("log4j.appender.oozie", "org.blah.blah"); fos = new FileOutputStream(propsFile); props.store(fos, ""); assertTrue(doStreamDisabledCheck()); // Test DailyRollingFileAppender but missing DatePattern props.setProperty("log4j.appender.oozie", "org.apache.log4j.DailyRollingFileAppender"); props.setProperty("log4j.appender.oozie.DatePattern", ""); fos = new FileOutputStream(propsFile); props.store(fos, ""); assertTrue(doStreamDisabledCheck()); // Test DailyRollingFileAppender but DatePattern that doesn't end with 'HH' or 'dd' props.setProperty("log4j.appender.oozie.DatePattern", "'.'yyyy-MM"); fos = new FileOutputStream(propsFile); props.store(fos, ""); assertTrue(doStreamDisabledCheck()); // Test DailyRollingFileAppender with everything correct (dd) props.setProperty("log4j.appender.oozie.DatePattern", "'.'yyyy-MM-dd"); fos = new FileOutputStream(propsFile); props.store(fos, ""); assertFalse(doStreamDisabledCheck()); // Test DailyRollingFileAppender with everything correct (HH) props.setProperty("log4j.appender.oozie.DatePattern", "'.'yyyy-MM-dd-HH"); fos = new FileOutputStream(propsFile); props.store(fos, ""); assertFalse(doStreamDisabledCheck()); // Test RollingFileAppender but missing FileNamePattern props.setProperty("log4j.appender.oozie", "org.apache.log4j.rolling.RollingFileAppender"); props.setProperty("log4j.appender.oozie.RollingPolicy.FileNamePattern", ""); fos = new FileOutputStream(propsFile); props.store(fos, ""); assertTrue(doStreamDisabledCheck()); // Test RollingFileAppender but FileNamePattern with incorrect ending props.setProperty("log4j.appender.oozie.RollingPolicy.FileNamePattern", "${oozie.log.dir}/oozie.log-blah"); fos = new FileOutputStream(propsFile); props.store(fos, ""); assertTrue(doStreamDisabledCheck()); // Test RollingFileAppender but FileNamePattern with incorrect beginning props.setProperty("log4j.appender.oozie.RollingPolicy.FileNamePattern", "${oozie.log.dir}/blah.log-%d{yyyy-MM-dd-HH}"); fos = new FileOutputStream(propsFile); props.store(fos, ""); assertTrue(doStreamDisabledCheck()); // Test RollingFileAppender with everything correct props.setProperty("log4j.appender.oozie.RollingPolicy.FileNamePattern", "${oozie.log.dir}/oozie.log-%d{yyyy-MM-dd-HH}"); fos = new FileOutputStream(propsFile); props.store(fos, ""); assertFalse(doStreamDisabledCheck()); // Test RollingFileAppender with everything correct (gz) props.setProperty("log4j.appender.oozie.RollingPolicy.FileNamePattern", "${oozie.log.dir}/oozie.log-%d{yyyy-MM-dd-HH}.gz"); fos = new FileOutputStream(propsFile); props.store(fos, ""); assertFalse(doStreamDisabledCheck()); } public void testNoDashInConversionPattern() throws Exception { XLogFilter.reset(); XLogFilter.defineParameter("USER"); XLogFilter.defineParameter("GROUP"); XLogFilter.defineParameter("TOKEN"); XLogFilter.defineParameter("APP"); XLogFilter.defineParameter("JOB"); XLogFilter.defineParameter("ACTION"); XLogFilter xf = new XLogFilter(); xf.setParameter("USER", "oozie"); xf.setLogLevel("DEBUG|INFO"); // Previously, a dash ("-") was always required somewhere in a line in order for that line to pass the filter; this test // checks that this condition is no longer required for log streaming to work File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties"); ClassLoader cl = Thread.currentThread().getContextClassLoader(); InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties"); Properties log4jProps = new Properties(); log4jProps.load(is); // prevent conflicts with other tests by changing the log file location log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log"); log4jProps.store(new FileOutputStream(log4jFile), ""); setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName()); assertFalse(doStreamDisabledCheck()); LogFactory.getLog("a").info( "2009-06-24 02:43:14,505 INFO _L1_:317 - SERVER[foo] USER[oozie] GROUP[oozie] TOKEN[-] APP[-] " + "JOB[-] ACTION[-] Released Lock"); LogFactory.getLog("a") .info("2009-06-24 02:43:14,505 INFO _L2_:317 - SERVER[foo] USER[blah] GROUP[oozie] TOKEN[-] APP[-] " + "JOB[-] ACTION[-] Released Lock"); LogFactory.getLog("a") .info("2009-06-24 02:43:14,505 INFO _L3_:317 SERVER[foo] USER[oozie] GROUP[oozie] TOKEN[-] APP[-] " + "JOB[-] ACTION[-] Released Lock"); LogFactory.getLog("a") .info("2009-06-24 02:43:14,505 INFO _L4_:317 SERVER[foo] USER[blah] GROUP[oozie] TOKEN[-] APP[-] " + "JOB[-] ACTION[-] Released Lock"); String out = doStreamLog(xf); String outArr[] = out.split("\n"); // Lines 2 and 4 are filtered out because they have the wrong user assertEquals(2, outArr.length); assertTrue(outArr[0].contains("_L1_")); assertFalse(out.contains("_L2_")); assertTrue(outArr[1].contains("_L3_")); assertFalse(out.contains("_L4_")); } private boolean doStreamDisabledCheck() throws Exception { Services.get().get(XLogService.class).init(Services.get()); return doStreamLog(new XLogFilter()).equals("Log streaming disabled!!"); } protected String doStreamLog(XLogFilter xf) throws Exception { return doStreamLog(xf, new HashMap<String, String[]>()); } protected String doStreamLog(XLogFilter xf, Date startTime, Date endTime) throws Exception { return doStreamLog(xf, new HashMap<String, String[]>(), false, startTime, endTime); } protected String doStreamErrorLog(XLogFilter xf) throws Exception { return doStreamLog(xf, new HashMap<String, String[]>(), true); } private String doStreamErrorLog(XLogFilter xf, Date startDate, Date endDate) throws Exception { return doStreamLog(xf, new HashMap<String, String[]>(), true, startDate, startDate); } protected String doStreamLog(XLogFilter xf, Map<String, String[]> param) throws Exception { return doStreamLog(xf, param, false); } protected String doStreamLog(XLogFilter xf, Map<String, String[]> param, boolean isErrorLog) throws Exception { return doStreamLog(xf, param, isErrorLog, null, null); } protected String doStreamLog(XLogFilter xf, Map<String, String[]> param, boolean isErrorLog, Date startTime, Date endTime) throws Exception { StringWriter w = new StringWriter(); ZKXLogStreamingService zkxlss = new ZKXLogStreamingService(); try { Services services = Services.get(); services.setService(ZKJobsConcurrencyService.class); zkxlss.init(services); sleep(1000); // Sleep to allow ZKUtils ServiceCache to update if (isErrorLog) { zkxlss.streamLog(new XLogErrorStreamer(xf, param), startTime, endTime, w); } else { zkxlss.streamLog(new XLogStreamer(xf, param), startTime, endTime, w); } } finally { zkxlss.destroy(); } String wStr = w.toString(); System.out.println("\ndoStreamLog:\n" + wStr + "\n"); return wStr; } public void testStreamingWithMultipleOozieServers() throws Exception { XLogFilter.reset(); XLogFilter.defineParameter("USER"); XLogFilter.defineParameter("GROUP"); XLogFilter.defineParameter("TOKEN"); XLogFilter.defineParameter("APP"); XLogFilter.defineParameter("JOB"); XLogFilter.defineParameter("ACTION"); XLogFilter xf = new XLogFilter(); xf.setParameter("JOB", "0000003-130610102426873-oozie-rkan-W"); xf.setLogLevel("WARN|INFO"); File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties"); ClassLoader cl = Thread.currentThread().getContextClassLoader(); InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties"); Properties log4jProps = new Properties(); log4jProps.load(is); // prevent conflicts with other tests by changing the log file location log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log"); log4jProps.store(new FileOutputStream(log4jFile), ""); setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName()); assertFalse(doStreamDisabledCheck()); File logFile = new File(Services.get().get(XLogService.class).getOozieLogPath(), Services.get().get(XLogService.class).getOozieLogName()); logFile.getParentFile().mkdirs(); FileWriter logWriter = new FileWriter(logFile); // local logs logWriter.append( "2013-06-10 10:25:44,008 WARN HiveActionExecutor:542 SERVER[foo] USER[rkanter] GROUP[-] TOKEN[] " + "APP[hive-wf] JOB[0000003-130610102426873-oozie-rkan-W] ACTION[0000003-130610102426873-oozie-rkan-W@hive-node] " + "credentials is null for the action _L3_") .append("\n") .append("2013-06-10 10:26:10,008 INFO HiveActionExecutor:539 SERVER[foo] USER[rkanter] GROUP[-] TOKEN[] " + "APP[hive-wf] JOB[0000003-130610102426873-oozie-rkan-W] ACTION[0000003-130610102426873-oozie-rkan-W@hive-node] " + "action completed, external ID [job_201306101021_0005] _L4_") .append("\n") .append("2013-06-10 10:26:10,341 WARN ActionStartXCommand:542 USER[rkanter] GROUP[-] TOKEN[] " + "APP[hive-wf] JOB[0000003-130610102426873-oozie-rkan-W] ACTION[0000003-130610102426873-oozie-rkan-W@end] " + "[***0000003-130610102426873-oozie-rkan-W@end***]Action updated in DB! _L6_") .append("\n"); logWriter.close(); // logs to be returned by another "Oozie server" DummyLogStreamingServlet.logs = "2013-06-10 10:25:43,575 WARN ActionStartXCommand:542 SERVER[foo] USER[rkanter] GROUP[-] TOKEN[] APP[hive-wf] " + "JOB[0000003-130610102426873-oozie-rkan-W] ACTION[0000003-130610102426873-oozie-rkan-W@:start:] " + "[***0000003-130610102426873-oozie-rkan-W@:start:***]Action status=DONE _L1_" + "\n" + "2013-06-10 10:25:43,575 WARN ActionStartXCommand:542 SERVER[foo] USER[rkanter] GROUP[-] TOKEN[] APP[hive-wf] " + "JOB[0000003-130610102426873-oozie-rkan-W] ACTION[0000003-130610102426873-oozie-rkan-W@:start:] " + "[***0000003-130610102426873-oozie-rkan-W@:start:***]Action updated in DB! _L2_" + "\n" + "2013-06-10 10:26:10,148 INFO HiveActionExecutor:539 SERVER[foo] USER[rkanter] GROUP[-] TOKEN[] APP[hive-wf] " + "JOB[0000003-130610102426873-oozie-rkan-W] ACTION[0000003-130610102426873-oozie-rkan-W@hive-node] action produced" + " output _L5_" + "\n" // a multiline message with a stack trace + "2013-06-10 10:26:30,202 WARN ActionStartXCommand:542 - SERVER[foo] USER[rkanter] GROUP[-] TOKEN[] APP[hive-wf] " + "JOB[0000003-130610102426873-oozie-rkan-W] ACTION[0000003-130610102426873-oozie-rkan-W@hive-node] Error starting " + "action [hive-node]. ErrorType [TRANSIENT], ErrorCode [JA009], Message [JA009: java.io.IOException: Unknown " + "protocol to name node: org.apache.hadoop.mapred.JobSubmissionProtocol _L7_\n" + " at org.apache.hadoop.hdfs.server.namenode.NameNode.getProtocolVersion(NameNode.java:156) _L8_\n" + " at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)_L9_\n" + " at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1190) _L10_\n" + " at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1426) _L11_\n" + "] _L12_\n" + "org.apache.oozie.action.ActionExecutorException: JA009: java.io.IOException: Unknown protocol to name node: " + "org.apache.hadoop.mapred.JobSubmissionProtocol _L13_\n" + " at org.apache.hadoop.hdfs.server.namenode.NameNode.getProtocolVersion(NameNode.java:156) _L14_\n" + " at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) _L15_\n" + " at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) _L16_\n"; String out = doStreamLog(xf); String[] outArr = out.split("\n"); assertEquals(3, outArr.length); assertFalse(out.contains("_L1_")); assertFalse(out.contains("_L2_")); assertTrue(outArr[0].contains("_L3_")); assertTrue(outArr[1].contains("_L4_")); assertFalse(out.contains("_L5_")); assertTrue(outArr[2].contains("_L6_")); // We'll use a DummyZKOozie to create an entry in ZK and then set its url to an (unrelated) servlet that will simply return // some log messages DummyZKOozie dummyOozie = null; EmbeddedServletContainer container = new EmbeddedServletContainer("oozie"); container.addServletEndpoint("/other-oozie-server/*", DummyLogStreamingServlet.class); try { container.start(); dummyOozie = new DummyZKOozie("9876", container.getServletURL("/other-oozie-server/*")); DummyLogStreamingServlet.lastQueryString = null; out = doStreamLog(xf); outArr = out.split("\n"); assertEquals(16, outArr.length); assertTrue(outArr[0].contains("_L1_")); assertTrue(outArr[1].contains("_L2_")); assertTrue(outArr[2].contains("_L3_")); assertTrue(outArr[3].contains("_L4_")); assertTrue(outArr[4].contains("_L5_")); assertTrue(outArr[5].contains("_L6_")); assertTrue(outArr[6].contains("_L7_")); assertTrue(outArr[7].contains("_L8_")); assertTrue(outArr[8].contains("_L9_")); assertTrue(outArr[9].contains("_L10_")); assertTrue(outArr[10].contains("_L11_")); assertTrue(outArr[11].contains("_L12_")); assertTrue(outArr[12].contains("_L13_")); assertTrue(outArr[13].contains("_L14_")); assertTrue(outArr[14].contains("_L15_")); assertTrue(outArr[15].contains("_L16_")); assertEquals("show=log&allservers=false", DummyLogStreamingServlet.lastQueryString); // If we stop the container but leave the DummyZKOozie running, it will simulate if that server is down but still has // info in ZK; we should be able to get the logs from other servers (in this case, this server) and a message about // which servers it couldn't reach container.stop(); out = doStreamLog(xf); outArr = out.split("\n"); assertEquals(6, outArr.length); assertTrue(outArr[0].startsWith("Unable")); assertEquals("9876", outArr[1].trim()); assertEquals("", outArr[2]); assertFalse(out.contains("_L1_")); assertFalse(out.contains("_L2_")); assertTrue(outArr[3].contains("_L3_")); assertTrue(outArr[4].contains("_L4_")); assertFalse(out.contains("_L5_")); assertTrue(outArr[5].contains("_L6_")); } finally { if (dummyOozie != null) { dummyOozie.teardown(); } container.stop(); } } public void testStreamingWithMultipleOozieServers_coordActionList() throws Exception { XLogFilter.reset(); File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties"); ClassLoader cl = Thread.currentThread().getContextClassLoader(); InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties"); Properties log4jProps = new Properties(); log4jProps.load(is); // prevent conflicts with other tests by changing the log file location log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log"); log4jProps.store(new FileOutputStream(log4jFile), ""); setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName()); Services.get().get(XLogService.class).init(Services.get()); File logFile = new File(Services.get().get(XLogService.class).getOozieLogPath(), Services.get().get(XLogService.class).getOozieLogName()); logFile.getParentFile().mkdirs(); FileWriter logWriter = new FileWriter(logFile); // local logs StringBuffer bf = new StringBuffer(); bf.append( "2014-02-06 00:26:56,126 DEBUG CoordActionInputCheckXCommand:545 [pool-2-thread-26] - USER[-] GROUP[-] " + "TOKEN[-] APP[-] JOB[0000003-140205233038063-oozie-oozi-C] ACTION[0000003-140205233038063-oozie-oozi-C@1] " + "checking for the file ~:8020/user/purushah/examples/input-data/rawLogs/2010/01/01/01/00/_SUCCESS\n") .append("2014-02-06 00:26:56,150 INFO CoordActionInputCheckXCommand:539 [pool-2-thread-26] - USER[-] GROUP[-] " + "TOKEN[-] APP[-] JOB[0000003-140205233038063-oozie-oozi-C] ACTION[0000003-140205233038063-oozie-oozi-C@1] " + "[0000003-140205233038063-oozie-oozi-C@1]::ActionInputCheck:: File::8020/user/purushah/examples/input-data/" + "rawLogs/2010/01/01/01/00/_SUCCESS, Exists? :false" + "Action updated in DB! _L1_") .append("\n") .append("2014-02-06 00:27:56,126 DEBUG CoordActionInputCheckXCommand:545 [pool-2-thread-26] - USER[-] GROUP[-] " + "TOKEN[-] APP[-] JOB[0000003-140205233038063-oozie-oozi-C] ACTION[0000003-140205233038063-oozie-oozi-C@2] " + "checking for the file ~:8020/user/purushah/examples/input-data/rawLogs/2010/01/01/01/00/_SUCCESS\n") .append("2014-02-06 00:27:56,150 INFO CoordActionInputCheckXCommand:539 [pool-2-thread-26] - USER[-] GROUP[-] " + "TOKEN[-] APP[-] JOB[0000003-140205233038063-oozie-oozi-C] ACTION[0000003-140205233038063-oozie-oozi-C@2] " + "[0000003-140205233038063-oozie-oozi-C@2]::ActionInputCheck:: File::8020/user/purushah/examples/input-data/" + "rawLogs/2010/01/01/01/00/_SUCCESS, Exists? :false" + "Action updated in DB! _L2_") .append("\n"); logWriter.append(bf); logWriter.close(); XLogFilter.reset(); XLogFilter.defineParameter("USER"); XLogFilter.defineParameter("GROUP"); XLogFilter.defineParameter("TOKEN"); XLogFilter.defineParameter("APP"); XLogFilter.defineParameter("JOB"); XLogFilter.defineParameter("ACTION"); XLogFilter xf = new XLogFilter(); xf.setLogLevel("DEBUG|INFO"); xf.setParameter("USER", ".*"); xf.setParameter("GROUP", ".*"); xf.setParameter("TOKEN", ".*"); xf.setParameter("APP", ".*"); xf.setParameter("JOB", "0000003-140205233038063-oozie-oozi-C"); xf.setParameter(DagXLogInfoService.ACTION, "0000003-140205233038063-oozie-oozi-C@1"); String out = doStreamLog(xf); String[] outArr = out.split("\n"); assertEquals(2, outArr.length); assertTrue(out.contains("_L1_")); assertFalse(out.contains("_L2_")); // We'll use a DummyZKOozie to create an entry in ZK and then set its // url to an (unrelated) servlet that will simply return // some log messages DummyZKOozie dummyOozie = null; EmbeddedServletContainer container = new EmbeddedServletContainer("oozie"); container.addServletEndpoint("/other-oozie-server/*", DummyLogStreamingServlet.class); try { container.start(); dummyOozie = new DummyZKOozie("9876", container.getServletURL("/other-oozie-server/*")); DummyLogStreamingServlet.logs = ""; DummyLogStreamingServlet.lastQueryString = null; Map<String, String[]> param = new HashMap<String, String[]>(); param.put(RestConstants.JOB_COORD_RANGE_TYPE_PARAM, new String[] { RestConstants.JOB_LOG_ACTION }); param.put(RestConstants.JOB_COORD_SCOPE_PARAM, new String[] { "1" }); out = doStreamLog(xf, param); assertTrue(DummyLogStreamingServlet.lastQueryString.contains("show=log&allservers=false")); assertTrue(DummyLogStreamingServlet.lastQueryString.contains("type=" + RestConstants.JOB_LOG_ACTION)); assertTrue( DummyLogStreamingServlet.lastQueryString.contains(RestConstants.JOB_COORD_SCOPE_PARAM + "=1")); param.clear(); param.put(RestConstants.JOB_COORD_RANGE_TYPE_PARAM, new String[] { RestConstants.JOB_LOG_ACTION }); param.put(RestConstants.JOB_COORD_SCOPE_PARAM, new String[] { "1-4,5" }); out = doStreamLog(xf, param); assertTrue(DummyLogStreamingServlet.lastQueryString.contains("show=log&allservers=false")); assertTrue(DummyLogStreamingServlet.lastQueryString.contains("type=" + RestConstants.JOB_LOG_ACTION)); assertTrue(DummyLogStreamingServlet.lastQueryString .contains(RestConstants.JOB_COORD_SCOPE_PARAM + "=1-4,5")); param.clear(); Date endDate = new Date(); Date createdDate = new Date(endDate.getTime() / 2); String date = DateUtils.formatDateOozieTZ(createdDate) + "::" + DateUtils.formatDateOozieTZ(endDate); param.put(RestConstants.JOB_COORD_RANGE_TYPE_PARAM, new String[] { RestConstants.JOB_LOG_DATE }); param.put(RestConstants.JOB_COORD_SCOPE_PARAM, new String[] { date }); out = doStreamLog(xf, param); assertTrue(DummyLogStreamingServlet.lastQueryString.contains("show=log&allservers=false")); assertTrue(DummyLogStreamingServlet.lastQueryString.contains("type=" + RestConstants.JOB_LOG_DATE)); assertTrue(DummyLogStreamingServlet.lastQueryString .contains(RestConstants.JOB_COORD_SCOPE_PARAM + "=" + date)); container.stop(); } finally { if (dummyOozie != null) { dummyOozie.teardown(); } container.stop(); } } public void testStreamingWithMultipleOozieServers_errorLog() throws Exception { XLogFilter.reset(); File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties"); ClassLoader cl = Thread.currentThread().getContextClassLoader(); InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties"); Properties log4jProps = new Properties(); log4jProps.load(is); // prevent conflicts with other tests by changing the log file location log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log"); log4jProps.setProperty("log4j.appender.oozieError.File", getTestCaseDir() + "/oozie-error.log"); log4jProps.store(new FileOutputStream(log4jFile), ""); setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName()); Services.get().get(XLogService.class).init(Services.get()); File logFile = new File(Services.get().get(XLogService.class).getOozieErrorLogPath(), Services.get().get(XLogService.class).getOozieErrorLogName()); logFile.getParentFile().mkdirs(); FileWriter logWriter = new FileWriter(logFile); // local logs StringBuffer bf = new StringBuffer(); bf.append( "2014-02-06 00:26:56,126 WARN CoordActionInputCheckXCommand:545 [pool-2-thread-26] - USER[-] GROUP[-] " + "TOKEN[-] APP[-] JOB[0000003-140205233038063-oozie-oozi-C] ACTION[0000003-140205233038063-oozie-oozi-C@1] " + "checking for the file ~:8020/user/purushah/examples/input-data/rawLogs/2010/01/01/01/00/_SUCCESS\n") .append("2014-02-06 00:26:56,150 WARN CoordActionInputCheckXCommand:539 [pool-2-thread-26] - USER[-] GROUP[-] " + "TOKEN[-] APP[-] JOB[0000003-140205233038063-oozie-oozi-C] ACTION[0000003-140205233038063-oozie-oozi-C@1] " + "[0000003-140205233038063-oozie-oozi-C@1]::ActionInputCheck::File::8020/user/purushah/examples/input-data/" + "rawLogs/2010/01/01/01/00/_SUCCESS, Exists? :false" + "Action updated in DB! _L1_") .append("\n") .append("2014-02-06 00:27:56,126 WARN CoordActionInputCheckXCommand:545 [pool-2-thread-26] - USER[-] GROUP[-] " + "TOKEN[-] APP[-] JOB[0000003-140205233038063-oozie-oozi-C] ACTION[0000003-140205233038063-oozie-oozi-C@2] " + "checking for the file ~:8020/user/purushah/examples/input-data/rawLogs/2010/01/01/01/00/_SUCCESS\n") .append("2014-02-06 00:27:56,150 WARN CoordActionInputCheckXCommand:539 [pool-2-thread-26] - USER[-] GROUP[-] " + "TOKEN[-] APP[-] JOB[0000003-140205233038063-oozie-oozi-C] ACTION[0000003-140205233038063-oozie-oozi-C@2] " + "[0000003-140205233038063-oozie-oozi-C@2]::ActionInputCheck::File::8020/user/purushah/examples/input-data/" + "rawLogs/2010/01/01/01/00/_SUCCESS, Exists? :false" + "Action updated in DB! _L2_") .append("\n"); logWriter.append(bf); logWriter.close(); XLogFilter.reset(); XLogFilter.defineParameter("USER"); XLogFilter.defineParameter("GROUP"); XLogFilter.defineParameter("TOKEN"); XLogFilter.defineParameter("APP"); XLogFilter.defineParameter("JOB"); XLogFilter.defineParameter("ACTION"); XLogFilter xf = new XLogFilter(); xf.setParameter("USER", ".*"); xf.setParameter("GROUP", ".*"); xf.setParameter("TOKEN", ".*"); xf.setParameter("APP", ".*"); xf.setParameter("JOB", "0000003-140205233038063-oozie-oozi-C"); xf.setParameter(DagXLogInfoService.ACTION, "0000003-140205233038063-oozie-oozi-C@1"); String out = doStreamErrorLog(xf); String[] outArr = out.split("\n"); assertEquals(2, outArr.length); assertTrue(out.contains("_L1_")); assertFalse(out.contains("_L2_")); // We'll use a DummyZKOozie to create an entry in ZK and then set its // url to an (unrelated) servlet that will simply return // some log messages DummyZKOozie dummyOozie = null; EmbeddedServletContainer container = new EmbeddedServletContainer("oozie"); container.addServletEndpoint("/other-oozie-server/*", DummyLogStreamingServlet.class); try { container.start(); dummyOozie = new DummyZKOozie("9876", container.getServletURL("/other-oozie-server/*")); StringBuffer newLog = new StringBuffer(); newLog.append( "2014-02-07 00:26:56,126 WARN CoordActionInputCheckXCommand:545 [pool-2-thread-26] - USER[-] GROUP[-] " + "TOKEN[-] APP[-] JOB[0000003-140205233038063-oozie-oozi-C] ACTION[0000003-140205233038063-oozie-oozi-C@1] " + "checking for the file ~:8020/user/purushah/examples/input-data/rawLogs/2010/01/01/01/00/_SUCCESS\n") .append("2014-02-07 00:26:56,150 WARN CoordActionInputCheckXCommand:539 [pool-2-thread-26] - USER[-] GROUP[-] " + "TOKEN[-] APP[-] JOB[0000003-140205233038063-oozie-oozi-C] ACTION[0000003-140205233038063-oozie-oozi-C@1] " + "[0000003-140205233038063-oozie-oozi-C@1]::ActionInputCheck::File::8020/user/purushah/examples/input-data/" + "rawLogs/2010/01/01/01/00/_SUCCESS, Exists? :false" + "Action updated in DB! _L3_") .append("\n"); DummyLogStreamingServlet.logs = newLog.toString(); out = doStreamErrorLog(xf); outArr = out.split("\n"); assertEquals(4, outArr.length); assertTrue(out.contains("_L1_")); assertTrue(out.contains("_L3_")); assertFalse(out.contains("_L2_")); container.stop(); } finally { if (dummyOozie != null) { dummyOozie.teardown(); } container.stop(); } } public void testTuncateLog() throws Exception { XLogFilter.reset(); File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties"); ClassLoader cl = Thread.currentThread().getContextClassLoader(); InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties"); Properties log4jProps = new Properties(); log4jProps.load(is); // prevent conflicts with other tests by changing the log file location log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log"); log4jProps.store(new FileOutputStream(log4jFile), ""); setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName()); assertFalse(doStreamDisabledCheck()); File logFile = new File(Services.get().get(XLogService.class).getOozieLogPath(), Services.get().get(XLogService.class).getOozieLogName()); logFile.getParentFile().mkdirs(); ConfigurationService.set(XLogFilter.MAX_SCAN_DURATION, "1"); Date startDate = new Date(); Date endDate = new Date(startDate.getTime() + 60 * 60 * 1000 * 15); String log = doStreamLog(new XLogFilter(), startDate, endDate); assertTrue(log.contains("Truncated logs to max log scan duration")); String logError = doStreamErrorLog(new XLogFilter(), startDate, endDate); assertFalse(logError.contains("Truncated logs to max log scan duration")); } }