Java tutorial
/* * Copyright 2013 LinkedIn Corp. All rights reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package com.linkedin.databus.core; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import com.linkedin.databus.core.TrailFilePositionSetter.FileFilter; import com.linkedin.databus.core.TrailFilePositionSetter.FilePositionResult; import com.linkedin.databus.core.TrailFilePositionSetter.FilePositionResult.Status; import com.linkedin.databus.core.util.InvalidConfigException; import com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder; import com.linkedin.databus2.test.TestUtil; public class TestTrailFilePositionSetter { public static final String MODULE = TestTrailFilePositionSetter.class.getName(); public static final Logger LOG = Logger.getLogger(MODULE); private static final String TRAIL_FILENAME_PREFIX = "x3"; private static final String[] _txnPattern = { "<transaction timestamp=\"2013-03-09:02:54:34.000000\">", " <dbupdate table=\"TASKMGR.TASKCTL_JOBS_1\" type=\"update\">", " <columns>", " <column name=\"JOB_ID\" key=\"true\">1621745</column>", " <column name=\"GG_MODI_TS\">2013-03-09:02:54:33.996072000</column>", " <column name=\"GG_STATUS\">o</column>", " </columns>", " <tokens>", " <token name=\"TK-XID\">42.8.2681282</token>", " <token name=\"TK-CSN\">${SCN}</token>", " </tokens>", " </dbupdate>", " <dbupdate table=\"TASKMGR.TASKCTL_JOBS_2\" type=\"update\">", " <columns>", " <column name=\"JOB_ID\" key=\"true\">1621745</column>", " <column name=\"GG_MODI_TS\">2013-03-09:02:54:33.996072000</column>", " <column name=\"GG_STATUS\">o</column>", " </columns>", " <tokens>", " <token name=\"TK-XID\">42.8.2681283</token>", " <token name=\"TK-CSN\">${SCN}</token>", " </tokens>", " </dbupdate>", "</transaction>" }; // currently 24 lines; if startLine > 0, probably want it less than this private static final String DONE_STRING = "done\n----------------------------------------------------------------------------\n"; private ArrayList<File> _dirsToDelete = new ArrayList<File>(100); private BufferedWriter createWriter(String dir, String file) throws IOException { BufferedWriter w = new BufferedWriter(new FileWriter(new File(dir + "/" + file))); return w; } private void createTrailFiles(String dir, String prefix, long numTxns, long numLinesPerFile, long numLinesPerNewline, String newlineChar, int startLine, long corruptedScn, String corruption, boolean addAlternateCorruption, String altCorruption) throws IOException { HashSet<Long> corruptedScns = new HashSet<Long>(1); corruptedScns.add(new Long(corruptedScn)); createTrailFiles(dir, prefix, numTxns, numLinesPerFile, numLinesPerNewline, newlineChar, startLine, corruptedScns, corruption, addAlternateCorruption, altCorruption); } private void createTrailFiles(String dir, String prefix, long numTxns, long numLinesPerFile, long numLinesPerNewline, String newlineChar, int startLine, Set<Long> corruptedScns, String corruption, boolean addAlternateCorruption, String altCorruption) throws IOException { createTrailFiles(dir, prefix, numTxns, numLinesPerFile, numLinesPerNewline, newlineChar, startLine, corruptedScns, corruption, addAlternateCorruption, altCorruption, _txnPattern, 1, 100); } private void createTrailFiles(String dir, String prefix, long numTxns, long numLinesPerFile, long numLinesPerNewline, String newlineChar, int startLine, String[] txnPattern, int numDbUpdatesWithSameScn, long startScn) throws IOException { createTrailFiles(dir, prefix, numTxns, numLinesPerFile, numLinesPerNewline, newlineChar, startLine, new HashSet<Long>(), "", false, "", txnPattern, numDbUpdatesWithSameScn, startScn); } private void createTrailFiles(String dir, String prefix, long numTxns, long numLinesPerFile, long numLinesPerNewline, String newlineChar, int startLine, Set<Long> corruptedScns, String corruption, boolean addAlternateCorruption, String altCorruption, String[] txnPattern, int numDbUpdatesWithSameScn, long currScn) throws IOException { long numFiles = ((numTxns * (txnPattern.length)) / numLinesPerFile) + 1; long numDigits = new Double(Math.log10(numFiles)).longValue() + 1; long currFileNum = 0; String currFile = prefix + toFixedLengthString(currFileNum, numDigits); long lineCount = 0; BufferedWriter w = createWriter(dir, currFile); int start = startLine; int dbUpdates = 0; for (long txn = 0; txn < numTxns; ++txn) { boolean corruptNextTokensEndTag = false; if (txn > 0) start = 0; for (int j = 0; j < txnPattern.length; ++j) { lineCount++; String txnLine = txnPattern[j]; if (txnLine.contains("${SCN}")) { dbUpdates++; txnLine = txnLine.replace("${SCN}", new Long(currScn).toString() + (corruptedScns.contains(currScn) ? corruption : "")); if (addAlternateCorruption && corruptedScns.contains(currScn)) corruptNextTokensEndTag = true; if (dbUpdates >= numDbUpdatesWithSameScn) { currScn++; dbUpdates = 0; } } if (corruptNextTokensEndTag && txnLine.contains("</tokens>")) { //txnLine = txnLine.append(newlineChar).append(" ").append(altCorruption); txnLine = txnLine + newlineChar + " " + altCorruption; corruptNextTokensEndTag = false; } if (j >= start) { w.append(txnLine); if (lineCount % numLinesPerNewline == 0) w.append(newlineChar); } if ((lineCount % numLinesPerFile) == 0) { w.close(); currFileNum++; currFile = prefix + toFixedLengthString(currFileNum, numDigits); w = createWriter(dir, currFile); } } } if (w != null) w.close(); } private static String toFixedLengthString(Long num, long numDigits) { String n = num.toString(); long numDigitsToBeZeroed = numDigits - n.length(); StringBuilder bld = new StringBuilder(); for (int i = 0; i < numDigitsToBeZeroed; i++) bld.append('0'); bld.append(n); return bld.toString(); } @BeforeClass public void setUpClass() throws InvalidConfigException { //set up logging // TODO Setting the first argument to 'true' and make DDSDBUS-3577 not happen TestUtil.setupLoggingWithTimestampedFile(false, "/tmp/TestTrailFilePositionSetter_", ".log", Level.INFO); // FATAL } @AfterClass // optionally could do this after every test instead public void tearDownClass() { for (File dir : _dirsToDelete) { FileUtils.deleteQuietly(dir); } _dirsToDelete.clear(); } private File createTempDir() throws IOException { File dir = File.createTempFile("dir_", null); if (!(dir.delete())) throw new IOException("Unable to delete temp file " + dir.getAbsolutePath()); if (!dir.mkdir()) throw new IOException("Unable to create tempDir: " + dir.getAbsolutePath()); _dirsToDelete.add(dir); // deleteOnExit() deletes only if dir empty => use Apache FileUtils helper instead return dir; } /** * DDSDBUS-2603 NUllPointerException in GG Relay in Experimental CLuster * @throws Exception */ @Test public void testDeletedDirectory() throws Exception { final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testDeletedDirectory"); log.info("starting"); File dir = createTempDir(); TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder(); // Now delete the directory to make the locateFilePosition() see null for listFiles() call. boolean deleted = dir.delete(); Assert.assertTrue(deleted, "Deleted the trail directory successfully"); FilePositionResult res = posSetter.locateFilePosition(100, finder); Assert.assertEquals(res.getStatus(), Status.NO_TXNS_FOUND, "File Position Result Status"); log.info(DONE_STRING); } /** * Verify that corruption that occurs early in the file causes USE_EARLIEST_SCN to return the * first SCN of the first uncorrupted transaction. */ @Test public void testFailureMode_MalformedFirstTransactionInFirstFile() throws Exception { final Logger log = Logger .getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedFirstTransactionInFirstFile"); log.info("starting"); File dir = createTempDir(); // first SCN is hardcoded always to be 100 createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */, 1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0, 100 /* corrupt first SCN */, "xyzzy", false, ""); TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder(); FilePositionResult res; // SCN 100 is corrupted, so 101 is the effective oldest SCN => 100 treated as error: res = posSetter.locateFilePosition(100, finder); Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "expected error for exact-match SCN that's corrupted and oldest in all trail files."); // SCN 101 is OK (regexQuery() doesn't fully validate XML): finder.reset(); res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_EARLIEST_SCN, finder); assertFilePositionResult(res, dir, 101, FilePositionResult.Status.FOUND); log.info(DONE_STRING); } /** * Verify that corruption that occurs after the desired SCN/transaction doesn't cause errors * when requesting an exact SCN. */ @Test public void testFailureMode_MalformedLastTransactionInMiddleFile() throws Exception { final Logger log = Logger .getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedLastTransactionInMiddleFile"); log.info("starting"); File dir = createTempDir(); // first SCN is hardcoded always to be 100 createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */, 1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0, 307 /* corrupt last SCN in 2nd file */, "plugh", false, ""); TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder(); FilePositionResult res; // corruption at SCN 307 occurs after SCN 299, so latter should be found OK: res = posSetter.locateFilePosition(299, finder); assertFilePositionResult(res, dir, 299, FilePositionResult.Status.FOUND); // SCN 306 is in same transaction as 307, but regexQuery() doesn't fully validate XML => OK finder.reset(); res = posSetter.locateFilePosition(306, finder); assertFilePositionResult(res, dir, 306, FilePositionResult.Status.FOUND); log.info(DONE_STRING); } /** * Verify that corruption in the very last transaction causes USE_LATEST_SCN to return * the last SCN from the previous transaction. */ @Test public void testFailureMode_MalformedLastTransactionInLastFile() throws Exception { final Logger log = Logger .getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedLastTransactionInLastFile"); log.info("starting"); File dir = createTempDir(); // first SCN is hardcoded always to be 100 createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines and 2 SCNs each */, 1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0, 399 /* corrupt last SCN in 3rd file */, "quux", false, ""); TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder(); FilePositionResult res; // corruption at SCN 399 => should get 398 back (same transaction, but again, regexQuery() doesn't fully validate) res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_LATEST_SCN, finder); assertFilePositionResult(res, dir, 398, FilePositionResult.Status.FOUND); log.info(DONE_STRING); } /** * Verify that a non-corrupted SCN value within a corrupted transaction causes an exception to be thrown * when the exact SCN is requested. */ @Test public void testFailureMode_MalformedTransactionForExactScn() throws Exception { final Logger log = Logger .getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedTransactionForExactScn"); log.info("starting"); File dir = createTempDir(); // first SCN is hardcoded always to be 100 // this adds a non-XML string near the end of the dbupdate for SCN 377, but the SCN value itself is clean: createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */, 1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0, 377 /* corrupt a txn in 3rd file */, "", true, "kilroy"); TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder(); FilePositionResult res; // with a full XML parser, this should throw an exception; with regexQuery(), it's fine: finder.reset(); res = posSetter.locateFilePosition(377, finder); assertFilePositionResult(res, dir, 377, FilePositionResult.Status.FOUND); log.info(DONE_STRING); } /** * Verify that a transaction with corruption in all of its SCNs doesn't cause problems. * Case 1: bad transaction at the beginning of the first trail file. * Case 2: bad transaction at the beginning of the middle trail file (partial transaction). * Case 3: bad transaction at the beginning of the middle trail file (first full transaction; preceding partial bad). * Case 4: bad transaction in the middle of the middle trail file. * Case 5: bad transaction at the beginning of the last trail file (first full transaction; preceding partial is OK). * Case 6: bad transaction at the end of the last trail file. */ @Test public void testFailureMode_MalformedTransactionNoValidScnsInMiddleOfMiddleFile() throws Exception { final Logger log = Logger .getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedTransactionNoValidScns"); log.info("starting"); File dir = createTempDir(); // corrupt both SCNs in each of six transactions: HashSet<Long> corruptedScns = new HashSet<Long>(10); corruptedScns.add(new Long(100)); // case 1 corruptedScns.add(new Long(101)); // case 1 corruptedScns.add(new Long(204)); // case 2 corruptedScns.add(new Long(205)); // case 2 corruptedScns.add(new Long(206)); // case 3 corruptedScns.add(new Long(207)); // case 3 corruptedScns.add(new Long(250)); // case 4 corruptedScns.add(new Long(251)); // case 4 corruptedScns.add(new Long(310)); // case 5 corruptedScns.add(new Long(311)); // case 5 corruptedScns.add(new Long(398)); // case 6 corruptedScns.add(new Long(399)); // case 6 createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */, 1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0, corruptedScns, "blargh", false, ""); TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder(); FilePositionResult res; // SCN 101 is before the earliest (valid) SCN present, so expect ERROR: res = posSetter.locateFilePosition(101, finder); Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "expected error for exact-match SCN that's 'too old'."); // For SCN <= the earliest transactions maxSCN, we throw error finder.reset(); res = posSetter.locateFilePosition(102, finder); Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "expected error for exact-match SCN that's 'too old'."); // expect first non-corrupted SCN here, not first "transaction SCN": finder.reset(); res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_EARLIEST_SCN, finder); assertFilePositionResult(res, dir, 102, FilePositionResult.Status.FOUND); // 107 = max SCN of its transaction = "transaction SCN" => should get FOUND finder.reset(); res = posSetter.locateFilePosition(107, finder); assertFilePositionResult(res, dir, 107, FilePositionResult.Status.FOUND); // 203 = last valid SCN in first file = max SCN of its transaction = "transaction SCN" // => should be FOUND finder.reset(); res = posSetter.locateFilePosition(203, finder); assertFilePositionResult(res, dir, 203, FilePositionResult.Status.FOUND); // SCN 204 is invalid and is part of a transaction split across first/second files; // 209 = next "transaction SCN" and is near the top of the middle file finder.reset(); res = posSetter.locateFilePosition(204, finder); assertFilePositionResult(res, dir, 209, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); // SCN 250 is invalid (as is 251); expect 253 since max SCN of following transaction finder.reset(); res = posSetter.locateFilePosition(250, finder); assertFilePositionResult(res, dir, 253, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); // SCN 251 is invalid (as is 250); expect 253 since max SCN of following transaction finder.reset(); res = posSetter.locateFilePosition(251, finder); assertFilePositionResult(res, dir, 253, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); // SCN 252 is valid and present, but weird corner case => still EXACT_SCN_NOT_FOUND finder.reset(); res = posSetter.locateFilePosition(252, finder); assertFilePositionResult(res, dir, 252, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); // SCN 253 is valid and present and max SCN of its transaction => FOUND finder.reset(); res = posSetter.locateFilePosition(253, finder); assertFilePositionResult(res, dir, 253, FilePositionResult.Status.FOUND); // SCN 309 is valid and present and max SCN of its transaction => FOUND (even though // split across second/third files, and following transaction is corrupted) finder.reset(); res = posSetter.locateFilePosition(309, finder); assertFilePositionResult(res, dir, 309, FilePositionResult.Status.FOUND); // SCN 310 is invalid (as is 311); expect 313 finder.reset(); res = posSetter.locateFilePosition(310, finder); assertFilePositionResult(res, dir, 313, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); // SCN 311 is invalid (as is 310); expect 313 finder.reset(); res = posSetter.locateFilePosition(311, finder); assertFilePositionResult(res, dir, 313, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); // SCN 398 is invalid (as is 399) and is in last transaction of last file, but since // trail file is expected to continue growing (i.e., eventually to have a valid SCN // that's larger than the request), expect EXACT_SCN_NOT_FOUND rather than ERROR. SCN // returned will be that of last valid transaction, i.e., 397. // [checks beginning of last valid transaction == 396/397 one at byte offset 35650] finder.reset(); res = posSetter.locateFilePosition(398, finder); assertFilePositionResult(res, dir, 397, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); // SCN 405 is completely missing (would be after last transaction of last file); expect // same behavior as previous case finder.reset(); res = posSetter.locateFilePosition(405, finder); assertFilePositionResult(res, dir, 397, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); // last valid transaction-SCN is 397 finder.reset(); res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_LATEST_SCN, finder); assertFilePositionResult(res, dir, 397, FilePositionResult.Status.FOUND); log.info(DONE_STRING); } /** * Verify that requesting an SCN that's older than the oldest trail file causes ERROR * to be returned. Also check various valid states: SCN that isn't the max for its * transaction; SCN that is the max for its transaction; earliest SCN; etc. */ @Test public void testRequestedScnOlderThanOldestTransaction() throws Exception { final Logger log = Logger .getLogger("TestTrailFilePositionSetter.testRequestedScnOlderThanOldestTransaction"); //log.setLevel(Level.DEBUG); log.info("starting"); File dir = createTempDir(); // first SCN is hardcoded always to be 100: createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150 /* numTxns, 24 lines each */, 1250 /* numLinesPerFile */, 1 /* numLinesPerNewline */, "\n", 0, -1 /* no corruption */, "", false, ""); TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder(); FilePositionResult res; // SCN 99 is before the earliest SCN present in the trail files (100), so expect ERROR: res = posSetter.locateFilePosition(99, finder); Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "expected error for exact-match SCN that's too old."); // "Transaction-SCN" semantics: max SCN is 121 in this transaction and 119 in the // previous one, so those are the values used for most of the "is it found?" logic. // Since 120 falls between the two but doesn't equal either one, it's not considered // to be found exactly (even though it's actually present), but the state is still // valid, so processing can continue (i.e., it doesn't matter). finder.reset(); res = posSetter.locateFilePosition(120, finder); assertFilePositionResult(res, dir, 120, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); // For SCN <= the earliest transactions maxSCN, we throw error finder.reset(); res = posSetter.locateFilePosition(100, finder); Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "expected error for exact-match SCN that's too old."); // Related, weird corner case: USE_EARLIEST_SCN uses the min SCN in the oldest transaction, // so even though an exact match for 100 doesn't return FOUND, this does: finder.reset(); res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_EARLIEST_SCN, finder); assertFilePositionResult(res, dir, 100, FilePositionResult.Status.FOUND); // For SCN <= the earliest transactions maxSCN, we throw error finder.reset(); res = posSetter.locateFilePosition(101, finder); Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "expected error for exact-match SCN that's too old."); log.info(DONE_STRING); } @Test public void testScnPositionLocator() throws Exception { final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testScnPositionLocator"); log.info("starting"); testScnPositionLocator(0, 100, 200, 102, log); // case where all txn are complete log.info(DONE_STRING); } private void testScnPositionLocator(int startLine, int startScn, int endScn, int beginFoundScn, Logger log) throws Exception { File dir = createTempDir(); // TODO/FIXME: why are startScn and endScn being used for numTxns and numLinesPerFile, respectively? createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, startScn, endScn, 1, "\n", startLine, -1, "", false, ""); log.info("Directory is: " + dir); TrailFilePositionSetter posSetter = null; //GoldenGateTransactionSCNFinder finder; GGXMLTrailTransactionFinder finder; //less than minScn for (long i = 0; i < beginFoundScn; i++) { posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); //finder = new GoldenGateTransactionSCNFinder(); finder = new GGXMLTrailTransactionFinder(); FilePositionResult res = posSetter.locateFilePosition(i, finder); Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "Result Status for SCN: " + i + ", Result: " + res); } //Found Case for (long i = beginFoundScn; i < (startScn + endScn); i++) { posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); //finder = new GoldenGateTransactionSCNFinder(); finder = new GGXMLTrailTransactionFinder(); FilePositionResult res = posSetter.locateFilePosition(i, finder); log.info("For scn (" + i + "): the result is: " + res); if (i % 2 == 0) assertFilePositionResult(res, dir, i + 1, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); else assertFilePositionResult(res, dir, i, FilePositionResult.Status.FOUND); } //Found Case for (long i = (startScn + endScn); i < (startScn + endScn) + 20; i++) { posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); //finder = new GoldenGateTransactionSCNFinder(); finder = new GGXMLTrailTransactionFinder(); FilePositionResult res = posSetter.locateFilePosition(i, finder); log.info("For scn (" + i + "): the result is: " + res); assertFilePositionResult(res, dir, 299, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); } } @Test public void testRepeatSCN() throws Exception { final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testRepeatSCN"); File dir = createTempDir(); long startScn = 100; int numTxnsPerFile = 10; /** * Create trail files with the following SCN range * * Trail-File Number of Txns SCN Range * x300 10 100-100 * x301 10 100-100 * x302 10 101-101 * x303 10 101-101 * x304 10 102-102 * x305 10 102-102 * x306 10 103-103 * x307 10 103-103 * x308 10 104-104 * x309 10 104-104 * */ createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 100L, _txnPattern.length * numTxnsPerFile, 1L, "\n", 0, _txnPattern, 2 * numTxnsPerFile * 2, // Transactions in 2 files have the same SCN startScn); log.info("Directory is: " + dir); TrailFilePositionSetter posSetter = null; // GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder(); GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder(); posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); // SCN 100 is not found because this is the first SCN in the trail file. FilePositionResult res = posSetter.locateFilePosition(100, finder); Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "Result Status for SCN: " + 100 + ", Result: " + res); // SCN 101 is found res = posSetter.locateFilePosition(101, finder); Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status"); Assert.assertEquals(res.getTxnPos().getFile(), "x302", "File found"); Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found"); Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found"); Assert.assertEquals(res.getTxnPos().getMinScn(), 101, "MinScn check"); Assert.assertEquals(res.getTxnPos().getMaxScn(), 101, "MinScn check"); Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank"); // SCN 102 is found res = posSetter.locateFilePosition(102, finder); Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status"); Assert.assertEquals(res.getTxnPos().getFile(), "x304", "File found"); Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found"); Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found"); Assert.assertEquals(res.getTxnPos().getMinScn(), 102, "MinScn check"); Assert.assertEquals(res.getTxnPos().getMaxScn(), 102, "MinScn check"); Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank"); // SCN 103 is found res = posSetter.locateFilePosition(103, finder); Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status"); Assert.assertEquals(res.getTxnPos().getFile(), "x306", "File found"); Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found"); Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found"); Assert.assertEquals(res.getTxnPos().getMinScn(), 103, "MinScn check"); Assert.assertEquals(res.getTxnPos().getMaxScn(), 103, "MinScn check"); Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank"); // SCN 104 is found res = posSetter.locateFilePosition(104, finder); Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status"); Assert.assertEquals(res.getTxnPos().getFile(), "x308", "File found"); Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found"); Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found"); Assert.assertEquals(res.getTxnPos().getMinScn(), 104, "MinScn check"); Assert.assertEquals(res.getTxnPos().getMaxScn(), 104, "MinScn check"); Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank"); } @Test public void testScnPositionSetter1() throws Exception { final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testScnPositionSetter1"); log.info("starting"); testScnPositionSetter(0, 100, 200, 100, log); // case where all txn are complete testScnPositionSetter(10, 100, 200, 102, log); // case where initial txn is incomplete log.info(DONE_STRING); } private void testScnPositionSetter(int startLine, int startScn, int endScn, int beginFoundScn, Logger log) throws Exception { String[] newLines = { "\n", "\r", "\r\n" }; for (String newLine : newLines) { File dir = createTempDir(); createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, startScn, endScn, 1, newLine, startLine, -1, "", false, ""); log.info("Directory is: " + dir); TrailFilePositionSetter posSetter = null; //GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder(); GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder(); //less than minScn for (long i = 0; i < beginFoundScn; i++) { posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); Logger log2 = posSetter._log; log2.setLevel(Level.INFO); log2.info("Created a TrailFilePositionSetter with suffix x3"); //finder = new GoldenGateTransactionSCNFinder(); finder = new GGXMLTrailTransactionFinder(); FilePositionResult res = posSetter.getFilePosition(i, finder); Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "Result Status for SCN: " + i + ", Result: " + res); } //Found Case for (long i = beginFoundScn; i < (startScn + endScn); i++) { posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); //finder = new GoldenGateTransactionSCNFinder(); finder = new GGXMLTrailTransactionFinder(); FilePositionResult res = posSetter.getFilePosition(i, finder); log.info("For scn (" + i + "): the result is: " + res); if (i % 2 == 0) assertFilePositionResult(res, dir, i + 1, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); else assertFilePositionResult(res, dir, i, FilePositionResult.Status.FOUND); } //Found Case for (long i = (startScn + endScn); i < (startScn + endScn) + 20; i++) { posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); //finder = new GoldenGateTransactionSCNFinder(); finder = new GGXMLTrailTransactionFinder(); FilePositionResult res = posSetter.getFilePosition(i, finder); log.info("For scn (" + i + "): the result is: " + res); assertFilePositionResult(res, dir, 299, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); } } } /** * Case where XML snippets have newlines at different positions. * NOTE: This test is EXCEEDINGLY slow... * * @throws Exception */ @Test public void testScnPositionSetter2() throws Exception { final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testScnPositionSetter2"); log.info("starting"); String[] newLines = { "\r", "\n", "\r\n" }; for (String newLine : newLines) { // Iterate by Number of Tags per line for (int j = 10; j < 32; j++) { log.info("NumPerLine: " + j); File dir = createTempDir(); createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 100, 200, j, newLine, 0, -1, "", false, ""); log.info("Directory is: " + dir); TrailFilePositionSetter posSetter = null; //GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder(); GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder(); //less than minScn log.info("less than MinScn case started !!"); for (long i = 0; i < 100; i++) { if (true) break; posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); //finder = new GoldenGateTransactionSCNFinder(); finder = new GGXMLTrailTransactionFinder(); FilePositionResult res = posSetter.getFilePosition(i, finder); Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "Result Status"); } log.info("less than MinScn case passed !!"); //Found Case for (long i = 100; i < 300; i++) { posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); finder = new GGXMLTrailTransactionFinder(); log.info("SCN: " + i); FilePositionResult res = posSetter.getFilePosition(i, finder); log.info("For scn (" + i + "): the result is: " + res); if (i % 2 == 0) assertFilePositionResult(res, dir, i + 1, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); else assertFilePositionResult(res, dir, i, FilePositionResult.Status.FOUND); } //Found Case FilePositionResult res = null; for (long i = 300; i < 320; i++) { posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); //finder = new GoldenGateTransactionSCNFinder(); finder = new GGXMLTrailTransactionFinder(); res = posSetter.getFilePosition(i, finder); //log.info("For scn (" + i + "): the result is: " + res); assertFilePositionResult(res, dir, 299, FilePositionResult.Status.EXACT_SCN_NOT_FOUND); } // USE Latest SCN (-1) posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX); //finder = new GoldenGateTransactionSCNFinder(); finder = new GGXMLTrailTransactionFinder(); res = posSetter.getFilePosition(-1, finder); //log.info("For scn (" + i + "): the result is: " + res); assertFilePositionResult(res, dir, 299, FilePositionResult.Status.FOUND); } } log.info(DONE_STRING); } private void assertFilePositionResult(FilePositionResult res, File dir, long expScn, FilePositionResult.Status expStatus) throws Exception { Assert.assertEquals(res.getStatus(), expStatus, "Result Status for scn: " + expScn); ConcurrentAppendableCompositeFileInputStream c = new ConcurrentAppendableCompositeFileInputStream( dir.getAbsolutePath(), res.getTxnPos().getFile(), res.getTxnPos().getFileOffset(), new TrailFilePositionSetter.FileFilter(dir, TRAIL_FILENAME_PREFIX), true); ReadFirstSCN scnFetcher = new ReadFirstSCN(new BufferedReader(new InputStreamReader(c)), expScn); scnFetcher.start(); scnFetcher.awaitShutdown(); if (c != null) c.close(); LOG.info("Result is: " + res); Assert.assertEquals(scnFetcher.isPatternMatched(), true, "expected SCN " + expScn + " not found at byte offset " + res.getTxnPos().getFileOffset() + " of file " + res.getTxnPos().getFile() + "."); LOG.info("Finding result txn (" + res + ") passed !!"); } public static class ReadFirstSCN extends DatabusThreadBase { private final BufferedReader _reader; private boolean _patternMatched = false; private String _errorMessage = null; private long _expScn = -1; public ReadFirstSCN(BufferedReader reader, long expScn) { super("ReadFirSCN"); _reader = reader; _expScn = expScn; } @Override public void run() { boolean beginSeen = false; String line = null; try { String pattern = "<token name=\"tk-csn\">" + _expScn + "</token>"; while ((line = _reader.readLine()) != null) { //LOG.info("Line: (" + line + ")"); line = line.toLowerCase(); if (!beginSeen) { beginSeen = true; if (!line.startsWith(GGXMLTrailTransactionFinder.TRANSACTION_BEGIN_PREFIX)) { _errorMessage = "Stream not starting with transaction begin string !! Line is: " + line; return; } } if (line.contains(pattern) && line.contains(GGXMLTrailTransactionFinder.TRANSACTION_END_PREFIX)) { int index1 = line.indexOf(pattern); int index2 = line.indexOf(GGXMLTrailTransactionFinder.TRANSACTION_END_PREFIX); if (index1 < index2) { _patternMatched = true; } break; } if (line.contains(pattern)) _patternMatched = true; if (line.contains(GGXMLTrailTransactionFinder.TRANSACTION_END_PREFIX)) break; } } catch (IOException e) { e.printStackTrace(); } finally { if (null != _reader) try { _reader.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } doShutdownNotify(); } } public boolean isPatternMatched() { return _patternMatched; } } @Test public void testSplitBytesByNewLines() { final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testSplitBytesByNewLines"); log.info("starting"); String s = "abc\r\ndef\r\n"; List<String> l = new ArrayList<String>(); List<Integer> endPos = new ArrayList<Integer>(); String lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, null, l, endPos); Assert.assertNull(lastLineStr, "Returned String should be null"); Assert.assertEquals(2, l.size(), "Length"); Assert.assertEquals(endPos.size(), 2, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "def", "Second in " + l); Assert.assertEquals(endPos.get(0).intValue(), 5, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 10, "Second in " + endPos); s = "abc\r\ndef\r\n"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 0, null, l, endPos); Assert.assertNull(lastLineStr, "Returned String should be null"); Assert.assertEquals(3, l.size(), "Length"); Assert.assertEquals(endPos.size(), 3, "Length"); Assert.assertEquals(l.get(0), "", "First in " + l); Assert.assertEquals(l.get(1), "abc", "Second in " + l); Assert.assertEquals(l.get(2), "def", "Third in " + l); Assert.assertEquals(endPos.get(0).intValue(), -1, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 5, "First in " + endPos); Assert.assertEquals(endPos.get(2).intValue(), 10, "Second in " + endPos); s = "abc\r\ndef\r\n"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 1, null, l, endPos); Assert.assertNull(lastLineStr, "Returned String should be null"); Assert.assertEquals(3, l.size(), "Length"); Assert.assertEquals(endPos.size(), 3, "Length"); Assert.assertEquals(l.get(0), "a", "First in " + l); Assert.assertEquals(l.get(1), "bc", "Second in " + l); Assert.assertEquals(l.get(2), "def", "Third in " + l); Assert.assertEquals(endPos.get(0).intValue(), -1, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 5, "First in " + endPos); Assert.assertEquals(endPos.get(2).intValue(), 10, "Second in " + endPos); s = "abc\r\ndef\r\n"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 2, null, l, endPos); Assert.assertNull(lastLineStr, "Returned String should be null"); Assert.assertEquals(3, l.size(), "Length"); Assert.assertEquals(endPos.size(), 3, "Length"); Assert.assertEquals(l.get(0), "ab", "First in " + l); Assert.assertEquals(l.get(1), "c", "Second in " + l); Assert.assertEquals(l.get(2), "def", "Third in " + l); Assert.assertEquals(endPos.get(0).intValue(), -1, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 5, "First in " + endPos); Assert.assertEquals(endPos.get(2).intValue(), 10, "Second in " + endPos); s = "abc\r\ndef\r\n"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 3, null, l, endPos); Assert.assertNull(lastLineStr, "Returned String should be null"); Assert.assertEquals(2, l.size(), "Length"); Assert.assertEquals(endPos.size(), 2, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "def", "Second in " + l); Assert.assertEquals(endPos.get(0).intValue(), 5, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 10, "Second in " + endPos); s = "abc\r\ndef\r\n"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 5, null, l, endPos); Assert.assertNull(lastLineStr, "Returned String should be null"); Assert.assertEquals(l.size(), 3, "Length"); Assert.assertEquals(endPos.size(), 3, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "", "Second in " + l); Assert.assertEquals(l.get(2), "def", "Third in " + l); Assert.assertEquals(endPos.get(0).intValue(), 5, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), -1, "First in " + endPos); Assert.assertEquals(endPos.get(2).intValue(), 10, "Second in " + endPos); s = "abc\r\ndef\r\n"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 6, null, l, endPos); Assert.assertNull(lastLineStr, "Returned String should be null"); Assert.assertEquals(l.size(), 3, "Length"); Assert.assertEquals(endPos.size(), 3, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "d", "Second in " + l); Assert.assertEquals(l.get(2), "ef", "Third in " + l); Assert.assertEquals(endPos.get(0).intValue(), 5, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), -1, "First in " + endPos); Assert.assertEquals(endPos.get(2).intValue(), 10, "Second in " + endPos); s = "abc\r\ndef\r\n"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); //log.info("Starting !!"); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 7, null, l, endPos); Assert.assertNull(lastLineStr, "Returned String should be null"); Assert.assertEquals(l.size(), 3, "Length"); Assert.assertEquals(endPos.size(), 3, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "de", "Second in " + l); Assert.assertEquals(l.get(2), "f", "Third in " + l); Assert.assertEquals(endPos.get(0).intValue(), 5, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), -1, "Second in " + endPos); Assert.assertEquals(endPos.get(2).intValue(), 10, "Third in " + endPos); s = "abc\r\ndef\r\n"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 4, null, l, endPos); Assert.assertNull(lastLineStr, "Returned String should be null"); Assert.assertEquals(2, l.size(), "Length"); Assert.assertEquals(endPos.size(), 2, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "def", "Second in " + l); Assert.assertEquals(endPos.get(0).intValue(), 5, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 10, "Second in " + endPos); s = "abc\r\ndef\r\n"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 8, null, l, endPos); Assert.assertNull(lastLineStr, "Returned String should be null"); Assert.assertEquals(2, l.size(), "Length"); Assert.assertEquals(endPos.size(), 2, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "def", "Second in " + l); Assert.assertEquals(endPos.get(0).intValue(), 5, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 10, "Second in " + endPos); s = "abc\ndef\n"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 3, null, l, endPos); Assert.assertNull(lastLineStr, "Returned String should be null"); Assert.assertEquals(2, l.size(), "Length"); Assert.assertEquals(endPos.size(), 2, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "def", "Second in " + l); Assert.assertEquals(endPos.get(0).intValue(), 4, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 8, "Second in " + endPos); s = "abc\ndef\n"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, null, l, endPos); Assert.assertNull(lastLineStr, "Returned String should be null"); Assert.assertEquals(2, l.size(), "Length"); Assert.assertEquals(endPos.size(), 2, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "def", "Second in " + l); Assert.assertEquals(endPos.get(0).intValue(), 4, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 8, "Second in " + endPos); s = "abc\r\ndef\n"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, null, l, endPos); Assert.assertNull(lastLineStr, "Returned String should be null"); Assert.assertEquals(2, l.size(), "Length"); Assert.assertEquals(endPos.size(), 2, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "def", "Second in " + l); Assert.assertEquals(endPos.get(0).intValue(), 5, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 9, "Second in " + endPos); s = "abc\r\ndef"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, null, l, endPos); Assert.assertEquals(lastLineStr, "def", "Returned String"); Assert.assertEquals(l.size(), 1, "Length"); Assert.assertEquals(endPos.size(), 1, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(endPos.get(0).intValue(), 5, "First in " + endPos); s = "abc\r\ndef\r"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, null, l, endPos); //log.info("Length: " + lastLineStr.length()); Assert.assertEquals(lastLineStr, "def\r", "Returned String"); Assert.assertEquals(l.size(), 1, "Length"); Assert.assertEquals(endPos.size(), 1, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(endPos.get(0).intValue(), 5, "First in " + endPos); lastLineStr = "abc"; s = "def\r\nghi"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, lastLineStr, l, endPos); Assert.assertEquals(lastLineStr, "ghi", "Returned String"); Assert.assertEquals(l.size(), 1, "Length"); Assert.assertEquals(endPos.size(), 1, "Length"); Assert.assertEquals(l.get(0), "abcdef", "First in " + l); Assert.assertEquals(endPos.get(0).intValue(), 8, "First in " + endPos); lastLineStr = "abc\r"; s = "def\r\nghi"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, lastLineStr, l, endPos); Assert.assertEquals(lastLineStr, "ghi", "Returned String"); Assert.assertEquals(l.size(), 2, "Length: " + l); Assert.assertEquals(endPos.size(), 2, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "def", "First in " + l); Assert.assertEquals(endPos.get(0).intValue(), 4, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 9, "Second in " + endPos); lastLineStr = "abc\r"; s = "\ndef\r\nghi"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, lastLineStr, l, endPos); Assert.assertEquals(lastLineStr, "ghi", "Returned String"); Assert.assertEquals(l.size(), 2, "Length: " + l); Assert.assertEquals(endPos.size(), 2, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "def", "First in " + l); Assert.assertEquals(endPos.get(0).intValue(), 5, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 10, "Second in " + endPos); lastLineStr = "abc"; s = "\ndef\r\nghi"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, lastLineStr, l, endPos); Assert.assertEquals(lastLineStr, "ghi", "Returned String"); Assert.assertEquals(l.size(), 2, "Length"); Assert.assertEquals(endPos.size(), 2, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "def", "First in " + l); Assert.assertEquals(endPos.get(0).intValue(), 4, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 9, "Second in " + endPos); lastLineStr = "abc"; s = "\rdef\r\nghi"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, lastLineStr, l, endPos); Assert.assertEquals(lastLineStr, "ghi", "Returned String"); Assert.assertEquals(l.size(), 2, "Length"); Assert.assertEquals(endPos.size(), 2, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "def", "Second in " + l); Assert.assertEquals(endPos.get(0).intValue(), 4, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 9, "Second in " + endPos); lastLineStr = "xyz"; s = "abcdef"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, lastLineStr, l, endPos); Assert.assertEquals(lastLineStr, "xyzabcdef", "Returned String"); Assert.assertEquals(l.size(), 0, "Length"); Assert.assertEquals(endPos.size(), 0, "Length"); s = "abcdef"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, null, l, endPos); Assert.assertEquals(lastLineStr, "abcdef", "Returned String"); Assert.assertEquals(l.size(), 0, "Length"); Assert.assertEquals(endPos.size(), 0, "Length"); s = "abc\n\n def"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, null, l, endPos); Assert.assertEquals(lastLineStr, " def", "Returned String"); Assert.assertEquals(l.size(), 2, "Length"); Assert.assertEquals(endPos.size(), 2, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "", "Second in " + l); Assert.assertEquals(endPos.get(0).intValue(), 4, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 5, "Second in " + endPos); s = "abc\n\n def\n ghi"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), true, 4, null, l, endPos); Assert.assertEquals(lastLineStr, " ghi", "Returned String"); Assert.assertEquals(l.size(), 3, "Length"); Assert.assertEquals(endPos.size(), 3, "Length"); Assert.assertEquals(l.get(0), "abc", "First in " + l); Assert.assertEquals(l.get(1), "", "Second in " + l); Assert.assertEquals(l.get(2), " def", "Third in " + l); Assert.assertEquals(endPos.get(0).intValue(), 4, "First in " + endPos); Assert.assertEquals(endPos.get(1).intValue(), 5, "Second in " + endPos); Assert.assertEquals(endPos.get(2).intValue(), 11, "Third in " + endPos); s = "\n"; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, null, l, endPos); Assert.assertEquals(lastLineStr, null, "Returned String"); Assert.assertEquals(l.size(), 1, "Length"); Assert.assertEquals(endPos.size(), 1, "Length"); Assert.assertEquals(l.get(0), "", "First in " + l); Assert.assertEquals(endPos.get(0).intValue(), 1, "First in " + endPos); s = " "; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, null, l, endPos); Assert.assertEquals(lastLineStr, " ", "Returned String"); Assert.assertEquals(l.size(), 0, "Length"); Assert.assertEquals(endPos.size(), 0, "Length"); s = ""; l = new ArrayList<String>(); endPos = new ArrayList<Integer>(); lastLineStr = TrailFilePositionSetter.splitBytesByNewLines(s.getBytes(Charset.defaultCharset()), s.length(), false, -1, null, l, endPos); Assert.assertEquals(lastLineStr, null, "Returned String"); Assert.assertEquals(l.size(), 0, "Length"); Assert.assertEquals(endPos.size(), 0, "Length"); log.info(DONE_STRING); } @Test public void testTrailFileComparator() { final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testTrailFileComparator"); log.info("starting"); String prefix = "x4"; File dir = new File("/tmp"); FileFilter f = new TrailFilePositionSetter.FileFilter(dir, prefix); //isTrailFile Check Assert.assertFalse(f.isTrailFile(null), "Null trail file"); Assert.assertFalse(f.isTrailFile(dir), "Dir as trail file"); Assert.assertFalse(f.isTrailFile(new File("")), "Empty trail file name"); Assert.assertFalse(f.isTrailFile(new File("/tmp/x12222")), "trail file with different prefix"); Assert.assertFalse(f.isTrailFile(new File("/tmp/42222")), "trail file with different prefix"); Assert.assertFalse(f.isTrailFile(new File("/tmp/x412222.xml")), "trail file with bad suffix"); Assert.assertFalse(f.isTrailFile(new File("/tmp/x412222x")), "trail file with bad suffix"); Assert.assertTrue(f.isTrailFile(new File("/tmp/x4122222")), "correct trail file"); Assert.assertTrue(f.isTrailFile(new File("x4122222")), "correct trail file"); //compareTo Check Assert.assertEquals(-1, f.compareFileName(new File("/tmp/x400001"), new File("/tmp/x400002"))); Assert.assertEquals(-1, f.compareFileName(new File("/tmp/x400001"), new File("/tmp/x400002"))); // DUPLICATE? Assert.assertEquals(1, f.compareFileName(new File("/tmp/x410001"), new File("/tmp/x400002"))); Assert.assertEquals(1, f.compareFileName(new File("/tmp/x410009"), new File("/tmp/x410003"))); Assert.assertEquals(0, f.compareFileName(new File("/tmp/x410009"), new File("/tmp/x410009"))); //isNextFileInSeq Assert.assertFalse(f.isNextFileInSequence(new File("/tmp/x400000"), new File("/tmp/x400000"))); Assert.assertFalse(f.isNextFileInSequence(new File("/tmp/x410000"), new File("/tmp/x400000"))); Assert.assertFalse(f.isNextFileInSequence(new File("/tmp/x400001"), new File("/tmp/x400000"))); Assert.assertTrue(f.isNextFileInSequence(new File("/tmp/x400000"), new File("/tmp/x400001"))); log.info(DONE_STRING); } }