Java tutorial
/** * * Copyright 2009-2013 The MITRE Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * ************************************************************************** * NOTICE * This software was produced for the U. S. Government under Contract No. * W15P7T-12-C-F600, and is subject to the Rights in Noncommercial Computer * Software and Noncommercial Computer Software Documentation Clause * 252.227-7014 (JUN 1995) * * (c) 2012 The MITRE Corporation. All Rights Reserved. * ************************************************************************** */ package org.mitre.xcoord; import java.io.IOException; import java.io.InputStreamReader; import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.io.FilenameUtils; import org.supercsv.io.CsvMapReader; import org.supercsv.prefs.CsvPreference; import org.mitre.flexpat.PatternTestCase; import org.mitre.flexpat.TextMatch; import org.mitre.flexpat.TextMatchResultSet; import org.mitre.opensextant.util.FileUtility; import org.mitre.opensextant.util.TextUtils; /** * * @author ubaldino */ public class TestScript { static Logger log = LoggerFactory.getLogger(TestUtility.class); private XCoord xcoord = null; /** * * @param xc */ public TestScript(XCoord xc) { xcoord = xc; } /** * Test one input file, reporting output as a CSV * * @param file */ public void test(String file) { systemTests(); fileTests(file); } /** * * @param file */ public void fileTests(String file) { log.info("\n\n=== TEXT FILE TESTS ===\n\n"); TestUtility tester = null; String buffer = null; try { String _file = file.trim(); buffer = FileUtility.readFile(_file); String fname = FilenameUtils.getBaseName(_file); tester = new TestUtility("./results/xcoord_" + fname + ".csv"); } catch (IOException err) { log.error("Failed to open test file", err); return; } xcoord.matchAll(); String jobid = TextUtils.text_id(buffer); log.info("Extract coordinates; All patterns enabled"); try { TextMatchResultSet results = xcoord.extract_coordinates(buffer, jobid); tester.save_result(null, results); } catch (Exception err) { log.error("Failed to write report", err); } tester.close_report(); log.info("=== TEXT FILE TESTS DONE ==="); } /** * Using the TestUtility, all patterns are tested and reported to the * results folder. */ public void systemTests() { log.info("\n\n=== SYSTEM TESTS ===\n\n"); if (!xcoord.patterns.testing) { log.info("TESTING OFF -- TURN ON DEBUG in LOG4J"); return; } xcoord.match_UTM(true); xcoord.match_MGRS(true); xcoord.match_DD(true); xcoord.match_DMS(true); xcoord.match_DM(true); try { TestUtility tester = new TestUtility("./results/xcoord_System.csv"); for (PatternTestCase tst : xcoord.patterns.testcases) { TextMatchResultSet results = xcoord.extract_coordinates(tst.text, tst.id, tst.family_id); results.add_trace("Test Payload: " + tst.text); if (!results.evaluated) { continue; } log.info("=========SYSTEM TEST " + tst.id + " FOUND:" + (results.matches.isEmpty() ? "NOTHING" : results.matches.size())); tester.save_result(null, results); } tester.close_report(); } catch (Exception err) { log.error("Not finishing tests", err); return; } log.info("=== SYSTEM TESTS DONE ==="); } /** * This will accomodate any test file that has at least the following style: * * FAMILY-XXX COORDINATE TEXT "FAIL" * * Where the first FAMILY token is * * @param coordfile */ public void fileTestByLines(String coordfile) { xcoord.match_UTM(true); xcoord.match_MGRS(true); xcoord.match_DD(true); xcoord.match_DMS(true); xcoord.match_DM(true); try { String _file = coordfile.trim(); String fname = FilenameUtils.getBaseName(_file); TestUtility tester = new TestUtility("./results/xcoord_" + fname + "-lines.csv"); java.io.LineNumberReader in = FileUtility.getLineReader(coordfile); String line = null; while ((line = in.readLine()) != null) { String text = line.trim(); if (text.startsWith("#")) { continue; } if (text.isEmpty()) { continue; } String fam = find_family(line); int famx = XConstants.get_CCE_family(fam); if (famx == XConstants.UNK_PATTERN) { log.error("Unknown test pattern TEXT=" + text); continue; } TestCase tst = new TestCase("#" + in.getLineNumber(), fam, text); TextMatchResultSet results = xcoord.extract_coordinates(tst.text, tst.id); /** * tst.family_id */ results.add_trace("Test Payload: " + tst.text); if (!results.evaluated) { continue; } log.info("=========FILE TEST " + tst.id + " FOUND:" + (results.matches.isEmpty() ? "NOTHING" : results.matches.size())); tester.save_result(tst, results); } tester.close_report(); log.info("=== FILE TESTS DONE ==="); } catch (Exception err) { log.error("TEST BY LINES", err); } } /** * Create a typical CSV writer -- Excel compliant * * @param file * @return * @throws IOException */ public CsvMapReader open(String file) throws IOException { InputStreamReader rdr = FileUtility.getInputStream(file, "UTF-8"); CsvMapReader R = new CsvMapReader(rdr, CsvPreference.STANDARD_PREFERENCE); return R; } /** * This will accomodate any test file that has at least the following style: * * FAMILY-XXX COORDINATE TEXT "FAIL" * * Where the first FAMILY token is * * @param coordfile */ public void fileTruth(String coordfile) { xcoord.match_UTM(true); xcoord.match_MGRS(true); xcoord.match_DD(true); xcoord.match_DMS(true); xcoord.match_DM(true); try { String _file = coordfile.trim(); String fname = FilenameUtils.getBaseName(_file); TestUtility tester = new TestUtility("./results/xcoord_" + fname + "-rows.csv"); // tester.full_report = false; CsvMapReader in = open(coordfile); String text = null; int linenum = 0; String[] columns = in.getHeader(true); Map<String, String> testRow = null; // id, enumeration, test, true_lat, true_lon, remark while ((testRow = in.read(columns)) != null) { String patid = testRow.get("id"); if (patid == null) { continue; } patid = patid.trim(); if (patid.startsWith("#")) { continue; } if (patid.isEmpty()) { continue; } String fam = find_family(patid); int famx = XConstants.get_CCE_family(fam); if (famx == XConstants.UNK_PATTERN) { log.error("Unknown test pattern TEXT=" + text); continue; } text = testRow.get("enumeration"); linenum = Integer.parseInt(text); text = testRow.get("test"); text = text.replace("$NL", "\n"); String rmks = testRow.get("remark"); // "Patid # rowid" == test instance id // DMS07#12 -- 12th example of DMS07 test. // TestCase tst = new TestCase(patid + "#" + linenum, fam, text); tst.match.setLatitude(testRow.get("true_lat")); tst.match.setLongitude(testRow.get("true_lon")); tst.setRemarks(rmks); TextMatchResultSet results = xcoord.extract_coordinates(tst.text, tst.id); /** * tst.family_id */ results.add_trace("Test Payload: " + tst.text); if (!results.evaluated) { continue; } log.info("=========FILE TEST " + tst.id + " FOUND:" + (results.matches.isEmpty() ? "NOTHING" : results.matches.size())); tester.save_result(tst, results); } tester.close_report(); log.info("=== FILE TESTS DONE ==="); } catch (Exception err) { log.error("TEST BY LINES", err); } } private String find_family(String line) { char ch = line.charAt(0); if (!(ch == 'M' || ch == 'U' || ch == 'D')) { return null; } for (String fam : XConstants.familyInt.keySet()) { if (line.startsWith("DMS")) { return "DMS"; } if (line.startsWith(fam)) { return fam; } } return null; } /** * Use for limited developmen testing. */ protected void focusedTests() { log.info("=== ADHOC TESTS ==="); log.info("Trying some specific DD tests now:\n========================="); xcoord.match_DD(false); xcoord.match_DMS(false); xcoord.match_DM(false); xcoord.match_MGRS(false); xcoord.match_UTM(false); TextMatchResultSet results = null; // = xcoord.extract_coordinates("text before " + "17S 699999 3335554" + " and after", "UTM"); boolean dd = false; boolean dms = false; boolean dm = false; boolean mgrs = true; // xcoord.match_MGRS(mgrs); String[] mgrstest = { "4 jul 2008", "10 Jan 1994", // edge case, bare minimum. "10 Jan 13", // edge case, bare minimum. "10 Jan 94", // no, this is the real bare minimum. "38SMB 461136560", "38SMB 461103656", "38SMB 46110 3656", "38SMB 4611 03656", // 0-padded Northing/Easting? 7 4 or 0007 0004 "38SMB 46110365 60", "38SMB 46110365\n60", // even, but whitespace "38SMB 4611035\n60", // odd, and whitespace "38 SMB 4611 3656", "42 RPR 4611 3656", "10 Jan 2005 02", // MGRS 01, 10JAN 200502 "10 Jan 1995 02" }; xcoord.match_DD(dd); String[] ddtest = { "15S5E", "TARGET [1] LATITUDE: +32.3345 LONGITUDE: -179.3412", //DD04 "TARGET [1] LATITUDE= +32.3345 LONGITUDE= -179.3412", //DD04 "42.3N; 102.4W", "42.3 N; 102.4 W", "23.34N 88.22E", //"34.00N 44E", // Expected to fail as this matches DD06 -- 44E anchors this as a straight degree pattern. "N32.3345:W179.3412", // DD01 "+32.3345:-179.3412", // DD03 " 32.3345:-179.3412", // DD03 " 32.3345;-179.3412", // DD03 "032.3345;-179.3412", // DD03 leading 0 on lat; "N32.3345:W179.3412", // DD01 "032.3345N;-179.3412W", // DD03 leading 0 on lat; "N32.3345:E179.3412", // DD01 "32.3345N/179.3412E", // DD02 "32.33N 179.34E" // DD02 }; xcoord.match_DMS(dms); xcoord.match_DM(dm); String[] dmtest = { "15S5E", //"01-02-03-04 005-06-07-08", "12 kts; B 14:28N - 053:00E", " 79.22.333N, 100.22.333W", " N 01 44' E 101 22'", "+42 18.0 x -102 24.0", "42 DEG 18.0N 102 DEG 24.0W", "#TEST DM 01b 01DEG 44 N 101DEG 44 E", "03bv 4218N 10224W", "03bv 4218'N 10224'W", "03bv 42 18'N 102 24'W", "N 01 44' E 101 22'", "1122N-00 11122W-00", "01DEG 44N 101DEG 44E", "42 9-00 N 102 6-00W", "N42 18-00 x W102 24-00", "N01 44' 55.5\" E101 22' 33.0\"", "N 01 44' 55\" E 101 22'33.0\"", "33-04-05 12:11:10", "3124' 7021'", "4055'23.2\" 943'51.1\"", // No HEMI "-4055'23.2\" +943'51.1\"", // with HEMI "42 9-00 N 102 6-00W; ", "42 18-009 N 102 24-009W; ", "0829.067' 1314.067'", // No HEMI "0829.067'N 1314.067'W", "0829.067'N 11314.067'W", "4055'23.2\"N 943'51\"E", "42 18' 00\" 102 24' 00", "(42 18' 00\" 102 24' 00", "01 44' 55.5\" 101 22' 33.0\"", "7755'33.22\"N 12733'22.11\"W", "40:26:46.123N,79:56:55.000W", "43-04-30.2720N 073-34-58.4170W", "31 53 45.55N 54 16 38.99E", "42.18.009N x 102.24.003W", "42.18.009N 102.24.003W", "42.18.009 N x 102.24.003 W", "014455666N1012233444E", "N7922333W10022333", "0144'55.5\"N 10122'33.0\"E;", "N0144'55.5\" E10122'33.0\"", "4025131234N 12015191234W", "5113N 00425E", "27 37' 45N, 82 42' 10W", // original "27 37' 45N, 82 42' 10W", // single second hash sym "27 37' 45N 82 42' 10W", // no lat/lon sep "27 37 45N, 82 42 10W" // no min hash. }; int count = 0; List<String> tests = new ArrayList<>(); if (dd) { tests.addAll(Arrays.asList(ddtest)); } if (dms | dm) { tests.addAll(Arrays.asList(dmtest)); } if (mgrs) { tests.addAll(Arrays.asList(mgrstest)); } for (String testcase : tests) { ++count; String test_id = "" + count; results = xcoord.extract_coordinates("text before " + testcase + " and after", test_id); log.info("TEST (" + count + ") " + testcase + " FOUND:" + (results.matches.isEmpty() ? "NOTHING" : results.matches.size())); if (results.matches != null) { for (TextMatch m : results.matches) { log.info("\t" + m.toString()); GeocoordMatch g = (GeocoordMatch) m; log.info("\t" + g.formatLatitude() + ", " + g.formatLongitude()); } } } log.info("=== ADHOC TESTS DONE ==="); } }