List of usage examples for java.util BitSet BitSet
public BitSet()
From source file:org.roaringbitmap.TestRoaringBitmap.java
@Test public void flipTest7() { // within 1 word, first container System.out.println("FlipTest7"); final RoaringBitmap rb = new RoaringBitmap(); rb.flip(650, 132000);//from ww w.j a v a2 s .c o m rb.flip(648, 651); final int rbcard = rb.getCardinality(); // 648, 649, 651-131999 Assert.assertEquals(132000 - 651 + 2, rbcard); final BitSet bs = new BitSet(); bs.set(648); bs.set(649); for (int i = 651; i < 132000; ++i) bs.set(i); Assert.assertTrue(equals(bs, rb)); }
From source file:org.roaringbitmap.TestRoaringBitmap.java
@Test public void flipTest7A() { // within 1 word, first container System.out.println("FlipTest7A"); final RoaringBitmap rb = new RoaringBitmap(); final RoaringBitmap rb1 = RoaringBitmap.flip(rb, 650, 132000); final RoaringBitmap rb2 = RoaringBitmap.flip(rb1, 648, 651); final int rbcard = rb2.getCardinality(); // 648, 649, 651-131999 Assert.assertEquals(132000 - 651 + 2, rbcard); final BitSet bs = new BitSet(); bs.set(648);// w w w . j a v a 2s . c o m bs.set(649); for (int i = 651; i < 132000; ++i) bs.set(i); Assert.assertTrue(equals(bs, rb2)); }
From source file:org.roaringbitmap.TestRoaringBitmap.java
@Test public void flipTest8() { final RoaringBitmap rb = new RoaringBitmap(); rb.add(0);//from ww w . java 2 s .c om rb.add(2); final RoaringBitmap rb2 = RoaringBitmap.flip(rb, 0, 3); final BitSet bs = new BitSet(); bs.set(1); Assert.assertTrue(equals(bs, rb2)); }
From source file:org.roaringbitmap.TestRoaringBitmap.java
@Test public void flipTestBig() { final int numCases = 1000; System.out.println("flipTestBig for " + numCases + " tests"); final RoaringBitmap rb = new RoaringBitmap(); final BitSet bs = new BitSet(); final Random r = new Random(3333); int checkTime = 2; for (int i = 0; i < numCases; ++i) { final int start = r.nextInt(65536 * 20); int end = r.nextInt(65536 * 20); if (r.nextDouble() < 0.1) end = start + r.nextInt(100); rb.flip(start, end);/*w ww. ja v a 2 s . c o m*/ if (start < end) bs.flip(start, end); // throws exception // otherwise // insert some more ANDs to keep things sparser if (r.nextDouble() < 0.2) { final RoaringBitmap mask = new RoaringBitmap(); final BitSet mask1 = new BitSet(); final int startM = r.nextInt(65536 * 20); final int endM = startM + 100000; mask.flip(startM, endM); mask1.flip(startM, endM); mask.flip(0, 65536 * 20 + 100000); mask1.flip(0, 65536 * 20 + 100000); rb.and(mask); bs.and(mask1); } // see if we can detect incorrectly shared containers if (r.nextDouble() < 0.1) { final RoaringBitmap irrelevant = RoaringBitmap.flip(rb, 10, 100000); irrelevant.flip(5, 200000); irrelevant.flip(190000, 260000); } if (i > checkTime) { Assert.assertTrue(equals(bs, rb)); checkTime *= 1.5; } } }
From source file:org.roaringbitmap.TestRoaringBitmap.java
@Test public void flipTestBigA() { final int numCases = 1000; final BitSet bs = new BitSet(); final Random r = new Random(3333); int checkTime = 2; RoaringBitmap rb1 = new RoaringBitmap(), rb2 = null; // alternate // between//from w ww. jav a2s . c om // them for (int i = 0; i < numCases; ++i) { final int start = r.nextInt(65536 * 20); int end = r.nextInt(65536 * 20); if (r.nextDouble() < 0.1) end = start + r.nextInt(100); if ((i & 1) == 0) { rb2 = RoaringBitmap.flip(rb1, start, end); // tweak the other, catch bad sharing int r1 = r.nextInt(65536 * 20); int r2 = r.nextInt(65536 * 20); rb1.flip(r1, r2); } else { rb1 = RoaringBitmap.flip(rb2, start, end); int r1 = r.nextInt(65536 * 20); int r2 = r.nextInt(65536 * 20); rb2.flip(r1, r2); } if (start < end) { bs.flip(start, end); // throws exception // otherwise } // insert some more ANDs to keep things sparser if (r.nextDouble() < 0.2 && (i & 1) == 0) { final RoaringBitmap mask = new RoaringBitmap(); final BitSet mask1 = new BitSet(); final int startM = r.nextInt(65536 * 20); final int endM = startM + 100000; mask.flip(startM, endM); mask1.flip(startM, endM); mask.flip(0, 65536 * 20 + 100000); mask1.flip(0, 65536 * 20 + 100000); rb2.and(mask); bs.and(mask1); } if (i > checkTime) { System.out.println("check after " + i + ", card = " + rb2.getCardinality()); final RoaringBitmap rb = (i & 1) == 0 ? rb2 : rb1; final boolean status = equals(bs, rb); Assert.assertTrue(status); checkTime *= 1.5; } } }
From source file:org.roaringbitmap.TestRoaringBitmap.java
public void rTest(final int N) { System.out.println("rtest N=" + N); for (int gap = 1; gap <= 65536; gap *= 2) { final BitSet bs1 = new BitSet(); final RoaringBitmap rb1 = new RoaringBitmap(); for (int x = 0; x <= N; x += gap) { bs1.set(x);/*from w ww. j a v a 2 s .c o m*/ rb1.add(x); } if (bs1.cardinality() != rb1.getCardinality()) throw new RuntimeException("different card"); if (!equals(bs1, rb1)) throw new RuntimeException("basic bug"); for (int offset = 1; offset <= gap; offset *= 2) { final BitSet bs2 = new BitSet(); final RoaringBitmap rb2 = new RoaringBitmap(); for (int x = 0; x <= N; x += gap) { bs2.set(x + offset); rb2.add(x + offset); } if (bs2.cardinality() != rb2.getCardinality()) throw new RuntimeException("different card"); if (!equals(bs2, rb2)) throw new RuntimeException("basic bug"); BitSet clonebs1; // testing AND clonebs1 = (BitSet) bs1.clone(); clonebs1.and(bs2); if (!equals(clonebs1, RoaringBitmap.and(rb1, rb2))) throw new RuntimeException("bug and"); { final RoaringBitmap t = rb1.clone(); t.and(rb2); if (!equals(clonebs1, t)) throw new RuntimeException("bug inplace and"); if (!t.equals(RoaringBitmap.and(rb1, rb2))) { System.out.println(t.highLowContainer.getContainerAtIndex(0).getClass().getCanonicalName()); System.out.println(RoaringBitmap.and(rb1, rb2).highLowContainer.getContainerAtIndex(0) .getClass().getCanonicalName()); throw new RuntimeException("bug inplace and"); } } // testing OR clonebs1 = (BitSet) bs1.clone(); clonebs1.or(bs2); if (!equals(clonebs1, RoaringBitmap.or(rb1, rb2))) throw new RuntimeException("bug or"); { final RoaringBitmap t = rb1.clone(); t.or(rb2); if (!equals(clonebs1, t)) throw new RuntimeException("bug or"); if (!t.equals(RoaringBitmap.or(rb1, rb2))) throw new RuntimeException("bug or"); if (!t.toString().equals(RoaringBitmap.or(rb1, rb2).toString())) throw new RuntimeException("bug or"); } // testing XOR clonebs1 = (BitSet) bs1.clone(); clonebs1.xor(bs2); if (!equals(clonebs1, RoaringBitmap.xor(rb1, rb2))) { throw new RuntimeException("bug xor"); } { final RoaringBitmap t = rb1.clone(); t.xor(rb2); if (!equals(clonebs1, t)) throw new RuntimeException("bug xor"); if (!t.equals(RoaringBitmap.xor(rb1, rb2))) throw new RuntimeException("bug xor"); } // testing NOTAND clonebs1 = (BitSet) bs1.clone(); clonebs1.andNot(bs2); if (!equals(clonebs1, RoaringBitmap.andNot(rb1, rb2))) { throw new RuntimeException("bug andnot"); } clonebs1 = (BitSet) bs2.clone(); clonebs1.andNot(bs1); if (!equals(clonebs1, RoaringBitmap.andNot(rb2, rb1))) { throw new RuntimeException("bug andnot"); } { final RoaringBitmap t = rb2.clone(); t.andNot(rb1); if (!equals(clonebs1, t)) { throw new RuntimeException("bug inplace andnot"); } final RoaringBitmap g = RoaringBitmap.andNot(rb2, rb1); if (!equals(clonebs1, g)) { throw new RuntimeException("bug andnot"); } if (!t.equals(g)) throw new RuntimeException("bug"); } clonebs1 = (BitSet) bs1.clone(); clonebs1.andNot(bs2); if (!equals(clonebs1, RoaringBitmap.andNot(rb1, rb2))) { throw new RuntimeException("bug andnot"); } { final RoaringBitmap t = rb1.clone(); t.andNot(rb2); if (!equals(clonebs1, t)) { throw new RuntimeException("bug andnot"); } final RoaringBitmap g = RoaringBitmap.andNot(rb1, rb2); if (!equals(clonebs1, g)) { throw new RuntimeException("bug andnot"); } if (!t.equals(g)) throw new RuntimeException("bug"); } } } }
From source file:org.apache.carbondata.core.scan.filter.FilterUtil.java
/** * Below method will be used to get all the include filter values in case of range filters when * blocklet is encoded with local dictionary * @param expression/*from w ww.ja v a 2s.c o m*/ * filter expression * @param dictionary * dictionary * @return include filter bitset * @throws FilterUnsupportedException */ private static BitSet getIncludeDictFilterValuesForRange(Expression expression, CarbonDictionary dictionary) throws FilterUnsupportedException { ConditionalExpression conExp = (ConditionalExpression) expression; ColumnExpression columnExpression = conExp.getColumnList().get(0); BitSet includeFilterBitSet = new BitSet(); for (int i = 2; i < dictionary.getDictionarySize(); i++) { if (null == dictionary.getDictionaryValue(i)) { continue; } try { RowIntf row = new RowImpl(); String stringValue = new String(dictionary.getDictionaryValue(i), Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(stringValue, columnExpression.getCarbonColumn().getDataType()) }); Boolean rslt = expression.evaluate(row).getBoolean(); if (null != rslt) { if (rslt) { includeFilterBitSet.set(i); } } } catch (FilterIllegalMemberException e) { LOGGER.debug(e.getMessage()); } } return includeFilterBitSet; }
From source file:FourByFour.java
public Positions() { // Define colors for lighting Color3f white = new Color3f(1.0f, 1.0f, 1.0f); Color3f black = new Color3f(0.0f, 0.0f, 0.0f); Color3f red = new Color3f(0.9f, 0.1f, 0.2f); Color3f blue = new Color3f(0.3f, 0.3f, 0.8f); Color3f yellow = new Color3f(1.0f, 1.0f, 0.0f); Color3f ambRed = new Color3f(0.3f, 0.03f, 0.03f); Color3f ambBlue = new Color3f(0.03f, 0.03f, 0.3f); Color3f ambYellow = new Color3f(0.3f, 0.3f, 0.03f); Color3f ambWhite = new Color3f(0.3f, 0.3f, 0.3f); Color3f specular = new Color3f(1.0f, 1.0f, 1.0f); // Create the red appearance node redMat = new Material(ambRed, black, red, specular, 100.f); redMat.setLightingEnable(true);// w ww .j a v a 2 s.com redApp = new Appearance(); redApp.setMaterial(redMat); // Create the blue appearance node blueMat = new Material(ambBlue, black, blue, specular, 100.f); blueMat.setLightingEnable(true); blueApp = new Appearance(); blueApp.setMaterial(blueMat); // Create the yellow appearance node yellowMat = new Material(ambYellow, black, yellow, specular, 100.f); yellowMat.setLightingEnable(true); yellowApp = new Appearance(); yellowApp.setMaterial(yellowMat); // Create the white appearance node whiteMat = new Material(ambWhite, black, white, specular, 100.f); whiteMat.setLightingEnable(true); whiteApp = new Appearance(); whiteApp.setMaterial(whiteMat); // Load the point array with the offset (coordinates) for each of // the 64 positions. point = new Vector3f[64]; int count = 0; for (int i = -30; i < 40; i += 20) { for (int j = -30; j < 40; j += 20) { for (int k = -30; k < 40; k += 20) { point[count] = new Vector3f((float) k, (float) j, (float) i); count++; } } } // Create the switch nodes posSwitch = new Switch(Switch.CHILD_MASK); humanSwitch = new Switch(Switch.CHILD_MASK); machineSwitch = new Switch(Switch.CHILD_MASK); // Set the capability bits posSwitch.setCapability(Switch.ALLOW_SWITCH_READ); posSwitch.setCapability(Switch.ALLOW_SWITCH_WRITE); humanSwitch.setCapability(Switch.ALLOW_SWITCH_READ); humanSwitch.setCapability(Switch.ALLOW_SWITCH_WRITE); machineSwitch.setCapability(Switch.ALLOW_SWITCH_READ); machineSwitch.setCapability(Switch.ALLOW_SWITCH_WRITE); // Create the bit masks posMask = new BitSet(); humanMask = new BitSet(); machineMask = new BitSet(); // Create the small white spheres that mark unoccupied // positions. posSphere = new Sphere[64]; for (int i = 0; i < 64; i++) { Transform3D transform3D = new Transform3D(); transform3D.set(point[i]); TransformGroup transformGroup = new TransformGroup(transform3D); posSphere[i] = new Sphere(2.0f, Sphere.GENERATE_NORMALS | Sphere.ENABLE_APPEARANCE_MODIFY, 12, whiteApp); Shape3D shape = posSphere[i].getShape(); ID id = new ID(i); shape.setUserData(id); transformGroup.addChild(posSphere[i]); posSwitch.addChild(transformGroup); posMask.set(i); } // Create the red spheres that mark the user's positions. for (int i = 0; i < 64; i++) { Transform3D transform3D = new Transform3D(); transform3D.set(point[i]); TransformGroup transformGroup = new TransformGroup(transform3D); transformGroup.addChild(new Sphere(7.0f, redApp)); humanSwitch.addChild(transformGroup); humanMask.clear(i); } // Create the blue cubes that mark the computer's positions. for (int i = 0; i < 64; i++) { Transform3D transform3D = new Transform3D(); transform3D.set(point[i]); TransformGroup transformGroup = new TransformGroup(transform3D); BigCube cube = new BigCube(blueApp); transformGroup.addChild(cube.getChild()); machineSwitch.addChild(transformGroup); machineMask.clear(i); } // Set the positions mask posSwitch.setChildMask(posMask); humanSwitch.setChildMask(humanMask); machineSwitch.setChildMask(machineMask); // Throw everything into a single group group = new Group(); group.addChild(posSwitch); group.addChild(humanSwitch); group.addChild(machineSwitch); }
From source file:gov.noaa.pfel.erddap.util.EDStatic.java
/** * This returns the Oceanic/Atmospheric Acronyms table as a Table: * col0=acronym, col1=fullName.//from w w w. j a va 2s . c om * THIS IS ONLY FOR GenerateDatasetsXml THREADS -- a few common acronyms are removed. * * @return the oceanic/atmospheric variable names table with some common acronyms removed * @throws Exception if trouble (e.g., file not found) */ public static Table gdxAcronymsTable() throws Exception { if (gdxAcronymsTable == null) { Table table = oceanicAtmosphericAcronymsTable(); StringArray acronymSA = (StringArray) (table.getColumn(0)); StringArray fullNameSA = (StringArray) (table.getColumn(1)); //remove some really common acronyms I don't want to expand BitSet keep = new BitSet(); keep.set(0, acronymSA.size()); String common[] = { //"DOC", "DOD", "DOE", "USDOC", "USDOD", "USDOE", "NOAA", "NASA", "US" }; for (int c = 0; c < common.length; c++) { int po = acronymSA.indexOf(common[c]); if (po >= 0) keep.clear(po); } table.justKeep(keep); gdxAcronymsTable = table; //swap into place } return gdxAcronymsTable; }
From source file:gov.noaa.pfel.erddap.dataset.EDDTableFromNcFiles.java
/** NOT FOR GENERAL USE. Bob uses this to consolidate the individual GTSPP * data files into 30 x 30 x 1 month files (tiles). * 30 x 30 leads to 12x6=72 files for a given time point, so a request * for a short time but entire world opens ~72 files. * There are ~240 months worth of data, so a request for a small lon lat * range for all time opens ~240 files.// w ww . java2 s. c o m * * <p>Why tile? Because there are ~10^6 profiles/year now, so ~10^7 total. * And if 100 bytes of info per file for EDDTableFromFiles fileTable, that's 1 GB!. * So there needs to be fewer files. * We want to balance number of files for 1 time point (all region tiles), * and number of time point files (I'll stick with their use of 1 month). * The tiling size selected is ok, but searches for single profile (by name) * are slow since a given file may have a wide range of station_ids. * * <p>Quality flags * <br>https://www.nodc.noaa.gov/GTSPP/document/qcmans/GTSPP_RT_QC_Manual_20090916.pdf * <br>http://www.ifremer.fr/gosud/formats/gtspp_qcflags.htm * <br>CODE SIGNIFICATION * <br>0 NOT CONTROLLED VALUE * <br>1 CORRECT VALUE * <br>2 VALUE INCONSISTENT WITH STATISTICS * <br>3 DOUBTFUL VALUE (spike, ...) * <br>4 FALSE VALUE (out of scale, constant profile, vertical instability, ...) * <br>5 VALUE MODIFIED DURING QC (only for interpolate location or date) * <br>6-8 Not USED * <br>9 NO VALUE * <br> * <br>I interpret as: okay values are 1, 2, 5 * * @param firstYear e.g., 1990 * @param firstMonth e.g., 1 (1..) * @param lastYear e.g., 2010 * @param lastMonth e.g., 12 (1..) * @param testMode if true, this just processes .nc files * already in testTempDir f:/data/gtspp/testTemp/ * and puts results in testDestDir f:/data/gtspp/testDest/. * So the first/last/Year/Month params are ignored. */ public static void bobConsolidateGtsppTgz(int firstYear, int firstMonth, int lastYear, int lastMonth, boolean testMode) throws Throwable { int chunkSize = 45; //lon width, lat height of a tile, in degrees int minLat = -90; int maxLat = 90; int minLon = -180; int maxLon = 180; String today = Calendar2.getCurrentISODateTimeStringZulu().substring(0, 10); //to nearest day String sevenZip = "c:\\progra~1\\7-Zip\\7z"; String zipDir = "c:\\data\\gtspp\\bestNcZip\\"; //gtspp_at199001.tgz String destDir = "c:\\data\\gtspp\\bestNcConsolidated\\"; String tempDir = "c:\\data\\gtspp\\temp\\"; String testTempDir = "c:\\data\\gtspp\\testTemp\\"; //tempDir if testMode=true String testDestDir = "c:\\data\\gtspp\\testDest\\"; //destDir if testMode=true String logFile = "c:\\data\\gtspp\\log" + String2.replaceAll(today, "-", "") + ".txt"; File2.makeDirectory(tempDir); //https://www.nodc.noaa.gov/GTSPP/document/qcmans/qcflags.htm //1=correct, 2=probably correct, 5=modified (so now correct) //pre 2012-04-15 was {1,2,5} //pre 2012-05-25 was {1,2} int okQF[] = { 1, 2, 5 }; String okQFCsv = String2.toCSSVString(okQF); float depthMV = 99999; //was -99; float temperatureMV = 99999; //was -99; float salinityMV = 99999; //was -99; int qMV = 9; String timeUnits = "days since 1900-01-01 00:00:00"; //causes roundoff error(!) double timeBaseAndFactor[] = Calendar2.getTimeBaseAndFactor(timeUnits); //impossible values: float minDepth = -0.4f, maxDepth = 10000; //-0.4 allows for imprecise values float minTemperature = -4, maxTemperature = 40; float minSalinity = 0, maxSalinity = 41; if (testMode) { firstYear = 1990; firstMonth = 1; lastYear = 1990; lastMonth = 1; } SSR.verbose = false; String2.setupLog(true, false, logFile, false, 1000000000); String2.log("*** starting bobConsolidateGtsppTgz " + Calendar2.getCurrentISODateTimeStringLocalTZ() + "\n" + "logFile=" + String2.logFileName() + "\n" + String2.standardHelpAboutMessage()); long elapsedTime = System.currentTimeMillis(); //q_pos (position quality flag), q_date_time (time quality flag) int stationCol = -1, organizationCol = -1, dataTypeCol = -1, platformCol = -1, cruiseCol = -1, longitudeCol = -1, latitudeCol = -1, timeCol = -1, depthCol = -1, temperatureCol = -1, salinityCol = -1; int totalNGoodStation = 0, totalNGoodPos = 0, totalNGoodTime = 0, totalNGoodDepth = 0, totalNGoodTemperature = 0, totalNGoodSalinity = 0; int totalNBadStation = 0, totalNBadPos = 0, totalNBadTime = 0, totalNBadDepth = 0, totalNBadTemperature = 0, totalNBadSalinity = 0, totalNWarnings = 0, totalNExceptions = 0; long totalNGoodRows = 0, totalNBadRows = 0; StringArray impossibleNanLat = new StringArray(); StringArray impossibleMinLat = new StringArray(); StringArray impossibleMaxLat = new StringArray(); StringArray impossibleNanLon = new StringArray(); StringArray impossibleMinLon = new StringArray(); StringArray impossibleMaxLon = new StringArray(); //StringArray impossibleNaNDepth = new StringArray(); StringArray impossibleMinDepth = new StringArray(); StringArray impossibleMaxDepth = new StringArray(); //StringArray impossibleNanTemperature = new StringArray(); StringArray impossibleMinTemperature = new StringArray(); StringArray impossibleMaxTemperature = new StringArray(); //StringArray impossibleNanSalinity = new StringArray(); StringArray impossibleMinSalinity = new StringArray(); StringArray impossibleMaxSalinity = new StringArray(); int nLons = 0, nLats = 0, nFiles = 0; int lonSum = 0, latSum = 0; long profilesSum = 0; long rowsSum = 0; //*** process a month's data int year = firstYear; int month = firstMonth; long chunkTime = System.currentTimeMillis(); while (year <= lastYear) { String2.log("\n*** " + Calendar2.getCurrentISODateTimeStringLocalTZ() + " start processing year=" + year + " month=" + month); String zMonth = String2.zeroPad("" + month, 2); String zMonth1 = String2.zeroPad("" + (month + 1), 2); double minEpochSeconds = Calendar2.isoStringToEpochSeconds(year + "-" + zMonth + "-01"); double maxEpochSeconds = Calendar2.isoStringToEpochSeconds(year + "-" + zMonth1 + "-01"); //destination directory String tDestDir = testMode ? testDestDir : destDir + year + "\\" + zMonth + "\\"; File2.makeDirectory(tDestDir); HashMap tableHashMap = new HashMap(); //make sure all files are deleted int waitSeconds = 2; int nAttempts = 10; long cmdTime = System.currentTimeMillis(); String cmd = "del/q " + tDestDir + "*.*"; for (int attempt = 0; attempt < nAttempts; attempt++) { if (attempt % 8 == 0) { String2.log(cmd); SSR.dosShell(cmd, 30 * 60); //10 minutes*60 seconds //File2.deleteAllFiles(tempDir); //previous method } Math2.gc(waitSeconds * 1000); //gtspp: give OS time to settle File destDirFile = new File(tDestDir); File files[] = destDirFile.listFiles(); String2.log(" nRemainingFiles=" + files.length); if (files.length == 0) break; waitSeconds = 2 * nAttempts; } String2.log(" cmd total time=" + Calendar2.elapsedTimeString(System.currentTimeMillis() - cmdTime)); //unzip all atlantic, indian, and pacific .zip files for that month String region2[] = { "at", "in", "pa" }; int nRegions = testMode ? 1 : 3; for (int region = 0; region < nRegions; region++) { String sourceBaseName = "gtspp4_" + region2[region] + year + zMonth; String sourceZipJustFileName = sourceBaseName + ".tgz"; String sourceZipName = zipDir + sourceZipJustFileName; if (!testMode) { //delete all files in tempDir waitSeconds = 2; nAttempts = 10; cmdTime = System.currentTimeMillis(); cmd = "del/q " + tempDir + "*.*"; String2.log(""); //blank line for (int attempt = 0; attempt < nAttempts; attempt++) { String2.log(cmd); SSR.dosShell(cmd, 30 * 60); //30 minutes*60 seconds //File2.deleteAllFiles(tempDir); //previous method //delete dirs too File2.deleteAllFiles(tempDir, true, true); Math2.gc(waitSeconds * 1000); //gtspp: give OS time to settle String2.log(" " + Math2.memoryString()); File tempDirFile = new File(tempDir); File files[] = tempDirFile.listFiles(); String2.log(" nRemainingFiles=" + files.length); if (files.length == 0) break; waitSeconds = 2 * nAttempts; } String2.log(" cmd total time=" + Calendar2.elapsedTimeString(System.currentTimeMillis() - cmdTime)); //unzip file into tempDir //gtspp_at199001.zip cmd = sevenZip + " -y e " + sourceZipName + " -o" + tempDir + " -r"; cmdTime = System.currentTimeMillis(); String2.log("\n*** " + cmd); if (File2.isFile(sourceZipName)) { try { SSR.dosShell(cmd, 30 * 60); //10 minutes*60 seconds String2.log(" cmd time=" + Calendar2.elapsedTimeString(System.currentTimeMillis() - cmdTime)); //extract from the .tar file //gtspp4_at199001.tar cmd = sevenZip + " -y e " + tempDir + sourceBaseName + ".tar -o" + tempDir + " -r"; cmdTime = System.currentTimeMillis(); String2.log("\n*** " + cmd); SSR.dosShell(cmd, 120 * 60); //120 minutes*60 seconds String2.log(" cmd time=" + Calendar2.elapsedTimeString(System.currentTimeMillis() - cmdTime)); } catch (Exception e) { String2.log("Caught exception: " + MustBe.throwableToString(e)); } } //previous method //SSR.unzip(sourceZipName, // tempDir, true, 100 * 60, null); //ignoreZipDirectories, timeOutSeconds 100 minutes } //read each file and put data in proper table String tTempDir = testMode ? testTempDir : tempDir; File tTempDirAsFile = new File(tTempDir); String sourceFileNames[] = tTempDirAsFile.list(); //just the file names String2.log("\nunzipped " + sourceFileNames.length + " files"); int nSourceFileNames = //testMode? 100 : sourceFileNames.length; int nGoodStation = 0, nGoodPos = 0, nGoodTime = 0, nGoodDepth = 0, nGoodTemperature = 0, nGoodSalinity = 0, nGoodRows = 0; int nBadStation = 0, nBadPos = 0, nBadTime = 0, nBadDepth = 0, nBadTemperature = 0, nBadSalinity = 0, nBadRows = 0, nWarnings = 0, nExceptions = 0; long fileReadTime = System.currentTimeMillis(); profilesSum += nSourceFileNames; for (int sfi = 0; sfi < nSourceFileNames; sfi++) { String sourceFileName = sourceFileNames[sfi]; if (sfi % 10000 == 0) { //if (sfi > 0) //2012-12-13 commented out. Let Java handle it. // Math2.gc(3 * 1000); //gtspp: give OS time to settle //high water mark is ~160 MB, so memory not a problem String2.log("file #" + sfi + " " + Math2.memoryString()); } if (!sourceFileName.endsWith(".nc")) { //String2.log("ERROR: not a .nc file: " + sourceFileName); continue; } NetcdfFile ncFile = null; try { //get the station name //gtspp_13635162_te_111.nc gtspp_10313692_cu_111.nc if (!sourceFileName.matches("gtspp_[0-9]+_.*\\.nc")) { //was "\\d+")) {//all digits nBadStation++; throw new SimpleException("Invalid sourceFileName=" + sourceFileName); } int po = sourceFileName.indexOf('_', 6); if (po < 0) { nBadStation++; throw new SimpleException("Invalid sourceFileName=" + sourceFileName); } int station = String2.parseInt(sourceFileName.substring(6, po)); nGoodStation++; String key = sourceZipJustFileName + " " + sourceFileName; //open the file ncFile = NcHelper.openFile(tTempDir + sourceFileName); Variable var; Attributes tVarAtts = new Attributes(); String tUnits; //get all of the data //stream_ident var = ncFile.findVariable("stream_ident"); String organization = ""; String dataType = ""; if (var == null) { nWarnings++; String2.log("WARNING: No stream_ident in " + sourceFileName); } else { PrimitiveArray streamPA = NcHelper.getPrimitiveArray(var); if (streamPA instanceof StringArray && streamPA.size() > 0) { String stream = streamPA.getString(0); if (stream.length() >= 4) { organization = stream.substring(0, 2).trim(); dataType = stream.substring(2, 4).trim(); } else { String2.log("WARNING: stream_ident isn't a 4 char string: " + stream); } } else { String2.log("WARNING: stream_ident isn't a StringArray: " + streamPA.toString()); } } //platform_code var = ncFile.findVariable("gtspp_platform_code"); String platform = ""; if (var == null) { //a small percentage have this problem //nWarnings++; //String2.log("WARNING: No gtspp_platform_code in " + sourceFileName); } else { PrimitiveArray pa = NcHelper.getPrimitiveArray(var); if (pa instanceof StringArray && pa.size() > 0) { platform = pa.getString(0).trim(); //String2.log("platform_code=" + platform_code); } else { String2.log("WARNING: gtspp_platform_code isn't a StringArray: " + pa.toString()); } } //cruise var = ncFile.findVariable("cruise_id"); String cruise = ""; if (var == null) { nWarnings++; String2.log("WARNING: No cruise_id in " + sourceFileName); } else { PrimitiveArray cruisePA = NcHelper.getPrimitiveArray(var); if (cruisePA instanceof StringArray && cruisePA.size() > 0) { cruise = cruisePA.getString(0).trim(); } else { String2.log("WARNING: cruise_id isn't a StringArray: " + cruisePA.toString()); } } //prof_type is TEMP or PSAL so don't save it. /*var = ncFile.findVariable("prof_type"); String prof_type = ""; if (var == null) { nWarnings++; String2.log("WARNING: No prof_type in " + sourceFileName); } else { PrimitiveArray pa = NcHelper.getPrimitiveArray(var); if (pa instanceof StringArray && pa.size() > 0) { prof_type = pa.getString(0).trim(); String2.log("prof_type=" + prof_type); } else { String2.log("WARNING: prof_type isn't a StringArray: " + pa.toString()); } }*/ //position quality flag var = ncFile.findVariable("position_quality_flag"); //was "q_pos"); if (var == null) { nWarnings++; String2.log("WARNING: No position_quality_flag in " + sourceFileName); } else { PrimitiveArray q_pos = NcHelper.getPrimitiveArray(var); if (!(q_pos instanceof IntArray) || q_pos.size() != 1) throw new SimpleException("Invalid position_quality_flag=" + q_pos); int ti = q_pos.getInt(0); if (String2.indexOf(okQF, ti) < 0) { nBadPos++; continue; } //nGoodPos++; is below } //time quality flag var = ncFile.findVariable("time_quality_flag"); //q_date_time"); if (var == null) { nWarnings++; String2.log("WARNING: No time_quality_flag in " + sourceFileName); } else { PrimitiveArray q_date_time = NcHelper.getPrimitiveArray(var); if (!(q_date_time instanceof IntArray) || q_date_time.size() != 1) throw new SimpleException("Invalid time_quality_flag=" + q_date_time); int ti = q_date_time.getInt(0); if (String2.indexOf(okQF, ti) < 0) { nBadTime++; continue; } //nGoodTime is below } //time var = ncFile.findVariable("time"); if (var == null) throw new SimpleException("No time!"); tVarAtts.clear(); NcHelper.getVariableAttributes(var, tVarAtts); tUnits = tVarAtts.getString("units"); if (!timeUnits.equals(tUnits)) throw new SimpleException("Invalid time units=" + tUnits); PrimitiveArray time = NcHelper.getPrimitiveArray(var); if (!(time instanceof DoubleArray) || time.size() != 1) throw new SimpleException("Invalid time=" + time); double tTime = Calendar2.unitsSinceToEpochSeconds(timeBaseAndFactor[0], timeBaseAndFactor[1], time.getDouble(0)); if (tTime < minEpochSeconds || tTime > maxEpochSeconds) throw new SimpleException( "Invalid tTime=" + Calendar2.safeEpochSecondsToIsoStringTZ(tTime, "")); //original times (that I looked at) are to nearest second //so round to nearest second (fix .99999 problems) tTime = Math.rint(tTime); nGoodTime++; //longitude (position qFlag is good) var = ncFile.findVariable("longitude"); if (var == null) { impossibleNanLon.add(key + " lon=null"); continue; } PrimitiveArray longitude = NcHelper.getPrimitiveArray(var); if (!(longitude instanceof FloatArray) || longitude.size() != 1) { impossibleNanLon.add(key + " lon=wrongTypeOrSize"); continue; } float lon = longitude.getFloat(0); if (Float.isNaN(lon)) { impossibleNanLon.add(key + " lon=NaN"); continue; } else if (lon < minLon) { impossibleMinLon.add(key + " lon=" + lon); //fall through } else if (lon > maxLon) { impossibleMaxLon.add(key + " lon=" + lon); //fall through } lon = (float) Math2.anglePM180(lon); //latitude (position qFlag is good) var = ncFile.findVariable("latitude"); if (var == null) { impossibleNanLat.add(key + " lat=null"); continue; } PrimitiveArray latitude = NcHelper.getPrimitiveArray(var); if (!(latitude instanceof FloatArray) || latitude.size() != 1) { impossibleNanLat.add(key + " lat=wrongTypeOrSize"); continue; } float lat = latitude.getFloat(0); if (Float.isNaN(lat)) { impossibleNanLat.add(key + " lat=NaN"); continue; } else if (lat < minLat) { impossibleMinLat.add(key + " lat=" + lat); continue; } else if (lat > maxLat) { impossibleMaxLat.add(key + " lat=" + lat); continue; } nGoodPos++; //depth var = ncFile.findVariable("z"); if (var == null) throw new SimpleException("No z!"); PrimitiveArray depth = NcHelper.getPrimitiveArray(var); if (!(depth instanceof FloatArray) || depth.size() == 0) throw new SimpleException("Invalid z=" + depth); int nDepth = depth.size(); //DEPH_qparm var = ncFile.findVariable("z_variable_quality_flag"); //DEPH_qparm"); if (var == null) throw new SimpleException("No z_variable_quality_flag!"); PrimitiveArray DEPH_qparm = NcHelper.getPrimitiveArray(var); if (!(DEPH_qparm instanceof IntArray) || DEPH_qparm.size() != nDepth) throw new SimpleException("Invalid z_variable_quality_flag=" + DEPH_qparm); //nGoodDepth is below //temperature var = ncFile.findVariable("temperature"); PrimitiveArray temperature; PrimitiveArray TEMP_qparm; float temperatureFV = temperatureMV; if (var == null) { //nWarnings++; //String2.log("WARNING: No temperature in " + sourceFileName); reasonably common temperature = PrimitiveArray.factory(float.class, nDepth, "" + temperatureMV); TEMP_qparm = PrimitiveArray.factory(int.class, nDepth, "" + qMV); } else { temperature = NcHelper.getPrimitiveArray(var); if (!(temperature instanceof FloatArray) || temperature.size() != nDepth) throw new SimpleException("Invalid temperature=" + temperature); tVarAtts.clear(); NcHelper.getVariableAttributes(var, tVarAtts); temperatureFV = tVarAtts.getFloat("_FillValue"); if (!Float.isNaN(temperatureFV) && temperatureFV != temperatureMV) throw new SimpleException("Invalid temperature _FillValue=" + temperatureFV); //TEMP_qparm var = ncFile.findVariable("temperature_quality_flag"); //TEMP_qparm"); if (var == null) { nWarnings++; String2.log("WARNING: No temperature_quality_flag in " + sourceFileName); TEMP_qparm = PrimitiveArray.factory(int.class, nDepth, "" + qMV); } else { TEMP_qparm = NcHelper.getPrimitiveArray(var); if (!(TEMP_qparm instanceof IntArray) || TEMP_qparm.size() != nDepth) throw new SimpleException("Invalid temperature_quality_flag=" + TEMP_qparm); } } //salinity var = ncFile.findVariable("salinity"); PrimitiveArray salinity; PrimitiveArray PSAL_qparm; float salinityFV = salinityMV; if (var == null) { //String2.log("WARNING: No salinity in " + sourceFileName); //very common salinity = PrimitiveArray.factory(float.class, nDepth, "" + salinityMV); PSAL_qparm = PrimitiveArray.factory(int.class, nDepth, "" + qMV); } else { salinity = NcHelper.getPrimitiveArray(var); if (!(salinity instanceof FloatArray) || salinity.size() != nDepth) throw new SimpleException("Invalid salinity=" + salinity); tVarAtts.clear(); NcHelper.getVariableAttributes(var, tVarAtts); salinityFV = tVarAtts.getFloat("_FillValue"); if (!Float.isNaN(salinityFV) && salinityFV != salinityMV) throw new SimpleException("Invalid salinity _FillValue=" + salinityFV); //PSAL_qparm var = ncFile.findVariable("salinity_quality_flag"); //PSAL_qparm"); if (var == null) { nWarnings++; String2.log("WARNING: No salinity_quality_flag in " + sourceFileName); PSAL_qparm = PrimitiveArray.factory(int.class, nDepth, "" + qMV); } else { PSAL_qparm = NcHelper.getPrimitiveArray(var); if (!(PSAL_qparm instanceof IntArray) || PSAL_qparm.size() != nDepth) throw new SimpleException("Invalid salinity_quality_flag=" + PSAL_qparm); } } //clean the data //(good to do it here so memory usage is low -- table remains as small as possible) //Change "impossible" data to NaN //(from https://www.nodc.noaa.gov/GTSPP/document/qcmans/GTSPP_RT_QC_Manual_20090916.pdf //pg 61 has Table 2.1: Global Impossible Parameter Values). BitSet keep = new BitSet(); keep.set(0, nDepth); //all true //find worst impossible depth/temperature/salinity for this station //boolean tImpossibleNanDepth = false; //boolean tImpossibleNanTemperature = false; //boolean tImpossibleNanSalinity = false; float tImpossibleMinDepth = minDepth; float tImpossibleMaxDepth = maxDepth; float tImpossibleMinTemperature = minTemperature; float tImpossibleMaxTemperature = maxTemperature; float tImpossibleMinSalinity = minSalinity; float tImpossibleMaxSalinity = maxSalinity; for (int row = 0; row < nDepth; row++) { //DEPH_qparm int qs = DEPH_qparm.getInt(row); float f = depth.getFloat(row); if (String2.indexOf(okQF, qs) < 0) { nBadDepth++; keep.clear(row); continue; } else if (Float.isNaN(f) || f == depthMV) { //"impossible" depth //tImpossibleNanDepth = true; nBadDepth++; keep.clear(row); continue; } else if (f < minDepth) { tImpossibleMinDepth = Math.min(tImpossibleMinDepth, f); nBadDepth++; keep.clear(row); continue; } else if (f > maxDepth) { tImpossibleMaxDepth = Math.max(tImpossibleMaxDepth, f); nBadDepth++; keep.clear(row); continue; } nGoodDepth++; boolean hasData = false; //temperature qs = TEMP_qparm.getInt(row); f = temperature.getFloat(row); if (String2.indexOf(okQF, qs) < 0) { temperature.setString(row, ""); //so bad value is now NaN nBadTemperature++; } else if (Float.isNaN(f) || f == temperatureMV) { temperature.setString(row, ""); //so missing value is now NaN nBadTemperature++; } else if (f < minTemperature) { //"impossible" water temperature tImpossibleMinTemperature = Math.min(tImpossibleMinTemperature, f); temperature.setString(row, ""); //so impossible value is now NaN nBadTemperature++; } else if (f > maxTemperature) { //"impossible" water temperature tImpossibleMaxTemperature = Math.max(tImpossibleMaxTemperature, f); temperature.setString(row, ""); //so impossible value is now NaN nBadTemperature++; } else { nGoodTemperature++; hasData = true; } //salinity qs = PSAL_qparm.getInt(row); f = salinity.getFloat(row); if (String2.indexOf(okQF, qs) < 0) { salinity.setString(row, ""); //so bad value is now NaN nBadSalinity++; } else if (Float.isNaN(f) || f == salinityMV) { salinity.setString(row, ""); //so missing value is now NaN nBadSalinity++; } else if (f < minSalinity) { //"impossible" salinity tImpossibleMinSalinity = Math.min(tImpossibleMinSalinity, f); salinity.setString(row, ""); //so impossible value is now NaN nBadSalinity++; } else if (f > maxSalinity) { //"impossible" salinity tImpossibleMaxSalinity = Math.max(tImpossibleMaxSalinity, f); salinity.setString(row, ""); //so impossible value is now NaN nBadSalinity++; } else { nGoodSalinity++; hasData = true; } //no valid temperature or salinity data? if (!hasData) { keep.clear(row); } } //ensure sizes still correct Test.ensureEqual(depth.size(), nDepth, "depth.size changed!"); Test.ensureEqual(temperature.size(), nDepth, "temperature.size changed!"); Test.ensureEqual(salinity.size(), nDepth, "salinity.size changed!"); //actually remove the bad rows int tnGood = keep.cardinality(); if (testMode && verbose) String2.log( sourceFileName + ": nGoodRows=" + tnGood + " nBadRows=" + (nDepth - tnGood)); nGoodRows += tnGood; nBadRows += nDepth - tnGood; depth.justKeep(keep); temperature.justKeep(keep); salinity.justKeep(keep); nDepth = depth.size(); //impossible //if (tImpossibleNanDepth) // impossibleNanDepth.add(key + " hasNaN=true"); //if (tImpossibleNanTemperature) // impossibleNanTemperature.add(key + " hasNaN=true"); //if (tImpossibleNanSalinity) // impossibleNanSalinity.add(key + " hasNaN=true"); if (tImpossibleMinDepth < minDepth) impossibleMinDepth.add(key + " worst = " + tImpossibleMinDepth); if (tImpossibleMaxDepth > maxDepth) impossibleMaxDepth.add(key + " worst = " + tImpossibleMaxDepth); if (tImpossibleMinTemperature < minTemperature) impossibleMinTemperature.add(key + " worst = " + tImpossibleMinTemperature); if (tImpossibleMaxTemperature > maxTemperature) impossibleMaxTemperature.add(key + " worst = " + tImpossibleMaxTemperature); if (tImpossibleMinSalinity < minSalinity) impossibleMinSalinity.add(key + " worst = " + tImpossibleMinSalinity); if (tImpossibleMaxSalinity > maxSalinity) impossibleMaxSalinity.add(key + " worst = " + tImpossibleMaxSalinity); //which table if (tnGood == 0) continue; int loni = Math2 .roundToInt(Math.floor((Math.min(lon, maxLon - 0.1f) - minLon) / chunkSize)); int lati = Math2 .roundToInt(Math.floor((Math.min(lat, maxLat - 0.1f) - minLat) / chunkSize)); String outTableName = (minLon + loni * chunkSize) + "E_" + (minLat + lati * chunkSize) + "N"; //String2.replaceAll(cruise + "_" + organization + dataType, ' ', '_'); //too many: 3000+/month in 2011 Table tTable = (Table) tableHashMap.get(outTableName); if (tTable == null) { Attributes ncGlobalAtts = new Attributes(); NcHelper.getGlobalAttributes(ncFile, ncGlobalAtts); String tHistory = ncGlobalAtts.getString("history"); tHistory = tHistory != null && tHistory.length() > 0 ? tHistory + "\n" : ""; //make a table for this platform tTable = new Table(); Attributes ga = tTable.globalAttributes(); String ack = "These data were acquired from the US NOAA National Oceanographic Data Center (NODC) on " + today + " from https://www.nodc.noaa.gov/GTSPP/."; ga.add("acknowledgment", ack); ga.add("license", "These data are openly available to the public. " + "Please acknowledge the use of these data with:\n" + ack + "\n\n" + "[standard]"); ga.add("history", tHistory + ".tgz files from ftp.nodc.noaa.gov /pub/gtspp/best_nc/ (https://www.nodc.noaa.gov/GTSPP/)\n" + today + " Most recent ingest, clean, and reformat at ERD (bob.simons at noaa.gov)."); ga.add("infoUrl", "https://www.nodc.noaa.gov/GTSPP/"); ga.add("institution", "NOAA NODC"); ga.add("title", "Global Temperature and Salinity Profile Programme (GTSPP) Data"); String attName = "gtspp_ConventionVersion"; String attValue = ncGlobalAtts.getString(attName); if (attValue != null && attValue.length() > 0) ga.add(attName, attValue); attName = "gtspp_program"; attValue = ncGlobalAtts.getString(attName); if (attValue != null && attValue.length() > 0) ga.add(attName, attValue); attName = "gtspp_programVersion"; attValue = ncGlobalAtts.getString(attName); if (attValue != null && attValue.length() > 0) ga.add(attName, attValue); attName = "gtspp_handbook_version"; attValue = ncGlobalAtts.getString(attName); if (attValue != null && attValue.length() > 0) ga.add(attName, attValue); organizationCol = tTable.addColumn(tTable.nColumns(), "org", new StringArray(), new Attributes()); platformCol = tTable.addColumn(tTable.nColumns(), "platform", new StringArray(), new Attributes()); dataTypeCol = tTable.addColumn(tTable.nColumns(), "type", new StringArray(), new Attributes()); cruiseCol = tTable.addColumn(tTable.nColumns(), "cruise", new StringArray(), new Attributes()); stationCol = tTable.addColumn(tTable.nColumns(), "station_id", new IntArray(), new Attributes()); longitudeCol = tTable.addColumn(tTable.nColumns(), "longitude", new FloatArray(), (new Attributes()).add("units", EDV.LON_UNITS)); latitudeCol = tTable.addColumn(tTable.nColumns(), "latitude", new FloatArray(), (new Attributes()).add("units", EDV.LAT_UNITS)); timeCol = tTable.addColumn(tTable.nColumns(), "time", new DoubleArray(), (new Attributes()).add("units", EDV.TIME_UNITS)); depthCol = tTable.addColumn(tTable.nColumns(), "depth", new FloatArray(), (new Attributes()).add("units", "m")); temperatureCol = tTable.addColumn(tTable.nColumns(), "temperature", new FloatArray(), (new Attributes()).add("units", "degree_C")); salinityCol = tTable.addColumn(tTable.nColumns(), "salinity", new FloatArray(), (new Attributes()).add("units", "1e-3")); //PSU changed to 1e-3 with CF std names 25 tableHashMap.put(outTableName, tTable); } //put data in tTable int oNRows = tTable.nRows(); ((StringArray) tTable.getColumn(organizationCol)).addN(nDepth, organization); ((StringArray) tTable.getColumn(platformCol)).addN(nDepth, platform); ((StringArray) tTable.getColumn(dataTypeCol)).addN(nDepth, dataType); ((StringArray) tTable.getColumn(cruiseCol)).addN(nDepth, cruise); ((IntArray) tTable.getColumn(stationCol)).addN(nDepth, station); ((FloatArray) tTable.getColumn(longitudeCol)).addN(nDepth, lon); ((FloatArray) tTable.getColumn(latitudeCol)).addN(nDepth, lat); ((DoubleArray) tTable.getColumn(timeCol)).addN(nDepth, tTime); ((FloatArray) tTable.getColumn(depthCol)).append(depth); ((FloatArray) tTable.getColumn(temperatureCol)).append(temperature); ((FloatArray) tTable.getColumn(salinityCol)).append(salinity); //ensure the table is valid (same size for each column) tTable.ensureValid(); } catch (Throwable t) { nExceptions++; String2.log( "ERROR while processing " + sourceFileName + "\n " + MustBe.throwableToString(t)); } finally { //always close the ncFile if (ncFile != null) { try { ncFile.close(); } catch (Throwable t) { String2.log("ERROR: unable to close " + sourceFileName + "\n" + MustBe.getShortErrorMessage(t)); } } } } String2.log("\n time to read all those files = " + Calendar2.elapsedTimeString(System.currentTimeMillis() - fileReadTime)); //end of region loop String2.log("\nIn zip=" + sourceZipName + "\n nExceptions= " + nExceptions + " nWarnings=" + nWarnings + "\n nBadStation= " + nBadStation + " nGoodStation=" + nGoodStation + "\n nBadPos= " + nBadPos + " nGoodPos=" + nGoodPos + "\n nBadTime= " + nBadTime + " nGoodTime=" + nGoodTime + "\n nBadDepth= " + nBadDepth + " nGoodDepth=" + nGoodDepth + "\n nBadTemperature=" + nBadTemperature + " nGoodTemperature=" + nGoodTemperature + "\n nBadSalinity= " + nBadSalinity + " nGoodSalinity=" + nGoodSalinity); totalNGoodStation += nGoodStation; totalNGoodPos += nGoodPos; totalNGoodTime += nGoodTime; totalNGoodDepth += nGoodDepth; totalNGoodTemperature += nGoodTemperature; totalNGoodSalinity += nGoodSalinity; totalNGoodRows += nGoodRows; totalNBadPos += nBadPos; totalNBadTime += nBadTime; totalNBadDepth += nBadDepth; totalNBadTemperature += nBadTemperature; totalNBadSalinity += nBadSalinity; totalNBadRows += nBadRows; totalNWarnings += nWarnings; totalNExceptions += nExceptions; } //end of region loop //save by outTableName boolean filePrinted = false; Object keys[] = tableHashMap.keySet().toArray(); int nKeys = keys.length; String2.log("\n*** saving nFiles=" + nKeys); for (int keyi = 0; keyi < nKeys; keyi++) { String key = keys[keyi].toString(); Table tTable = (Table) tableHashMap.remove(key); if (tTable == null || tTable.nRows() == 0) { String2.log("Unexpected: no table for key=" + key); continue; } //sort by time, station, depth //depth matches the source files: from surface to deepest tTable.sort(new int[] { timeCol, stationCol, depthCol }, new boolean[] { true, true, true }); //is this saving a small lat lon range? double stationStats[] = tTable.getColumn(stationCol).calculateStats(); //double lonStats[] = tTable.getColumn(longitudeCol).calculateStats(); //double latStats[] = tTable.getColumn(latitudeCol).calculateStats(); //nLats++; //double latRange = latStats[PrimitiveArray.STATS_MAX] - latStats[PrimitiveArray.STATS_MIN]; //latSum += latRange; rowsSum += tTable.nRows(); String2.log(" stationRange=" + Math2.roundToInt( stationStats[PrimitiveArray.STATS_MAX] - stationStats[PrimitiveArray.STATS_MIN]) + //" lonRange=" + Math2.roundToInt(lonStats[ PrimitiveArray.STATS_MAX] - lonStats[ PrimitiveArray.STATS_MIN]) + //" latRange=" + Math2.roundToInt(latRange) + " nRows=" + tTable.nRows()); //save it String tName = tDestDir + String2.encodeFileNameSafe(key); /*if (lonStats[PrimitiveArray.STATS_MAX] > 45 && lonStats[PrimitiveArray.STATS_MIN] < -45) { //NO MORE: This happened with 1 file/cruise, // but won't happen now with lon/lat tiles. //crosses dateline (or widely across lon=0)? split into 2 files Table ttTable = (Table)tTable.clone(); ttTable.oneStepApplyConstraint(0, "longitude", "<", "0"); ttTable.saveAsFlatNc(tName + "_W.nc", "row", false); double lonStatsW[] = ttTable.getColumn(longitudeCol).calculateStats(); nLons++; double lonRangeW = lonStatsW[PrimitiveArray.STATS_MAX] - lonStatsW[PrimitiveArray.STATS_MIN]; lonSum += lonRangeW; ttTable = (Table)tTable.clone(); ttTable.oneStepApplyConstraint(0, "longitude", ">=", "0"); ttTable.saveAsFlatNc(tName + "_E.nc", "row", false); double lonStatsE[] = ttTable.getColumn(longitudeCol).calculateStats(); nLons++; double lonRangeE = lonStatsE[PrimitiveArray.STATS_MAX] - lonStatsE[PrimitiveArray.STATS_MIN]; lonSum += lonRangeE; String2.log(" westLonRange=" + Math2.roundToInt(lonRangeW) + " eastLonRange=" + Math2.roundToInt(lonRangeE)); } else */ { //nLons++; nFiles++; //create trajectory variable: platform + cruise StringArray pl = (StringArray) tTable.getColumn("platform"); StringArray cr = (StringArray) tTable.getColumn("cruise"); StringArray or = (StringArray) tTable.getColumn("org"); StringArray ty = (StringArray) tTable.getColumn("type"); StringArray tr = new StringArray(); int n = pl.size(); for (int i = 0; i < n; i++) { pl.set(i, String2.whitespacesToSpace(pl.get(i))); cr.set(i, String2.whitespacesToSpace(cr.get(i))); or.set(i, String2.whitespacesToSpace(or.get(i))); ty.set(i, String2.whitespacesToSpace(ty.get(i))); tr.add(or.getString(i) + "_" + ty.getString(i) + "_" + pl.getString(i) + "_" + cr.getString(i)); } tTable.addColumn(0, "trajectory", tr, new Attributes()); tTable.saveAsFlatNc(tName + ".nc", "row", false); //convertToFakeMissingValues (keep mv's as NaNs) } //print a file if (testMode && !filePrinted) { filePrinted = true; String2.log(NcHelper.dumpString(tName, true)); } } String2.log("\ncumulative nProfiles=" + profilesSum + " nRows=" + rowsSum + " mean nRows/file=" + (rowsSum / Math.max(1, nFiles))); //if (nLats > 0) // String2.log( "cumulative nLats=" + nLats + " meanLatRange=" + (float)(latSum / nLats)); //if (nLons > 0) { // String2.log( "cumulative nLons=" + nLons + " meanLonRange=" + (float)(lonSum / nLons)); // String2.log("mean nRows per saved file = " + (rowsSum / nLons)); //} //print list of impossible at end of year or end of run if (month == 12 || (year == lastYear && month == lastMonth)) { String2.log("\n*** " + Calendar2.getCurrentISODateTimeStringLocalTZ() + " bobConsolidateGtsppTgz finished the chunk ending " + year + "-" + month + "\n" + "chunkTime=" + Calendar2.elapsedTimeString(System.currentTimeMillis() - chunkTime)); chunkTime = System.currentTimeMillis(); //print impossible statistics String2.log("\nCumulative number of stations with:\n" + "impossibleNanLon = " + impossibleNanLon.size() + "\n" + "impossibleMinLon = " + impossibleMinLon.size() + "\n" + "impossibleMaxLon = " + impossibleMaxLon.size() + "\n" + "impossibleNanLat = " + impossibleNanLat.size() + "\n" + "impossibleMinLat = " + impossibleMinLat.size() + "\n" + "impossibleMaxLat = " + impossibleMaxLat.size() + "\n" + "impossibleMinDepth = " + impossibleMinDepth.size() + "\n" + "impossibleMaxDepth = " + impossibleMaxDepth.size() + "\n" + //"impossibleLatLon = " + impossibleLatLon.size() + "\n" + "impossibleMinTemperature = " + impossibleMinTemperature.size() + "\n" + "impossibleMaxTemperature = " + impossibleMaxTemperature.size() + "\n" + "impossibleMinSalinity = " + impossibleMinSalinity.size() + "\n" + "impossibleMaxSalinity = " + impossibleMaxSalinity.size() + "\n"); //lon String2.log("\n*** " + impossibleNanLon.size() + " stations had invalid lon" + " and good pos quality flags (" + okQFCsv + ")."); impossibleNanLon.sortIgnoreCase(); String2.log(impossibleNanLon.toNewlineString()); String2.log("\n*** " + impossibleMinLon.size() + " stations had lon<" + minLon + " and good pos quality flags (" + okQFCsv + ")."); impossibleMinLon.sortIgnoreCase(); String2.log(impossibleMinLon.toNewlineString()); String2.log("\n*** " + impossibleMaxLon.size() + " stations had lon>" + maxLon + " and good pos quality flags (" + okQFCsv + ")."); impossibleMaxLon.sortIgnoreCase(); String2.log(impossibleMaxLon.toNewlineString()); //lat String2.log("\n*** " + impossibleNanLat.size() + " stations had invalid lat" + " and good pos quality flags (" + okQFCsv + ")."); impossibleNanLat.sortIgnoreCase(); String2.log(impossibleNanLat.toNewlineString()); String2.log("\n*** " + impossibleMinLat.size() + " stations had lat<" + minLat + " and good pos quality flags (" + okQFCsv + ")."); impossibleMinLat.sortIgnoreCase(); String2.log(impossibleMinLat.toNewlineString()); String2.log("\n*** " + impossibleMaxLat.size() + " stations had lat>" + maxLat + " and good pos quality flags (" + okQFCsv + ")."); impossibleMaxLat.sortIgnoreCase(); String2.log(impossibleMaxLat.toNewlineString()); //depth String2.log("\n*** " + impossibleMinDepth.size() + " stations had depth<" + minDepth + " and good depth quality flags (" + okQFCsv + ")."); impossibleMinDepth.sortIgnoreCase(); String2.log(impossibleMinDepth.toNewlineString()); String2.log("\n*** " + impossibleMaxDepth.size() + " stations had depth>" + maxDepth + " and good depth quality flags (" + okQFCsv + ")."); impossibleMaxDepth.sortIgnoreCase(); String2.log(impossibleMaxDepth.toNewlineString()); //sa = impossibleLatLon.toArray(); //Arrays.sort(sa); //String2.log("\n*** " + sa.length + " stations had impossible latitude or longitude values" + // " and good q_pos quality flags."); //String2.log(String2.toNewlineString(sa)); String2.log("\n*** " + impossibleMinTemperature.size() + " stations had temperature<" + minTemperature + " and good temperature quality flags (" + okQFCsv + ")."); impossibleMinTemperature.sortIgnoreCase(); String2.log(impossibleMinTemperature.toNewlineString()); String2.log("\n*** " + impossibleMaxTemperature.size() + " stations had temperature>" + maxTemperature + " and good temperature quality flags (" + okQFCsv + ")."); impossibleMaxTemperature.sortIgnoreCase(); String2.log(impossibleMaxTemperature.toNewlineString()); String2.log("\n*** " + impossibleMinSalinity.size() + " stations had salinity<" + minSalinity + " and good salinity quality flags (" + okQFCsv + ")."); impossibleMinSalinity.sortIgnoreCase(); String2.log(impossibleMinSalinity.toNewlineString()); String2.log("\n*** " + impossibleMaxSalinity.size() + " stations had salinity>" + maxSalinity + " and good salinity quality flags (" + okQFCsv + ")."); impossibleMaxSalinity.sortIgnoreCase(); String2.log(impossibleMaxSalinity.toNewlineString()); } //are we done? if (year == lastYear && month == lastMonth) break; //increment the month month++; if (month == 13) { year++; month = 1; } } //end of month/year loop String2.log("\n*** bobConsolidateGtspp completely finished " + firstYear + "-" + firstMonth + " through " + lastYear + "-" + lastMonth); String2.log("\n***" + "\ntotalNExceptions= " + totalNExceptions + " totalNWarnings= " + totalNWarnings + "\ntotalNBadStation= " + totalNBadStation + " totalNGoodStation= " + totalNGoodStation + "\ntotalNBadPos= " + totalNBadPos + " totalNGoodPos= " + totalNGoodPos + "\ntotalNBadTime= " + totalNBadTime + " totalNGoodTime= " + totalNGoodTime + "\ntotalNBadDepth= " + totalNBadDepth + " totalNGoodDepth= " + totalNGoodDepth + "\ntotalNBadTemperature=" + totalNBadTemperature + " totalNGoodTemperature=" + totalNGoodTemperature + "\ntotalNBadSalinity= " + totalNBadSalinity + " totalNGoodSalinity= " + totalNGoodSalinity + "\ntotalNBadRows= " + totalNBadRows + " totalNGoodRows= " + totalNGoodRows + "\nlogFile=F:/data/gtspp/log.txt" + "\n\n*** all finished time=" + Calendar2.elapsedTimeString(System.currentTimeMillis() - elapsedTime)); String2.returnLoggingToSystemOut(); }