List of usage examples for java.util.concurrent ForkJoinPool isQuiescent
public boolean isQuiescent()
From source file:com.hygenics.parser.ParseDispatcher.java
/** * Fork/Join Pool Solution Maximizes Speed. JSon increases ease of use * /*from w ww .ja va 2 s . co m*/ */ public void run() { log.info("Starting Clock and Parsing @" + Calendar.getInstance().getTime().toString()); long t = Calendar.getInstance().getTimeInMillis(); int pid = 0; int id = 0; int checkattempts = 0; String add = null; this.schema = Properties.getProperty(this.schema); this.select = Properties.getProperty(this.select); this.extracondition = Properties.getProperty(this.extracondition); this.column = Properties.getProperty(this.column); ArrayList<String> parsedrows = new ArrayList<String>(); Set<Callable<String>> collect = new HashSet<Callable<String>>(); List<Future<String>> futures; List<Future<ArrayList<String>>> qfutures; Set<Callable<ArrayList<String>>> qcollect = new HashSet<Callable<ArrayList<String>>>(4); ForkJoinPool fjp = new ForkJoinPool((int) Math.ceil(Runtime.getRuntime().availableProcessors() * procnum)); if (schema != null) { createTables(); } boolean run = true; String condition; int w = 0; int start = offset; int chunksize = (int) Math.ceil(pullsize / qnum); // attempt to query the database from multiple threads do { // query for pages pages = new ArrayList<String>(pullsize); log.info("Looking for Pages."); for (int conn = 0; conn < qnum; conn++) { // create condition condition = " WHERE " + pullid + " >= " + (start + (conn * chunksize)) + " AND " + pullid + " < " + Integer.toString(start + (chunksize * (conn + 1))); if (extracondition != null) { condition += " " + extracondition.trim(); } // get queries qcollect.add(new SplitQuery(template, (select + condition))); log.info("Fetching " + select + condition); } start += (chunksize * qnum); qfutures = fjp.invokeAll(qcollect); w = 0; while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) { w++; } log.info("Waited for " + w + " cycles"); for (Future<ArrayList<String>> f : qfutures) { try { ArrayList<String> test = f.get(); if (test != null) { if (test.size() > 0) { pages.addAll(test); } } if (f.isDone() == false) { f.cancel(true); } f = null; } catch (Exception e) { log.warn("Encoding Error!"); e.printStackTrace(); } } qcollect = new HashSet<Callable<ArrayList<String>>>(4); qfutures = null; log.info("Finished Getting Pages"); // if no records then get records that may have been dropped if (pages.size() == 0 && checkstring != null && checkstring.trim().length() > 0 && checkattempts < reattempts) { checkattempts += 1; log.info("Checking for Drops"); qcollect.add(new SplitQuery(template, (checkstring))); qfutures = fjp.invokeAll(qcollect); w = 0; while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) { w++; } log.info("Waited for " + w + " cycles"); for (Future<ArrayList<String>> f : qfutures) { try { ArrayList<String> test = f.get(); if (test != null) { if (test.size() > 0) { pages.addAll(test); } } if (f.isDone() == false) { f.cancel(true); } f = null; } catch (Exception e) { log.warn("Encoding Error!"); e.printStackTrace(); } } qfutures = null; qcollect = new HashSet<Callable<ArrayList<String>>>(4); } else if (checkattempts >= reattempts) { pages.clear(); } log.info("Found " + pages.size() + " records!"); // get hashes if necessary if (getHash) { log.info("Hashing " + pages.size() + " Records"); ArrayList<String> hashedrows = new ArrayList<String>(); for (String row : pages) { collect.add(new CreateHash(row, pid)); pid++; } log.info("Invoking"); futures = fjp.invokeAll(collect); w = 0; while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) { w++; } log.info("Waited " + w + " Cycles!"); for (Future<String> f : futures) { if (f != null) { String json; try { json = f.get(termtime, TimeUnit.MILLISECONDS); if (json != null) { hashedrows.add(json); } } catch (Exception e) { log.warn("Encoding Error!"); e.printStackTrace(); } } } log.info("Hashed " + hashedrows.size() + " Records!"); pages = hashedrows; collect = new HashSet<Callable<String>>(pullsize); futures.clear(); log.info("Completed Hashing"); } log.info("Performing Regex"); // handle single patterns int i = 0; if (singlepats != null) { log.info("Found Singlepats"); int subs = 0; int rows = 0; for (String row : pages) { rows += 1; String inrow = row; try { inrow = inrow.replaceAll("\t|\r|\r\n|\n", ""); Map<String, Json> jmap = Json.read(inrow).asJsonMap(); if (singlepats.containsKey("table")) { subs += 1; if (fjp.isShutdown()) { fjp = new ForkJoinPool((Runtime.getRuntime().availableProcessors() * procnum)); } if (jmap.get(column) != null) { if (test) { System.out.println("//////////////////////HTML////////////////////////\n" + jmap.get(column).asString() + "\n///////////////////////////////END///////////////////////////\n\n"); } if (mustcontain != null) { if (jmap.get(column).asString().contains(mustcontain)) { if (cannotcontain != null) { if (jmap.get(column).asString().contains(cannotcontain) == false) collect.add(new ParsePage(unescape, replacementPattern, singlepats.get("table"), jmap.get(column).asString().replaceAll("\\s\\s", " "), singlepats, Calendar.getInstance().getTime().toString(), jmap.get("offenderhash").asString())); } else { collect.add(new ParsePage(unescape, replacementPattern, singlepats.get("table"), jmap.get(column).asString().replaceAll("\\s\\s", " "), singlepats, Calendar.getInstance().getTime().toString(), jmap.get("offenderhash").asString())); } } } else if (cannotcontain != null) { if (jmap.get(column).asString().contains(cannotcontain) == false) { collect.add( new ParsePage(unescape, replacementPattern, singlepats.get("table"), jmap.get(column).asString().replaceAll("\\s\\s", " "), singlepats, Calendar.getInstance().getTime().toString(), jmap.get("offenderhash").asString())); } } else { collect.add(new ParsePage(unescape, replacementPattern, singlepats.get("table"), jmap.get(column).asString().replaceAll("\\s\\s", " "), singlepats, Calendar.getInstance().getTime().toString(), jmap.get("offenderhash").asString())); } } } i++; if (((i % commit_size) == 0 & i != 0) || i == pages.size() || pages.size() == 1 && singlepats != null) { log.info("Getting Regex Results"); log.info("Getting Tasks"); futures = fjp.invokeAll(collect); w = 0; while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) { w++; } log.info("Waited for " + w + " cycles"); for (Future<String> r : futures) { try { add = r.get(); if (add.contains("No Data") == false) { parsedrows.add(add); } add = null; } catch (Exception e) { log.warn("Encoding Error!"); e.printStackTrace(); } } futures = null; collect = new HashSet<Callable<String>>(); if (parsedrows.size() >= commit_size) { log.info("INSERTING " + parsedrows.size() + " records!"); if (parsedrows.size() >= SPLITSIZE) { sendToDb(parsedrows, true); } else { sendToDb(parsedrows, false); } parsedrows = new ArrayList<String>(pullsize); } // hint to the gc in case it actually pays off; use // -X:compactexplicitgc to improve odds and // -XX:UseConcMarkSweepGC for improving odds on // older generation strings // (think if i were a gambling man) System.gc(); Runtime.getRuntime().gc(); } } catch (Exception e) { log.warn("Encoding Error!"); e.printStackTrace(); } } log.info("Submitted " + subs + " records. Found " + rows + " rows"); } log.info("REMAINING ROWS TO COMMIT " + parsedrows.size()); log.info("Rows Left" + parsedrows.size()); if (parsedrows.size() > 0) { if (parsedrows.size() >= SPLITSIZE) { sendToDb(parsedrows, true); } else { sendToDb(parsedrows, false); } parsedrows = new ArrayList<String>(); } // handle multi patterns if (multipats != null) { // parse multiple pages for the run int subs = 0; for (String row : pages) { try { for (String k : multipats.keySet()) { if (fjp.isShutdown()) { fjp = new ForkJoinPool(Runtime.getRuntime().availableProcessors()); } Map<String, Json> jmap = Json.read(row).asJsonMap(); if (jmap.get(column) != null) { subs += 1; if (test) { System.out.println("//////////////////////HTML////////////////////////\n" + jmap.get(column).asString() + "\n///////////////////////////////END///////////////////////////\n\n"); } if (mustcontain != null) { if (jmap.get(column).asString().contains(mustcontain)) { if (cannotcontain != null) { if (jmap.get(column).asString().contains(cannotcontain) == false) { collect.add( new ParseMultiPage(unescape, replacementPattern, k, jmap.get(column).asString().replaceAll("\\s\\s", " "), jmap.get("offenderhash").asString(), Calendar.getInstance().getTime().toString(), multipats.get(k))); } } else { collect.add(new ParseMultiPage(unescape, replacementPattern, k, jmap.get(column).asString(), jmap.get("offenderhash").asString().replaceAll("\\s\\s", " "), Calendar.getInstance().getTime().toString(), multipats.get(k))); } } } else if (cannotcontain != null) { if (jmap.get(column).asString().contains(cannotcontain) == false) { collect.add(new ParseMultiPage(unescape, replacementPattern, k, jmap.get(column).asString().replaceAll("\\s\\s", " "), jmap.get("offenderhash").asString(), Calendar.getInstance().getTime().toString(), multipats.get(k))); } } else { collect.add(new ParseMultiPage(unescape, replacementPattern, k, jmap.get(column).asString().replaceAll("\\s\\s", " "), jmap.get("offenderhash").asString(), Calendar.getInstance().getTime().toString(), multipats.get(k))); } } i++; if (((i % commit_size) == 0 & i != 0) || i == pages.size() || pages.size() == 1 && multipats != null) { futures = fjp.invokeAll(collect); w = 0; while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) { w++; } log.info("Waited " + w + " Cycles"); for (Future<String> r : futures) { try { add = r.get(); if (add.contains("No Data") == false) { for (String js : add.split("~")) { parsedrows.add(js); } } add = null; if (r.isDone() == false) { r.cancel(true); } r = null; } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ExecutionException e) { // TODO Auto-generated catch block e.printStackTrace(); } } futures = null; collect = new HashSet<Callable<String>>(); if (parsedrows.size() >= commit_size) { log.info("INSERTING " + parsedrows.size() + " records!"); if (parsedrows.size() >= SPLITSIZE) { sendToDb(parsedrows, true); } else { sendToDb(parsedrows, false); } parsedrows = new ArrayList<String>(pullsize); } // hint to the gc in case it actually pays off System.gc(); Runtime.getRuntime().gc(); } } } catch (Exception e) { log.warn("Encoding Error!"); } } log.info("Submitted " + subs + " records."); } // handle looped patterns if (loopedpats != null) { log.info("Looped Patterns Found"); int subs = 0; if (fjp.isShutdown()) { fjp = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * procnum); } for (String row : pages) { try { for (String k : loopedpats.keySet()) { if (fjp.isShutdown()) { fjp = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * procnum); } Map<String, Json> jmap = Json.read(row).asJsonMap(); if (jmap.get(column) != null) { subs += 1; if (mustcontain != null) { if (jmap.get(column).asString().contains(mustcontain)) { if (cannotcontain != null) { if (jmap.get(column).asString().contains(cannotcontain) == false) { collect.add( new LoopRegex(unescape, jmap.get(column).asString().replaceAll("\\s\\s", " "), jmap.get("offenderhash").asString(), Calendar.getInstance().getTime().toString(), k, replacementPattern, loopedpats.get(k), test)); } } else { collect.add(new LoopRegex(unescape, jmap.get(column).asString().replaceAll("\\s\\s", " "), jmap.get("offenderhash").asString(), Calendar.getInstance().getTime().toString(), k, replacementPattern, loopedpats.get(k), test)); } } } else if (cannotcontain != null) { if (jmap.get(column).asString().contains(cannotcontain) == false) { collect.add(new LoopRegex(unescape, jmap.get(column).asString().replaceAll("\\s\\s", " "), jmap.get("offenderhash").asString(), Calendar.getInstance().getTime().toString(), k, replacementPattern, loopedpats.get(k), test)); } } else { collect.add(new LoopRegex(unescape, jmap.get(column).asString().replaceAll("\\s\\s", " "), jmap.get("offenderhash").asString(), Calendar.getInstance().getTime().toString(), k, replacementPattern, loopedpats.get(k), test)); } jmap.remove(k); } i++; if (((i % commit_size) == 0 & i != 0) || (i % (pages.size() - 1)) == 0 || pages.size() == 1) { futures = fjp.invokeAll(collect); w = 0; while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) { w++; } log.info("Waited " + w + " Cycles"); for (Future<String> r : futures) { try { add = r.get(); if (add.contains("No Data") == false) { for (String toarr : add.split("~")) { parsedrows.add(toarr); } } if (r.isDone() == false) { r.cancel(true); } add = null; } catch (Exception e) { log.warn("Encoding Error!"); e.printStackTrace(); } } futures = null; collect = new HashSet<Callable<String>>(); // hint to the gc in case it actually pays off System.gc(); Runtime.getRuntime().gc(); } } if (parsedrows.size() >= this.commit_size) { log.info("INSERTING " + parsedrows.size() + " records!"); if (parsedrows.size() >= SPLITSIZE) { sendToDb(parsedrows, true); } else { sendToDb(parsedrows, false); } parsedrows = new ArrayList<String>(pullsize); } } catch (Exception e) { log.warn("Encoding Error!"); } } log.info("Submitted " + subs + " records."); } if (collect.size() > 0) { log.info("Getting Last Regex Results for Iteration"); log.info("Getting Tasks"); futures = fjp.invokeAll(collect); w = 0; while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) { w++; } log.info("Waited for " + w + " cycles"); for (Future<String> r : futures) { try { add = r.get(); if (add.contains("No Data") == false) { parsedrows.add(add); } add = null; } catch (Exception e) { log.warn("Encoding Error!"); e.printStackTrace(); } } futures = null; collect = new HashSet<Callable<String>>(pullsize); // hint to the gc in case it actually pays off; use // -X:compactexplicitgc to improve odds and // -XX:UseConcMarkSweepGC for improving odds on older generation // strings // (think if i were a gambling man) System.gc(); Runtime.getRuntime().gc(); } log.info("REMAINING ROWS TO COMMIT " + parsedrows.size()); log.info("Rows Left" + parsedrows.size()); if (parsedrows.size() > 0) { if (parsedrows.size() >= SPLITSIZE) { sendToDb(parsedrows, true); } else { sendToDb(parsedrows, false); } parsedrows = new ArrayList<String>(); } } while (pages != null && pages.size() > 0); // ensure that nothing is still caught in limbo // final parser to ensure that nothing is left out if (collect.size() > 0) { log.info("More Rows Caught in FJP, Completing Process"); futures = fjp.invokeAll(collect); w = 0; while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) { w++; } log.info("Waited " + w + " Cycles"); for (Future<String> r : futures) { try { add = r.get(); if (add.contains("No Data") == false) { for (String js : add.split("~")) { parsedrows.add(js); } } add = null; if (r.isDone() == false) { r.cancel(true); } r = null; } catch (InterruptedException e) { e.printStackTrace(); } catch (ExecutionException e) { e.printStackTrace(); } } futures = null; collect = null; } // send any remaining parsed rows to the db if (parsedrows.size() > 0) { if (parsedrows.size() >= SPLITSIZE) { sendToDb(parsedrows, true); } else { sendToDb(parsedrows, false); } parsedrows = new ArrayList<String>(); } log.info("Shutting Down Fork Join Pool"); if (fjp.isShutdown() == false) { fjp.shutdownNow(); } fjp = null; log.info("Complete @" + Calendar.getInstance().getTime().toString()); log.info("Total Runtime(seconds): " + Double.toString((double) (Calendar.getInstance().getTimeInMillis() - t) / 1000)); // hint to the gc in case it actually pays off System.gc(); Runtime.getRuntime().gc(); }
From source file:com.hygenics.parser.GetImages.java
private void addFromFile() { File f = new File(fpath); ForkJoinPool fjp = new ForkJoinPool(Runtime.getRuntime().availableProcessors()); ArrayList<String> imageData = new ArrayList<String>(); int size = (int) Math.ceil(commitsize / numqueries); if (f.exists()) { // get the image data File[] list = f.listFiles(); int curr = 0; if (list != null) { for (File img : list) { curr += 1;//from w w w . j ava 2 s . c o m if (img.isDirectory() == false && (img.getName().contains(".bmp") || img.getName().toLowerCase().contains(".jpg") || img.getName().toLowerCase().contains(".png") || img.getName().toLowerCase().contains("jpeg"))) { try { if (dbcondition == null || template .getJsonData(dbcondition.replace("$IMAGE$", img.getName().replaceAll("(?mis)" + imagepostfix, ""))) .size() > 0) { BufferedImage bi = ImageIO.read(img);// only // used // to // ensure // that // this // is an // image JsonObject jobj = new JsonObject(); jobj.add("offenderhash", img.getName().replaceAll("(?mis)" + imagepostfix, ""));// assumes // hash // is // file // name+postfix jobj.add("image", img.getName().replaceAll("(?mis)" + imagepostfix, "")); jobj.add("image_path", img.getName()); jobj.add("table", table); jobj.add("date", Calendar.getInstance().getTime().toString()); imageData.add(jobj.toString()); } } catch (IOException e) { log.info(img.getName() + " is not an Image!"); e.printStackTrace(); } catch (Exception e) { log.info("Error in Posting to Database."); e.printStackTrace(); } } // post if > commitsize if (imageData.size() >= commitsize || curr == list.length) { log.info("Posting to DB @ " + Calendar.getInstance().getTime().toString()); for (int i = 0; i < numqueries; i++) { if (((i + 1) * size) < imageData.size()) { fjp.execute(new ImagePost(imageData.subList((i * size), ((i + 1) * size)))); } else { fjp.execute(new ImagePost(imageData.subList((i * size), imageData.size()))); } } int w = 0; while (fjp.isQuiescent() == false && fjp.getActiveThreadCount() > 0) { w++; } log.info("Waited for " + w + " cycles"); imageData.clear(); log.info("Finished Posting to DB @ " + Calendar.getInstance().getTime().toString()); } } } } else { log.error("File Path does Not Exist.Please Check Image Pull!"); } fjp.shutdown(); fjp = null; }
From source file:com.hygenics.parser.JDump.java
private void toFile() { ArrayList<String> archs = new ArrayList<String>(); List<Future<ArrayList<String>>> qfutures; Set<Callable<ArrayList<String>>> qcollect = new HashSet<Callable<ArrayList<String>>>(4); ForkJoinPool fjp = new ForkJoinPool((int) Math.ceil(Runtime.getRuntime().availableProcessors() * procnum)); int dumped = 0; if (archive) { log.info("Cleaning"); for (String k : fpaths.keySet()) { String fpath = ""; for (String ofp : fpaths.get(k).keySet()) { fpath = ofp;/*from w ww . jav a2 s .c o m*/ } if (fpath.length() > 0) { String[] barr = fpath.split("\\/"); String basefile = ""; Archiver zip = new Archiver(); for (int i = 0; i > barr.length - 1; i++) { basefile += (i == 0) ? barr[i] : "/" + barr[i]; } if (basefile.trim().length() > 0) { zip.setBasedirectory(basefile); zip.setZipDirectory(basefile + "archive.zip"); zip.setAvoidanceString(".zip|archive"); zip.setDelFiles(true); zip.run(); } } } } log.info("Dumping"); for (String table : fpaths.keySet()) { int offset = 0; if (template.checkTable(table, table.split("\\.")[0])) { if (template.getCount(table) > 0) { log.info("Dumping for " + table); // get header String select = "SELECT * FROM " + table; String fpath = null; ArrayList<String> jsons; String condition; int w = 0; int start = offset; int chunksize = (int) Math.ceil(pullsize / qnum); // get fpath for (String ofp : fpaths.get(table).keySet()) { start = fpaths.get(table).get(ofp); fpath = ofp; } // perform write if (headers != null && fpath != null) { List<String> headersList = headers.get(table); String output = null; boolean existed = true; if (addFileDate) { fpath = fpath + Calendar.getInstance().getTime().toString().trim().replaceAll(":|\\s", "") + ".txt"; } // check to see if file should be created if (!new File(fpath).exists()) { try { new File(fpath).createNewFile(); } catch (IOException e) { e.printStackTrace(); } existed = false; } // check to see if file must be recreated if (!append) { File f = new File(fpath); f.delete(); try { f.createNewFile(); } catch (IOException e) { e.printStackTrace(); } } if (headersList != null && (append == false || existed == false)) { for (String header : headersList) { output = (output == null) ? StringEscapeUtils.unescapeXml(header) : output + delimeter + StringEscapeUtils.unescapeXml(header); } } do { // get records jsons = new ArrayList<String>(pullsize); log.info("Looking for Pages."); for (int conn = 0; conn < qnum; conn++) { // create condition condition = " WHERE " + pullid + " >= " + (start + (conn * chunksize)) + " AND " + pullid + " < " + Integer.toString(start + (chunksize * (conn + 1))); if (extracondition != null) { condition += " " + extracondition.trim(); } // get queries qcollect.add(new SplitQuery(template, (select + condition))); log.info("Fetching " + select + condition); } start += (chunksize * qnum); qfutures = fjp.invokeAll(qcollect); w = 0; while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) { w++; } log.info("Waited for " + w + " cycles"); for (Future<ArrayList<String>> f : qfutures) { try { ArrayList<String> test = f.get(); if (test != null) { if (test.size() > 0) { jsons.addAll(test); } } if (f.isDone() == false) { f.cancel(true); } f = null; } catch (Exception e) { log.warn("Encoding Error!"); e.printStackTrace(); } } qcollect = new HashSet<Callable<ArrayList<String>>>(4); qfutures = null; log.info("Finished Getting Pages"); // post records to the file try (FileWriter fw = new FileWriter(new File(fpath), true)) { // get and write headers if (jsons.size() > 0) { fw.write(output + "\n"); // write data for (String json : jsons) { output = null; JsonObject jo = JsonObject.readFrom(json); if (jo.size() >= headersList.size()) {// allows // trimming // of // table // to // key // aspects output = null; for (String key : headers.get(table)) { if (jo.get(key.toLowerCase()) != null) { String data = StringEscapeUtils .unescapeXml(jo.get(key.toLowerCase()).asString()); if (replacementPattern != null) { data = data.replaceAll(replacementPattern, ""); data = data.replace(delimeter, delimreplace); } output = (output == null) ? data.replaceAll("[^\u0020-\u0070 ]+", "") : output + delimeter + data.replaceAll("[^\u0020-\u0070 ]+", ""); } else { output += delimeter; } } if (output != null && output.trim().length() > headersList.size()) { fw.write(output + "\n"); } } else { if (jsons.size() == 0) { Log.info( "Number of Headers and Keys from Json Array and Headers List Impossible to Match"); try { throw new MismatchException( "Number of Headers: " + headersList.size() + " && Number of Keys: " + jo.size()); } catch (MismatchException e) { e.printStackTrace(); } } } output = null; } } else { log.info("EOF FOUND! No New Records in This Iteration....Stopping."); } } catch (IOException e) { e.printStackTrace(); } } while (jsons.size() > 0); } else { try { throw new NullPointerException( "No Headers Input to Class. Please Create the Requisite Map."); } catch (NullPointerException e) { e.printStackTrace(); } } dumped += 1; } else { try { throw new NoDataException("No Data Found in Table " + table); } catch (NoDataException e) { e.printStackTrace(); } } } else { log.info("Missing Table " + table); try { throw new NullPointerException("Table " + table + " Does Not Exist!!!"); } catch (NullPointerException e) { e.printStackTrace(); } } } // end LOOP if (!fjp.isShutdown()) { fjp.shutdownNow(); } if (dumped == 0) { log.error("No Data Found in Any Table"); System.exit(-1); } }
From source file:com.hygenics.parser.JDumpWithReference.java
private void toFile() { List<Future<ArrayList<String>>> qfutures; Set<Callable<ArrayList<String>>> qcollect = new HashSet<Callable<ArrayList<String>>>(4); ForkJoinPool fjp = new ForkJoinPool((int) Math.ceil(Runtime.getRuntime().availableProcessors() * procnum)); int dumped = 0; if (archive) { log.info("Cleaning"); for (String k : fpaths.keySet()) { String fpath = ""; for (String ofp : fpaths.get(k).keySet()) { fpath = ofp;//from ww w. j a v a 2 s . c om } if (fpath.length() > 0) { String[] barr = fpath.split("\\/"); String basefile = ""; Archiver zip = new Archiver(); for (int i = 0; i > barr.length - 1; i++) { basefile += (i == 0) ? barr[i] : "/" + barr[i]; } if (basefile.trim().length() > 0) { zip.setBasedirectory(basefile); zip.setZipDirectory(basefile + "archive.zip"); zip.setAvoidanceString(".zip|archive"); zip.setDelFiles(true); zip.run(); } } } } log.info("Dumping"); for (String table : fpaths.keySet()) { int offset = 0; if (template.checkTable(this.baseschema + "." + table, this.baseschema)) { if (template.getCount(this.baseschema + "." + table) > 0) { log.info("Dumping for " + table); // get header String select = "SELECT * FROM " + this.baseschema + "." + table; String fpath = null; ArrayList<String> jsons; String condition; int w = 0; int start = offset; int chunksize = (int) Math.ceil(pullsize / qnum); // get fpath for (String ofp : fpaths.get(table).keySet()) { start = fpaths.get(table).get(ofp); fpath = ofp; } // perform write if (headers != null && fpath != null) { List<String> headersList = headers.get(table); String output = null; boolean existed = true; if (addFileDate) { fpath = fpath + Calendar.getInstance().getTime().toString().trim().replaceAll(":|\\s", "") + ".txt"; } // check to see if file should be created if (!new File(fpath).exists()) { try { new File(this.baseFilePath + fpath).createNewFile(); } catch (IOException e) { e.printStackTrace(); } existed = false; } // check to see if file must be recreated if (!append) { File f = new File(this.baseFilePath + fpath); f.delete(); try { f.createNewFile(); } catch (IOException e) { e.printStackTrace(); } } if (headersList != null && (append == false || existed == false)) { for (String header : headersList) { output = (output == null) ? StringEscapeUtils.unescapeXml(header) : output + delimeter + StringEscapeUtils.unescapeXml(header); } } do { // get records jsons = new ArrayList<String>(pullsize); log.info("Looking for Pages."); for (int conn = 0; conn < qnum; conn++) { // create condition condition = " WHERE " + pullid + " >= " + (start + (conn * chunksize)) + " AND " + pullid + " < " + Integer.toString(start + (chunksize * (conn + 1))); if (extracondition != null) { condition += " " + extracondition.trim(); } // get queries qcollect.add(new SplitQuery(template, (select + condition))); log.info("Fetching " + select + condition); } start += (chunksize * qnum); qfutures = fjp.invokeAll(qcollect); w = 0; while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) { w++; } log.info("Waited for " + w + " cycles"); for (Future<ArrayList<String>> f : qfutures) { try { ArrayList<String> test = f.get(); if (test != null) { if (test.size() > 0) { jsons.addAll(test); } } if (f.isDone() == false) { f.cancel(true); } f = null; } catch (Exception e) { log.warn("Encoding Error!"); e.printStackTrace(); } } qcollect = new HashSet<Callable<ArrayList<String>>>(4); qfutures = null; log.info("Finished Getting Pages"); // post records to the file try (FileWriter fw = new FileWriter(new File(this.baseFilePath + fpath), true)) { // get and write headers if (jsons.size() > 0) { fw.write(output + "\n"); // write data for (String json : jsons) { output = null; JsonObject jo = JsonObject.readFrom(json); if (jo.size() >= headersList.size()) {// allows // trimming // of // table // to // key // aspects output = null; for (String key : headers.get(table)) { if (jo.get(key.toLowerCase()) != null) { String data = StringEscapeUtils .unescapeXml(jo.get(key.toLowerCase()).asString()); if (replacementPattern != null) { data = data.replaceAll(replacementPattern, ""); data = data.replace(delimeter, delimreplace); } output = (output == null) ? data.replaceAll("[^\u0020-\u007E ]+", "") : output + delimeter + data.replaceAll("[^\u0020-\u007E ]+", ""); } else { output += delimeter; } } if (output != null && output.trim().length() > headersList.size()) { fw.write(output + "\n"); } } else { if (jsons.size() == 0) { Log.info( "Number of Headers and Keys from Json Array and Headers List Impossible to Match"); try { throw new MismatchException( "Number of Headers: " + headersList.size() + " && Number of Keys: " + jo.size()); } catch (MismatchException e) { e.printStackTrace(); } } } output = null; } } else { log.info("EOF FOUND! No New Records in This Iteration....Stopping."); } } catch (IOException e) { e.printStackTrace(); } } while (jsons.size() > 0); } else { try { throw new NullPointerException( "No Headers Input to Class. Please Create the Requisite Map."); } catch (NullPointerException e) { e.printStackTrace(); } } dumped += 1; } else { try { throw new NoDataException("No Data in Table " + table); } catch (NoDataException e) { e.printStackTrace(); } } } else { log.info("Missing Table " + table); try { throw new NullPointerException("Table " + table + " Does Not Exist!!!"); } catch (NullPointerException e) { e.printStackTrace(); } } } // end LOOP if (!fjp.isShutdown()) { fjp.shutdownNow(); } if (dumped == 0) { log.error("No Data found in Any Tables"); System.exit(-1); } }