Example usage for java.util.concurrent ForkJoinPool shutdownNow

Introduction

In this page you can find the example usage for java.util.concurrent ForkJoinPool shutdownNow.

Prototype

public List<Runnable> shutdownNow() 

Source Link

Document

Possibly attempts to cancel and/or stop all tasks, and reject all subsequently submitted tasks.

Usage

From source file:edu.usu.sdl.openstorefront.report.ExternalLinkValidationReport.java

private void checkLinks() {
    int timeOutTime = MAX_CONNECTION_TIME_MILLIS;
    if (report.getReportOption() != null) {
        if (report.getReportOption().getMaxWaitSeconds() != null) {
            timeOutTime = report.getReportOption().getMaxWaitSeconds() * 1000;
        }/*from w w  w .  j  a  v  a  2s. c om*/
    }

    ForkJoinPool forkJoinPool = new ForkJoinPool(MAX_CHECKPOOL_SIZE);

    Map<String, LinkCheckModel> linkMap = new HashMap();
    List<ForkJoinTask<LinkCheckModel>> tasks = new ArrayList<>();
    for (LinkCheckModel link : links) {
        linkMap.put(link.getId(), link);
        tasks.add(forkJoinPool.submit(new CheckLinkTask(link, timeOutTime)));
    }

    int completedCount = 0;
    for (ForkJoinTask<LinkCheckModel> task : tasks) {
        try {
            LinkCheckModel processed;
            try {
                processed = task.get(timeOutTime, TimeUnit.MILLISECONDS);
                if (processed != null) {
                    LinkCheckModel reportModel = linkMap.get(processed.getId());
                    reportModel.setStatus(processed.getStatus());
                    reportModel.setCheckResults(processed.getCheckResults());
                    reportModel.setHttpStatus(processed.getHttpStatus());
                } else {
                    //This shouldn't occur, however if it does at least show a message.
                    log.log(Level.WARNING, MessageFormat.format(
                            "A link check task failed to return results.  Status at Completed Abnormally? {0}",
                            task.isCompletedAbnormally()));
                }
            } catch (TimeoutException e) {
                task.cancel(true);
            }

            completedCount++;
        } catch (InterruptedException | ExecutionException ex) {
            log.log(Level.WARNING, "Check task  was interrupted.  Report results may be not complete.", ex);
        }
        log.log(Level.FINE, MessageFormat.format("Complete Checking Link Count: {0} out of {1}",
                new Object[] { completedCount, links.size() }));
    }

    for (LinkCheckModel checkModel : links) {
        if (StringUtils.isBlank(checkModel.getStatus())) {
            checkModel.setStatus("Unable to verify.  Timed out while waiting.");
        }
    }

    forkJoinPool.shutdownNow();
    try {
        forkJoinPool.awaitTermination(1000, TimeUnit.MILLISECONDS);
    } catch (InterruptedException ex) {
        log.log(Level.WARNING,
                "Check task shutdown was interrupted.  The application will recover and continue.", ex);
    }
}

---

From source file:com.hygenics.parser.KVParser.java

public void run() {
    log.info("Starting Parse @ " + Calendar.getInstance().getTime().toString());
    ForkJoinPool fjp = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * procs);
    Set<Callable<ArrayList<String>>> collection;
    List<Future<ArrayList<String>>> futures;
    ArrayList<String> data = new ArrayList<String>((commitsize + 10));
    ArrayList<String> outdata = new ArrayList<String>(((commitsize + 10) * 3));

    int currpos = 0;
    boolean run = true;

    while (run) {
        log.info("Getting Pages");
        // get pages
        String query = select;/*from   w  w  w  .j a va  2 s . c o  m*/

        if (data.size() > 0) {
            data.clear();
        }

        if (extracondition != null) {
            query += " " + extracondition;
        }

        if (extracondition != null) {
            query += " WHERE " + extracondition + " AND ";
        } else {
            query += " WHERE ";
        }

        collection = new HashSet<Callable<ArrayList<String>>>(qnums);
        for (int i = 0; i < qnums; i++) {

            if (currpos + (Math.round(commitsize / qnums * (i + 1))) < currpos + commitsize) {
                collection.add(new SplitQuery((query + pullid + " >= "
                        + Integer.toString(currpos + (Math.round(commitsize / qnums * (i)))) + " AND " + pullid
                        + " < " + Integer.toString(currpos + (Math.round(commitsize / qnums * (i + 1)))))));
            } else {
                collection.add(new SplitQuery((query + pullid + " >= "
                        + Integer.toString(currpos + (Math.round(commitsize / qnums * (i)))) + " AND " + pullid
                        + " < " + Integer.toString(currpos + commitsize))));
            }
        }

        currpos += commitsize;

        if (collection.size() > 0) {

            futures = fjp.invokeAll(collection);

            int w = 0;

            while (fjp.isQuiescent() == false && fjp.getActiveThreadCount() > 0) {
                w++;
            }

            for (Future<ArrayList<String>> f : futures) {
                try {
                    ArrayList<String> darr = f.get();
                    if (darr != null && darr.size() > 0) {
                        data.addAll(darr);
                    }
                } catch (NullPointerException e) {
                    log.info("Some Data Returned Null");
                } catch (InterruptedException e) {
                    e.printStackTrace();
                } catch (ExecutionException e) {
                    e.printStackTrace();
                }
            }

        }

        if (data.size() == 0 && checkString != null) {
            collection = new HashSet<Callable<ArrayList<String>>>(1);
            collection.add(new SplitQuery(checkString));

            futures = fjp.invokeAll(collection);
            int w = 0;
            while (fjp.isQuiescent() == false && fjp.getActiveThreadCount() > 0) {
                w++;
            }

            for (Future<ArrayList<String>> f : futures) {
                try {
                    ArrayList<String> arr = f.get();

                    if (arr != null) {
                        for (String a : arr) {
                            if (a != null) {
                                data.add(a);
                            }
                        }
                    }
                    if (!f.isDone()) {
                        f.cancel(true);
                    }
                    f = null;
                } catch (NullPointerException e) {
                    log.info("Some Data Returned Null");
                } catch (InterruptedException e) {
                    e.printStackTrace();
                } catch (ExecutionException e) {
                    e.printStackTrace();
                }
            }
        }

        // parse pages
        if (data.size() > 0) {
            log.info("Parsing " + Integer.toString(data.size()) + " Records");
            collection = new HashSet<Callable<ArrayList<String>>>(data.size());

            for (String json : data) {
                Map<String, Object> jmap = Json.read(json).asMap();

                // for each table in the tags Map which is a key
                for (String k : tags.keySet()) {

                    collection.add(new Parser(tags.get(k), jmap.get(htmlColumn).toString(), replacePattern,
                            replacement, jmap.get(hashColumn).toString(), hashColumn, k));

                    if (collection.size() + 1 == data.size()
                            || (collection.size() % commitsize == 0 && collection.size() >= commitsize)) {
                        log.info("Waiting for Tasks to Complete");
                        futures = fjp.invokeAll(collection);

                        // post data
                        int w = 0;
                        while (fjp.isQuiescent() == false && fjp.getActiveThreadCount() > 0) {
                            w++;
                        }

                        for (Future<ArrayList<String>> future : futures) {
                            try {
                                outdata.addAll(future.get());
                            } catch (NullPointerException e) {
                                log.info("Some Data Returned Null");
                            } catch (InterruptedException e) {
                                e.printStackTrace();
                            } catch (ExecutionException e) {
                                e.printStackTrace();
                            }
                        }

                        log.info("Parsed " + outdata.size() + " records!");
                        // post data
                        int cp = 0;
                        if (outdata.size() > 0) {
                            checkTables(outdata);
                            this.sendToDb(outdata, true);
                            outdata = new ArrayList<String>(commitsize);
                        }

                    }

                }
            }
            data = new ArrayList<String>(commitsize);
        } else {
            log.info("No Records Found. Terminating!");
            run = false;
        }

    }

    if (outdata.size() > 0) {
        log.info("Posting Last Records");
        // post remaining pages for the iteration
        if (outdata.size() > 0) {
            int cp = 0;
            if (outdata.size() > 0) {
                checkTables(outdata);
                this.sendToDb(outdata, true);
            }
            data.clear();
            outdata.clear();
        }
    }

    // shutdown
    log.info("Complete! Shutting Down FJP.");
    fjp.shutdownNow();

    log.info("Finished Parse @ " + Calendar.getInstance().getTime().toString());
}

---

From source file:com.hygenics.parser.GetImages.java

private void getImages() {
    // controls the web process from a removed method
    log.info("Setting Up Pull");
    String[] proxyarr = (proxies == null) ? null : proxies.split(",");
    // cleanup/*from  w ww. j a v a  2 s.  c  om*/
    if (cleanup) {
        cleanupDir(fpath);
    }

    // image grab
    CookieManager cm = new CookieManager();
    cm.setCookiePolicy(CookiePolicy.ACCEPT_ALL);
    CookieHandler.setDefault(cm);
    int numimages = 0;
    InputStream is;
    byte[] bytes;
    int iter = 0;
    int found = 0;

    // set proxy if needed
    if (proxyuser != null) {
        proxy(proxyhost, proxyport, https, proxyuser, proxypass);
    }

    int i = 0;
    ArrayList<String> postImages = new ArrayList<String>();
    ForkJoinPool fjp = new ForkJoinPool(Runtime.getRuntime().availableProcessors());
    Set<Callable<String>> pulls = new HashSet<Callable<String>>();
    Set<Callable<ArrayList<String>>> sqls = new HashSet<Callable<ArrayList<String>>>();
    List<Future<String>> imageFutures;

    ArrayList<String> images;
    int chunksize = (int) Math.ceil(commitsize / numqueries);
    log.info("Chunksize: " + chunksize);
    if (baseurl != null || baseurlcolumn != null) {
        do {
            log.info("Offset: " + offset);
            log.info("Getting Images");
            images = new ArrayList<String>(commitsize);
            log.info("Getting Columns");
            for (int n = 0; n < numqueries; n++) {
                String tempsql = sql + " WHERE " + idString + " >= " + offset + " AND " + idString + " < "
                        + (offset + chunksize);

                if (conditions != null) {
                    tempsql += conditions;
                }

                sqls.add(new QueryDatabase(
                        ((extracondition != null) ? tempsql + " " + extracondition : tempsql)));

                offset += chunksize;
            }

            List<Future<ArrayList<String>>> futures = fjp.invokeAll(sqls);

            int w = 0;
            while (fjp.isQuiescent() && fjp.getActiveThreadCount() > 0) {
                w++;
            }

            for (Future<ArrayList<String>> f : futures) {
                try {
                    ArrayList<String> fjson;
                    fjson = f.get();
                    if (fjson.size() > 0) {
                        images.addAll(fjson);
                    }

                    if (f.isDone() == false) {
                        f.cancel(true);
                    }
                } catch (InterruptedException e) {
                    e.printStackTrace();
                } catch (ExecutionException e) {
                    e.printStackTrace();
                }
            }
            log.info(Integer.toString(images.size()) + " image links found. Pulling.");

            ArrayList<String> tempproxies = new ArrayList<String>();

            if (proxyarr != null) {
                for (String proxy : proxyarr) {
                    tempproxies.add(proxy.trim());
                }
            }

            if (maxproxies > 0) {
                maxproxies -= 1;// 0 and 1 should be equivalent conditions
                // --num is not like most 0 based still due
                // to >=
            }

            // get images
            for (int num = 0; num < images.size(); num++) {
                String icols = images.get(num);
                int proxnum = (int) Math.random() * (tempproxies.size() - 1);
                String proxy = (tempproxies.size() == 0) ? null : tempproxies.get(proxnum);

                // add grab
                pulls.add(new ImageGrabber(icols, proxy));

                if (proxy != null) {
                    tempproxies.remove(proxy);
                }

                // check for execution
                if (num + 1 == images.size() || pulls.size() >= commitsize || tempproxies.size() == 0) {
                    if (tempproxies.size() == 0 && proxies != null) {
                        tempproxies = new ArrayList<String>(proxyarr.length);

                        for (String p : proxyarr) {
                            tempproxies.add(p.trim());
                        }
                    }

                    imageFutures = fjp.invokeAll(pulls);
                    w = 0;

                    while (fjp.isQuiescent() == false && fjp.getActiveThreadCount() > 0) {
                        w++;
                    }

                    for (Future<String> f : imageFutures) {
                        String add;
                        try {
                            add = f.get();

                            if (add != null) {
                                postImages.add(add);
                            }
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        } catch (ExecutionException e) {
                            e.printStackTrace();
                        }
                    }
                    imageFutures = null;// garbage collect elligible
                    pulls = new HashSet<Callable<String>>(commitsize);
                }

                if (postImages.size() >= commitsize && addtoDB == true) {
                    if (addtoDB) {
                        log.info("Posting to Database");
                        log.info("Found " + postImages.size() + " images");
                        numimages += postImages.size();
                        int size = (int) Math.floor(postImages.size() / numqueries);
                        for (int n = 0; n < numqueries; n++) {
                            if (((n + 1) * size) < postImages.size() && (n + 1) < numqueries) {
                                fjp.execute(new ImagePost(postImages.subList(n * size, (n + 1) * size)));
                            } else {
                                fjp.execute(new ImagePost(postImages.subList(n * size, postImages.size() - 1)));
                            }
                        }

                        w = 0;
                        while (fjp.isQuiescent() && fjp.getActiveThreadCount() > 0) {
                            w++;
                        }
                    }
                    found += postImages.size();
                    postImages.clear();
                }

            }

            if (postImages.size() > 0 && addtoDB == true) {
                log.info("Posting to Database");
                numimages += postImages.size();
                int size = (int) Math.floor(postImages.size() / numqueries);
                for (int n = 0; n < numqueries; n++) {
                    if (((n + 1) * size) < postImages.size()) {
                        fjp.execute(new ImagePost(postImages.subList(n * size, (n + 1) * size)));
                    } else {
                        fjp.execute(new ImagePost(postImages.subList(n * size, postImages.size())));
                    }
                }

                w = 0;
                while (fjp.isQuiescent() && fjp.getActiveThreadCount() > 0) {
                    w++;
                }

                found += postImages.size();
                postImages.clear();
            }

            // handle iterations specs
            iter += 1;
            log.info("Iteration: " + iter);
            if ((iter < iterations && found < images.size()) || tillfound == true) {
                log.info("Not All Images Obtained Trying Iteration " + iter + " of " + iterations);
                offset -= commitsize;
            } else if ((iter < iterations && found >= images.size()) && tillfound == false) {
                log.info("Images Obtained in " + iter + " iterations. Continuing.");
                iter = 0;
            } else {
                // precautionary
                log.info("Images Obtained in " + iter + " iterations. Continuing");
                iter = 0;
            }

        } while (images.size() > 0 && iter < iterations);

        if (fjp.isShutdown()) {
            fjp.shutdownNow();
        }
    }

    log.info("Complete. Check for Errors \n " + numimages + " Images Found");
}

---

From source file:com.hygenics.parser.ParseJSoup.java

/**
 * Runs the Program// ww w  . j a  v  a 2 s.c  o m
 */
public void run() {
    int its = 0;

    this.select = Properties.getProperty(this.select);
    this.extracondition = Properties.getProperty(this.extracondition);
    this.column = Properties.getProperty(this.column);

    createTables();
    log.info("Starting Parse via JSoup @ " + Calendar.getInstance().getTime().toString());

    ForkJoinPool fjp = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * procs);
    Set<Callable<ArrayList<String>>> collection;
    List<Future<ArrayList<String>>> futures;
    ArrayList<String> data = new ArrayList<String>((commitsize + 10));
    ArrayList<String> outdata = new ArrayList<String>(((commitsize + 10) * 3));
    int offenderhash = offset;

    boolean run = true;
    int iteration = 0;

    int currpos = 0;
    do {
        collection = new HashSet<Callable<ArrayList<String>>>(qnums);
        log.info("Getting Data");
        // get data
        currpos = iteration * commitsize + offset;
        iteration += 1;
        String query = select;

        if (extracondition != null) {
            query += " " + extracondition;
        }

        if (extracondition != null) {
            query += " WHERE " + extracondition + " AND ";
        } else {
            query += " WHERE ";
        }

        for (int i = 0; i < qnums; i++) {

            if (currpos + (Math.round(commitsize / qnums * (i + 1))) < currpos + commitsize) {
                collection.add(new SplitQuery((query + pullid + " >= "
                        + Integer.toString(currpos + (Math.round(commitsize / qnums * (i)))) + " AND " + pullid
                        + " < " + Integer.toString(currpos + (Math.round(commitsize / qnums * (i + 1)))))));
            } else {
                collection.add(new SplitQuery((query + pullid + " >= "
                        + Integer.toString(currpos + (Math.round(commitsize / qnums * (i)))) + " AND " + pullid
                        + " < " + Integer.toString(currpos + commitsize))));
            }
        }

        if (collection.size() > 0) {

            futures = fjp.invokeAll(collection);

            int w = 0;

            while (fjp.isQuiescent() == false && fjp.getActiveThreadCount() > 0) {
                w++;
            }

            for (Future<ArrayList<String>> f : futures) {
                try {
                    // TODO Get Pages to Parse
                    data.addAll(f.get());
                } catch (NullPointerException e) {
                    log.info("Some Data Returned Null");
                } catch (InterruptedException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (ExecutionException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }

        }

        collection = new HashSet<Callable<ArrayList<String>>>(data.size());
        // checkstring
        if (data.size() == 0 && checkstring != null && its <= maxchecks) {
            its++;
            collection.add(new SplitQuery(checkstring));

            futures = fjp.invokeAll(collection);

            int w = 0;
            while (fjp.isQuiescent() == false && fjp.getActiveThreadCount() > 0) {
                w++;
            }

            for (Future<ArrayList<String>> f : futures) {
                try {
                    // TODO Get Pages to Parse
                    data.addAll(f.get());
                } catch (NullPointerException e) {
                    log.info("Some Data Returned Null");
                } catch (InterruptedException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                } catch (ExecutionException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }

        }

        if (data.size() == 0) {
            // set to stop if size is0
            log.info("No Pages to Parse. Will Terminate");
            run = false;
        } else {
            // parse
            log.info("Starting JSoup Parse @ " + Calendar.getInstance().getTime().toString());
            for (String json : data) {
                // faster json reader is minimal json but faster parser is
                // Simple Json
                Map<String, Json> jMap = Json.read(json).asJsonMap();

                if (jMap.containsKey("offenderhash")) {
                    // string to int in case it is a string and has some
                    // extra space
                    offenderhash = Integer.parseInt(jMap.get("offenderhash").asString().trim());
                }

                boolean allow = true;

                if (mustcontain != null) {
                    if (jMap.get(column).asString().contains(mustcontain) == false) {
                        allow = false;
                    }
                }

                if (cannotcontain != null) {
                    if (jMap.get(column).asString().contains(cannotcontain)) {
                        allow = false;
                    }
                }

                // this is the fastest way. I was learning before and will
                // rewrite when time permits.
                if (allow == true) {
                    if (jMap.containsKey("offenderhash")) {
                        if (this.singlepaths != null) {
                            collection.add(new ParseSingle(Integer.toString(offenderhash), header, footer,
                                    pagenarrow, singlepaths,
                                    StringEscapeUtils.unescapeXml(jMap.get(column).asString()), replace,
                                    replaceSequence));
                        }

                        if (this.multipaths != null) {
                            collection.add(new ParseRows(Integer.toString(offenderhash), header, footer,
                                    pagenarrow, multipaths,
                                    StringEscapeUtils.unescapeXml(jMap.get(column).asString()), replace,
                                    replaceSequence));
                        }

                        if (this.recordpaths != null) {
                            collection.add(new ParseLoop(Integer.toString(offenderhash), header, footer,
                                    pagenarrow, recordpaths,
                                    StringEscapeUtils.unescapeXml(jMap.get(column).asString()), replace,
                                    replaceSequence));
                        }
                    }
                }
                offenderhash += 1;

            }

            // complete parse
            log.info("Waiting for Parsing to Complete.");
            if (collection.size() > 0) {
                futures = fjp.invokeAll(collection);

                int w = 0;
                while (fjp.isQuiescent() && fjp.getActiveThreadCount() > 0) {
                    w++;
                }

                log.info("Waited for " + Integer.toString(w) + " Cycles!");
                for (Future<ArrayList<String>> f : futures) {
                    try {
                        outdata.addAll(f.get());
                    } catch (InterruptedException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                    } catch (ExecutionException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                    }
                }

            }
            log.info("Finished Parsing @ " + Calendar.getInstance().getTime().toString());

            int cp = 0;
            // post data
            log.info("Posting Data @ " + Calendar.getInstance().getTime().toString());
            if (outdata.size() > 0) {

                for (int i = 0; i < qnums; i++) {

                    ArrayList<String> od = new ArrayList<String>(
                            ((cp + (Math.round(outdata.size() / qnums) - cp))));

                    if (cp + (Math.round(outdata.size() / qnums)) < outdata.size()) {
                        od.addAll(outdata.subList(cp, (cp + (Math.round(outdata.size() / qnums)))));
                    } else {
                        od.addAll(outdata.subList(cp, (outdata.size() - 1)));
                    }
                    fjp.execute(new SplitPost(template, od));
                    cp += Math.round(outdata.size() / qnums);
                }

                int w = 0;
                while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) {
                    w++;
                }
                log.info("Waited for " + Integer.toString(w) + " cycles!");

            }
            log.info("Finished Posting to DB @ " + Calendar.getInstance().getTime().toString());

            // size should remain same with 10 slot buffer room
            data.clear();
            outdata.clear();
        }

        // my favorite really desperate attempt to actually invoke garbage
        // collection because of MASSIVE STRINGS
        System.gc();
        Runtime.getRuntime().gc();

    } while (run);

    log.info("Shutting Down FJP");
    // shutdown fjp
    if (fjp.isShutdown() == false) {
        fjp.shutdownNow();
    }

    log.info("Finished Parsing @ " + Calendar.getInstance().getTime().toString());

}

---

From source file:com.hygenics.parser.JDump.java

private void toFile() {
    ArrayList<String> archs = new ArrayList<String>();
    List<Future<ArrayList<String>>> qfutures;
    Set<Callable<ArrayList<String>>> qcollect = new HashSet<Callable<ArrayList<String>>>(4);

    ForkJoinPool fjp = new ForkJoinPool((int) Math.ceil(Runtime.getRuntime().availableProcessors() * procnum));

    int dumped = 0;

    if (archive) {
        log.info("Cleaning");
        for (String k : fpaths.keySet()) {
            String fpath = "";

            for (String ofp : fpaths.get(k).keySet()) {
                fpath = ofp;/*from  w ww .ja v a 2s  .c  o m*/
            }

            if (fpath.length() > 0) {
                String[] barr = fpath.split("\\/");
                String basefile = "";
                Archiver zip = new Archiver();
                for (int i = 0; i > barr.length - 1; i++) {
                    basefile += (i == 0) ? barr[i] : "/" + barr[i];
                }
                if (basefile.trim().length() > 0) {
                    zip.setBasedirectory(basefile);
                    zip.setZipDirectory(basefile + "archive.zip");
                    zip.setAvoidanceString(".zip|archive");
                    zip.setDelFiles(true);
                    zip.run();
                }
            }
        }
    }

    log.info("Dumping");
    for (String table : fpaths.keySet()) {
        int offset = 0;
        if (template.checkTable(table, table.split("\\.")[0])) {
            if (template.getCount(table) > 0) {
                log.info("Dumping for " + table);
                // get header
                String select = "SELECT * FROM " + table;
                String fpath = null;
                ArrayList<String> jsons;
                String condition;
                int w = 0;
                int start = offset;
                int chunksize = (int) Math.ceil(pullsize / qnum);

                // get fpath
                for (String ofp : fpaths.get(table).keySet()) {
                    start = fpaths.get(table).get(ofp);
                    fpath = ofp;
                }

                // perform write
                if (headers != null && fpath != null) {
                    List<String> headersList = headers.get(table);

                    String output = null;
                    boolean existed = true;

                    if (addFileDate) {
                        fpath = fpath
                                + Calendar.getInstance().getTime().toString().trim().replaceAll(":|\\s", "")
                                + ".txt";
                    }

                    // check to see if file should be created
                    if (!new File(fpath).exists()) {

                        try {
                            new File(fpath).createNewFile();
                        } catch (IOException e) {
                            e.printStackTrace();
                        }
                        existed = false;
                    }

                    // check to see if file must be recreated
                    if (!append) {

                        File f = new File(fpath);
                        f.delete();
                        try {
                            f.createNewFile();
                        } catch (IOException e) {
                            e.printStackTrace();
                        }
                    }

                    if (headersList != null && (append == false || existed == false)) {
                        for (String header : headersList) {
                            output = (output == null) ? StringEscapeUtils.unescapeXml(header)
                                    : output + delimeter + StringEscapeUtils.unescapeXml(header);
                        }
                    }

                    do {

                        // get records
                        jsons = new ArrayList<String>(pullsize);
                        log.info("Looking for Pages.");
                        for (int conn = 0; conn < qnum; conn++) {
                            // create condition
                            condition = " WHERE " + pullid + " >= " + (start + (conn * chunksize)) + " AND "
                                    + pullid + " < " + Integer.toString(start + (chunksize * (conn + 1)));

                            if (extracondition != null) {
                                condition += " " + extracondition.trim();
                            }

                            // get queries
                            qcollect.add(new SplitQuery(template, (select + condition)));
                            log.info("Fetching " + select + condition);
                        }

                        start += (chunksize * qnum);

                        qfutures = fjp.invokeAll(qcollect);

                        w = 0;
                        while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) {
                            w++;
                        }
                        log.info("Waited for " + w + " cycles");

                        for (Future<ArrayList<String>> f : qfutures) {
                            try {

                                ArrayList<String> test = f.get();
                                if (test != null) {
                                    if (test.size() > 0) {
                                        jsons.addAll(test);
                                    }
                                }

                                if (f.isDone() == false) {
                                    f.cancel(true);
                                }

                                f = null;
                            } catch (Exception e) {
                                log.warn("Encoding Error!");
                                e.printStackTrace();
                            }
                        }
                        qcollect = new HashSet<Callable<ArrayList<String>>>(4);
                        qfutures = null;
                        log.info("Finished Getting Pages");

                        // post records to the file
                        try (FileWriter fw = new FileWriter(new File(fpath), true)) {
                            // get and write headers

                            if (jsons.size() > 0) {
                                fw.write(output + "\n");
                                // write data
                                for (String json : jsons) {
                                    output = null;
                                    JsonObject jo = JsonObject.readFrom(json);
                                    if (jo.size() >= headersList.size()) {// allows
                                        // trimming
                                        // of
                                        // table
                                        // to
                                        // key
                                        // aspects
                                        output = null;

                                        for (String key : headers.get(table)) {

                                            if (jo.get(key.toLowerCase()) != null) {
                                                String data = StringEscapeUtils
                                                        .unescapeXml(jo.get(key.toLowerCase()).asString());

                                                if (replacementPattern != null) {
                                                    data = data.replaceAll(replacementPattern, "");
                                                    data = data.replace(delimeter, delimreplace);
                                                }

                                                output = (output == null)
                                                        ? data.replaceAll("[^\u0020-\u0070 ]+", "")
                                                        : output + delimeter
                                                                + data.replaceAll("[^\u0020-\u0070 ]+", "");
                                            } else {
                                                output += delimeter;
                                            }
                                        }

                                        if (output != null && output.trim().length() > headersList.size()) {
                                            fw.write(output + "\n");
                                        }
                                    } else {
                                        if (jsons.size() == 0) {
                                            Log.info(
                                                    "Number of Headers and Keys from Json Array and Headers List Impossible to Match");
                                            try {
                                                throw new MismatchException(
                                                        "Number of Headers: " + headersList.size()
                                                                + " && Number of Keys: " + jo.size());
                                            } catch (MismatchException e) {
                                                e.printStackTrace();
                                            }
                                        }
                                    }

                                    output = null;
                                }
                            } else {
                                log.info("EOF FOUND! No New Records in This Iteration....Stopping.");
                            }
                        } catch (IOException e) {
                            e.printStackTrace();
                        }

                    } while (jsons.size() > 0);

                } else {
                    try {
                        throw new NullPointerException(
                                "No Headers Input to Class. Please Create the Requisite Map.");
                    } catch (NullPointerException e) {
                        e.printStackTrace();
                    }
                }
                dumped += 1;
            } else {
                try {
                    throw new NoDataException("No Data Found in Table " + table);
                } catch (NoDataException e) {
                    e.printStackTrace();
                }
            }
        } else {
            log.info("Missing Table " + table);
            try {
                throw new NullPointerException("Table " + table + " Does Not Exist!!!");
            } catch (NullPointerException e) {
                e.printStackTrace();
            }
        }
    } // end LOOP

    if (!fjp.isShutdown()) {
        fjp.shutdownNow();
    }

    if (dumped == 0) {
        log.error("No Data Found in Any Table");
        System.exit(-1);
    }
}

---

From source file:com.hygenics.parser.JDumpWithReference.java

private void toFile() {
    List<Future<ArrayList<String>>> qfutures;
    Set<Callable<ArrayList<String>>> qcollect = new HashSet<Callable<ArrayList<String>>>(4);

    ForkJoinPool fjp = new ForkJoinPool((int) Math.ceil(Runtime.getRuntime().availableProcessors() * procnum));
    int dumped = 0;

    if (archive) {
        log.info("Cleaning");
        for (String k : fpaths.keySet()) {
            String fpath = "";

            for (String ofp : fpaths.get(k).keySet()) {
                fpath = ofp;/*ww  w.j a  v a 2 s  .co  m*/
            }

            if (fpath.length() > 0) {
                String[] barr = fpath.split("\\/");
                String basefile = "";
                Archiver zip = new Archiver();
                for (int i = 0; i > barr.length - 1; i++) {
                    basefile += (i == 0) ? barr[i] : "/" + barr[i];
                }
                if (basefile.trim().length() > 0) {
                    zip.setBasedirectory(basefile);
                    zip.setZipDirectory(basefile + "archive.zip");
                    zip.setAvoidanceString(".zip|archive");
                    zip.setDelFiles(true);
                    zip.run();
                }
            }
        }
    }

    log.info("Dumping");
    for (String table : fpaths.keySet()) {
        int offset = 0;
        if (template.checkTable(this.baseschema + "." + table, this.baseschema)) {
            if (template.getCount(this.baseschema + "." + table) > 0) {
                log.info("Dumping for " + table);
                // get header
                String select = "SELECT * FROM " + this.baseschema + "." + table;
                String fpath = null;
                ArrayList<String> jsons;
                String condition;
                int w = 0;
                int start = offset;
                int chunksize = (int) Math.ceil(pullsize / qnum);

                // get fpath
                for (String ofp : fpaths.get(table).keySet()) {
                    start = fpaths.get(table).get(ofp);
                    fpath = ofp;
                }

                // perform write
                if (headers != null && fpath != null) {
                    List<String> headersList = headers.get(table);

                    String output = null;
                    boolean existed = true;

                    if (addFileDate) {
                        fpath = fpath
                                + Calendar.getInstance().getTime().toString().trim().replaceAll(":|\\s", "")
                                + ".txt";
                    }

                    // check to see if file should be created
                    if (!new File(fpath).exists()) {

                        try {
                            new File(this.baseFilePath + fpath).createNewFile();
                        } catch (IOException e) {
                            e.printStackTrace();
                        }
                        existed = false;
                    }

                    // check to see if file must be recreated
                    if (!append) {

                        File f = new File(this.baseFilePath + fpath);
                        f.delete();
                        try {
                            f.createNewFile();
                        } catch (IOException e) {
                            e.printStackTrace();
                        }
                    }

                    if (headersList != null && (append == false || existed == false)) {
                        for (String header : headersList) {
                            output = (output == null) ? StringEscapeUtils.unescapeXml(header)
                                    : output + delimeter + StringEscapeUtils.unescapeXml(header);
                        }
                    }

                    do {

                        // get records
                        jsons = new ArrayList<String>(pullsize);
                        log.info("Looking for Pages.");
                        for (int conn = 0; conn < qnum; conn++) {
                            // create condition
                            condition = " WHERE " + pullid + " >= " + (start + (conn * chunksize)) + " AND "
                                    + pullid + " < " + Integer.toString(start + (chunksize * (conn + 1)));

                            if (extracondition != null) {
                                condition += " " + extracondition.trim();
                            }

                            // get queries
                            qcollect.add(new SplitQuery(template, (select + condition)));
                            log.info("Fetching " + select + condition);
                        }

                        start += (chunksize * qnum);

                        qfutures = fjp.invokeAll(qcollect);

                        w = 0;
                        while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) {
                            w++;
                        }
                        log.info("Waited for " + w + " cycles");

                        for (Future<ArrayList<String>> f : qfutures) {
                            try {

                                ArrayList<String> test = f.get();
                                if (test != null) {
                                    if (test.size() > 0) {
                                        jsons.addAll(test);
                                    }
                                }

                                if (f.isDone() == false) {
                                    f.cancel(true);
                                }

                                f = null;
                            } catch (Exception e) {
                                log.warn("Encoding Error!");
                                e.printStackTrace();
                            }
                        }
                        qcollect = new HashSet<Callable<ArrayList<String>>>(4);
                        qfutures = null;
                        log.info("Finished Getting Pages");

                        // post records to the file
                        try (FileWriter fw = new FileWriter(new File(this.baseFilePath + fpath), true)) {
                            // get and write headers

                            if (jsons.size() > 0) {
                                fw.write(output + "\n");
                                // write data
                                for (String json : jsons) {
                                    output = null;
                                    JsonObject jo = JsonObject.readFrom(json);
                                    if (jo.size() >= headersList.size()) {// allows
                                        // trimming
                                        // of
                                        // table
                                        // to
                                        // key
                                        // aspects
                                        output = null;

                                        for (String key : headers.get(table)) {

                                            if (jo.get(key.toLowerCase()) != null) {
                                                String data = StringEscapeUtils
                                                        .unescapeXml(jo.get(key.toLowerCase()).asString());

                                                if (replacementPattern != null) {
                                                    data = data.replaceAll(replacementPattern, "");
                                                    data = data.replace(delimeter, delimreplace);
                                                }

                                                output = (output == null)
                                                        ? data.replaceAll("[^\u0020-\u007E ]+", "")
                                                        : output + delimeter
                                                                + data.replaceAll("[^\u0020-\u007E ]+", "");
                                            } else {
                                                output += delimeter;
                                            }
                                        }

                                        if (output != null && output.trim().length() > headersList.size()) {
                                            fw.write(output + "\n");
                                        }
                                    } else {
                                        if (jsons.size() == 0) {
                                            Log.info(
                                                    "Number of Headers and Keys from Json Array and Headers List Impossible to Match");
                                            try {
                                                throw new MismatchException(
                                                        "Number of Headers: " + headersList.size()
                                                                + " && Number of Keys: " + jo.size());
                                            } catch (MismatchException e) {
                                                e.printStackTrace();
                                            }
                                        }
                                    }

                                    output = null;
                                }
                            } else {
                                log.info("EOF FOUND! No New Records in This Iteration....Stopping.");
                            }
                        } catch (IOException e) {
                            e.printStackTrace();
                        }

                    } while (jsons.size() > 0);

                } else {
                    try {
                        throw new NullPointerException(
                                "No Headers Input to Class. Please Create the Requisite Map.");
                    } catch (NullPointerException e) {
                        e.printStackTrace();
                    }
                }
                dumped += 1;
            } else {
                try {
                    throw new NoDataException("No Data in Table " + table);
                } catch (NoDataException e) {
                    e.printStackTrace();
                }
            }
        } else {
            log.info("Missing Table " + table);
            try {
                throw new NullPointerException("Table " + table + " Does Not Exist!!!");
            } catch (NullPointerException e) {
                e.printStackTrace();
            }
        }
    } // end LOOP

    if (!fjp.isShutdown()) {
        fjp.shutdownNow();
    }

    if (dumped == 0) {
        log.error("No Data found in Any Tables");
        System.exit(-1);
    }
}

---

From source file:com.hygenics.parser.BreakMultiple.java

/**
 * run the class//from   w  w  w.  j  a v  a 2 s  .c  o  m
 */
public void run() {
    int j = 0;
    checkTable();
    rows = new ArrayList<String>();
    log.info("Starting Break");

    // the pool
    ForkJoinPool fjp = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * procnum);

    // for returned results
    List<Future<ArrayList<String>>> futures = new ArrayList<Future<ArrayList<String>>>();

    // for parsing
    Set<Callable<ArrayList<String>>> collect = new HashSet<Callable<ArrayList<String>>>();

    // for querying
    Set<Callable<ArrayList<String>>> qcollect = new HashSet<Callable<ArrayList<String>>>();

    // results
    ArrayList<String> jsons = new ArrayList<String>();

    String condition = null;
    int size = (int) Math.ceil(pullsize / qnum);
    // get initial data from user
    for (int i = 0; i < qnum; i++) {
        condition = " WHERE " + idcolumn + " > " + Integer.toString(offset + (Math.round(pullsize / qnum) * i))
                + " AND " + idcolumn + " <= "
                + Integer.toString(offset + (Math.round(pullsize / qnum) * (i + 1)));

        if (extracondition != null) {
            condition += " " + extracondition.trim();
        }

        qcollect.add(new GetFromDB((select + condition), template));
        log.info("SELECTING " + select + " " + condition);
    }

    log.info("Getting From DB @" + Calendar.getInstance().getTime().toString());
    futures = fjp.invokeAll(qcollect);

    int w = 0;
    while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) {
        w++;
    }

    log.info("Waited for " + w + "Cycles");

    for (Future<ArrayList<String>> f : futures) {
        try {
            rows.addAll(f.get());
            f.cancel(true);
        } catch (InterruptedException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (ExecutionException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

    qcollect = new HashSet<Callable<ArrayList<String>>>();
    futures = null;

    log.info("Breaking");
    // process while there is still data to process
    while (rows.size() > 0) {
        log.info("Iteration Contains " + rows.size() + " Rows");
        // add to the commit size for future processing
        offset += pullsize;
        log.info("Submitting Tasks");
        // submit for breaking apart

        for (String r : rows) {

            if (fjp.isShutdown()) {
                fjp = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * procnum);
            }

            if (r != null) {

                if (mustcontain != null) {
                    if (r.contains(mustcontain)) {
                        if (cannotcontain != null) {
                            if (r.contains(cannotcontain) == false) {
                                Map<String, Json> rowmap = Json.read(r).asJsonMap();

                                // final getDAOTemplate template, final
                                // String row, final String token, final
                                // String replacementPattern, final
                                // Map<String,String> positions,final String
                                // date, final String table, final String
                                // offenderhash
                                if (rowmap.size() > 0) {
                                    collect.add(new Break(unescape, repeatkeys, template,
                                            rowmap.get(rowcolumn).asString(), token, replacementPattern,
                                            positions, (Calendar.getInstance().getTime().toString()),
                                            targettable, rowmap.get("offenderhash").asString(), maxpos,
                                            genhash));
                                }
                            }
                        } else {
                            Map<String, Json> rowmap = Json.read(r).asJsonMap();

                            // final getDAOTemplate template, final String
                            // row, final String token, final String
                            // replacementPattern, final Map<String,String>
                            // positions,final String date, final String
                            // table, final String offenderhash
                            if (rowmap.size() > 0) {
                                collect.add(new Break(unescape, repeatkeys, template,
                                        rowmap.get(rowcolumn).asString(), token, replacementPattern, positions,
                                        (Calendar.getInstance().getTime().toString()), targettable,
                                        rowmap.get("offenderhash").asString(), maxpos, genhash));
                            }
                        }
                    }
                } else {

                    if (cannotcontain != null) {
                        if (r.contains(cannotcontain) == false) {
                            Map<String, Json> rowmap = Json.read(r).asJsonMap();

                            // to ascend you must die, to die you must be
                            // crucified; so get off your -- cross so that
                            // we can nail down the nex martyr
                            // final getDAOTemplate template, final String
                            // row, final String token, final String
                            // replacementPattern, final Map<String,String>
                            // positions,final String date, final String
                            // table, final String offenderhash
                            if (rowmap.size() > 0) {
                                collect.add(new Break(unescape, repeatkeys, template,
                                        rowmap.get(rowcolumn).asString(), token, replacementPattern, positions,
                                        (Calendar.getInstance().getTime().toString()), targettable,
                                        rowmap.get("offenderhash").asString(), maxpos, genhash));
                            }
                        }
                    } else {
                        Map<String, Json> rowmap = Json.read(r).asJsonMap();

                        // final getDAOTemplate template, final String row,
                        // final String token, final String
                        // replacementPattern, final Map<String,String>
                        // positions,final String date, final String table,
                        // final String offenderhash
                        if (rowmap.size() > 0) {
                            collect.add(new Break(unescape, repeatkeys, template,
                                    rowmap.get(rowcolumn).asString(), token, replacementPattern, positions,
                                    (Calendar.getInstance().getTime().toString()), targettable,
                                    rowmap.get("offenderhash").asString(), maxpos, genhash));
                        }
                    }
                }
            }
        }

        log.info("SUBMITTED " + collect.size() + " tasks");

        futures = fjp.invokeAll(collect);

        w = 0;

        while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) {
            w++;
        }

        log.info("Waited for " + w + " Cycles");

        jsons.clear();
        log.info("Getting Strings");
        try {

            for (Future<ArrayList<String>> p : futures) {
                ArrayList<String> retlist = p.get();

                if (retlist != null) {
                    if (retlist.size() > 0) {
                        jsons.addAll(retlist);
                    }

                    if (jsons.size() >= commit_size) {
                        // send to db
                        if (jsons.size() > SPLITSIZE) {
                            log.info("Split True: Sending to DB @ "
                                    + Calendar.getInstance().getTime().toString());

                            postToDb(jsons, true);
                            jsons = new ArrayList<String>();
                            log.info("Posted to DB @ " + Calendar.getInstance().getTime().toString());
                        } else {
                            log.info("Split False: Sending to DB @ "
                                    + Calendar.getInstance().getTime().toString());
                            postToDb(jsons, false);
                            jsons = new ArrayList<String>();
                            log.info("Posted to DB @ " + Calendar.getInstance().getTime().toString());
                        }
                    }
                }
                p.cancel(true);
            }
        } catch (InterruptedException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (ExecutionException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

        futures = null;
        collect = new HashSet<Callable<ArrayList<String>>>();

        // send to db
        if (jsons.size() > SPLITSIZE) {
            log.info("Split True: Sending to DB @" + Calendar.getInstance().getTime().toString());
            postToDb(jsons, true);
            jsons = new ArrayList<String>();
            log.info("Posted to DB @ " + Calendar.getInstance().getTime().toString());
        } else {
            log.info("Split False: Sending to DB @" + Calendar.getInstance().getTime().toString());
            postToDb(jsons, false);
            jsons = new ArrayList<String>();
            log.info("Posted to DB @ " + Calendar.getInstance().getTime().toString());
        }

        // get more information
        rows = new ArrayList<String>();

        if (Runtime.getRuntime().freeMemory() < 500000 | ((loops % waitloops) == 0 & waitloops != 0)) {
            log.info("Paused Free Memory Left: " + Runtime.getRuntime().freeMemory());
            System.gc();
            Runtime.getRuntime().gc();

            try {
                Thread.sleep(2000);
            } catch (InterruptedException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }

            while (Runtime.getRuntime().freeMemory() < 500000) {
                try {
                    Thread.sleep(2000);
                } catch (InterruptedException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }

            log.info("Restart Free Memory Left: " + Runtime.getRuntime().freeMemory());
        }

        rows = new ArrayList<String>();

        // attempt to query the database from multiple threads
        for (int conn = 1; conn <= qnum; conn++) {
            // change condition
            condition = " WHERE " + idcolumn + " > "
                    + Integer.toString(offset + (Math.round(pullsize / qnum) * conn)) + " AND " + idcolumn
                    + " <= " + Integer.toString(offset + (Math.round(pullsize / qnum) * (conn + 1)));

            if (extracondition != null) {
                condition += " " + extracondition.trim();
            }

            qcollect.add(new GetFromDB((select + condition), template));
            log.info("SELECTING " + select + " " + condition);
        }

        futures = fjp.invokeAll(qcollect);

        w = 0;

        while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) {
            w++;
        }

        log.info("Waited for " + w + " Cycles");

        for (Future<ArrayList<String>> f : futures) {
            try {

                ArrayList<String> test = f.get();

                if (test != null) {
                    if (test.size() > 0) {
                        rows.addAll(test);
                    }
                }

                f.cancel(true);

            } catch (InterruptedException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            } catch (ExecutionException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }

        futures = null;
        qcollect = new HashSet<Callable<ArrayList<String>>>(4);

        j++;

        Runtime.getRuntime().gc();
        System.gc();

    }

    // send to db
    if (jsons.size() > SPLITSIZE) {
        log.info("Split True: Sending to DB @" + Calendar.getInstance().getTime().toString());
        postToDb(jsons, true);
        jsons = new ArrayList<String>();
    } else if (jsons.size() > 0) {
        log.info("Split False: Sending to DB @" + Calendar.getInstance().getTime().toString());
        postToDb(jsons, false);
        jsons = new ArrayList<String>();
    }

    Runtime.getRuntime().gc();
    System.gc();

    log.info("Shutting Down Forkjoin Pool");
    if (fjp.isShutdown() == false) {
        fjp.shutdownNow();
    }
}

---

From source file:com.hygenics.parser.ParseDispatcher.java

/**
 * Fork/Join Pool Solution Maximizes Speed. JSon increases ease of use
 * /*from w w  w  . j  a  v  a 2 s . c o m*/
 */
public void run() {
    log.info("Starting Clock and Parsing @" + Calendar.getInstance().getTime().toString());
    long t = Calendar.getInstance().getTimeInMillis();
    int pid = 0;
    int id = 0;
    int checkattempts = 0;
    String add = null;

    this.schema = Properties.getProperty(this.schema);
    this.select = Properties.getProperty(this.select);
    this.extracondition = Properties.getProperty(this.extracondition);
    this.column = Properties.getProperty(this.column);

    ArrayList<String> parsedrows = new ArrayList<String>();

    Set<Callable<String>> collect = new HashSet<Callable<String>>();
    List<Future<String>> futures;

    List<Future<ArrayList<String>>> qfutures;
    Set<Callable<ArrayList<String>>> qcollect = new HashSet<Callable<ArrayList<String>>>(4);

    ForkJoinPool fjp = new ForkJoinPool((int) Math.ceil(Runtime.getRuntime().availableProcessors() * procnum));

    if (schema != null) {
        createTables();
    }

    boolean run = true;
    String condition;
    int w = 0;
    int start = offset;
    int chunksize = (int) Math.ceil(pullsize / qnum);

    // attempt to query the database from multiple threads
    do {
        // query for pages
        pages = new ArrayList<String>(pullsize);
        log.info("Looking for Pages.");
        for (int conn = 0; conn < qnum; conn++) {
            // create condition
            condition = " WHERE " + pullid + " >= " + (start + (conn * chunksize)) + " AND " + pullid + " < "
                    + Integer.toString(start + (chunksize * (conn + 1)));

            if (extracondition != null) {
                condition += " " + extracondition.trim();
            }

            // get queries
            qcollect.add(new SplitQuery(template, (select + condition)));
            log.info("Fetching " + select + condition);
        }
        start += (chunksize * qnum);

        qfutures = fjp.invokeAll(qcollect);

        w = 0;
        while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) {
            w++;
        }
        log.info("Waited for " + w + " cycles");

        for (Future<ArrayList<String>> f : qfutures) {
            try {

                ArrayList<String> test = f.get();
                if (test != null) {
                    if (test.size() > 0) {
                        pages.addAll(test);
                    }
                }

                if (f.isDone() == false) {
                    f.cancel(true);
                }

                f = null;
            } catch (Exception e) {
                log.warn("Encoding Error!");
                e.printStackTrace();
            }
        }
        qcollect = new HashSet<Callable<ArrayList<String>>>(4);
        qfutures = null;
        log.info("Finished Getting Pages");

        // if no records then get records that may have been dropped
        if (pages.size() == 0 && checkstring != null && checkstring.trim().length() > 0
                && checkattempts < reattempts) {
            checkattempts += 1;
            log.info("Checking for Drops");
            qcollect.add(new SplitQuery(template, (checkstring)));
            qfutures = fjp.invokeAll(qcollect);

            w = 0;
            while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) {
                w++;
            }
            log.info("Waited for " + w + " cycles");

            for (Future<ArrayList<String>> f : qfutures) {
                try {

                    ArrayList<String> test = f.get();
                    if (test != null) {
                        if (test.size() > 0) {
                            pages.addAll(test);
                        }
                    }

                    if (f.isDone() == false) {
                        f.cancel(true);
                    }

                    f = null;
                } catch (Exception e) {
                    log.warn("Encoding Error!");
                    e.printStackTrace();
                }
            }
            qfutures = null;
            qcollect = new HashSet<Callable<ArrayList<String>>>(4);

        } else if (checkattempts >= reattempts) {
            pages.clear();
        }

        log.info("Found " + pages.size() + " records!");

        // get hashes if necessary
        if (getHash) {
            log.info("Hashing " + pages.size() + " Records");

            ArrayList<String> hashedrows = new ArrayList<String>();
            for (String row : pages) {

                collect.add(new CreateHash(row, pid));
                pid++;

            }

            log.info("Invoking");
            futures = fjp.invokeAll(collect);

            w = 0;
            while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) {
                w++;
            }

            log.info("Waited " + w + " Cycles!");

            for (Future<String> f : futures) {
                if (f != null) {
                    String json;
                    try {
                        json = f.get(termtime, TimeUnit.MILLISECONDS);

                        if (json != null) {
                            hashedrows.add(json);
                        }

                    } catch (Exception e) {
                        log.warn("Encoding Error!");
                        e.printStackTrace();
                    }
                }

            }
            log.info("Hashed " + hashedrows.size() + " Records!");
            pages = hashedrows;

            collect = new HashSet<Callable<String>>(pullsize);
            futures.clear();
            log.info("Completed Hashing");
        }

        log.info("Performing Regex");
        // handle single patterns
        int i = 0;
        if (singlepats != null) {

            log.info("Found Singlepats");
            int subs = 0;
            int rows = 0;
            for (String row : pages) {
                rows += 1;
                String inrow = row;
                try {

                    inrow = inrow.replaceAll("\t|\r|\r\n|\n", "");

                    Map<String, Json> jmap = Json.read(inrow).asJsonMap();

                    if (singlepats.containsKey("table")) {
                        subs += 1;

                        if (fjp.isShutdown()) {
                            fjp = new ForkJoinPool((Runtime.getRuntime().availableProcessors() * procnum));
                        }

                        if (jmap.get(column) != null) {

                            if (test) {
                                System.out.println("//////////////////////HTML////////////////////////\n"
                                        + jmap.get(column).asString()
                                        + "\n///////////////////////////////END///////////////////////////\n\n");
                            }

                            if (mustcontain != null) {
                                if (jmap.get(column).asString().contains(mustcontain)) {
                                    if (cannotcontain != null) {
                                        if (jmap.get(column).asString().contains(cannotcontain) == false)
                                            collect.add(new ParsePage(unescape, replacementPattern,
                                                    singlepats.get("table"),
                                                    jmap.get(column).asString().replaceAll("\\s\\s", " "),
                                                    singlepats, Calendar.getInstance().getTime().toString(),
                                                    jmap.get("offenderhash").asString()));
                                    } else {
                                        collect.add(new ParsePage(unescape, replacementPattern,
                                                singlepats.get("table"),
                                                jmap.get(column).asString().replaceAll("\\s\\s", " "),
                                                singlepats, Calendar.getInstance().getTime().toString(),
                                                jmap.get("offenderhash").asString()));
                                    }
                                }
                            } else if (cannotcontain != null) {
                                if (jmap.get(column).asString().contains(cannotcontain) == false) {
                                    collect.add(
                                            new ParsePage(unescape, replacementPattern, singlepats.get("table"),
                                                    jmap.get(column).asString().replaceAll("\\s\\s", " "),
                                                    singlepats, Calendar.getInstance().getTime().toString(),
                                                    jmap.get("offenderhash").asString()));
                                }
                            } else {
                                collect.add(new ParsePage(unescape, replacementPattern, singlepats.get("table"),
                                        jmap.get(column).asString().replaceAll("\\s\\s", " "), singlepats,
                                        Calendar.getInstance().getTime().toString(),
                                        jmap.get("offenderhash").asString()));
                            }
                        }
                    }
                    i++;

                    if (((i % commit_size) == 0 & i != 0) || i == pages.size()
                            || pages.size() == 1 && singlepats != null) {
                        log.info("Getting Regex Results");

                        log.info("Getting Tasks");

                        futures = fjp.invokeAll(collect);

                        w = 0;

                        while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) {
                            w++;
                        }

                        log.info("Waited for " + w + " cycles");

                        for (Future<String> r : futures) {
                            try {

                                add = r.get();
                                if (add.contains("No Data") == false) {
                                    parsedrows.add(add);
                                }

                                add = null;

                            } catch (Exception e) {
                                log.warn("Encoding Error!");
                                e.printStackTrace();
                            }
                        }

                        futures = null;
                        collect = new HashSet<Callable<String>>();

                        if (parsedrows.size() >= commit_size) {
                            log.info("INSERTING " + parsedrows.size() + " records!");
                            if (parsedrows.size() >= SPLITSIZE) {
                                sendToDb(parsedrows, true);
                            } else {
                                sendToDb(parsedrows, false);
                            }

                            parsedrows = new ArrayList<String>(pullsize);
                        }

                        // hint to the gc in case it actually pays off; use
                        // -X:compactexplicitgc to improve odds and
                        // -XX:UseConcMarkSweepGC for improving odds on
                        // older generation strings
                        // (think if i were a gambling man)
                        System.gc();
                        Runtime.getRuntime().gc();
                    }
                } catch (Exception e) {
                    log.warn("Encoding Error!");
                    e.printStackTrace();
                }
            }
            log.info("Submitted " + subs + " records. Found " + rows + " rows");
        }

        log.info("REMAINING ROWS TO COMMIT " + parsedrows.size());
        log.info("Rows Left" + parsedrows.size());
        if (parsedrows.size() > 0) {

            if (parsedrows.size() >= SPLITSIZE) {
                sendToDb(parsedrows, true);
            } else {
                sendToDb(parsedrows, false);
            }

            parsedrows = new ArrayList<String>();
        }

        // handle multi patterns
        if (multipats != null) {
            // parse multiple pages for the run
            int subs = 0;
            for (String row : pages) {
                try {
                    for (String k : multipats.keySet()) {
                        if (fjp.isShutdown()) {

                            fjp = new ForkJoinPool(Runtime.getRuntime().availableProcessors());
                        }

                        Map<String, Json> jmap = Json.read(row).asJsonMap();

                        if (jmap.get(column) != null) {
                            subs += 1;
                            if (test) {
                                System.out.println("//////////////////////HTML////////////////////////\n"
                                        + jmap.get(column).asString()
                                        + "\n///////////////////////////////END///////////////////////////\n\n");
                            }

                            if (mustcontain != null) {
                                if (jmap.get(column).asString().contains(mustcontain)) {
                                    if (cannotcontain != null) {
                                        if (jmap.get(column).asString().contains(cannotcontain) == false) {
                                            collect.add(
                                                    new ParseMultiPage(unescape, replacementPattern, k,
                                                            jmap.get(column).asString().replaceAll("\\s\\s",
                                                                    " "),
                                                            jmap.get("offenderhash").asString(),
                                                            Calendar.getInstance().getTime().toString(),
                                                            multipats.get(k)));
                                        }
                                    } else {
                                        collect.add(new ParseMultiPage(unescape, replacementPattern, k,
                                                jmap.get(column).asString(),
                                                jmap.get("offenderhash").asString().replaceAll("\\s\\s", " "),
                                                Calendar.getInstance().getTime().toString(), multipats.get(k)));
                                    }
                                }
                            } else if (cannotcontain != null) {
                                if (jmap.get(column).asString().contains(cannotcontain) == false) {
                                    collect.add(new ParseMultiPage(unescape, replacementPattern, k,
                                            jmap.get(column).asString().replaceAll("\\s\\s", " "),
                                            jmap.get("offenderhash").asString(),
                                            Calendar.getInstance().getTime().toString(), multipats.get(k)));
                                }

                            } else {
                                collect.add(new ParseMultiPage(unescape, replacementPattern, k,
                                        jmap.get(column).asString().replaceAll("\\s\\s", " "),
                                        jmap.get("offenderhash").asString(),
                                        Calendar.getInstance().getTime().toString(), multipats.get(k)));
                            }
                        }

                        i++;
                        if (((i % commit_size) == 0 & i != 0) || i == pages.size()
                                || pages.size() == 1 && multipats != null) {
                            futures = fjp.invokeAll(collect);
                            w = 0;
                            while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) {
                                w++;
                            }

                            log.info("Waited " + w + " Cycles");

                            for (Future<String> r : futures) {
                                try {
                                    add = r.get();

                                    if (add.contains("No Data") == false) {

                                        for (String js : add.split("~")) {
                                            parsedrows.add(js);
                                        }
                                    }
                                    add = null;

                                    if (r.isDone() == false) {
                                        r.cancel(true);
                                    }
                                    r = null;

                                } catch (InterruptedException e) {
                                    // TODO Auto-generated catch block
                                    e.printStackTrace();
                                } catch (ExecutionException e) {
                                    // TODO Auto-generated catch block
                                    e.printStackTrace();
                                }
                            }

                            futures = null;
                            collect = new HashSet<Callable<String>>();

                            if (parsedrows.size() >= commit_size) {
                                log.info("INSERTING " + parsedrows.size() + " records!");
                                if (parsedrows.size() >= SPLITSIZE) {
                                    sendToDb(parsedrows, true);
                                } else {
                                    sendToDb(parsedrows, false);
                                }
                                parsedrows = new ArrayList<String>(pullsize);
                            }

                            // hint to the gc in case it actually pays off
                            System.gc();
                            Runtime.getRuntime().gc();
                        }
                    }

                } catch (Exception e) {
                    log.warn("Encoding Error!");
                }

            }
            log.info("Submitted " + subs + " records.");
        }

        // handle looped patterns
        if (loopedpats != null) {
            log.info("Looped Patterns Found");
            int subs = 0;
            if (fjp.isShutdown()) {
                fjp = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * procnum);
            }

            for (String row : pages) {
                try {

                    for (String k : loopedpats.keySet()) {
                        if (fjp.isShutdown()) {
                            fjp = new ForkJoinPool(Runtime.getRuntime().availableProcessors() * procnum);
                        }
                        Map<String, Json> jmap = Json.read(row).asJsonMap();

                        if (jmap.get(column) != null) {
                            subs += 1;
                            if (mustcontain != null) {
                                if (jmap.get(column).asString().contains(mustcontain)) {
                                    if (cannotcontain != null) {
                                        if (jmap.get(column).asString().contains(cannotcontain) == false) {
                                            collect.add(
                                                    new LoopRegex(unescape,
                                                            jmap.get(column).asString().replaceAll("\\s\\s",
                                                                    " "),
                                                            jmap.get("offenderhash").asString(),
                                                            Calendar.getInstance().getTime().toString(), k,
                                                            replacementPattern, loopedpats.get(k), test));
                                        }
                                    } else {
                                        collect.add(new LoopRegex(unescape,
                                                jmap.get(column).asString().replaceAll("\\s\\s", " "),
                                                jmap.get("offenderhash").asString(),
                                                Calendar.getInstance().getTime().toString(), k,
                                                replacementPattern, loopedpats.get(k), test));
                                    }
                                }
                            } else if (cannotcontain != null) {
                                if (jmap.get(column).asString().contains(cannotcontain) == false) {
                                    collect.add(new LoopRegex(unescape,
                                            jmap.get(column).asString().replaceAll("\\s\\s", " "),
                                            jmap.get("offenderhash").asString(),
                                            Calendar.getInstance().getTime().toString(), k, replacementPattern,
                                            loopedpats.get(k), test));
                                }
                            } else {
                                collect.add(new LoopRegex(unescape,
                                        jmap.get(column).asString().replaceAll("\\s\\s", " "),
                                        jmap.get("offenderhash").asString(),
                                        Calendar.getInstance().getTime().toString(), k, replacementPattern,
                                        loopedpats.get(k), test));
                            }
                            jmap.remove(k);
                        }
                        i++;
                        if (((i % commit_size) == 0 & i != 0) || (i % (pages.size() - 1)) == 0
                                || pages.size() == 1) {

                            futures = fjp.invokeAll(collect);

                            w = 0;

                            while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) {
                                w++;
                            }
                            log.info("Waited " + w + " Cycles");

                            for (Future<String> r : futures) {
                                try {
                                    add = r.get();
                                    if (add.contains("No Data") == false) {
                                        for (String toarr : add.split("~")) {
                                            parsedrows.add(toarr);
                                        }
                                    }

                                    if (r.isDone() == false) {
                                        r.cancel(true);
                                    }
                                    add = null;

                                } catch (Exception e) {
                                    log.warn("Encoding Error!");
                                    e.printStackTrace();
                                }
                            }

                            futures = null;
                            collect = new HashSet<Callable<String>>();

                            // hint to the gc in case it actually pays off
                            System.gc();
                            Runtime.getRuntime().gc();
                        }
                    }

                    if (parsedrows.size() >= this.commit_size) {
                        log.info("INSERTING " + parsedrows.size() + " records!");
                        if (parsedrows.size() >= SPLITSIZE) {
                            sendToDb(parsedrows, true);
                        } else {
                            sendToDb(parsedrows, false);
                        }

                        parsedrows = new ArrayList<String>(pullsize);
                    }

                } catch (Exception e) {
                    log.warn("Encoding Error!");
                }
            }
            log.info("Submitted " + subs + " records.");
        }

        if (collect.size() > 0) {
            log.info("Getting Last Regex Results for Iteration");

            log.info("Getting Tasks");

            futures = fjp.invokeAll(collect);

            w = 0;

            while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) {
                w++;
            }

            log.info("Waited for " + w + " cycles");

            for (Future<String> r : futures) {
                try {

                    add = r.get();
                    if (add.contains("No Data") == false) {
                        parsedrows.add(add);
                    }

                    add = null;

                } catch (Exception e) {
                    log.warn("Encoding Error!");
                    e.printStackTrace();
                }
            }

            futures = null;
            collect = new HashSet<Callable<String>>(pullsize);
            // hint to the gc in case it actually pays off; use
            // -X:compactexplicitgc to improve odds and
            // -XX:UseConcMarkSweepGC for improving odds on older generation
            // strings
            // (think if i were a gambling man)
            System.gc();
            Runtime.getRuntime().gc();
        }

        log.info("REMAINING ROWS TO COMMIT " + parsedrows.size());
        log.info("Rows Left" + parsedrows.size());
        if (parsedrows.size() > 0) {

            if (parsedrows.size() >= SPLITSIZE) {
                sendToDb(parsedrows, true);
            } else {
                sendToDb(parsedrows, false);
            }

            parsedrows = new ArrayList<String>();
        }

    } while (pages != null && pages.size() > 0);

    // ensure that nothing is still caught in limbo
    // final parser to ensure that nothing is left out
    if (collect.size() > 0) {
        log.info("More Rows Caught in FJP, Completing Process");
        futures = fjp.invokeAll(collect);

        w = 0;

        while (fjp.getActiveThreadCount() > 0 && fjp.isQuiescent() == false) {
            w++;
        }
        log.info("Waited " + w + " Cycles");

        for (Future<String> r : futures) {
            try {
                add = r.get();

                if (add.contains("No Data") == false) {

                    for (String js : add.split("~")) {
                        parsedrows.add(js);
                    }
                }
                add = null;

                if (r.isDone() == false) {
                    r.cancel(true);
                }
                r = null;

            } catch (InterruptedException e) {
                e.printStackTrace();
            } catch (ExecutionException e) {
                e.printStackTrace();
            }
        }

        futures = null;
        collect = null;
    }

    // send any remaining parsed rows to the db
    if (parsedrows.size() > 0) {

        if (parsedrows.size() >= SPLITSIZE) {
            sendToDb(parsedrows, true);
        } else {
            sendToDb(parsedrows, false);
        }

        parsedrows = new ArrayList<String>();
    }

    log.info("Shutting Down Fork Join Pool");
    if (fjp.isShutdown() == false) {
        fjp.shutdownNow();
    }

    fjp = null;

    log.info("Complete @" + Calendar.getInstance().getTime().toString());
    log.info("Total Runtime(seconds): "
            + Double.toString((double) (Calendar.getInstance().getTimeInMillis() - t) / 1000));

    // hint to the gc in case it actually pays off
    System.gc();
    Runtime.getRuntime().gc();
}

---

From source file:uniol.apt.analysis.synthesize.FindWords.java

/**
 * Generate Petri net solvable words with the given characteristics.
 * @param properties The properties that should be considered.
 * @param alphabet The alphabet from which words should be generated.
 * @param quickFail Should quick-fail synthesis be done or should full synthesis be attempted?
 * @param wordCallback Callback that should be called for each word that is found.
 * @param lengthDoneCallback Callback that should be called when all words of a given length were handled.
 * @throws PreconditionFailedException If a combination of properties is specified for which there is no
 * sensible definition of 'minimally unsolvable word', i.e. plain+k-marking.
 *//* w  ww .j a  v a  2 s .c o m*/
static public void generateList(PNProperties properties, SortedSet<Character> alphabet, boolean quickFail,
        WordCallback wordCallback, LengthDoneCallback lengthDoneCallback) throws PreconditionFailedException {
    // Java 8 provides ForkJoinPool.commonPool(). Java 7 does not, so we need to create our own pool.
    ForkJoinPool executor = new ForkJoinPool();
    try {
        generateList(properties, alphabet, quickFail, wordCallback, lengthDoneCallback, executor);
    } finally {
        executor.shutdownNow();
    }
}

---