Example usage for java.util LinkedList size

List of usage examples for java.util LinkedList size

Introduction

In this page you can find the example usage for java.util LinkedList size.

Prototype

int size

To view the source code for java.util LinkedList size.

Click Source Link

Usage

From source file:com.act.lcms.v2.MZCollisionCounter.java

public static void main(String[] args) throws Exception {
    CLIUtil cliUtil = new CLIUtil(MassChargeCalculator.class, HELP_MESSAGE, OPTION_BUILDERS);
    CommandLine cl = cliUtil.parseCommandLine(args);

    File inputFile = new File(cl.getOptionValue(OPTION_INPUT_INCHI_LIST));
    if (!inputFile.exists()) {
        cliUtil.failWithMessage("Input file at does not exist at %s", inputFile.getAbsolutePath());
    }//  w  w  w.j a v  a2 s  . co  m

    List<MassChargeCalculator.MZSource> sources = new ArrayList<>();
    try (BufferedReader reader = new BufferedReader(new FileReader(inputFile))) {
        String line;
        while ((line = reader.readLine()) != null) {
            line = line.trim();
            sources.add(new MassChargeCalculator.MZSource(line));
            if (sources.size() % 1000 == 0) {
                LOGGER.info("Loaded %d sources from input file", sources.size());
            }
        }
    }

    Set<String> considerIons = Collections.emptySet();
    if (cl.hasOption(OPTION_ONLY_CONSIDER_IONS)) {
        List<String> ions = Arrays.asList(cl.getOptionValues(OPTION_ONLY_CONSIDER_IONS));
        LOGGER.info("Only considering ions for m/z calculation: %s", StringUtils.join(ions, ", "));
        considerIons = new HashSet<>(ions);
    }

    TSVWriter<String, Long> tsvWriter = new TSVWriter<>(Arrays.asList("collisions", "count"));
    tsvWriter.open(new File(cl.getOptionValue(OPTION_OUTPUT_FILE)));

    try {
        LOGGER.info("Loaded %d sources in total from input file", sources.size());

        MassChargeCalculator.MassChargeMap mzMap = MassChargeCalculator.makeMassChargeMap(sources,
                considerIons);

        if (!cl.hasOption(OPTION_COUNT_WINDOW_INTERSECTIONS)) {
            // Do an exact analysis of the m/z collisions if windowing is not specified.

            LOGGER.info("Computing precise collision histogram.");
            Iterable<Double> mzs = mzMap.ionMZIter();
            Map<Integer, Long> collisionHistogram = histogram(
                    StreamSupport.stream(mzs.spliterator(), false).map(mz -> { // See comment about Iterable below.
                        try {
                            return mzMap.ionMZToMZSources(mz).size();
                        } catch (NoSuchElementException e) {
                            LOGGER.error("Caught no such element exception for mz %f: %s", mz, e.getMessage());
                            throw e;
                        }
                    }));
            List<Integer> sortedCollisions = new ArrayList<>(collisionHistogram.keySet());
            Collections.sort(sortedCollisions);
            for (Integer collision : sortedCollisions) {
                tsvWriter.append(new HashMap<String, Long>() {
                    {
                        put("collisions", collision.longValue());
                        put("count", collisionHistogram.get(collision));
                    }
                });
            }
        } else {
            /* After some deliberation (thanks Gil!), the windowed variant of this calculation counts the number of
             * structures whose 0.01 Da m/z windows (for some set of ions) overlap with each other.
             *
             * For example, let's assume we have five total input structures, and are only searching for one ion.  Let's
             * also assume that three of those structures have m/z A and the remaining two have m/z B.  The windows might
             * look like this in the m/z domain:
             * |----A----|
             *        |----B----|
             * Because A represents three structures and overlaps with B, which represents two, we assign A a count of 5--
             * this is the number of structures we believe could fall into the range of A given our current peak calling
             * approach.  Similarly, B is assigned a count of 5, as the possibility for collision/confusion is symmetric.
             *
             * Note that this is an over-approximation of collisions, as we could more precisely only consider intersections
             * when the exact m/z of B falls within the window around A and vice versa.  However, because we have observed
             * cases where the MS sensor doesn't report structures at exactly the m/z we predict, we employ this weaker
             * definition of intersection to give a slightly pessimistic view of what confusions might be possible. */
            // Compute windows for every m/z.  We don't care about the original mz values since we just want the count.
            List<Double> mzs = mzMap.ionMZsSorted();

            final Double windowHalfWidth;
            if (cl.hasOption(OPTION_WINDOW_HALFWIDTH)) {
                // Don't use get with default for this option, as we want the exact FP value of the default tolerance.
                windowHalfWidth = Double.valueOf(cl.getOptionValue(OPTION_WINDOW_HALFWIDTH));
            } else {
                windowHalfWidth = DEFAULT_WINDOW_TOLERANCE;
            }

            /* Window = (lower bound, upper bound), counter of represented m/z's that collide with this window, and number
             * of representative structures (which will be used in counting collisions). */
            LinkedList<CollisionWindow> allWindows = new LinkedList<CollisionWindow>() {
                {
                    for (Double mz : mzs) {
                        // CPU for memory trade-off: don't re-compute the window bounds over and over and over and over and over.
                        try {
                            add(new CollisionWindow(mz, windowHalfWidth, mzMap.ionMZToMZSources(mz).size()));
                        } catch (NoSuchElementException e) {
                            LOGGER.error("Caught no such element exception for mz %f: %s", mz, e.getMessage());
                            throw e;
                        }
                    }
                }
            };

            // Sweep line time!  The window ranges are the interesting points.  We just accumulate overlap counts as we go.
            LinkedList<CollisionWindow> workingSet = new LinkedList<>();
            List<CollisionWindow> finished = new LinkedList<>();

            while (allWindows.size() > 0) {
                CollisionWindow thisWindow = allWindows.pop();
                // Remove any windows from the working set that don't overlap with the next window.
                while (workingSet.size() > 0 && workingSet.peekFirst().getMaxMZ() < thisWindow.getMinMZ()) {
                    finished.add(workingSet.pop());
                }

                for (CollisionWindow w : workingSet) {
                    /* Add the size of the new overlapping window's structure count to each of the windows in the working set,
                     * which represents the number of possible confused structures that fall within the overlapping region.
                     * We exclude the window itself as it should already have counted the colliding structures it represents. */
                    w.getAccumulator().add(thisWindow.getStructureCount());

                    /* Reciprocally, add the structure counts of all windows with which the current window overlaps to it. */
                    thisWindow.getAccumulator().add(w.getStructureCount());
                }

                // Now that accumulation is complete, we can safely add the current window.
                workingSet.add(thisWindow);
            }

            // All the interesting events are done, so drop the remaining windows into the finished set.
            finished.addAll(workingSet);

            Map<Long, Long> collisionHistogram = histogram(
                    finished.stream().map(w -> w.getAccumulator().longValue()));
            List<Long> sortedCollisions = new ArrayList<>(collisionHistogram.keySet());
            Collections.sort(sortedCollisions);
            for (Long collision : sortedCollisions) {
                tsvWriter.append(new HashMap<String, Long>() {
                    {
                        put("collisions", collision);
                        put("count", collisionHistogram.get(collision));
                    }
                });
            }
        }
    } finally {
        if (tsvWriter != null) {
            tsvWriter.close();
        }
    }
}

From source file:AndroidUninstallStock.java

@SuppressWarnings("static-access")
public static void main(String[] args) {
    try {//from  w  w w . ja v a  2  s  . c  o m
        String lang = Locale.getDefault().getLanguage();
        GnuParser cmdparser = new GnuParser();
        Options cmdopts = new Options();
        for (String fld : Arrays.asList("shortOpts", "longOpts", "optionGroups")) {
            // hack for printOptions
            java.lang.reflect.Field fieldopt = cmdopts.getClass().getDeclaredField(fld);
            fieldopt.setAccessible(true);
            fieldopt.set(cmdopts, new LinkedHashMap<>());
        }
        cmdopts.addOption("h", "help", false, "Help");
        cmdopts.addOption("t", "test", false, "Show only report");
        cmdopts.addOption(OptionBuilder.withLongOpt("adb").withArgName("file").hasArg()
                .withDescription("Path to ADB from Android SDK").create("a"));
        cmdopts.addOption(OptionBuilder.withLongOpt("dev").withArgName("device").hasArg()
                .withDescription("Select device (\"adb devices\")").create("d"));
        cmdopts.addOption(null, "restore", false,
                "If packages have not yet removed and are disabled, " + "you can activate them again");
        cmdopts.addOption(null, "google", false, "Delete packages are in the Google section");
        cmdopts.addOption(null, "unapk", false, "Delete /system/app/ *.apk *.odex *.dex"
                + System.lineSeparator() + "(It is required to repeat command execution)");
        cmdopts.addOption(null, "unlib", false, "Delete /system/lib/[libs in apk]");
        //cmdopts.addOption(null, "unfrw", false, "Delete /system/framework/ (special list)");
        cmdopts.addOption(null, "scanlibs", false,
                "(Dangerous!) Include all the libraries of selected packages." + " Use with --unlib");

        cmdopts.addOptionGroup(new OptionGroup() {
            {
                addOption(OptionBuilder.withLongOpt("genfile").withArgName("file").hasArg().isRequired()
                        .withDescription("Create file with list packages").create());
                addOption(OptionBuilder.withLongOpt("lang").withArgName("ISO 639").hasArg().create());
            }
        });
        cmdopts.getOption("lang").setDescription(
                "See hl= in Google URL (default: " + lang + ") " + "for description from Google Play Market");
        CommandLine cmd = cmdparser.parse(cmdopts, args);

        if (args.length == 0 || cmd.hasOption("help")) {
            PrintWriter console = new PrintWriter(System.out);
            HelpFormatter cmdhelp = new HelpFormatter();
            cmdhelp.setOptionComparator(new Comparator<Option>() {
                @Override
                public int compare(Option o1, Option o2) {
                    return 0;
                }
            });
            console.println("WARNING: Before use make a backup with ClockworkMod Recovery!");
            console.println();
            console.println("AndroidUninstallStock [options] [AndroidListSoft.xml]");
            cmdhelp.printOptions(console, 80, cmdopts, 3, 2);
            console.flush();
            return;
        }

        String adb = cmd.getOptionValue("adb", "adb");
        try {
            run(adb, "start-server");
        } catch (IOException e) {
            System.out.println("Error: Not found ADB! Use -a or --adb");
            return;
        }

        final boolean NotTest = !cmd.hasOption("test");

        String deverror = getDeviceStatus(adb, cmd.getOptionValue("dev"));
        if (!deverror.isEmpty()) {
            System.out.println(deverror);
            return;
        }

        System.out.println("Getting list packages:");
        LinkedHashMap<String, String> apklist = new LinkedHashMap<String, String>();
        for (String ln : run(adb, "-s", lastdevice, "shell", "pm list packages -s -f")) {
            // "pm list packages" give list sorted by packages ;)
            String pckg = ln.substring("package:".length());
            String pckgname = ln.substring(ln.lastIndexOf('=') + 1);
            pckg = pckg.substring(0, pckg.length() - pckgname.length() - 1);
            if (!pckgname.equals("android") && !pckgname.equals("com.android.vending")/*Google Play Market*/) {
                apklist.put(pckg, pckgname);
            }
        }
        for (String ln : run(adb, "-s", lastdevice, "shell", "ls /system/app/")) {
            String path = "/system/app/" + ln.replace(".odex", ".apk").replace(".dex", ".apk");
            if (!apklist.containsKey(path)) {
                apklist.put(path, "");
            }
        }
        apklist.remove("/system/app/mcRegistry");
        for (Map.Entry<String, String> info : sortByValues(apklist).entrySet()) {
            System.out.println(info.getValue() + " = " + info.getKey());
        }

        String genfile = cmd.getOptionValue("genfile");
        if (genfile != null) {
            Path genpath = Paths.get(genfile);
            try (BufferedWriter gen = Files.newBufferedWriter(genpath, StandardCharsets.UTF_8,
                    new StandardOpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING,
                            StandardOpenOption.WRITE })) {
                if (cmd.getOptionValue("lang") != null) {
                    lang = cmd.getOptionValue("lang");
                }

                LinkedHashSet<String> listsystem = new LinkedHashSet<String>() {
                    {
                        add("com.android");
                        add("com.google.android");
                        //add("com.sec.android.app");
                        add("com.monotype.android");
                        add("eu.chainfire.supersu");
                    }
                };

                // \r\n for Windows Notepad
                gen.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n");
                gen.write("<!-- & raplace with &amp; or use <![CDATA[ ]]> -->\r\n");
                gen.write("<AndroidUninstallStock>\r\n\r\n");
                gen.write("<Normal>\r\n");
                System.out.println();
                System.out.println("\tNormal:");
                writeInfo(gen, apklist, lang, listsystem, true);
                gen.write("\t<apk name=\"Exclude Google and etc\">\r\n");
                for (String exc : listsystem) {
                    gen.write("\t\t<exclude global=\"true\" in=\"package\" pattern=\"" + exc + "\" />\r\n");
                }
                gen.write("\t</apk>\r\n");
                gen.write("</Normal>\r\n\r\n");
                gen.write("<Google>\r\n");
                System.out.println();
                System.out.println("\tGoogle:");
                writeInfo(gen, apklist, lang, listsystem, false);
                gen.write("</Google>\r\n\r\n");
                gen.write("</AndroidUninstallStock>\r\n");
                System.out.println("File " + genpath.toAbsolutePath() + " created.");
            }
            return;
        }

        String[] FileName = cmd.getArgs();
        if (!(FileName.length > 0 && Files.isReadable(Paths.get(FileName[0])))) {
            System.out.println("Error: File " + FileName[0] + " not found!");
            return;
        }

        DocumentBuilderFactory xmlfactory = getXmlDocFactory();

        // DocumentBuilder.setErrorHandler() for print errors
        Document xml = xmlfactory.newDocumentBuilder().parse(new File(FileName[0]));

        LinkedList<AusInfo> Normal = new LinkedList<AusInfo>();
        LinkedList<AusInfo> Google = new LinkedList<AusInfo>();

        NodeList ndaus = xml.getElementsByTagName("AndroidUninstallStock").item(0).getChildNodes();
        for (int ndausx = 0, ndausc = ndaus.getLength(); ndausx < ndausc; ndausx++) {
            Node ndnow = ndaus.item(ndausx);
            NodeList nd = ndnow.getChildNodes();
            String ndname = ndnow.getNodeName();
            for (int ndx = 0, ndc = nd.getLength(); ndx < ndc; ndx++) {
                if (!nd.item(ndx).getNodeName().equalsIgnoreCase("apk")) {
                    continue;
                }
                if (ndname.equalsIgnoreCase("Normal")) {
                    Normal.add(getApkInfo(nd.item(ndx)));
                } else if (ndname.equalsIgnoreCase("Google")) {
                    Google.add(getApkInfo(nd.item(ndx)));
                }
            }
        }

        // FIXME This part must be repeated until the "pm uninstall" will not issue "Failure" on all packages.
        //       Now requires a restart.
        System.out.println();
        System.out.println("Include and Exclude packages (Normal):");
        LinkedHashMap<String, String> apkNormal = getApkFromPattern(apklist, Normal, false);
        System.out.println();
        System.out.println("Global Exclude packages (Normal):");
        apkNormal = getApkFromPattern(apkNormal, Normal, true);
        System.out.println();
        System.out.println("Final list packages (Normal):");
        for (Map.Entry<String, String> info : sortByValues(apkNormal).entrySet()) {
            System.out.println(info.getValue() + " = " + info.getKey());
        }

        LinkedHashMap<String, String> apkGoogle = new LinkedHashMap<String, String>();
        if (cmd.hasOption("google")) {
            System.out.println();
            System.out.println("Include and Exclude packages (Google):");
            apkGoogle = getApkFromPattern(apklist, Google, false);
            System.out.println();
            System.out.println("Global Exclude packages (Google):");
            apkGoogle = getApkFromPattern(apkGoogle, Google, true);
            System.out.println();
            System.out.println("Final list packages (Google):");
            for (Map.Entry<String, String> info : sortByValues(apkGoogle).entrySet()) {
                System.out.println(info.getValue() + " = " + info.getKey());
            }
        }

        if (NotTest) {
            if (!hasRoot(adb)) {
                System.out.println("No Root");
                System.out.println();
                System.out.println("FINISH :)");
                return;
            }
        }

        if (cmd.hasOption("restore")) {
            System.out.println();
            System.out.println("Enable (Restore) packages (Normal):");
            damage(adb, "pm enable ", NotTest, apkNormal, 2);
            if (cmd.hasOption("google")) {
                System.out.println();
                System.out.println("Enable (Restore) packages (Google):");
                damage(adb, "pm enable ", NotTest, apkGoogle, 2);
            }
            System.out.println();
            System.out.println("FINISH :)");
            return;
        } else {
            System.out.println();
            System.out.println("Disable packages (Normal):");
            damage(adb, "pm disable ", NotTest, apkNormal, 2);
            if (cmd.hasOption("google")) {
                System.out.println();
                System.out.println("Disable packages (Google):");
                damage(adb, "pm disable ", NotTest, apkGoogle, 2);
            }
        }

        if (!cmd.hasOption("unapk") && !cmd.hasOption("unlib")) {
            System.out.println();
            System.out.println("FINISH :)");
            return;
        }

        // Reboot now not needed
        /*if (NotTest) {
        reboot(adb, "-s", lastdevice, "reboot");
        if (!hasRoot(adb)) {
            System.out.println("No Root");
            System.out.println();
            System.out.println("FINISH :)");
            return;
        }
        }*/

        if (cmd.hasOption("unlib")) {
            // "find" not found
            System.out.println();
            System.out.println("Getting list libraries:");
            LinkedList<String> liblist = new LinkedList<String>();
            liblist.addAll(run(adb, "-s", lastdevice, "shell", "ls -l /system/lib/"));
            String dircur = "/system/lib/";
            for (int x = 0; x < liblist.size(); x++) {
                if (liblist.get(x).startsWith("scan:")) {
                    dircur = liblist.get(x).substring("scan:".length());
                    liblist.remove(x);
                    x--;
                } else if (liblist.get(x).startsWith("d")) {
                    String dir = liblist.get(x).substring(liblist.get(x).lastIndexOf(':') + 4) + "/";
                    liblist.remove(x);
                    x--;
                    liblist.add("scan:/system/lib/" + dir);
                    liblist.addAll(run(adb, "-s", lastdevice, "shell", "ls -l /system/lib/" + dir));
                    continue;
                }
                liblist.set(x, dircur + liblist.get(x).substring(liblist.get(x).lastIndexOf(':') + 4));
                System.out.println(liblist.get(x));
            }

            final boolean scanlibs = cmd.hasOption("scanlibs");
            LinkedHashMap<String, String> libNormal = getLibFromPatternInclude(adb, liblist, apkNormal, Normal,
                    "Normal", scanlibs);
            libNormal = getLibFromPatternGlobalExclude(libNormal, Normal, "Normal");
            System.out.println();
            System.out.println("Final list libraries (Normal):");
            for (Map.Entry<String, String> info : sortByValues(libNormal).entrySet()) {
                System.out.println(info.getKey() + " = " + info.getValue());
            }

            LinkedHashMap<String, String> libGoogle = new LinkedHashMap<String, String>();
            if (cmd.hasOption("google")) {
                libGoogle = getLibFromPatternInclude(adb, liblist, apkGoogle, Google, "Google", scanlibs);
                libGoogle = getLibFromPatternGlobalExclude(libGoogle, Google, "Google");
                System.out.println();
                System.out.println("Final list libraries (Google):");
                for (Map.Entry<String, String> info : sortByValues(libGoogle).entrySet()) {
                    System.out.println(info.getKey() + " = " + info.getValue());
                }
            }

            LinkedHashMap<String, String> apkExclude = new LinkedHashMap<String, String>(apklist);
            for (String key : apkNormal.keySet()) {
                apkExclude.remove(key);
            }
            for (String key : apkGoogle.keySet()) {
                apkExclude.remove(key);
            }

            System.out.println();
            System.out.println("Include libraries from Exclude packages:");
            LinkedHashMap<String, String> libExclude = getLibFromPackage(adb, liblist, apkExclude);
            System.out.println();
            System.out.println("Enclude libraries from Exclude packages (Normal):");
            for (Map.Entry<String, String> info : sortByValues(libNormal).entrySet()) {
                if (libExclude.containsKey(info.getKey())) {
                    System.out.println("exclude: " + info.getKey() + " = " + libExclude.get(info.getKey()));
                    libNormal.remove(info.getKey());
                }
            }
            System.out.println();
            System.out.println("Enclude libraries from Exclude packages (Google):");
            for (Map.Entry<String, String> info : sortByValues(libGoogle).entrySet()) {
                if (libExclude.containsKey(info.getKey())) {
                    System.out.println("exclude: " + info.getKey() + " = " + libExclude.get(info.getKey()));
                    libGoogle.remove(info.getKey());
                }
            }

            System.out.println();
            System.out.println("Delete libraries (Normal):");
            damage(adb, "rm ", NotTest, libNormal, 1);
            if (cmd.hasOption("google")) {
                System.out.println();
                System.out.println("Delete libraries (Google):");
                damage(adb, "rm ", NotTest, libGoogle, 1);
            }
        }

        if (cmd.hasOption("unapk")) {
            System.out.println();
            System.out.println("Cleaning data packages (Normal):");
            damage(adb, "pm clear ", NotTest, apkNormal, 2);
            if (cmd.hasOption("google")) {
                System.out.println();
                System.out.println("Cleaning data packages (Google):");
                damage(adb, "pm clear ", NotTest, apkGoogle, 2);
            }

            System.out.println();
            System.out.println("Uninstall packages (Normal):");
            damage(adb, "pm uninstall ", NotTest, apkNormal, 2);
            if (cmd.hasOption("google")) {
                System.out.println();
                System.out.println("Uninstall packages (Google):");
                damage(adb, "pm uninstall ", NotTest, apkGoogle, 2);
            }
        }

        if (cmd.hasOption("unapk")) {
            System.out.println();
            System.out.println("Delete packages (Normal):");
            LinkedHashMap<String, String> dexNormal = new LinkedHashMap<String, String>();
            for (Map.Entry<String, String> apk : apkNormal.entrySet()) {
                dexNormal.put(apk.getKey(), apk.getValue());
                dexNormal.put(apk.getKey().replace(".apk", ".dex"), apk.getValue());
                dexNormal.put(apk.getKey().replace(".apk", ".odex"), apk.getValue());
            }
            damage(adb, "rm ", NotTest, dexNormal, 1);
            if (cmd.hasOption("google")) {
                System.out.println();
                System.out.println("Delete packages (Google):");
                LinkedHashMap<String, String> dexGoogle = new LinkedHashMap<String, String>();
                for (Map.Entry<String, String> apk : apkGoogle.entrySet()) {
                    dexGoogle.put(apk.getKey(), apk.getValue());
                    dexGoogle.put(apk.getKey().replace(".apk", ".dex"), apk.getValue());
                    dexGoogle.put(apk.getKey().replace(".apk", ".odex"), apk.getValue());
                }
                damage(adb, "rm ", NotTest, dexGoogle, 1);
            }
        }

        if (NotTest) {
            run(adb, "-s", lastdevice, "reboot");
        }
        System.out.println();
        System.out.println("FINISH :)");
    } catch (SAXException e) {
        System.out.println("Error parsing list: " + e);
    } catch (Throwable e) {
        e.printStackTrace();
    }
}

From source file:com.ikanow.aleph2.enrichment.utils.services.JsScriptEngineTestService.java

/** Entry point
 * @param args/*from  w  w w  .  j  av a 2s .c o m*/
 * @throws IOException 
 */
public static void main(String[] args) throws IOException {
    if (args.length < 3) {
        System.out
                .println("ARGS: <script-file> <input-file> <output-prefix> [{[len: <LEN>], [group: <GROUP>]}]");
    }

    // STEP 1: load script file

    final String user_script = Files.toString(new File(args[0]), Charsets.UTF_8);

    // STEP 2: get a stream for the JSON file

    final InputStream io_stream = new FileInputStream(new File(args[1]));

    // STEP 3: set up control if applicable

    Optional<JsonNode> json = Optional.of("").filter(__ -> args.length > 3).map(__ -> args[3])
            .map(Lambdas.wrap_u(j -> _mapper.readTree(j)));

    // STEP 4: set up the various objects

    final DataBucketBean bucket = Mockito.mock(DataBucketBean.class);

    final JsScriptEngineService service_under_test = new JsScriptEngineService();

    final LinkedList<ObjectNode> emitted = new LinkedList<>();
    final LinkedList<JsonNode> grouped = new LinkedList<>();
    final LinkedList<JsonNode> externally_emitted = new LinkedList<>();

    final IEnrichmentModuleContext context = Mockito.mock(IEnrichmentModuleContext.class, new Answer<Void>() {
        @SuppressWarnings("unchecked")
        public Void answer(InvocationOnMock invocation) {
            try {
                Object[] args = invocation.getArguments();
                if (invocation.getMethod().getName().equals("emitMutableObject")) {
                    final Optional<JsonNode> grouping = (Optional<JsonNode>) args[3];
                    if (grouping.isPresent()) {
                        grouped.add(grouping.get());
                    }
                    emitted.add((ObjectNode) args[1]);
                } else if (invocation.getMethod().getName().equals("externalEmit")) {
                    final DataBucketBean to = (DataBucketBean) args[0];
                    final Either<JsonNode, Map<String, Object>> out = (Either<JsonNode, Map<String, Object>>) args[1];
                    externally_emitted
                            .add(((ObjectNode) out.left().value()).put("__a2_bucket", to.full_name()));
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
            return null;
        }
    });

    final EnrichmentControlMetadataBean control = BeanTemplateUtils.build(EnrichmentControlMetadataBean.class)
            .with(EnrichmentControlMetadataBean::config,
                    new LinkedHashMap<String, Object>(
                            ImmutableMap.<String, Object>builder().put("script", user_script).build()))
            .done().get();

    service_under_test.onStageInitialize(context, bucket, control,
            Tuples._2T(ProcessingStage.batch, ProcessingStage.grouping), Optional.empty());

    final BeJsonParser json_parser = new BeJsonParser();

    // Run the file through

    final Stream<Tuple2<Long, IBatchRecord>> json_stream = StreamUtils
            .takeUntil(Stream.generate(() -> json_parser.getNextRecord(io_stream)), i -> null == i)
            .map(j -> Tuples._2T(0L, new BatchRecord(j)));

    service_under_test.onObjectBatch(json_stream, json.map(j -> j.get("len")).map(j -> (int) j.asLong(0L)),
            json.map(j -> j.get("group")));

    System.out.println("RESULTS: ");
    System.out.println("emitted: " + emitted.size());
    System.out.println("grouped: " + grouped.size());
    System.out.println("externally emitted: " + externally_emitted.size());
    Files.write(emitted.stream().map(j -> j.toString()).collect(Collectors.joining(";")),
            new File(args[2] + "emit.json"), Charsets.UTF_8);
    Files.write(grouped.stream().map(j -> j.toString()).collect(Collectors.joining(";")),
            new File(args[2] + "group.json"), Charsets.UTF_8);
    Files.write(externally_emitted.stream().map(j -> j.toString()).collect(Collectors.joining(";")),
            new File(args[2] + "external_emit.json"), Charsets.UTF_8);
}

From source file:eu.smartfp7.foursquare.AttendanceCrawler.java

/**
 * The main takes an undefined number of cities as arguments, then initializes
 * the specific crawling of all the trending venues of these cities.
 * The trending venues must have been previously identified using the `DownloadPages`
 * program.//from w  ww .j  ava2  s .  com
 * 
 * Current valid cities are: london, amsterdam, goldcoast, sanfrancisco.
 * 
 */
public static void main(String[] args) throws Exception {
    Settings settings = Settings.getInstance();
    String folder = settings.getFolder();

    // We keep info and error logs, so that we know what happened in case
    // of incoherence in the time series.
    Map<String, FileWriter> info_logs = new HashMap<String, FileWriter>();
    Map<String, FileWriter> error_logs = new HashMap<String, FileWriter>();

    // For each city we monitor, we store the venue IDs that we got from
    // a previous crawl.
    Map<String, Collection<String>> city_venues = new HashMap<String, Collection<String>>();

    // Contains the epoch time when the last API call has been made for each 
    // venue. Ensures that we get data only once each hour. 
    Map<String, Long> venue_last_call = new HashMap<String, Long>();

    // Contains the epoch time when we last checked if time series were broken
    // for each city.
    // We do these checks once every day before the batch forecasting begins.
    Map<String, Long> sanity_checks = new HashMap<String, Long>();

    // We also keep in memory the number of checkins for the last hour for
    // each venue.
    Map<String, Integer> venue_last_checkin = new HashMap<String, Integer>();

    Map<Long, Integer> APICallsCount = new HashMap<Long, Integer>();

    DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

    int total_venues = 0;
    long total_calls = 0;
    long time_spent_on_API = 0;

    for (String c : args) {
        settings.checkFileHierarchy(c);

        city_venues.put(c, loadVenues(c));
        total_venues += city_venues.get(c).size();

        info_logs.put(c,
                new FileWriter(folder + c + File.separator + "log" + File.separator + "info.log", true));
        error_logs.put(c,
                new FileWriter(folder + c + File.separator + "log" + File.separator + "error.log", true));

        Calendar cal = Calendar.getInstance();

        info_logs.get(c).write("[" + df.format(cal.getTime()) + "] Crawler initialization for " + c + ". "
                + city_venues.get(c).size() + " venues loaded.\n");
        info_logs.get(c).flush();

        // If we interrupted the program for some reason, we can get back
        // the in-memory data.
        // Important: the program must not be interrupted for more than one
        // hour, or we will lose time series data.
        for (String venue_id : city_venues.get(c)) {
            String ts_file = folder + c + File.separator + "attendances_crawl" + File.separator + venue_id
                    + ".ts";

            if (new File(ts_file).exists()) {
                BufferedReader buffer = new BufferedReader(new FileReader(ts_file));
                String mem = null, line = null;
                for (; (line = buffer.readLine()) != null; mem = line)
                    ;
                buffer.close();

                if (mem == null)
                    continue;

                String[] tmp = mem.split(",");
                venue_last_call.put(venue_id, df.parse(tmp[0]).getTime());
                venue_last_checkin.put(venue_id, Integer.parseInt(tmp[3]));

                VenueUtil.fixBrokenTimeSeriesVenue(new File(ts_file));
            } // if
        } // for

        sanity_checks.put(c, cal.getTimeInMillis());
    } // for

    if (total_venues > 5000) {
        System.out.println(
                "Too much venues for a single API account (max 5000).\nPlease create a new Foursquare API account and use these credentials.\nExiting now.");
        return;
    }

    while (true) {

        for (String c : args) {
            // We create a FIFO queue and pop venue IDs one at a time.
            LinkedList<String> city_venues_buffer = new LinkedList<String>(city_venues.get(c));
            String venue_id = null;

            // Artificial wait to avoid processors looping at 100% of their capacity
            // when there is no more venues to crawl for the current hour.
            Thread.sleep(3000);

            while ((venue_id = city_venues_buffer.pollFirst()) != null) {
                // We get the current time according to the city's time zone
                Calendar cal = Calendar.getInstance();
                cal.add(Calendar.MILLISECOND,
                        TimeZone.getTimeZone(settings.getCityTimezone(c)).getOffset(cal.getTime().getTime())
                                - Calendar.getInstance().getTimeZone().getOffset(cal.getTime().getTime()));
                //TimeZone.getTimeZone("Europe/London").getOffset(cal.getTime().getTime()));

                long current_time = DateUtils.truncate(cal.getTime(), Calendar.HOUR).getTime();

                // We query Foursquare only once per hour per venue.
                if (venue_last_call.get(venue_id) != null
                        && current_time < venue_last_call.get(venue_id) + 3600000)
                    continue;

                intelligentWait(total_venues, cal.getTime().getTime(),
                        (total_calls == 0 ? 0 : Math.round(time_spent_on_API / total_calls)));

                Venue venue = null;

                try {
                    long beforeCall = System.currentTimeMillis();
                    venue = new Venue(getFoursquareVenueById(venue_id, c));

                    // If there is no last call, this is the beginning of the time series
                    // for this venue. We get the number of people "here now" to initialize
                    // the series.
                    if (venue_last_call.get(venue_id) == null) {
                        /** TODO: by doing this, we keep a representation of the venue dating from the beginning
                         *       of the specific crawl. we might want to change this and update this file once
                         *      in a while.
                         */
                        FileWriter info = new FileWriter(folder + c + File.separator + "foursquare_venues"
                                + File.separator + venue_id + ".info");
                        info.write(venue.getFoursquareJson());
                        info.close();

                        FileWriter out = new FileWriter(folder + c + File.separator + "attendances_crawl"
                                + File.separator + venue_id + ".ts");
                        out.write("Date,here_now,hour_checkins,total_checkins\n");
                        out.write(df.format(current_time) + "," + venue.getHereNow() + "," + venue.getHereNow()
                                + "," + venue.getCheckincount() + "\n");
                        out.close();
                    } else {
                        FileWriter out = new FileWriter(folder + c + File.separator + "attendances_crawl"
                                + File.separator + venue_id + ".ts", true);
                        int checks = venue.getCheckincount() - venue_last_checkin.get(venue_id);
                        out.write(df.format(current_time) + "," + venue.getHereNow() + ","
                                + Integer.toString(checks) + "," + venue.getCheckincount() + "\n");
                        out.close();
                    }

                    if (APICallsCount.get(current_time) == null)
                        APICallsCount.put(current_time, 1);
                    else
                        APICallsCount.put(current_time, APICallsCount.get(current_time) + 1);

                    total_calls++;

                    venue_last_call.put(venue_id, current_time);
                    venue_last_checkin.put(venue_id, venue.getCheckincount());

                    time_spent_on_API += System.currentTimeMillis() - beforeCall;
                } catch (Exception e) {
                    // If something bad happens (crawler not available, IO error, ...), we put the
                    // venue_id in the FIFO queue so that it gets reevaluated later.
                    //e.printStackTrace();
                    error_logs.get(c)
                            .write("[" + df.format(cal.getTime().getTime()) + "] Error with venue " + venue_id
                                    + " (" + e.getMessage() + "). " + APICallsCount.get(current_time)
                                    + " API calls so far this hour, " + city_venues_buffer.size()
                                    + " venues remaining in the buffer.\n");
                    error_logs.get(c).flush();

                    System.out.println("[" + df.format(cal.getTime().getTime()) + "] " + c + " -- "
                            + APICallsCount.get(current_time) + " API calls // " + city_venues_buffer.size()
                            + " venues remaining " + " (" + e.getMessage() + ")");

                    if (e instanceof FoursquareAPIException)
                        if (((FoursquareAPIException) e).getHttp_code().equals("400")
                                && ((FoursquareAPIException) e).getError_detail()
                                        .equals("Venue " + venue_id + " has been deleted")) {
                            city_venues.get(c).remove(venue_id);
                            removeVenue(venue_id, c);
                        } else
                            city_venues_buffer.add(venue_id);

                    continue;
                }
            } // while

            // Every day between 0am and 2am, we repair all the broken time series (if there
            // is something to repair).
            Calendar cal = Calendar.getInstance();
            if (city_venues_buffer.peekFirst() == null
                    && (cal.getTimeInMillis() - sanity_checks.get(c)) >= 86400000
                    && cal.get(Calendar.HOUR_OF_DAY) < 2) {
                VenueUtil.fixBrokenTimeSeriesCity(c, folder);
                sanity_checks.put(c, cal.getTimeInMillis());
                info_logs.get(c).write("[" + df.format(cal.getTime()) + "] Sanity check OK.\n");
                info_logs.get(c).flush();
            }
        } // for
    } // while
}

From source file:Main.java

public static void add2Sequence(LinkedList<String> sequenceList, String url) {

    if (sequenceList.size() <= SEGMENTSIZE) {
        sequenceList.add(url);//from  w w w  .  j a v  a  2s . c om
        System.out.println("sequence no: " + sequenceList.size());
    } else {

    }
}

From source file:Main.java

public static Double[] toArray(LinkedList<Double> list) {
    Double[] result = new Double[list.size()];
    for (int i = 0; i < result.length; i++) {
        result[i] = list.get(i);//from  w  w w .  j a  va  2s .c o  m
    }
    return result;
}

From source file:Main.java

public static Byte[] toBytes(LinkedList<Byte> listBytes) {
    Byte[] result = new Byte[listBytes.size()];
    for (int i = 0; i < result.length; i++) {
        result[i] = listBytes.get(i);/*from   w  w  w .  j  a va  2  s.  c  o m*/
    }
    return result;
}

From source file:Main.java

public static byte[] toSimpleBytes(LinkedList<Byte> listBytes) {
    byte[] result = new byte[listBytes.size()];
    for (int i = 0; i < result.length; i++) {
        result[i] = listBytes.get(i);/*www  . j  av  a 2 s  .  c o  m*/
    }
    return result;
}

From source file:Main.java

public static LinkedList<byte[]> eatUpJavaHeap(LinkedList<byte[]> list, int eatUpSize, int atomSize) {
    try {//www .j ava  2  s.  com
        while (list.size() * atomSize < eatUpSize) {
            list.add(new byte[atomSize]);
        }
    } catch (Throwable t) {
        t.printStackTrace();
    } finally {
    }
    return list;
}

From source file:com.dsclab.loader.app.Loader.java

public static Produce getProduce() {
    TableTask firstTableTask = tableTask.getFirst();
    if (tableTask.size() != 0) {
        LinkedList<String> firstTaskSql = firstTableTask.getTaskSqlList();
        while (firstTaskSql.size() == 0) {
            tableTask.removeFirst();/*from  w w w  . j a  v  a  2 s. c  o m*/
            if (tableTask.size() == 0) {
                return null;
            }
            firstTableTask = tableTask.getFirst();
            firstTaskSql = firstTableTask.getTaskSqlList();
        }
        //firstTableTask = tableTask.getFirst();
        //firstTaskSql = firstTableTask.getTaskSqlList();
        Table firstTableInfo = firstTableTask.getTableInfo();
        System.out.println("Tablename:" + firstTableInfo.getTableName());

        System.out.println("next table:" + tableTask.getFirst().getTableInfo().getTableName());
        String taskSql = firstTaskSql.getFirst();
        firstTaskSql.removeFirst();
        System.out.println("TaskSql:" + taskSql);
        return new Produce(firstTableInfo, taskSql);
    }
    return null;
}