Example usage for java.util TreeMap entrySet

List of usage examples for java.util TreeMap entrySet

  1. HOME
  2. Java
  3. java
  4. java.util.*
  5. TreeMap
  6. entrySet

Introduction

In this page you can find the example usage for java.util TreeMap entrySet.

Prototype

EntrySet entrySet

To view the source code for java.util TreeMap entrySet.

Click Source Link

Document

Fields initialized to contain an instance of the entry set view the first time this view is requested.

Usage

From source file:org.apache.nutch.crawl.CrawlDbReader.java

public Object query(Map<String, String> args, Configuration conf, String type, String crawlId)
        throws Exception {

    Map<String, Object> results = new HashMap<>();
    String crawlDb = crawlId + "/crawldb";

    if (type.equalsIgnoreCase("stats")) {
        boolean sort = false;
        if (args.containsKey("sort")) {
            if (args.get("sort").equalsIgnoreCase("true"))
                sort = true;/*from   w  ww .java 2  s . c o m*/
        }
        TreeMap<String, Writable> stats = processStatJobHelper(crawlDb, NutchConfiguration.create(), sort);
        LongWritable totalCnt = (LongWritable) stats.get("T");
        stats.remove("T");
        results.put("totalUrls", String.valueOf(totalCnt.get()));
        Map<String, Object> statusMap = new HashMap<>();

        for (Map.Entry<String, Writable> entry : stats.entrySet()) {
            String k = entry.getKey();
            long val = 0L;
            double fval = 0.0;
            if (entry.getValue() instanceof LongWritable) {
                val = ((LongWritable) entry.getValue()).get();
            } else if (entry.getValue() instanceof FloatWritable) {
                fval = ((FloatWritable) entry.getValue()).get();
            } else if (entry.getValue() instanceof BytesWritable) {
                continue;
            }
            if (k.equals("scn")) {
                results.put("minScore", String.valueOf(fval));
            } else if (k.equals("scx")) {
                results.put("maxScore", String.valueOf(fval));
            } else if (k.equals("sct")) {
                results.put("avgScore", String.valueOf((fval / totalCnt.get())));
            } else if (k.startsWith("status")) {
                String[] st = k.split(" ");
                int code = Integer.parseInt(st[1]);
                if (st.length > 2) {
                    @SuppressWarnings("unchecked")
                    Map<String, Object> individualStatusInfo = (Map<String, Object>) statusMap
                            .get(String.valueOf(code));
                    Map<String, String> hostValues;
                    if (individualStatusInfo.containsKey("hostValues")) {
                        hostValues = (Map<String, String>) individualStatusInfo.get("hostValues");
                    } else {
                        hostValues = new HashMap<>();
                        individualStatusInfo.put("hostValues", hostValues);
                    }
                    hostValues.put(st[2], String.valueOf(val));
                } else {
                    Map<String, Object> individualStatusInfo = new HashMap<>();

                    individualStatusInfo.put("statusValue", CrawlDatum.getStatusName((byte) code));
                    individualStatusInfo.put("count", String.valueOf(val));

                    statusMap.put(String.valueOf(code), individualStatusInfo);
                }
            } else {
                results.put(k, String.valueOf(val));
            }
        }
        results.put("status", statusMap);
        return results;
    }
    if (type.equalsIgnoreCase("dump")) {
        String output = args.get("out_dir");
        String format = "normal";
        String regex = null;
        Integer retry = null;
        String status = null;
        String expr = null;
        Float sample = null;
        if (args.containsKey("format")) {
            format = args.get("format");
        }
        if (args.containsKey("regex")) {
            regex = args.get("regex");
        }
        if (args.containsKey("retry")) {
            retry = Integer.parseInt(args.get("retry"));
        }
        if (args.containsKey("status")) {
            status = args.get("status");
        }
        if (args.containsKey("expr")) {
            expr = args.get("expr");
        }
        if (args.containsKey("sample")) {
            sample = Float.parseFloat(args.get("sample"));
        }
        processDumpJob(crawlDb, output, conf, format, regex, status, retry, expr, sample);
        File dumpFile = new File(output + "/part-00000");
        return dumpFile;
    }
    if (type.equalsIgnoreCase("topN")) {
        String output = args.get("out_dir");
        long topN = Long.parseLong(args.get("nnn"));
        float min = 0.0f;
        if (args.containsKey("min")) {
            min = Float.parseFloat(args.get("min"));
        }
        processTopNJob(crawlDb, topN, min, output, conf);
        File dumpFile = new File(output + "/part-00000");
        return dumpFile;
    }

    if (type.equalsIgnoreCase("url")) {
        String url = args.get("url");
        CrawlDatum res = get(crawlDb, url, conf);
        results.put("status", res.getStatus());
        results.put("fetchTime", new Date(res.getFetchTime()));
        results.put("modifiedTime", new Date(res.getModifiedTime()));
        results.put("retriesSinceFetch", res.getRetriesSinceFetch());
        results.put("retryInterval", res.getFetchInterval());
        results.put("score", res.getScore());
        results.put("signature", StringUtil.toHexString(res.getSignature()));
        Map<String, String> metadata = new HashMap<>();
        if (res.getMetaData() != null) {
            for (Entry<Writable, Writable> e : res.getMetaData().entrySet()) {
                metadata.put(String.valueOf(e.getKey()), String.valueOf(e.getValue()));
            }
        }
        results.put("metadata", metadata);

        return results;
    }
    return results;
}

From source file:org.commoncrawl.service.listcrawler.CrawlHistoryManager.java

public static void writeCheckpoint(TreeMap<URLFP, ProxyCrawlHistoryItem> itemMap, Configuration conf,
        FileSystem remoteFileSystem, Path mapOutputPath, Path filterOutputPath) throws IOException {

    try {/*  ww  w .j a  va 2s . c  o m*/

        LOG.info("Generating Map File at Location:" + mapOutputPath + " Filter At:" + filterOutputPath);

        // open a temporary hdfs streams ...
        MapFile.Writer writer = new MapFile.Writer(conf, remoteFileSystem, mapOutputPath.toString(),
                URLFP.class, ProxyCrawlHistoryItem.class);

        // create a bloom filter
        URLFPBloomFilter filter = new URLFPBloomFilter(_checkpointThreshold * 2, 10, 11);

        try {

            for (Map.Entry<URLFP, ProxyCrawlHistoryItem> entry : itemMap.entrySet()) {
                LOG.info("Writing Key to Map. DomainHash:" + entry.getKey().getDomainHash() + " URLHash:"
                        + entry.getKey().getUrlHash());
                filter.add(entry.getKey());
                writer.append(entry.getKey(), entry.getValue());
            }
        } finally {
            writer.close();
        }
        LOG.info("Done generating Map File");

        LOG.info("Writing Bloom Filter Data");
        // ok now also flush the bloom filter
        FSDataOutputStream bloomFilterOutputStream = remoteFileSystem.create(filterOutputPath);
        try {
            filter.serialize(bloomFilterOutputStream);
        } finally {
            bloomFilterOutputStream.flush();
            bloomFilterOutputStream.close();
        }
    } catch (IOException e) {

        // delete all relevant files ...
        remoteFileSystem.delete(mapOutputPath, true);
        remoteFileSystem.delete(filterOutputPath, false);

        // throw exception back out ...
        throw e;
    }
}

From source file:org.apache.accumulo.shell.commands.ConfigCommand.java

@Override
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState)
        throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException,
        ClassNotFoundException, NamespaceNotFoundException {
    reader = shellState.getReader();//from www  .  j  av  a2  s. c om

    final String tableName = cl.getOptionValue(tableOpt.getOpt());
    if (tableName != null && !shellState.getConnector().tableOperations().exists(tableName)) {
        throw new TableNotFoundException(null, tableName, null);
    }
    final String namespace = cl.getOptionValue(namespaceOpt.getOpt());
    if (namespace != null && !shellState.getConnector().namespaceOperations().exists(namespace)) {
        throw new NamespaceNotFoundException(null, namespace, null);
    }
    if (cl.hasOption(deleteOpt.getOpt())) {
        // delete property from table
        String property = cl.getOptionValue(deleteOpt.getOpt());
        if (property.contains("=")) {
            throw new BadArgumentException("Invalid '=' operator in delete operation.", fullCommand,
                    fullCommand.indexOf('='));
        }
        if (tableName != null) {
            if (!Property.isValidTablePropertyKey(property)) {
                Shell.log.warn("Invalid per-table property : " + property
                        + ", still removing from zookeeper if it's there.");
            }
            shellState.getConnector().tableOperations().removeProperty(tableName, property);
            Shell.log.debug("Successfully deleted table configuration option.");
        } else if (namespace != null) {
            if (!Property.isValidTablePropertyKey(property)) {
                Shell.log.warn("Invalid per-table property : " + property
                        + ", still removing from zookeeper if it's there.");
            }
            shellState.getConnector().namespaceOperations().removeProperty(namespace, property);
            Shell.log.debug("Successfully deleted namespace configuration option.");
        } else {
            if (!Property.isValidZooPropertyKey(property)) {
                Shell.log.warn("Invalid per-table property : " + property
                        + ", still removing from zookeeper if it's there.");
            }
            shellState.getConnector().instanceOperations().removeProperty(property);
            Shell.log.debug("Successfully deleted system configuration option");
        }
    } else if (cl.hasOption(setOpt.getOpt())) {
        // set property on table
        String property = cl.getOptionValue(setOpt.getOpt()), value = null;
        if (!property.contains("=")) {
            throw new BadArgumentException("Missing '=' operator in set operation.", fullCommand,
                    fullCommand.indexOf(property));
        }
        final String pair[] = property.split("=", 2);
        property = pair[0];
        value = pair[1];

        if (tableName != null) {
            if (!Property.isValidTablePropertyKey(property)) {
                throw new BadArgumentException("Invalid per-table property.", fullCommand,
                        fullCommand.indexOf(property));
            }
            if (property.equals(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey())) {
                new ColumnVisibility(value); // validate that it is a valid expression
            }
            shellState.getConnector().tableOperations().setProperty(tableName, property, value);
            Shell.log.debug("Successfully set table configuration option.");
        } else if (namespace != null) {
            if (!Property.isValidTablePropertyKey(property)) {
                throw new BadArgumentException("Invalid per-table property.", fullCommand,
                        fullCommand.indexOf(property));
            }
            if (property.equals(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey())) {
                new ColumnVisibility(value); // validate that it is a valid expression
            }
            shellState.getConnector().namespaceOperations().setProperty(namespace, property, value);
            Shell.log.debug("Successfully set table configuration option.");
        } else {
            if (!Property.isValidZooPropertyKey(property)) {
                throw new BadArgumentException("Property cannot be modified in zookeeper", fullCommand,
                        fullCommand.indexOf(property));
            }
            shellState.getConnector().instanceOperations().setProperty(property, value);
            Shell.log.debug("Successfully set system configuration option");
        }
    } else {
        // display properties
        final TreeMap<String, String> systemConfig = new TreeMap<String, String>();
        systemConfig.putAll(shellState.getConnector().instanceOperations().getSystemConfiguration());

        final String outputFile = cl.getOptionValue(outputFileOpt.getOpt());
        final PrintFile printFile = outputFile == null ? null : new PrintFile(outputFile);

        final TreeMap<String, String> siteConfig = new TreeMap<String, String>();
        siteConfig.putAll(shellState.getConnector().instanceOperations().getSiteConfiguration());

        final TreeMap<String, String> defaults = new TreeMap<String, String>();
        for (Entry<String, String> defaultEntry : AccumuloConfiguration.getDefaultConfiguration()) {
            defaults.put(defaultEntry.getKey(), defaultEntry.getValue());
        }

        final TreeMap<String, String> namespaceConfig = new TreeMap<String, String>();
        if (tableName != null) {
            String n = Namespaces.getNamespaceName(shellState.getInstance(), Tables.getNamespaceId(
                    shellState.getInstance(), Tables.getTableId(shellState.getInstance(), tableName)));
            for (Entry<String, String> e : shellState.getConnector().namespaceOperations().getProperties(n)) {
                namespaceConfig.put(e.getKey(), e.getValue());
            }
        }

        Iterable<Entry<String, String>> acuconf = shellState.getConnector().instanceOperations()
                .getSystemConfiguration().entrySet();
        if (tableName != null) {
            acuconf = shellState.getConnector().tableOperations().getProperties(tableName);
        } else if (namespace != null) {
            acuconf = shellState.getConnector().namespaceOperations().getProperties(namespace);
        }
        final TreeMap<String, String> sortedConf = new TreeMap<String, String>();
        for (Entry<String, String> propEntry : acuconf) {
            sortedConf.put(propEntry.getKey(), propEntry.getValue());
        }

        for (Entry<String, String> propEntry : acuconf) {
            final String key = propEntry.getKey();
            // only show properties with similar names to that
            // specified, or all of them if none specified
            if (cl.hasOption(filterOpt.getOpt()) && !key.contains(cl.getOptionValue(filterOpt.getOpt()))) {
                continue;
            }
            if ((tableName != null || namespace != null) && !Property.isValidTablePropertyKey(key)) {
                continue;
            }
            COL2 = Math.max(COL2, propEntry.getKey().length() + 3);
        }

        final ArrayList<String> output = new ArrayList<String>();
        printConfHeader(output);

        for (Entry<String, String> propEntry : sortedConf.entrySet()) {
            final String key = propEntry.getKey();

            // only show properties with similar names to that
            // specified, or all of them if none specified
            if (cl.hasOption(filterOpt.getOpt()) && !key.contains(cl.getOptionValue(filterOpt.getOpt()))) {
                continue;
            }
            if ((tableName != null || namespace != null) && !Property.isValidTablePropertyKey(key)) {
                continue;
            }
            String siteVal = siteConfig.get(key);
            String sysVal = systemConfig.get(key);
            String curVal = propEntry.getValue();
            String dfault = defaults.get(key);
            String nspVal = namespaceConfig.get(key);
            boolean printed = false;

            if (dfault != null && key.toLowerCase().contains("password")) {
                siteVal = sysVal = dfault = curVal = curVal.replaceAll(".", "*");
            }
            if (sysVal != null) {
                if (defaults.containsKey(key) && !Property.getPropertyByKey(key).isExperimental()) {
                    printConfLine(output, "default", key, dfault);
                    printed = true;
                }
                if (!defaults.containsKey(key) || !defaults.get(key).equals(siteVal)) {
                    printConfLine(output, "site", printed ? "   @override" : key,
                            siteVal == null ? "" : siteVal);
                    printed = true;
                }
                if (!siteConfig.containsKey(key) || !siteVal.equals(sysVal)) {
                    printConfLine(output, "system", printed ? "   @override" : key, sysVal);
                    printed = true;
                }

            }
            if (nspVal != null) {
                if (!systemConfig.containsKey(key) || !sysVal.equals(nspVal)) {
                    printConfLine(output, "namespace", printed ? "   @override" : key, nspVal);
                    printed = true;
                }
            }

            // show per-table value only if it is different (overridden)
            if (tableName != null && !curVal.equals(nspVal)) {
                printConfLine(output, "table", printed ? "   @override" : key, curVal);
            } else if (namespace != null && !curVal.equals(sysVal)) {
                printConfLine(output, "namespace", printed ? "   @override" : key, curVal);
            }
        }
        printConfFooter(output);
        shellState.printLines(output.iterator(), !cl.hasOption(disablePaginationOpt.getOpt()), printFile);
        if (printFile != null) {
            printFile.close();
        }
    }
    return 0;
}

From source file:org.apache.accumulo.core.util.shell.commands.ConfigCommand.java

@Override
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState)
        throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException,
        ClassNotFoundException, NamespaceNotFoundException {
    reader = shellState.getReader();/*from   ww  w  . ja  v  a 2 s. c om*/

    final String tableName = cl.getOptionValue(tableOpt.getOpt());
    if (tableName != null && !shellState.getConnector().tableOperations().exists(tableName)) {
        throw new TableNotFoundException(null, tableName, null);
    }
    final String namespace = cl.getOptionValue(namespaceOpt.getOpt());
    if (namespace != null && !shellState.getConnector().namespaceOperations().exists(namespace)) {
        throw new NamespaceNotFoundException(null, namespace, null);
    }
    if (cl.hasOption(deleteOpt.getOpt())) {
        // delete property from table
        String property = cl.getOptionValue(deleteOpt.getOpt());
        if (property.contains("=")) {
            throw new BadArgumentException("Invalid '=' operator in delete operation.", fullCommand,
                    fullCommand.indexOf('='));
        }
        if (tableName != null) {
            if (!Property.isValidTablePropertyKey(property)) {
                Shell.log.warn("Invalid per-table property : " + property
                        + ", still removing from zookeeper if it's there.");
            }
            shellState.getConnector().tableOperations().removeProperty(tableName, property);
            Shell.log.debug("Successfully deleted table configuration option.");
        } else if (namespace != null) {
            if (!Property.isValidTablePropertyKey(property)) {
                Shell.log.warn("Invalid per-table property : " + property
                        + ", still removing from zookeeper if it's there.");
            }
            shellState.getConnector().namespaceOperations().removeProperty(namespace, property);
            Shell.log.debug("Successfully deleted namespace configuration option.");
        } else {
            if (!Property.isValidZooPropertyKey(property)) {
                Shell.log.warn("Invalid per-table property : " + property
                        + ", still removing from zookeeper if it's there.");
            }
            shellState.getConnector().instanceOperations().removeProperty(property);
            Shell.log.debug("Successfully deleted system configuration option");
        }
    } else if (cl.hasOption(setOpt.getOpt())) {
        // set property on table
        String property = cl.getOptionValue(setOpt.getOpt()), value = null;
        if (!property.contains("=")) {
            throw new BadArgumentException("Missing '=' operator in set operation.", fullCommand,
                    fullCommand.indexOf(property));
        }
        final String pair[] = property.split("=", 2);
        property = pair[0];
        value = pair[1];

        if (tableName != null) {
            if (!Property.isValidTablePropertyKey(property)) {
                throw new BadArgumentException("Invalid per-table property.", fullCommand,
                        fullCommand.indexOf(property));
            }
            if (property.equals(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey())) {
                new ColumnVisibility(value); // validate that it is a valid expression
            }
            shellState.getConnector().tableOperations().setProperty(tableName, property, value);
            Shell.log.debug("Successfully set table configuration option.");
        } else if (namespace != null) {
            if (!Property.isValidTablePropertyKey(property)) {
                throw new BadArgumentException("Invalid per-table property.", fullCommand,
                        fullCommand.indexOf(property));
            }
            if (property.equals(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey())) {
                new ColumnVisibility(value); // validate that it is a valid expression
            }
            shellState.getConnector().namespaceOperations().setProperty(namespace, property, value);
            Shell.log.debug("Successfully set table configuration option.");
        } else {
            if (!Property.isValidZooPropertyKey(property)) {
                throw new BadArgumentException("Property cannot be modified in zookeeper", fullCommand,
                        fullCommand.indexOf(property));
            }
            shellState.getConnector().instanceOperations().setProperty(property, value);
            Shell.log.debug("Successfully set system configuration option");
        }
    } else {
        // display properties
        final TreeMap<String, String> systemConfig = new TreeMap<String, String>();
        systemConfig.putAll(shellState.getConnector().instanceOperations().getSystemConfiguration());

        final String outputFile = cl.getOptionValue(outputFileOpt.getOpt());
        final PrintFile printFile = outputFile == null ? null : new PrintFile(outputFile);

        final TreeMap<String, String> siteConfig = new TreeMap<String, String>();
        siteConfig.putAll(shellState.getConnector().instanceOperations().getSiteConfiguration());

        final TreeMap<String, String> defaults = new TreeMap<String, String>();
        for (Entry<String, String> defaultEntry : AccumuloConfiguration.getDefaultConfiguration()) {
            defaults.put(defaultEntry.getKey(), defaultEntry.getValue());
        }

        final TreeMap<String, String> namespaceConfig = new TreeMap<String, String>();
        if (tableName != null) {
            String n = Namespaces.getNamespaceName(shellState.getInstance(), Tables.getNamespaceId(
                    shellState.getInstance(), Tables.getTableId(shellState.getInstance(), tableName)));
            for (Entry<String, String> e : shellState.getConnector().namespaceOperations().getProperties(n)) {
                namespaceConfig.put(e.getKey(), e.getValue());
            }
        }

        Iterable<Entry<String, String>> acuconf = shellState.getConnector().instanceOperations()
                .getSystemConfiguration().entrySet();
        if (tableName != null) {
            acuconf = shellState.getConnector().tableOperations().getProperties(tableName);
        } else if (namespace != null) {
            acuconf = shellState.getConnector().namespaceOperations().getProperties(namespace);
        }
        final TreeMap<String, String> sortedConf = new TreeMap<String, String>();
        for (Entry<String, String> propEntry : acuconf) {
            sortedConf.put(propEntry.getKey(), propEntry.getValue());
        }

        for (Entry<String, String> propEntry : acuconf) {
            final String key = propEntry.getKey();
            // only show properties with similar names to that
            // specified, or all of them if none specified
            if (cl.hasOption(filterOpt.getOpt()) && !key.contains(cl.getOptionValue(filterOpt.getOpt()))) {
                continue;
            }
            if ((tableName != null || namespace != null) && !Property.isValidTablePropertyKey(key)) {
                continue;
            }
            COL2 = Math.max(COL2, propEntry.getKey().length() + 3);
        }

        final ArrayList<String> output = new ArrayList<String>();
        printConfHeader(output);

        for (Entry<String, String> propEntry : sortedConf.entrySet()) {
            final String key = propEntry.getKey();

            // only show properties with similar names to that
            // specified, or all of them if none specified
            if (cl.hasOption(filterOpt.getOpt()) && !key.contains(cl.getOptionValue(filterOpt.getOpt()))) {
                continue;
            }
            if ((tableName != null || namespace != null) && !Property.isValidTablePropertyKey(key)) {
                continue;
            }
            String siteVal = siteConfig.get(key);
            String sysVal = systemConfig.get(key);
            String curVal = propEntry.getValue();
            String dfault = defaults.get(key);
            String nspVal = namespaceConfig.get(key);
            boolean printed = false;

            if (dfault != null && key.toLowerCase().contains("password")) {
                siteVal = sysVal = dfault = curVal = curVal.replaceAll(".", "*");
            }
            if (sysVal != null) {
                if (defaults.containsKey(key)) {
                    printConfLine(output, "default", key, dfault);
                    printed = true;
                }
                if (!defaults.containsKey(key) || !defaults.get(key).equals(siteVal)) {
                    printConfLine(output, "site", printed ? "   @override" : key,
                            siteVal == null ? "" : siteVal);
                    printed = true;
                }
                if (!siteConfig.containsKey(key) || !siteVal.equals(sysVal)) {
                    printConfLine(output, "system", printed ? "   @override" : key,
                            sysVal == null ? "" : sysVal);
                    printed = true;
                }

            }
            if (nspVal != null) {
                if (!systemConfig.containsKey(key) || !sysVal.equals(nspVal)) {
                    printConfLine(output, "namespace", printed ? "   @override" : key,
                            nspVal == null ? "" : nspVal);
                    printed = true;
                }
            }

            // show per-table value only if it is different (overridden)
            if (tableName != null && !curVal.equals(nspVal)) {
                printConfLine(output, "table", printed ? "   @override" : key, curVal);
            } else if (namespace != null && !curVal.equals(sysVal)) {
                printConfLine(output, "namespace", printed ? "   @override" : key, curVal);
            }
        }
        printConfFooter(output);
        shellState.printLines(output.iterator(), !cl.hasOption(disablePaginationOpt.getOpt()), printFile);
        if (printFile != null) {
            printFile.close();
        }
    }
    return 0;
}

From source file:org.apache.hadoop.hbase.extended.loadbalance.strategies.hotspot.HotSpotLoadBalancer.java

private boolean isHotSpot(TreeMap<HotSpotRegionLoad, HRegionInfo> regionLoadMap,
        List<HotSpotRegionLoad> exisitingRegionLoadList, double totalLoad) {
    // iterate in order on HotSpotRegionLoad
    boolean isHotSpot = false;
    double loadTillNow = 0.0;
    HotSpotRegionLoad load = null;/* ww w.j av a 2s.  c  o m*/
    List<HotSpotRegionLoad> listToMarkHotSpot = new ArrayList<HotSpotRegionLoad>();
    int counter = 0;
    double hotspotLoad = totalLoad * hotspotLoadPercentThreshold;
    LOG.debug("#################isHotSpot: hotspotLoad=" + hotspotLoad + " totalLoad= " + totalLoad
            + " hotspotLoadPercentThreshold = " + hotspotLoadPercentThreshold);
    int hotspotNumber = (int) Math.ceil(regionLoadMap.size() * hotspotLoadNumberRegionsThreshold);

    LOG.debug("#################isHotSpot: hotspotNumber=" + hotspotNumber + " regionLoadMap.size()= "
            + regionLoadMap.size() + " hotspotLoadNumberRegionsThreshold = "
            + hotspotLoadNumberRegionsThreshold);
    for (Map.Entry<HotSpotRegionLoad, HRegionInfo> regionLoadItem : regionLoadMap.entrySet()) {
        load = regionLoadItem.getKey();
        loadTillNow += load.getLoad();
        counter++;
        LOG.debug(String.format(
                "#################isHotSpot: load = %s;loadTillNow=%s; hotspotLoad=%s; counter=%s ", load,
                loadTillNow, hotspotLoad, counter));
        if (loadTillNow >= hotspotLoad) {
            LOG.debug(String.format("#################isHotSpot: counter = %s;hotspotNumber=%s; ", counter,
                    hotspotNumber));

            if (counter <= hotspotNumber) {
                // hotspot reached
                listToMarkHotSpot.add(load);
                isHotSpot = true;
                break;
            } else {
                break;
            }
        } else {
            LOG.debug(String.format("#################isHotSpot: Adding load = %s into potential hotspot list",
                    load));
            // potentially hotspot
            listToMarkHotSpot.add(load);
        }
    }
    LOG.debug(String.format(
            "#################isHotSpot: isHotSpot =%s ; ;;listToMarkHotSpot = %s;exisitingRegionLoadList=%s ",
            isHotSpot, listToMarkHotSpot, exisitingRegionLoadList));

    if (isHotSpot) {
        // need to mark the list as true.
        for (HotSpotRegionLoad item : listToMarkHotSpot) {
            int itemIndexIfExists = exisitingRegionLoadList.indexOf(item);
            LOG.debug(String.format(
                    "#################isHotSpot: Item =%s is in the exitisting list at index =%s ", item,
                    itemIndexIfExists));
            if (itemIndexIfExists >= 0) {
                exisitingRegionLoadList.get(itemIndexIfExists).setRegionHotspot(true);
            }
        }
    }

    LOG.debug("#################isHotSpot: listToMarkHotSpot List=" + listToMarkHotSpot);
    return isHotSpot;
}

From source file:org.wso2.carbon.apimgt.hostobjects.AssetStoreHostObject.java

public static NativeArray jsFunction_getAllTags(Context cx, Scriptable thisObj, Object[] args, Function funObj)
        throws ScriptException, APIManagementException {
    NativeArray tagArray = new NativeArray(0);
    try {//from   w w w.  ja v a2s .c om
        Collection collection = getRegistry(thisObj).executeQuery(
                RegistryConstants.QUERIES_COLLECTION_PATH + "/tags", Collections.<String, String>emptyMap());
        int i = 0;
        TreeMap<String, Integer> map = new TreeMap<String, Integer>();
        for (String fullTag : collection.getChildren()) {
            String tag = fullTag.split(";")[1].split(":")[1];
            map.put(tag, 1 + (map.containsKey(tag) ? map.get(tag) : 0));
        }
        for (Map.Entry<String, Integer> e : map.entrySet()) {
            NativeObject currentTag = new NativeObject();
            currentTag.put("name", currentTag, e.getKey());
            currentTag.put("count", currentTag, e.getValue());
            tagArray.put(i++, tagArray, currentTag);
        }
    } catch (Exception e) {
        log.error("Error while getting All Tags", e);
    }
    return tagArray;
}

From source file:org.wso2.carbon.appmgt.hostobjects.AssetStoreHostObject.java

public static NativeArray jsFunction_getAllTags(Context cx, Scriptable thisObj, Object[] args, Function funObj)
        throws ScriptException, AppManagementException {
    NativeArray tagArray = new NativeArray(0);
    try {//from  w  w w.  j  a  v  a 2s.c om
        Collection collection = getRegistry(thisObj).executeQuery(
                RegistryConstants.QUERIES_COLLECTION_PATH + "/tags", Collections.<String, String>emptyMap());
        int i = 0;
        TreeMap<String, Integer> map = new TreeMap<String, Integer>();
        for (String fullTag : collection.getChildren()) {
            String tag = fullTag.split(";")[1].split(":")[1];
            map.put(tag, 1 + (map.containsKey(tag) ? map.get(tag) : 0));
        }
        for (Map.Entry<String, Integer> e : map.entrySet()) {
            NativeObject currentTag = new NativeObject();
            currentTag.put("name", currentTag, e.getKey());
            currentTag.put("count", currentTag, e.getValue());
            tagArray.put(i++, tagArray, currentTag);
        }
    } catch (Exception e) {
        log.error("Error while getting All Tags", e);
    }
    return tagArray;
}

From source file:checkdb.CheckDb.java

private void makeChannelIndexTable() throws SQLException {
    Pattern ifoSubsysPat = Pattern.compile("(^.+):((.+?)[_-])?");

    for (Entry<String, TreeMap<String, ChanStat>> csl : chanstats.entrySet()) {
        ChanIndexInfo cii = new ChanIndexInfo();

        String cname = csl.getKey();
        cii.setName(cname);/*from   w  w  w .  j  av a  2s  .com*/

        TreeMap<String, ChanStat> serverList = csl.getValue();
        int n = serverList.size();
        cii.setnServers(n);

        Matcher ifoSubsys = ifoSubsysPat.matcher(cname);
        if (ifoSubsys.find()) {
            cii.setIfo(ifoSubsys.group(1));
            String subsys = ifoSubsys.group(3);
            subsys = subsys == null ? "" : subsys;
            cii.setSubsys(subsys);
        } else {
            cii.setIfo("");
            cii.setSubsys("");
        }
        float minRawRate = Float.MAX_VALUE;
        float maxRawRate = Float.MIN_VALUE;
        float minRdsRate = Float.MAX_VALUE;
        float maxRdsRate = Float.MIN_VALUE;

        boolean hasRaw = false;
        boolean hasRds = false;
        boolean hasOnline = false;
        boolean hasMtrends = false;
        boolean hasStrends = false;
        boolean hasTstpnt = false;
        boolean hasStatic = false;
        String cisAvail = " ";

        for (Entry<String, ChanStat> srv : serverList.entrySet()) {
            ChanStat cs = srv.getValue();
            minRawRate = Math.min(minRawRate, cs.getMinRawRate());
            maxRawRate = Math.max(maxRawRate, cs.getMaxRawRate());
            maxRdsRate = Math.max(maxRdsRate, cs.getMaxRdsRate());
            minRdsRate = Math.min(minRdsRate, cs.getMinRdsRate());
            hasRaw |= cs.hasRaw();
            hasRds |= cs.hasRds();
            hasMtrends |= cs.hasMtrends();
            hasStrends |= cs.hasStrend();
            hasStatic |= cs.hasStatic();
            hasOnline |= cs.hasOnline();
            String cis = cs.getCisAvail();
            if (cis.equalsIgnoreCase("d")) {
                cisAvail = cis;
            } else if (!cisAvail.equalsIgnoreCase("d") && cis.equalsIgnoreCase("a")) {
                cisAvail = cis;
            }
        }
        minRawRate = minRawRate == Float.MAX_VALUE ? 0 : minRawRate;
        cii.setMinRawRate(minRawRate);
        maxRawRate = maxRawRate == Float.MIN_VALUE ? 0 : maxRawRate;
        cii.setMaxRawRate(maxRawRate);
        minRdsRate = minRdsRate == Float.MAX_VALUE ? 0 : minRdsRate;
        cii.setMinRdsRate(minRdsRate);
        maxRdsRate = maxRdsRate == Float.MIN_VALUE ? 0 : maxRdsRate;
        cii.setMaxRdsRate(maxRdsRate);

        cii.setCisAvail(cisAvail);
        cii.setHasMtrends(hasMtrends);
        cii.setHasRaw(hasRaw);
        cii.setHasRds(hasRds);
        cii.setHasStrends(hasStrends);
        cii.setHasStatic(hasStatic);
        cii.setHasTestpoint(hasTstpnt);
        cii.setHasOnline(hasOnline);

        cidx.insertNewBulk(cii);
    }
    cidx.insertNewBulk(null); // flush any remaining
}

From source file:org.apache.nutch.crawl.CrawlDbReader.java

public void processStatJob(String crawlDb, Configuration config, boolean sort)
        throws IOException, InterruptedException, ClassNotFoundException {

    double quantiles[] = { .01, .05, .1, .2, .25, .3, .4, .5, .6, .7, .75, .8, .9, .95, .99 };
    if (config.get("db.stats.score.quantiles") != null) {
        List<Double> qs = new ArrayList<>();
        for (String s : config.getStrings("db.stats.score.quantiles")) {
            try {
                double d = Double.parseDouble(s);
                if (d >= 0.0 && d <= 1.0) {
                    qs.add(d);//from ww  w . ja  va  2  s.co m
                } else {
                    LOG.warn("Skipping quantile {} not in range in db.stats.score.quantiles: {}", s);
                }
            } catch (NumberFormatException e) {
                LOG.warn("Skipping bad floating point number {} in db.stats.score.quantiles: {}", s,
                        e.getMessage());
            }
            quantiles = new double[qs.size()];
            int i = 0;
            for (Double q : qs) {
                quantiles[i++] = q;
            }
            Arrays.sort(quantiles);
        }
    }

    if (LOG.isInfoEnabled()) {
        LOG.info("CrawlDb statistics start: " + crawlDb);
    }
    TreeMap<String, Writable> stats = processStatJobHelper(crawlDb, config, sort);

    if (LOG.isInfoEnabled()) {
        LOG.info("Statistics for CrawlDb: " + crawlDb);
        LongWritable totalCnt = new LongWritable(0);
        if (stats.containsKey("T")) {
            totalCnt = ((LongWritable) stats.get("T"));
            stats.remove("T");
        }
        LOG.info("TOTAL urls:\t" + totalCnt.get());
        for (Map.Entry<String, Writable> entry : stats.entrySet()) {
            String k = entry.getKey();
            long value = 0;
            double fvalue = 0.0;
            byte[] bytesValue = null;
            Writable val = entry.getValue();
            if (val instanceof LongWritable) {
                value = ((LongWritable) val).get();
            } else if (val instanceof FloatWritable) {
                fvalue = ((FloatWritable) val).get();
            } else if (val instanceof BytesWritable) {
                bytesValue = ((BytesWritable) val).getBytes();
            }
            if (k.equals("scn")) {
                LOG.info("min score:\t" + fvalue);
            } else if (k.equals("scx")) {
                LOG.info("max score:\t" + fvalue);
            } else if (k.equals("sct")) {
                LOG.info("avg score:\t" + (fvalue / totalCnt.get()));
            } else if (k.equals("scNaN")) {
                LOG.info("score == NaN:\t" + value);
            } else if (k.equals("ftn")) {
                LOG.info("earliest fetch time:\t" + new Date(1000 * 60 * value));
            } else if (k.equals("ftx")) {
                LOG.info("latest fetch time:\t" + new Date(1000 * 60 * value));
            } else if (k.equals("ftt")) {
                LOG.info("avg of fetch times:\t" + new Date(1000 * 60 * (value / totalCnt.get())));
            } else if (k.equals("fin")) {
                LOG.info("shortest fetch interval:\t{}", TimingUtil.secondsToDaysHMS(value));
            } else if (k.equals("fix")) {
                LOG.info("longest fetch interval:\t{}", TimingUtil.secondsToDaysHMS(value));
            } else if (k.equals("fit")) {
                LOG.info("avg fetch interval:\t{}", TimingUtil.secondsToDaysHMS(value / totalCnt.get()));
            } else if (k.startsWith("status")) {
                String[] st = k.split(" ");
                int code = Integer.parseInt(st[1]);
                if (st.length > 2)
                    LOG.info("   " + st[2] + " :\t" + val);
                else
                    LOG.info(st[0] + " " + code + " (" + CrawlDatum.getStatusName((byte) code) + "):\t" + val);
            } else if (k.equals("scd")) {
                MergingDigest tdigest = MergingDigest.fromBytes(ByteBuffer.wrap(bytesValue));
                for (double q : quantiles) {
                    LOG.info("score quantile {}:\t{}", q, tdigest.quantile(q));
                }
            } else {
                LOG.info(k + ":\t" + val);
            }
        }
    }
    if (LOG.isInfoEnabled()) {
        LOG.info("CrawlDb statistics: done");
    }

}

From source file:laboGrid.graphs.replication.ReplicationGraphHeuristicGenerator.java

public ReplicationGraph computeReplicationGraph(DAId[] das, GraphMapping cGraph, int backupDegree) {

    // Creating Peers dynamic structure
    TreeMap<String, Set<Integer>> dynPeers = new TreeMap<String, Set<Integer>>();
    Set<Integer>[] da2Sub = cGraph.getDa2Sub();
    for (int i = 0; i < das.length; ++i) {
        DAId c = das[i];//from  w  w w  .j a v a  2s .co  m
        if (da2Sub[i] != null && !da2Sub[i].isEmpty()) { // Da can be taken into account in backup graph
            //            String resourcePeer = c.getPeerId();
            String resourcePeer = "peerId";
            Set<Integer> resources = dynPeers.get(resourcePeer);
            if (resources == null) {
                System.out.println("Detected Peer: " + resourcePeer + ".");
                resources = new TreeSet<Integer>();
                dynPeers.put(resourcePeer, resources);
            }
            resources.add(i);
        }
    }

    if (dynPeers.size() == 1) {

        ReplicationGraphNaiveGenerator naiveGen = new ReplicationGraphNaiveGenerator();
        return naiveGen.computeReplicationGraph(das, cGraph, backupDegree);

    } else {

        // Convert dynamic structure into a static one
        Set<Integer>[] peers = new TreeSet[dynPeers.size()];
        Iterator<Entry<String, Set<Integer>>> it = dynPeers.entrySet().iterator();
        for (int i = 0; i < peers.length; ++i) {
            Entry<String, Set<Integer>> e = it.next();
            peers[i] = e.getValue();
        }

        return new ReplicationGraph(replicationGraph(das.length, backupDegree, peers));
    }

}