Example usage for com.google.common.collect Multimap keySet

List of usage examples for com.google.common.collect Multimap keySet

  1. HOME
  2. Java
  3. com.google.common
  4. com.google.common.collect.*
  5. Multimap
  6. keySet

Introduction

In this page you can find the example usage for com.google.common.collect Multimap keySet.

Prototype

Set<K> keySet();

Source Link

Document

Returns a view collection of all distinct keys contained in this multimap.

Usage

From source file:com.twitter.common.metrics.JvmStats.java

/**
 * Add a series of system and jvm-level stats to the given registry.
 *//* w  w  w.  j  a  v a 2 s  .c o  m*/
public static void register(MetricRegistry registry) {
    final MetricRegistry stats = registry.scope("jvm");
    final MemoryMXBean mem = ManagementFactory.getMemoryMXBean();

    // memory stats
    final MetricRegistry heapRegistry = stats.scope("heap");
    registerMemoryStats(heapRegistry, new MemoryReporter() {
        @Override
        public MemoryUsage getUsage() {
            return mem.getHeapMemoryUsage();
        }
    });
    final MetricRegistry nonHeapRegistry = stats.scope("nonheap");
    registerMemoryStats(nonHeapRegistry, new MemoryReporter() {
        @Override
        public MemoryUsage getUsage() {
            return mem.getNonHeapMemoryUsage();
        }
    });

    // threads
    final ThreadMXBean threads = ManagementFactory.getThreadMXBean();
    final MetricRegistry threadRegistry = stats.scope("thread");
    threadRegistry.register(new AbstractGauge<Integer>("daemon_count") {
        @Override
        public Integer read() {
            return threads.getDaemonThreadCount();
        }
    });
    threadRegistry.register(new AbstractGauge<Integer>("count") {
        @Override
        public Integer read() {
            return threads.getThreadCount();
        }
    });
    threadRegistry.register(new AbstractGauge<Integer>("peak_count") {
        @Override
        public Integer read() {
            return threads.getPeakThreadCount();
        }
    });

    // class loading bean
    final ClassLoadingMXBean classLoadingBean = ManagementFactory.getClassLoadingMXBean();
    stats.register(new AbstractGauge<Integer>("classes_loaded") {
        @Override
        public Integer read() {
            return classLoadingBean.getLoadedClassCount();
        }
    });
    stats.register(new AbstractGauge<Long>("total_classes_loaded") {
        @Override
        public Long read() {
            return classLoadingBean.getTotalLoadedClassCount();
        }
    });
    stats.register(new AbstractGauge<Long>("classes_unloaded") {
        @Override
        public Long read() {
            return classLoadingBean.getUnloadedClassCount();
        }
    });

    // runtime
    final RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
    stats.register(new AbstractGauge<Long>("start_time") {
        @Override
        public Long read() {
            return runtime.getStartTime();
        }
    });
    stats.register(new AbstractGauge<Long>("uptime") {
        @Override
        public Long read() {
            return runtime.getUptime();
        }
    });
    //stats.register(new AbstractGauge<String>)

    // os
    final OperatingSystemMXBean os = ManagementFactory.getOperatingSystemMXBean();
    stats.register(new AbstractGauge<Integer>("num_cpus") {
        @Override
        public Integer read() {
            return os.getAvailableProcessors();
        }
    });
    if (os instanceof com.sun.management.OperatingSystemMXBean) {
        final com.sun.management.OperatingSystemMXBean sunOsMbean = (com.sun.management.OperatingSystemMXBean) os;

        // if this is indeed an operating system
        stats.register(new AbstractGauge<Long>("free_physical_memory") {
            @Override
            public Long read() {
                return sunOsMbean.getFreePhysicalMemorySize();
            }
        });
        stats.register(new AbstractGauge<Long>("free_swap") {
            @Override
            public Long read() {
                return sunOsMbean.getFreeSwapSpaceSize();
            }
        });
        stats.register(new AbstractGauge<Long>("process_cpu_time") {
            @Override
            public Long read() {
                return sunOsMbean.getProcessCpuTime();
            }
        });
    }
    if (os instanceof com.sun.management.UnixOperatingSystemMXBean) {
        // it's a unix system... I know this!
        final UnixOperatingSystemMXBean unix = (UnixOperatingSystemMXBean) os;
        stats.register(new AbstractGauge<Long>("fd_count") {
            @Override
            public Long read() {
                return unix.getOpenFileDescriptorCount();
            }
        });
        stats.register(new AbstractGauge<Long>("fd_limit") {
            @Override
            public Long read() {
                return unix.getMaxFileDescriptorCount();
            }
        });
    }

    // mem
    final List<MemoryPoolMXBean> memPool = ManagementFactory.getMemoryPoolMXBeans();
    final MetricRegistry memRegistry = stats.scope("mem");
    final MetricRegistry currentMem = memRegistry.scope("current");
    final MetricRegistry postGCRegistry = memRegistry.scope("postGC");
    for (final MemoryPoolMXBean pool : memPool) {
        String name = normalizeName(pool.getName());
        registerMemoryStats(currentMem.scope(name), new MemoryReporter() {
            @Override
            public MemoryUsage getUsage() {
                return pool.getUsage();
            }
        });
        registerMemoryStats(postGCRegistry.scope(name), new MemoryReporter() {
            @Override
            public MemoryUsage getUsage() {
                return pool.getCollectionUsage();
            }
        });
    }
    currentMem.register(new AbstractGauge<Long>("used") {
        @Override
        public Long read() {
            long sum = 0;
            for (MemoryPoolMXBean pool : memPool) {
                MemoryUsage usage = pool.getUsage();
                if (usage != null) {
                    sum += usage.getUsed();
                }
            }
            return sum;
        }
    });
    AbstractGauge<Long> totalPostGCGauge = new AbstractGauge<Long>("used") {
        @Override
        public Long read() {
            long sum = 0;
            for (MemoryPoolMXBean pool : memPool) {
                MemoryUsage usage = pool.getCollectionUsage();
                if (usage != null) {
                    sum += usage.getUsed();
                }
            }
            return sum;
        }
    };
    postGCRegistry.register(totalPostGCGauge);

    // java 1.7 specific buffer pool gauges
    Multimap<String, AbstractGauge<Long>> java17gauges = buildJava17Gauges();
    if (!java17gauges.isEmpty()) {
        MetricRegistry bufferRegistry = stats.scope("buffer");
        for (String scope : java17gauges.keySet()) {
            MetricRegistry pool = bufferRegistry.scope(scope);
            for (AbstractGauge<Long> gauge : java17gauges.get(scope)) {
                pool.register(gauge);
            }
        }
    }

    // gc
    final List<GarbageCollectorMXBean> gcPool = ManagementFactory.getGarbageCollectorMXBeans();
    MetricRegistry gcRegistry = stats.scope("gc");
    for (final GarbageCollectorMXBean gc : gcPool) {
        String name = normalizeName(gc.getName());
        MetricRegistry scoped = memRegistry.scope(name);
        scoped.register(new AbstractGauge<Long>("cycles") {
            @Override
            public Long read() {
                return gc.getCollectionCount();
            }
        });
        scoped.register(new AbstractGauge<Long>("msec") {
            @Override
            public Long read() {
                return gc.getCollectionTime();
            }
        });
    }

    gcRegistry.register(new AbstractGauge<Long>("cycles") {
        @Override
        public Long read() {
            long sum = 0;
            for (GarbageCollectorMXBean pool : gcPool) {
                long count = pool.getCollectionCount();
                if (count > 0) {
                    sum += count;
                }
            }
            return sum;
        }
    });
    gcRegistry.register(new AbstractGauge<Long>("msec") {
        @Override
        public Long read() {
            long sum = 0;
            for (GarbageCollectorMXBean pool : gcPool) {
                long msec = pool.getCollectionTime();
                if (msec > 0) {
                    sum += msec;
                }
            }
            return sum;
        }
    });
}

From source file:org.crypto.sse.IEXRH2Lev.java

public static IEXRH2Lev setup(List<byte[]> keys, Multimap<String, String> lookup,
        Multimap<String, String> lookup2, int bigBlock, int smallBlock, int dataSize)
        throws InterruptedException, ExecutionException, IOException {

    // Instantiation of the object that contains Global MM, Local MMs and
    // the dictionary
    RH2Lev[] localMultiMap = new RH2Lev[lookup.keySet().size()];
    Multimap<String, Integer> dictionaryForMM = ArrayListMultimap.create();

    Printer.debugln("Number of (w, id) pairs " + lookup.size());

    Printer.debugln("Number of keywords " + lookup.keySet().size());

    Printer.debugln("Maximum size of |DB(w)| " + TextExtractPar.maxTupleSize);

    BufferedWriter writer = new BufferedWriter(new FileWriter("logs.txt", true));

    writer.write("\n *********************Stats******* \n");

    writer.write("\n Number of (w, id) pairs " + lookup2.size());
    writer.write("\n Number of keywords " + lookup.keySet().size());

    int counter = 0;

    ///////////////////// Computing Filtering Factor and exact needed data
    ///////////////////// size/////////////////////////////

    HashMap<Integer, Integer> histogram = new HashMap<Integer, Integer>();
    Printer.debugln("Number of documents " + lookup2.keySet().size());
    for (String keyword : lookup.keySet()) {
        if (histogram.get(lookup.get(keyword).size()) != null) {
            int tmp = histogram.get(lookup.get(keyword).size());
            histogram.put(lookup.get(keyword).size(), tmp + 1);
        } else {// ww  w .  j a  va 2 s  .  co  m
            histogram.put(lookup.get(keyword).size(), 1);
        }

        if (dataSize < lookup.get(keyword).size()) {
            dataSize = lookup.get(keyword).size();
        }

    }

    // Construction of the global multi-map
    Printer.debugln("\nBeginning of Global MM creation \n");

    long startTime1 = System.nanoTime();

    IEXRH2Lev disj2 = new IEXRH2Lev(
            RH2Lev.constructEMMParGMM(keys.get(0), lookup, bigBlock, smallBlock, dataSize), localMultiMap,
            dictionaryForMM);

    long endTime1 = System.nanoTime();

    writer.write("\n Time of MM global setup time #(w, id)/#DB " + (endTime1 - startTime1) / lookup2.size());
    writer.close();

    numberPairs = numberPairs + lookup.size();

    // Construction of the local multi-map

    Printer.debugln("Start of Local Multi-Map construction");

    long startTime = System.nanoTime();

    for (String keyword : lookup.keySet()) {

        // Stats for keeping track with the evaluation

        for (int j = 0; j < 100; j++) {

            if (counter == (int) ((j + 1) * lookup.keySet().size() / 100)) {
                BufferedWriter writer2 = new BufferedWriter(new FileWriter("temp-logs.txt", true));
                writer2.write("\n Number of local multi-maps created" + j + " %");
                writer2.close();

                break;
            }
        }

        // Filter setting optional. For a setup without any filtering set
        // filterParameter to 0
        if (((double) lookup.get(keyword).size() / TextExtractPar.maxTupleSize > filterParameter)) {

            // Stats
            Printer.debugln("Keyword in LMM " + keyword);
            BufferedWriter writer3 = new BufferedWriter(new FileWriter("words-logs.txt", true));
            writer3.write("\n Keyword in LMM " + keyword);
            writer3.close();

            for (int j = 0; j < 10; j++) {

                if (counter == (int) ((j + 1) * lookup.keySet().size() / 10)) {
                    Printer.statsln("Number of total keywords processed equals " + j + "0 % \n");
                    break;
                }
            }

            // First computing V_w. Determine Doc identifiers

            Set<String> VW = new TreeSet<String>();
            for (String idDoc : lookup.get(keyword)) {
                VW.addAll(lookup2.get(idDoc));
            }

            Multimap<String, String> secondaryLookup = ArrayListMultimap.create();

            // here we are only interested in documents in the intersection
            // between "keyword" and "word"
            for (String word : VW) {
                // Filter setting optional. For a setup without any
                // filtering set filterParameter to 0
                if (((double) lookup.get(word).size() / TextExtractPar.maxTupleSize > filterParameter)) {
                    Collection<String> l1 = new ArrayList<String>(lookup.get(word));
                    Collection<String> l2 = new ArrayList<String>(lookup.get(keyword));
                    l1.retainAll(l2);
                    secondaryLookup.putAll(word, l1);
                }
            }

            // End of VW construction
            RH2Lev.counter = 0;

            // dataSize = (int) filterParameter;
            EMM2Lev.eval = 4 + keyword;
            EMM2Lev.lmm = true;

            disj2.getLocalMultiMap()[counter] = RH2Lev.constructEMMParGMM(
                    CryptoPrimitives.generateCmac(keys.get(0), keyword), secondaryLookup, bigBlock, smallBlock,
                    dataSize);
            byte[] key3 = CryptoPrimitives.generateCmac(keys.get(1), 3 + keyword);
            numberPairs = numberPairs + secondaryLookup.size();
            dictionaryForMM.put(new String(key3), counter);

        }
        counter++;

    }

    long endTime = System.nanoTime();

    Printer.statsln("Time to construct LMM " + (endTime - startTime) / 1000000000);

    disj2.setDictionaryForMM(dictionaryForMM);
    return disj2;

}

From source file:org.opentripplanner.graph_builder.impl.osm.AreaGroup.java

public static List<AreaGroup> groupAreas(Map<Area, OSMLevel> areasLevels) {
    DisjointSet<Area> groups = new DisjointSet<Area>();
    Multimap<OSMNode, Area> areasForNode = LinkedListMultimap.create();
    for (Area area : areasLevels.keySet()) {
        for (Ring ring : area.outermostRings) {
            for (Ring inner : ring.holes) {
                for (OSMNode node : inner.nodes) {
                    areasForNode.put(node, area);
                }//from   ww  w . j av  a2 s  .  c o m
            }
            for (OSMNode node : ring.nodes) {
                areasForNode.put(node, area);
            }
        }
    }

    // areas that can be joined must share nodes and levels
    for (OSMNode osmNode : areasForNode.keySet()) {
        for (Area area1 : areasForNode.get(osmNode)) {
            OSMLevel level1 = areasLevels.get(area1);
            for (Area area2 : areasForNode.get(osmNode)) {
                OSMLevel level2 = areasLevels.get(area2);
                if ((level1 == null && level2 == null) || (level1 != null && level1.equals(level2))) {
                    groups.union(area1, area2);
                }
            }
        }
    }

    List<AreaGroup> out = new ArrayList<AreaGroup>();
    for (Set<Area> areaSet : groups.sets()) {
        try {
            out.add(new AreaGroup(areaSet));
        } catch (AreaGroup.RingConstructionException e) {
            for (Area area : areaSet) {
                LOG.debug("Failed to create merged area for " + area
                        + ".  This area might not be at fault; it might be one of the other areas in this list.");
                out.add(new AreaGroup(Arrays.asList(area)));
            }
        }
    }
    return out;
}

From source file:msi.gama.application.workspace.WorkspaceModelsManager.java

public static void loadModelsLibrary() {
    while (!GamaBundleLoader.LOADED && !GamaBundleLoader.ERRORED) {
        try {//  w w  w  .  j  a  v  a  2 s  . c o m
            Thread.sleep(100);
            DEBUG.OUT("Waiting for GAML subsystem to load...");
        } catch (final InterruptedException e) {
        }
    }
    if (GamaBundleLoader.ERRORED) {
        GAMA.getGui().tell("Error in loading GAML language subsystem. Please consult the logs");
        return;
    }
    DEBUG.OUT("Synchronous link of models library...");
    final Multimap<Bundle, String> pluginsWithModels = GamaBundleLoader.getPluginsWithModels();
    for (final Bundle plugin : pluginsWithModels.keySet()) {
        for (final String entry : pluginsWithModels.get(plugin)) {
            linkModelsToWorkspace(plugin, entry, false);
        }
    }
    final Multimap<Bundle, String> pluginsWithTests = GamaBundleLoader.getPluginsWithTests();
    for (final Bundle plugin : pluginsWithTests.keySet()) {
        for (final String entry : pluginsWithTests.get(plugin)) {
            linkModelsToWorkspace(plugin, entry, true);
        }
    }
}

From source file:de.csenk.gwt.commons.bean.rebind.observe.ObservableBeanModel.java

/**
 * @param logger /*from   ww w  . ja  v  a2s . co  m*/
 * @param methods
 * @param typeOracle 
 * @return
 * @throws UnableToCompleteException 
 */
private static Map<String, ObservableBeanPropertyModel> modelProperties(TreeLogger logger,
        Set<ObservableBeanMethodModel> methods, TypeOracle typeOracle) throws UnableToCompleteException {
    final Multimap<String, ObservableBeanMethodModel> associatedMethods = LinkedListMultimap.create();
    for (ObservableBeanMethodModel methodModel : methods) {
        final JBeanMethod action = methodModel.getAction();
        if (action != JBeanMethod.SET && action != JBeanMethod.GET)
            continue;

        final String propertyName = action.inferName(methodModel.getMethod());
        associatedMethods.put(propertyName, methodModel);
    }

    final Map<String, ObservableBeanPropertyModel> properties = Maps.newHashMap();
    for (String propertyName : associatedMethods.keySet()) {
        if (properties.containsKey(propertyName))
            die(logger, "Multiple getters/setters for property %s. Check spelling and for correct camel case.",
                    propertyName);

        final ObservableBeanMethodModel[] propertyAccessors = Iterables
                .toArray(associatedMethods.get(propertyName), ObservableBeanMethodModel.class);
        final JType propertyType = determinePropertyType(propertyAccessors[0], typeOracle);

        properties.put(propertyName,
                ObservableBeanPropertyModel.create(propertyName, propertyType, propertyAccessors));
    }

    return ImmutableMap.copyOf(properties);
}

From source file:com.android.tools.idea.editors.theme.attributes.AttributesGrouper.java

@NotNull
private static List<TableLabel> generateLabelsForGroup(final List<EditedStyleItem> source,
        final List<EditedStyleItem> sink) {
    // A TreeMap is used to ensure the keys are sorted in alphabetical order
    // ArrayLists are used for values to ensure that they stay in the same order they came in
    Multimap<String, EditedStyleItem> classes = Multimaps.newListMultimap(
            new TreeMap<String, Collection<EditedStyleItem>>(), new Supplier<List<EditedStyleItem>>() {
                @Override//from  ww  w  . j  av  a 2  s  . c om
                public List<EditedStyleItem> get() {
                    return new ArrayList<EditedStyleItem>();
                }
            });
    for (EditedStyleItem item : source) {
        String group = item.getAttrGroup();
        classes.put(group, item);
    }

    final List<TableLabel> labels = new ArrayList<TableLabel>();
    int offset = 0;
    sink.clear();
    for (String group : classes.keySet()) {
        final int size = classes.get(group).size();
        sink.addAll(classes.get(group));
        if (size != 0) {
            labels.add(new TableLabel(group, offset));
        }
        offset += size;
    }
    return labels;
}

From source file:eu.interedition.collatex.util.ParallelSegmentationApparatus.java

public static void generate(VariantGraphRanking ranking, GeneratorCallback callback) {

    callback.start();//from w  w w .  ja  v a  2s .c o  m

    final Set<Witness> allWitnesses = ranking.witnesses();
    for (Iterator<Map.Entry<Integer, Collection<VariantGraph.Vertex>>> rowIt = ranking.getByRank().asMap()
            .entrySet().iterator(); rowIt.hasNext();) {
        final Map.Entry<Integer, Collection<VariantGraph.Vertex>> row = rowIt.next();
        final int rank = row.getKey();
        final Collection<VariantGraph.Vertex> vertices = row.getValue();

        if (vertices.size() == 1 && Iterables.getOnlyElement(vertices).tokens().isEmpty()) {
            // skip start and end vertex
            continue;
        }

        // spreading vertices with same rank according to their registered transpositions
        final Multimap<Integer, VariantGraph.Vertex> verticesByTranspositionRank = HashMultimap.create();
        for (VariantGraph.Vertex v : vertices) {
            int transpositionRank = 0;
            for (VariantGraph.Transposition transposition : v.transpositions()) {
                for (VariantGraph.Vertex tv : transposition) {
                    transpositionRank += (ranking.apply(tv).intValue() - rank);
                }
            }
            verticesByTranspositionRank.put(transpositionRank, v);
        }

        // render segments
        for (Iterator<Integer> transpositionRankIt = Ordering.natural()
                .immutableSortedCopy(verticesByTranspositionRank.keySet()).iterator(); transpositionRankIt
                        .hasNext();) {
            final Multimap<Witness, Token> tokensByWitness = HashMultimap.create();
            for (VariantGraph.Vertex v : verticesByTranspositionRank.get(transpositionRankIt.next())) {
                for (Token token : v.tokens()) {
                    tokensByWitness.put(token.getWitness(), token);
                }
            }

            final SortedMap<Witness, Iterable<Token>> cellContents = Maps.newTreeMap(Witness.SIGIL_COMPARATOR);
            for (Witness witness : allWitnesses) {
                cellContents.put(witness,
                        tokensByWitness.containsKey(witness)
                                ? Iterables.unmodifiableIterable(tokensByWitness.get(witness))
                                : Collections.<Token>emptySet());
            }

            callback.segment(cellContents);
        }
    }

    callback.end();
}

From source file:org.jetbrains.jet.plugin.codeInsight.OverrideMethodsHandler.java

@NotNull
private static Set<CallableMemberDescriptor> collectSuperMethods(@NotNull ClassDescriptor classDescriptor) {
    Set<CallableMemberDescriptor> inheritedFunctions = new LinkedHashSet<CallableMemberDescriptor>();
    for (JetType supertype : classDescriptor.getTypeConstructor().getSupertypes()) {
        for (DeclarationDescriptor descriptor : supertype.getMemberScope().getAllDescriptors()) {
            if (descriptor instanceof CallableMemberDescriptor) {
                inheritedFunctions.add((CallableMemberDescriptor) descriptor);
            }/*w  ww.j a v  a 2  s. com*/
        }
    }

    // Only those actually inherited
    Set<CallableMemberDescriptor> filteredMembers = OverridingUtil.filterOutOverridden(inheritedFunctions);

    // Group members with "the same" signature
    Multimap<CallableMemberDescriptor, CallableMemberDescriptor> factoredMembers = LinkedHashMultimap.create();
    for (CallableMemberDescriptor one : filteredMembers) {
        if (factoredMembers.values().contains(one))
            continue;
        for (CallableMemberDescriptor another : filteredMembers) {
            //                if (one == another) continue;
            factoredMembers.put(one, one);
            if (OverridingUtil.isOverridableBy(one, another).getResult() == OVERRIDABLE
                    || OverridingUtil.isOverridableBy(another, one).getResult() == OVERRIDABLE) {
                factoredMembers.put(one, another);
            }
        }
    }

    return factoredMembers.keySet();
}

From source file:org.prebake.service.bake.WorkingDir.java

/**
 * Compute the list of files under the working directory that match a
 * product's output globs.//w  w w.  j  a v  a  2  s . c  om
 */
static ImmutableList<Path> matching(final Path workingDir, final Set<Path> exclusions,
        ImmutableGlobSet outputMatcher) throws IOException {
    // Get the prefix map so we only walk subtrees that are important.
    // E.g. for output globs
    //     [foo/lib/bar/*.lib, foo/lib/**.o, foo/lib/**.so, foo/bin/*.a]
    // this should yield the map
    //     "foo/lib" => [foo/lib/**.o, foo/lib/**.so]
    //     "foo/lib/bar" => [foo/lib/bar/*.lib]
    //     "foo/bin" => [foo/bin/*.a]
    // Note that the keys are sorted so that foo/lib always occurs before
    // foo/lib/bar so that the walker below does not do any unnecessary stating.
    final Map<String, List<Glob>> groupedByDir;
    {
        Multimap<String, Glob> byPrefix = outputMatcher.getGlobsGroupedByPrefix();
        groupedByDir = new TreeMap<String, List<Glob>>(new Comparator<String>() {
            // Sort so that shorter paths occur first.  That way we can start
            // walking the prefixes, and pick up the extra globs just in time when
            // we start walking those paths.
            public int compare(String a, String b) {
                long delta = ((long) a.length()) - b.length();
                return delta < 0 ? -1 : delta != 0 ? 1 : a.compareTo(b);
            }
        });
        Path root = workingDir.getRoot();
        for (String prefix : byPrefix.keySet()) {
            prefix = FsUtil.denormalizePath(root, prefix);
            String pathPrefix = workingDir.resolve(prefix).toString();
            groupedByDir.put(pathPrefix, ImmutableList.copyOf(byPrefix.get(prefix)));
        }
    }
    class Walker {
        final ImmutableList.Builder<Path> out = ImmutableList.builder();
        final Set<String> walked = Sets.newHashSet();

        void walk(Path p, GlobSet globs) throws IOException {
            // TODO: handle symbolic links
            String pStr = p.toString();
            List<Glob> extras = groupedByDir.get(pStr);
            if (extras != null) {
                globs = new MutableGlobSet(globs).addAll(extras);
                walked.add(pStr);
            }
            BasicFileAttributes attrs = Attributes.readBasicFileAttributes(p);
            if (attrs.isRegularFile()) {
                Path relPath = workingDir.relativize(p);
                if (globs.matches(relPath) && !exclusions.contains(relPath)) {
                    out.add(relPath);
                }
            } else {
                for (Path child : p.newDirectoryStream()) {
                    walk(child, globs);
                }
            }
        }
    }
    Walker w = new Walker();
    for (Map.Entry<String, List<Glob>> e : groupedByDir.entrySet()) {
        String prefix = e.getKey();
        if (w.walked.contains(prefix)) {
            continue;
        } // already walked
        Path p = workingDir.resolve(prefix);
        if (!p.notExists()) {
            w.walk(p, ImmutableGlobSet.empty());
        }
    }
    return w.out.build();
}

From source file:com.ikanow.aleph2.search_service.elasticsearch.utils.ElasticsearchSparkUtils.java

/** 
 * @param input_config - the input settings
 * @return//from w  ww. j  ava2  s.  c om
 */
public static IAnalyticsAccessContext<DataFrame> getDataFrame(final Client client,
        final AnalyticThreadJobBean.AnalyticThreadJobInputBean job_input) {
    final SetOnce<Map<String, String>> _es_options = new SetOnce<>();

    return new IAnalyticsAccessContext<DataFrame>() {

        @Override
        public String describe() {
            //(return the entire thing)
            return ErrorUtils.get("service_name={0} options={1}",
                    this.getAccessService().right().value().getSimpleName(), _es_options.optional());
        }

        /* (non-Javadoc)
         * @see com.ikanow.aleph2.data_model.interfaces.data_analytics.IAnalyticsAccessContext#getAccessService()
         */
        @SuppressWarnings("unchecked")
        @Override
        public Either<DataFrame, Class<DataFrame>> getAccessService() {
            return Either.right((Class<DataFrame>) (Class<?>) DataFrame.class);
        }

        @Override
        public Optional<Map<String, Object>> getAccessConfig() {

            // OK this is horrible but we're going to return a map of lambdas SparkContext -> SchemaRDD

            //TODO (XXX): going to start off with a simple version of this:

            final String index_resource = ElasticsearchContext.READ_PREFIX
                    + ElasticsearchIndexUtils.getBaseIndexName(BeanTemplateUtils.build(DataBucketBean.class)
                            .with(DataBucketBean::full_name, job_input.resource_name_or_id()).done().get(),
                            Optional.empty())
                    + "*";

            //TODO (ALEPH-72): support multi-buckets / buckets with non-standard indexes ... also use the tmin/tmax
            // (needs MDB to pull out - because need to get the full bucket ugh)

            // Currently need to add types: 
            //TODO (ALEPH-72): from elasticsearch-hadoop 2.2.0.m2 this will no longer be necessary (currently at 2.2.0.m1)
            final Multimap<String, String> index_type_mapping = ElasticsearchIndexUtils.getTypesForIndex(client,
                    index_resource);
            final String type_resource = index_type_mapping.values().stream().collect(Collectors.toSet())
                    .stream().collect(Collectors.joining(","));
            final String final_index = ElasticsearchHadoopUtils
                    .getTimedIndexes(job_input, index_type_mapping, new Date())
                    .map(s -> Stream
                            .concat(s,
                                    TimeSliceDirUtils
                                            .getUntimedDirectories(index_type_mapping.keySet().stream()))
                            .collect(Collectors.joining(",")))
                    .orElse(index_resource);

            //TODO (ALEPH-72): handle single/multiple types

            final Map<String, String> es_options = ImmutableMap.<String, String>of(
                    "es.index.read.missing.as.empty", "yes", "es.read.metadata", "true",
                    "es.read.metadata.field", Aleph2EsInputFormat.ALEPH2_META_FIELD, "es.resource",
                    final_index + "/" + type_resource);

            _es_options.set(es_options);
            final String table_name = Optional.ofNullable(job_input.name())
                    .orElse(BucketUtils.getUniqueSignature(job_input.resource_name_or_id(), Optional.empty()));

            Function<SQLContext, DataFrame> f = sql_context -> {
                final DataFrame df = JavaEsSparkSQL.esDF(sql_context, es_options);
                df.registerTempTable(table_name);
                return df;
            };
            return Optional.of(ImmutableMap.of(table_name, (Object) f));
        }
    };
}