List of usage examples for java.util.stream Collectors groupingBy
public static <T, K> Collector<T, ?, Map<K, List<T>>> groupingBy(Function<? super T, ? extends K> classifier)
From source file:com.thinkbiganalytics.nifi.feedmgr.TemplateCreationHelper.java
private void mergeControllerServices(TemplateInstance templateInstance) { final Map<String, ControllerServiceDTO> map = new HashMap<String, ControllerServiceDTO>(); final Map<String, List<ControllerServiceDTO>> serviceNameMap = new HashMap<>(); //first use the snapshotted servies as a baseline for (ControllerServiceDTO serviceDTO : snapshotControllerServices) { map.put(serviceDTO.getId(), serviceDTO); if (!serviceNameMap.containsKey(serviceDTO.getName())) { serviceNameMap.put(serviceDTO.getName(), new ArrayList<ControllerServiceDTO>()); }/*from w w w. j a v a2s. c o m*/ serviceNameMap.get(serviceDTO.getName()).add(serviceDTO); } java.util.function.Predicate<ControllerServiceDTO> matchingServiceFilter = ( cs) -> map.containsKey(cs.getId()) || serviceNameMap.containsKey(cs.getName()); List<ControllerServiceDTO> matchingControllerServices = newlyCreatedControllerServices.stream() .filter(matchingServiceFilter).collect(Collectors.toList()); List<ControllerServiceDTO> unmatchedServices = newlyCreatedControllerServices.stream() .filter(matchingServiceFilter.negate()).collect(Collectors.toList()); //if the service has additional propertyDescriptors that identify other services we need to fetch the service by its id. if (unmatchedServices != null && !unmatchedServices.isEmpty()) { Map<String, ControllerServiceDTO> updatedServices = unmatchedServices.stream().map(serviceToAdd -> { //if the service has additional propertyDescriptors that identify other services we need to fetch the service by its id if (serviceToAdd.getDescriptors() != null && serviceToAdd.getDescriptors().values().stream() .anyMatch(propertyDescriptorDTO -> StringUtils .isNotBlank(propertyDescriptorDTO.getIdentifiesControllerService()))) { try { Optional<ControllerServiceDTO> cs = restClient.getNiFiRestClient().controllerServices() .findById(serviceToAdd.getId()); if (cs.isPresent()) { return cs.get(); } else { return serviceToAdd; } } catch (Exception e) { return serviceToAdd; } } else { return serviceToAdd; } }).collect(Collectors.toMap(service -> service.getId(), service -> service)); map.putAll(updatedServices); //update the core item newlyCreatedControllerServices = newlyCreatedControllerServices.stream().map(controllerServiceDTO -> { if (map.containsKey(controllerServiceDTO.getId())) { return updatedServices.get(controllerServiceDTO.getId()); } else { return controllerServiceDTO; } }).collect(Collectors.toSet()); } //if match existing services, then delete the new ones if (matchingControllerServices != null && !matchingControllerServices.isEmpty()) { for (ControllerServiceDTO serviceToDelete : matchingControllerServices) { try { if (templateInstance != null) { templateInstance.addDeletedServiceMapping(serviceToDelete.getId(), serviceNameMap.get(serviceToDelete.getName())); } restClient.deleteControllerService(serviceToDelete.getId()); } catch (NifiClientRuntimeException e) { log.error( "Exception while attempting to mergeControllerServices. Unable to delete Service {}. {}", serviceToDelete.getId(), e.getMessage()); } } } mergedControllerServices = map; //validate //Create a map of the Controller Service Name to list of matching services this.serviceNameMap = mergedControllerServices.values().stream() .collect(Collectors.groupingBy(cs -> cs.getName())); this.enabledServiceNameMap = mergedControllerServices.values().stream() .filter(cs -> NifiProcessUtil.SERVICE_STATE.ENABLED.name().equalsIgnoreCase(cs.getState())) .collect(Collectors.groupingBy(cs -> cs.getName())); }
From source file:com.ikanow.aleph2.data_import_manager.analytics.utils.TestAnalyticTriggerCrudUtils.java
@Test public void test_getTriggersToCheck() throws InterruptedException { assertEquals(0, _test_crud.countObjects().join().intValue()); final DataBucketBean bucket = buildBucket("/test/check/triggers", true); // Just set the test up: {// w w w. j a va 2 s.com final Stream<AnalyticTriggerStateBean> test_stream = AnalyticTriggerBeanUtils .generateTriggerStateStream(bucket, false, Optional.empty()); final List<AnalyticTriggerStateBean> test_list = test_stream.collect(Collectors.toList()); assertEquals(8L, test_list.size());//(8 not 7 because we only dedup at the DB) final Map<Tuple2<String, String>, List<AnalyticTriggerStateBean>> grouped_triggers = test_list.stream() .collect(Collectors.groupingBy(t -> Tuples._2T(t.bucket_name(), null))); AnalyticTriggerCrudUtils.storeOrUpdateTriggerStage(bucket, _test_crud, grouped_triggers).join(); assertEquals(7L, _test_crud.countObjects().join().intValue()); } // Check the triggers: { final Map<Tuple2<String, String>, List<AnalyticTriggerStateBean>> res = AnalyticTriggerCrudUtils .getTriggersToCheck(_test_crud).join(); assertEquals("Just one bucket", 1, res.keySet().size()); final List<AnalyticTriggerStateBean> triggers = res.values().stream().findFirst().get(); assertEquals("One trigger for each resource", 3, triggers.size()); assertTrue("External triggers", triggers.stream().allMatch(trigger -> null != trigger.input_resource_combined())); // Save the triggers //DEBUG //this.printTriggerDatabase(); AnalyticTriggerCrudUtils.updateTriggerStatuses(_test_crud, triggers.stream(), Date.from(Instant.now().plusSeconds(2)), Optional.empty()).join(); //DEBUG //this.printTriggerDatabase(); } // Try again { final Map<Tuple2<String, String>, List<AnalyticTriggerStateBean>> res = AnalyticTriggerCrudUtils .getTriggersToCheck(_test_crud).join(); assertEquals("None this time", 0, res.keySet().size()); } // Activate the internal jobs and the external triggers, and set the times back // (this time will get the job deps but not the triggers) { // activates external with bucket_active AnalyticTriggerCrudUtils .updateTriggersWithBucketOrJobActivation(_test_crud, bucket, Optional.empty(), Optional.empty()) .join(); // activate internal with bucket and job active AnalyticTriggerCrudUtils.updateTriggersWithBucketOrJobActivation(_test_crud, bucket, Optional.of(bucket.analytic_thread().jobs()), Optional.empty()).join(); //(just update the next trigger time) _test_crud .updateObjectsBySpec(CrudUtils.allOf(AnalyticTriggerStateBean.class), Optional.empty(), CrudUtils.update(AnalyticTriggerStateBean.class).set( AnalyticTriggerStateBean::next_check, Date.from(Instant.now().minusSeconds(2)))) .join(); final Map<Tuple2<String, String>, List<AnalyticTriggerStateBean>> res = AnalyticTriggerCrudUtils .getTriggersToCheck(_test_crud).join(); final List<AnalyticTriggerStateBean> triggers = res.values().stream().findFirst().get(); assertEquals("One trigger for each job dep", 4, triggers.size()); assertFalse("Should be external triggers", triggers.stream().allMatch(trigger -> null == trigger.job_name())); AnalyticTriggerCrudUtils.updateTriggerStatuses(_test_crud, triggers.stream(), Date.from(Instant.now().plusSeconds(2)), Optional.empty()).join(); } // Try again { final Map<Tuple2<String, String>, List<AnalyticTriggerStateBean>> res = AnalyticTriggerCrudUtils .getTriggersToCheck(_test_crud).join(); assertEquals("None this time", 0, res.keySet().size()); } // Activate the jobs "properly" { AnalyticTriggerCrudUtils.createActiveBucketOrJobRecord(_test_crud, bucket, Optional.of(bucket.analytic_thread().jobs().stream().findFirst().get()), Optional.empty()) .join(); //DEBUG //this.printTriggerDatabase(); final Map<Tuple2<String, String>, List<AnalyticTriggerStateBean>> res2 = AnalyticTriggerCrudUtils .getTriggersToCheck(_test_crud).join(); assertEquals("Got the one active bucket", 1, res2.keySet().size()); final List<AnalyticTriggerStateBean> triggers = res2.values().stream().findFirst().get(); assertEquals("One trigger for the one active job + 1 for the bucket", 2, triggers.size()); } }
From source file:alfio.manager.system.ConfigurationManager.java
private static Collector<Configuration, ?, Map<ConfigurationKeys.SettingCategory, List<Configuration>>> groupByCategory() { return Collectors.groupingBy(c -> c.getConfigurationKey().getCategory()); }
From source file:com.ikanow.aleph2.analytics.services.DeduplicationService.java
@Override public void onObjectBatch(final Stream<Tuple2<Long, IBatchRecord>> batch, final Optional<Integer> batch_size, final Optional<JsonNode> grouping_key) { if (_deduplication_is_disabled.get()) { // no deduplication, generally shouldn't be here... //.. but if we are, make do the best we can batch.forEach(t2 -> _context.get().emitImmutableObject(t2._1(), t2._2().getJson(), Optional.empty(), Optional.empty(), Optional.empty())); return;//from w ww. ja va 2 s . c o m } // Create big query final Tuple3<QueryComponent<JsonNode>, List<Tuple2<JsonNode, Tuple2<Long, IBatchRecord>>>, Either<String, List<String>>> fieldinfo_dedupquery_keyfields = getDedupQuery( batch, _dedup_fields.get(), _db_mapper.get()); // Get duplicate results final Tuple2<List<String>, Boolean> fields_include = getIncludeFields(_policy.get(), _dedup_fields.get(), _timestamp_field.get()); final CompletableFuture<Iterator<JsonNode>> dedup_res = fieldinfo_dedupquery_keyfields._2().isEmpty() ? CompletableFuture.completedFuture(Collections.<JsonNode>emptyList().iterator()) : _dedup_context.get().getObjectsBySpec(fieldinfo_dedupquery_keyfields._1(), fields_include._1(), fields_include._2()).thenApply(cursor -> cursor.iterator()); // Wait for it to finsh //(create handy results structure if so) final LinkedHashMap<JsonNode, LinkedList<Tuple3<Long, IBatchRecord, ObjectNode>>> mutable_obj_map = fieldinfo_dedupquery_keyfields ._2().stream() .collect(Collector.of( () -> new LinkedHashMap<JsonNode, LinkedList<Tuple3<Long, IBatchRecord, ObjectNode>>>(), (acc, t2) -> { // (ie only the first element is added, duplicate elements are removed) final Tuple3<Long, IBatchRecord, ObjectNode> t3 = Tuples._3T(t2._2()._1(), t2._2()._2(), _mapper.createObjectNode()); acc.compute(t2._1(), (k, v) -> { final LinkedList<Tuple3<Long, IBatchRecord, ObjectNode>> new_list = (null == v) ? new LinkedList<>() : v; new_list.add(t3); return new_list; }); }, (map1, map2) -> { map1.putAll(map2); return map1; })); //TODO (ALEPH-20): add timestamps to annotation //TODO (ALEPH-20): support different timestamp fields for the different buckets //TODO (ALEPH-20): really need to support >1 current enrichment job // ^^(Really really longer term you should be able to decide what objects you want and what you don't <- NOTE: don't remember what i meant here) final Iterator<JsonNode> cursor = dedup_res.join(); // Handle the results final Stream<JsonNode> records_to_delete = Lambdas.get(() -> { if (isCustom(_doc_schema.get().deduplication_policy()) || _doc_schema.get().delete_unhandled_duplicates()) { return Optionals.streamOf(cursor, true) .collect(Collectors.groupingBy( ret_obj -> getKeyFieldsAgain(ret_obj, fieldinfo_dedupquery_keyfields._3()))) .entrySet().stream().<JsonNode>flatMap(kv -> { final Optional<JsonNode> maybe_key = kv.getKey(); final Optional<LinkedList<Tuple3<Long, IBatchRecord, ObjectNode>>> matching_records = maybe_key .map(key -> mutable_obj_map.get(key)); // Stats: _mutable_stats.duplicate_keys++; _mutable_stats.duplicates_existing += kv.getValue().size(); _mutable_stats.duplicates_incoming += matching_records.map(l -> l.size()).orElse(0); //DEBUG //System.out.println("?? " + kv.getValue().size() + " vs " + maybe_key + " vs " + matching_records.map(x -> Integer.toString(x.size())).orElse("(no match)")); return matching_records .<Stream<JsonNode>>map(records -> handleDuplicateRecord(_doc_schema.get(), _custom_handler.optional().map( handler -> Tuples._2T(handler, this._custom_context.get())), _timestamp_field.get(), records, kv.getValue(), maybe_key.get(), mutable_obj_map)) .orElse(Stream.empty()); }); } else { Optionals.streamOf(cursor, true).forEach(ret_obj -> { final Optional<JsonNode> maybe_key = getKeyFieldsAgain(ret_obj, fieldinfo_dedupquery_keyfields._3()); final Optional<LinkedList<Tuple3<Long, IBatchRecord, ObjectNode>>> matching_records = maybe_key .map(key -> mutable_obj_map.get(key)); //DEBUG //System.out.println("?? " + ret_obj + " vs " + maybe_key + " vs " + matching_record.map(x -> x._2().getJson().toString()).orElse("(no match)")); // Stats: _mutable_stats.duplicate_keys++; _mutable_stats.duplicates_existing++; _mutable_stats.duplicates_incoming += matching_records.map(l -> l.size()).orElse(0); matching_records.ifPresent(records -> handleDuplicateRecord(_doc_schema.get(), _custom_handler.optional() .map(handler -> Tuples._2T(handler, this._custom_context.get())), _timestamp_field.get(), records, Arrays.asList(ret_obj), maybe_key.get(), mutable_obj_map)); }); return Stream.<JsonNode>empty(); } }); final List<Object> ids = records_to_delete.map(j -> jsonToObject(j)).filter(j -> null != j) .collect(Collectors.toList()); if (!ids.isEmpty()) { // fire a bulk deletion request mutable_uncompleted_deletes.add( _dedup_context.get().deleteObjectsBySpec(CrudUtils.allOf().withAny(AnnotationBean._ID, ids))); _mutable_stats.deleted += ids.size(); //(quickly see if we can reduce the number of outstanding requests) final Iterator<CompletableFuture<Long>> it = mutable_uncompleted_deletes.iterator(); while (it.hasNext()) { final CompletableFuture<Long> cf = it.next(); if (cf.isDone()) { it.remove(); } else break; // ie stop as soon as we hit one that isn't complete) } } _mutable_stats.nonduplicate_keys += mutable_obj_map.size(); if (Optional.ofNullable(_doc_schema.get().custom_finalize_all_objects()).orElse(false)) { mutable_obj_map.entrySet().stream() .forEach(kv -> handleCustomDeduplication( _custom_handler.optional() .map(handler -> Tuples._2T(handler, this._custom_context.get())), kv.getValue(), Collections.emptyList(), kv.getKey())); } else { // Just emit the last element of each grouped object set mutable_obj_map.values().stream().map(t -> t.peekLast()) .forEach(t -> _context.get().emitImmutableObject(t._1(), t._2().getJson(), Optional.of(t._3()), Optional.empty(), Optional.empty())); } }
From source file:com.ikanow.aleph2.graph.titan.utils.TitanGraphBuildingUtils.java
/** Previously our edges were double-grouped by in and out keys - now we can group them by the (in/out) pair * @param key/*from ww w. j a v a2s . c om*/ * @param vertex_winner * @param mutable_edges * @return */ protected static Map<ObjectNode, List<ObjectNode>> finalEdgeGrouping(final ObjectNode key, final Vertex vertex_winner, final List<ObjectNode> mutable_edges) { final Map<ObjectNode, List<ObjectNode>> grouped_edges = mutable_edges.stream().filter(mutable_edge -> { final JsonNode in_key = mutable_edge.get(GraphAnnotationBean.inV); final JsonNode out_key = mutable_edge.get(GraphAnnotationBean.outV); final JsonNode matching_key = in_key.equals(key) // (has to be one of the 2 by construction) ? in_key : out_key; final JsonNode off_key = !in_key.equals(key) // (doesn't match this key) ? in_key : (!out_key.equals(key) ? out_key : null); // (a vertex can have an edge be to itself) if (null == off_key) { mutable_edge.put(GraphAnnotationBean.inV, (Long) vertex_winner.id()); mutable_edge.put(GraphAnnotationBean.outV, (Long) vertex_winner.id()); mutable_edge.set(GraphAnnotationBean.inVLabel, key); // (internal, see below) mutable_edge.set(GraphAnnotationBean.outVLabel, key); // (internal, see below) } else { mutable_edge.put((matching_key == in_key) ? GraphAnnotationBean.inV : GraphAnnotationBean.outV, (Long) vertex_winner.id()); mutable_edge.set( (matching_key == in_key) ? GraphAnnotationBean.inVLabel : GraphAnnotationBean.outVLabel, key); // (internal, see below) } return ((null == off_key) || off_key.isIntegralNumber()); }).collect(Collectors.groupingBy((ObjectNode mutable_edge) -> { // These edges are "promoted" - can call the user merge on them final ObjectNode edge_key = Optional.of(_mapper.createObjectNode()).map( o -> (ObjectNode) o.set(GraphAnnotationBean.label, mutable_edge.get(GraphAnnotationBean.label))) .map(o -> Optional.ofNullable(mutable_edge.remove(GraphAnnotationBean.inVLabel)) .map(k -> (ObjectNode) o.set(GraphAnnotationBean.inV, k)).orElse(o)) .map(o -> Optional.ofNullable(mutable_edge.remove(GraphAnnotationBean.outVLabel)) .map(k -> (ObjectNode) o.set(GraphAnnotationBean.outV, k)).orElse(o)) .get(); return edge_key; })); return grouped_edges; }
From source file:alfio.controller.api.admin.EventApiController.java
@RequestMapping("/events/{eventName}/additional-field") public List<TicketFieldConfigurationAndAllDescriptions> getAllAdditionalField( @PathVariable("eventName") String eventName) { final Map<Integer, List<TicketFieldDescription>> descById = ticketFieldRepository .findDescriptions(eventName).stream() .collect(Collectors.groupingBy(TicketFieldDescription::getTicketFieldConfigurationId)); return ticketFieldRepository.findAdditionalFieldsForEvent(eventName).stream() .map(field -> new TicketFieldConfigurationAndAllDescriptions(field, descById.getOrDefault(field.getId(), Collections.emptyList()))) .collect(toList());//from w w w . java 2 s. c o m }
From source file:com.thinkbiganalytics.feedmgr.rest.controller.NifiIntegrationRestController.java
@GET @Path("/controller-services/{serviceId}/references") @Produces(MediaType.APPLICATION_JSON)// w w w .j a v a2 s . c o m @ApiOperation(value = "Gets a controller service references in a map by component type. (i.e. Processor -> list, Controller Service -> list ...)", notes = "returns a map of the type and reference objects") @ApiResponses({ @ApiResponse(code = 200, message = "returns a map of the type and reference objects", response = ControllerServiceDTO.class), @ApiResponse(code = 500, message = "Unable to find the controller service", response = RestResponseStatus.class) }) public Response getControllerServiceReferencesMap(@PathParam("serviceId") String serviceId) { Map<String, List<ControllerServiceReferencingComponentDTO>> map = null; try { final ControllerServiceDTO controllerService = legacyNifiRestClient.getControllerService(null, serviceId); if (controllerService != null) { Optional<ControllerServiceReferencingComponentsEntity> optional = legacyNifiRestClient .getNiFiRestClient().controllerServices().getReferences(serviceId); if (optional.isPresent()) { ControllerServiceReferencingComponentsEntity entity = optional.get(); map = getReferencingComponents(entity.getControllerServiceReferencingComponents()).values() .stream().map(c -> c.getComponent()) .collect(Collectors.groupingBy(x -> x.getReferenceType())); } else { map = Collections.emptyMap(); } } return Response.ok(map).build(); } catch (Exception e) { RestResponseStatus error = new RestResponseStatus.ResponseStatusBuilder() .message("Unable to find controller service references for " + serviceId).buildError(); return Response.ok(error).build(); } }
From source file:com.ggvaidya.scinames.complexquery.ComplexQueryViewController.java
public void updateTableWithNameClusters(Project project, List<NameCluster> nameClusters, List<Dataset> datasets) { Table<NameCluster, String, Set<String>> precalc = HashBasedTable.create(); if (nameClusters == null) { dataTableView.setItems(FXCollections.emptyObservableList()); return;/*w ww. j a va 2 s .co m*/ } boolean flag_nameClustersAreTaxonConcepts = false; if (nameClusters.size() > 0 && TaxonConcept.class.isAssignableFrom(nameClusters.get(0).getClass())) flag_nameClustersAreTaxonConcepts = true; dataTableView.setItems(FXCollections.observableList(nameClusters)); // Precalculate. List<String> existingColNames = new ArrayList<>(); existingColNames.add("id"); existingColNames.add("name"); existingColNames.add("names_in_dataset"); existingColNames.add("all_names_in_cluster"); // If these are taxon concepts, there's three other columns we want // to emit. if (flag_nameClustersAreTaxonConcepts) { existingColNames.add("name_cluster_id"); existingColNames.add("starts_with"); existingColNames.add("ends_with"); existingColNames.add("is_ongoing"); } else { existingColNames.add("taxon_concept_count"); existingColNames.add("taxon_concepts"); } // Set<Name> recognizedNamesInDataset = namesDataset.getRecognizedNames(project).collect(Collectors.toSet()); for (NameCluster cluster : nameClusters) { precalc.put(cluster, "id", getOneElementSet(cluster.getId().toString())); // Okay, here's what we need to do: // - If names is ALL, then we can't do better than cluster.getName(). // if(namesDataset == ALL) { precalc.put(cluster, "names_in_dataset", cluster.getNames().stream().map(n -> n.getFullName()).collect(Collectors.toSet())); precalc.put(cluster, "name", getOneElementSet(cluster.getName().getFullName())); //} else { /* // hey, here's something cool we can do: figure out which name(s) // this dataset uses from this cluster! List<String> namesInDataset = cluster.getNames().stream() .filter(n -> recognizedNamesInDataset.contains(n)) .map(n -> n.getFullName()) .collect(Collectors.toList()); String firstName = ""; if(namesInDataset.size() > 0) firstName = namesInDataset.get(0); precalc.put(cluster, "names_in_dataset", new HashSet<>(namesInDataset)); precalc.put(cluster, "name", getOneElementSet(firstName)); }*/ precalc.put(cluster, "all_names_in_cluster", cluster.getNames().stream().map(n -> n.getFullName()).collect(Collectors.toSet())); // If it's a taxon concept, precalculate a few more columns. if (flag_nameClustersAreTaxonConcepts) { TaxonConcept tc = (TaxonConcept) cluster; precalc.put(cluster, "name_cluster_id", getOneElementSet(tc.getNameCluster().getId().toString())); precalc.put(cluster, "starts_with", tc.getStartsWith().stream().map(ch -> ch.toString()).collect(Collectors.toSet())); precalc.put(cluster, "ends_with", tc.getEndsWith().stream().map(ch -> ch.toString()).collect(Collectors.toSet())); precalc.put(cluster, "is_ongoing", getOneElementSet(tc.isOngoing(project) ? "yes" : "no")); } else { // If it's a true name cluster, then perhaps people will want // to know what taxon concepts are in here? Maybe for some sort // of PhD? List<TaxonConcept> tcs = cluster.getTaxonConcepts(project); precalc.put(cluster, "taxon_concept_count", getOneElementSet(String.valueOf(tcs.size()))); precalc.put(cluster, "taxon_concepts", tcs.stream().map(tc -> tc.toString()).collect(Collectors.toSet())); } // Okay, here's where we reconcile! for (Name n : cluster.getNames()) { // TODO: there's probably an optimization here, in which we should // loop on the smaller set (either loop on 'datasets' and compare // to cluster, or loop on cluster.foundIn and compare to 'datasets'). for (Dataset ds : datasets) { Map<Name, Set<DatasetRow>> rowsByName = ds.getRowsByName(); // Are we included in this name cluster? If not, skip! if (!cluster.getFoundIn().contains(ds)) continue; // Check to see if we have any rows for this name; if not, skip. if (!rowsByName.containsKey(n)) continue; Set<DatasetRow> matched = rowsByName.get(n); LOGGER.log(Level.FINER, "Adding {0} rows under name ''{1}''", new Object[] { matched.size(), n.getFullName() }); Map<Set<DatasetColumn>, List<DatasetRow>> rowsByCols = matched.stream() .collect(Collectors.groupingBy((DatasetRow row) -> row.getColumns())); for (Set<DatasetColumn> cols : rowsByCols.keySet()) { for (DatasetColumn col : cols) { String colName = col.getName(); if (existingColNames.contains(colName)) colName = "datasets." + colName; if (!precalc.contains(cluster, colName)) precalc.put(cluster, colName, new HashSet()); for (DatasetRow row : rowsByCols.get(cols)) { if (!row.hasColumn(col)) continue; precalc.get(cluster, colName).add(row.get(col)); } LOGGER.log(Level.FINER, "Added {0} rows under name ''{1}''", new Object[] { rowsByCols.get(cols).size(), n.getFullName() }); } } } } } dataTableView.getColumns().clear(); for (String colName : existingColNames) { dataTableView.getColumns().add(createColumnFromPrecalc(colName, precalc)); } // Get distinct column names. Stream<String> colNames = precalc.cellSet().stream().map(set -> set.getColumnKey()); // Eliminate columns that are in the existingColNames. colNames = colNames.filter(colName -> !existingColNames.contains(colName)); // And add tablecolumns for the rest. List<TableColumn<NameCluster, String>> cols = colNames.distinct().sorted() .map(colName -> createColumnFromPrecalc(colName, precalc)).collect(Collectors.toList()); dataTableView.getColumns().addAll(cols); dataTableView.refresh(); // Fill in status text field. statusTextField .setText(dataTableView.getItems().size() + " rows across " + cols.size() + " reconciled columns"); }
From source file:com.homeadvisor.kafdrop.service.CuratorKafkaMonitor.java
private Map<Integer, Long> getTopicPartitionSizes(TopicVO topic, long time) { try {//from ww w .jav a2 s. c o m PartitionOffsetRequestInfo requestInfo = new PartitionOffsetRequestInfo(time, 1); return threadPool.submit(() -> topic.getPartitions().parallelStream().filter(p -> p.getLeader() != null) .collect(Collectors.groupingBy(p -> p.getLeader().getId())) // Group partitions by leader broker id .entrySet().parallelStream().map(entry -> { final Integer brokerId = entry.getKey(); final List<TopicPartitionVO> brokerPartitions = entry.getValue(); try { // Get the size of the partitions for a topic from the leader. final OffsetResponse offsetResponse = sendOffsetRequest(brokerId, topic, requestInfo, brokerPartitions); // Build a map of partitionId -> topic size from the response return brokerPartitions.stream() .collect(Collectors.toMap(TopicPartitionVO::getId, partition -> Optional .ofNullable(offsetResponse.offsets(topic.getName(), partition.getId())) .map(Arrays::stream).orElse(LongStream.empty()).findFirst() .orElse(-1L))); } catch (Exception ex) { LOG.error("Unable to get partition log size for topic {} partitions ({})", topic.getName(), brokerPartitions.stream().map(TopicPartitionVO::getId) .map(String::valueOf).collect(Collectors.joining(",")), ex); // Map each partition to -1, indicating we got an error return brokerPartitions.stream() .collect(Collectors.toMap(TopicPartitionVO::getId, tp -> -1L)); } }).map(Map::entrySet).flatMap(Collection::stream) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))).get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw Throwables.propagate(e); } catch (ExecutionException e) { throw Throwables.propagate(e.getCause()); } }
From source file:com.vsct.dt.strowgr.admin.repository.consul.ConsulRepository.java
@Override public List<Map<String, String>> getHaproxyProperties() { List<Map<String, String>> result; try {/*from ww w .jav a2 s . c om*/ HttpGet getHaproxyURI = new HttpGet( "http://" + host + ":" + port + "/v1/kv/haproxy/" + "/?raw&recurse=true"); List<ConsulItem<String>> consulItems = client.execute(getHaproxyURI, httpResponse -> consulReader .parseHttpResponse(httpResponse, consulReader::parseConsulItemsFromHttpEntity) .orElseGet(ArrayList::new)); Map<String, List<ConsulItem<String>>> consulItemsById = consulItems.stream() .filter(consulItem -> consulItem.getKey().split("/").length > 1) //We evaluate twice consulItem.getKey().split("/"), no a big deal since haproxy dont have many properties yet. .collect(Collectors.groupingBy(consulItem -> consulItem.getKey().split("/")[1])); List<Map<String, String>> propertiesById = new ArrayList<>(consulItemsById.size()); for (Map.Entry<String, List<ConsulItem<String>>> entry : consulItemsById.entrySet()) { Map<String, String> haproxyItem = consulItemsToMap(entry.getValue()); haproxyItem.put("id", entry.getKey()); propertiesById.add(haproxyItem); } result = propertiesById; } catch (IOException e) { throw new RuntimeException(e); } return result; }