List of usage examples for java.util Set removeIf
default boolean removeIf(Predicate<? super E> filter)
From source file:com.haulmont.cuba.core.sys.FetchGroupManager.java
private void applyView(JpaQuery query, String queryString, AttributeGroup attrGroup, View view, boolean singleResultExpected) { boolean useFetchGroup = attrGroup instanceof FetchGroup; Set<FetchGroupField> fetchGroupFields = new LinkedHashSet<>(); processView(view, null, fetchGroupFields, useFetchGroup); Set<String> fetchGroupAttributes = new TreeSet<>(); Map<String, String> fetchHints = new TreeMap<>(); // sort hints by attribute path for (FetchGroupField field : fetchGroupFields) { fetchGroupAttributes.add(field.path()); }//from w w w . j a v a 2s . c o m if (attrGroup instanceof FetchGroup) ((FetchGroup) attrGroup).setShouldLoadAll(true); List<FetchGroupField> refFields = new ArrayList<>(); for (FetchGroupField field : fetchGroupFields) { if (field.metaProperty.getRange().isClass() && !metadataTools.isEmbedded(field.metaProperty)) refFields.add(field); } boolean hasBatches = false; MetaClass metaClass = metadata.getClassNN(view.getEntityClass()); if (!refFields.isEmpty()) { String alias = QueryTransformerFactory.createParser(queryString).getEntityAlias(); List<FetchGroupField> batchFields = new ArrayList<>(); List<FetchGroupField> joinFields = new ArrayList<>(); for (FetchGroupField refField : refFields) { if (refField.fetchMode == FetchMode.UNDEFINED) { if (refField.metaProperty.getRange().getCardinality().isMany()) { List<String> masterAttributes = getMasterEntityAttributes(fetchGroupFields, refField, useFetchGroup); fetchGroupAttributes.addAll(masterAttributes); } continue; } boolean selfRef = false; for (MetaProperty mp : refField.metaPropertyPath.getMetaProperties()) { if (!mp.getRange().getCardinality().isMany()) { MetaClass mpClass = mp.getRange().asClass(); if (metadataTools.isAssignableFrom(mpClass, metaClass) || metadataTools.isAssignableFrom(metaClass, mpClass)) { batchFields.add(refField); selfRef = true; break; } } } if (!selfRef) { if (refField.metaProperty.getRange().getCardinality().isMany()) { List<String> masterAttributes = getMasterEntityAttributes(fetchGroupFields, refField, useFetchGroup); fetchGroupAttributes.addAll(masterAttributes); if (refField.fetchMode == FetchMode.JOIN) { joinFields.add(refField); } else { batchFields.add(refField); } } else { if (refField.fetchMode == FetchMode.BATCH) { batchFields.add(refField); } else { joinFields.add(refField); } } } } for (FetchGroupField joinField : new ArrayList<>(joinFields)) { // adjust fetch mode according to parent attributes if (joinField.fetchMode == FetchMode.AUTO) { Optional<FetchMode> parentMode = refFields.stream() .filter(f -> joinField.metaPropertyPath.startsWith(f.metaPropertyPath) && joinField.fetchMode != FetchMode.JOIN) .sorted((f1, f2) -> f1.metaPropertyPath.getPath().length - f2.metaPropertyPath.getPath().length) .findFirst().map(f -> f.fetchMode); if (parentMode.isPresent() && parentMode.get() == FetchMode.UNDEFINED) { joinFields.remove(joinField); } else { for (FetchGroupField batchField : new ArrayList<>(batchFields)) { if (joinField.metaPropertyPath.startsWith(batchField.metaPropertyPath)) { joinFields.remove(joinField); batchFields.add(joinField); } } } } } QueryParser parser = QueryTransformerFactory.createParser(queryString); List<FetchGroupField> isNullFields = joinFields.stream() .filter(f -> f.fetchMode == FetchMode.AUTO && parser.hasIsNullCondition(f.path())) .collect(Collectors.toList()); if (!isNullFields.isEmpty()) { for (Iterator<FetchGroupField> fieldIt = joinFields.iterator(); fieldIt.hasNext();) { FetchGroupField joinField = fieldIt.next(); boolean isNullField = isNullFields.stream() .anyMatch(f -> joinField == f || f.fetchMode == FetchMode.AUTO && joinField.metaPropertyPath.startsWith(f.metaPropertyPath)); if (isNullField) { fieldIt.remove(); fetchGroupAttributes.removeIf(attr -> attr.startsWith(joinField.path() + ".")); } } } long toManyCount = refFields.stream().filter(f -> f.metaProperty.getRange().getCardinality().isMany()) .count(); // For query by ID, remove BATCH mode for to-many attributes that have no nested attributes if (singleResultExpected && toManyCount <= 1) { for (FetchGroupField batchField : new ArrayList<>(batchFields)) { if (batchField.metaProperty.getRange().getCardinality().isMany()) { boolean hasNested = refFields.stream().anyMatch( f -> f != batchField && f.metaPropertyPath.startsWith(batchField.metaPropertyPath)); if (!hasNested && batchField.fetchMode != FetchMode.BATCH) { batchFields.remove(batchField); } } } } //Find many-to-many fields with cycle loading same: {E}.b.a.b, where b of type {E}. //If {E}.b BATCH, {E}.b.a BATCH and {E}.b.a.b BATCH then same query used simultaneously //while loading {E}.b and {E}.b.a.b, so result of batch query is incorrect. //Remove this fields from BATCH processing for (FetchGroupField refField : refFields) { if (refField.fetchMode == FetchMode.AUTO && refField.metaProperty.getRange().getCardinality() == Range.Cardinality.MANY_TO_MANY) { //find property {E}.a.b for {E}.a where b of type {E} List<FetchGroupField> selfRefs = refFields.stream() .filter(f -> isTransitiveSelfReference(refField, f)).collect(Collectors.toList()); for (FetchGroupField selfRef : selfRefs) { List<FetchGroupField> secondLevelSelfRefs = refFields.stream() .filter(f -> isTransitiveSelfReference(selfRef, f)).collect(Collectors.toList()); for (FetchGroupField f : secondLevelSelfRefs) { batchFields.remove(f); batchFields.remove(selfRef); batchFields.remove(refField); } } } } for (FetchGroupField joinField : joinFields) { String attr = alias + "." + joinField.path(); fetchHints.put(attr, QueryHints.LEFT_FETCH); } for (FetchGroupField batchField : batchFields) { if (batchField.fetchMode == FetchMode.BATCH || !singleResultExpected || batchFields.size() > 1) { String attr = alias + "." + batchField.path(); fetchHints.put(attr, QueryHints.BATCH); hasBatches = true; } } } if (log.isTraceEnabled()) log.trace((useFetchGroup ? "Fetch" : "Load") + " group for " + view + ":\n" + fetchGroupAttributes.stream().collect(Collectors.joining("\n"))); for (String attribute : fetchGroupAttributes) { attrGroup.addAttribute(attribute); } if (!metadataTools.isCacheable(metaClass)) { query.setHint(useFetchGroup ? QueryHints.FETCH_GROUP : QueryHints.LOAD_GROUP, attrGroup); } if (log.isDebugEnabled()) { String fetchModes = fetchHints.entrySet().stream() .map(e -> e.getKey() + "=" + (e.getValue().equals(QueryHints.LEFT_FETCH) ? "JOIN" : "BATCH")) .collect(Collectors.joining(", ")); log.debug("Fetch modes for " + view + ": " + (fetchModes.equals("") ? "<none>" : fetchModes)); } for (Map.Entry<String, String> entry : fetchHints.entrySet()) { query.setHint(entry.getValue(), entry.getKey()); } if (hasBatches) { query.setHint(QueryHints.BATCH_TYPE, "IN"); } }
From source file:org.apache.geode.management.internal.cli.commands.CreateRegionCommand.java
@CliCommand(value = CliStrings.CREATE_REGION, help = CliStrings.CREATE_REGION__HELP) @CliMetaData(relatedTopic = CliStrings.TOPIC_GEODE_REGION) @ResourceOperation(resource = ResourcePermission.Resource.DATA, operation = ResourcePermission.Operation.MANAGE) public Result createRegion( @CliOption(key = CliStrings.CREATE_REGION__REGION, mandatory = true, help = CliStrings.CREATE_REGION__REGION__HELP) String regionPath, @CliOption(key = CliStrings.CREATE_REGION__REGIONSHORTCUT, help = CliStrings.CREATE_REGION__REGIONSHORTCUT__HELP) RegionShortcut regionShortcut, @CliOption(key = CliStrings.CREATE_REGION__USEATTRIBUTESFROM, optionContext = ConverterHint.REGION_PATH, help = CliStrings.CREATE_REGION__USEATTRIBUTESFROM__HELP) String useAttributesFrom, @CliOption(key = { CliStrings.GROUP, CliStrings.GROUPS }, optionContext = ConverterHint.MEMBERGROUP, help = CliStrings.CREATE_REGION__GROUP__HELP) String[] groups, @CliOption(key = CliStrings.CREATE_REGION__SKIPIFEXISTS, unspecifiedDefaultValue = "true", specifiedDefaultValue = "true", help = CliStrings.CREATE_REGION__SKIPIFEXISTS__HELP) boolean skipIfExists, // the following should all be in alphabetical order according to // their key string @CliOption(key = CliStrings.CREATE_REGION__ASYNCEVENTQUEUEID, help = CliStrings.CREATE_REGION__ASYNCEVENTQUEUEID__HELP) String[] asyncEventQueueIds, @CliOption(key = CliStrings.CREATE_REGION__CACHELISTENER, help = CliStrings.CREATE_REGION__CACHELISTENER__HELP) String[] cacheListener, @CliOption(key = CliStrings.CREATE_REGION__CACHELOADER, help = CliStrings.CREATE_REGION__CACHELOADER__HELP) String cacheLoader, @CliOption(key = CliStrings.CREATE_REGION__CACHEWRITER, help = CliStrings.CREATE_REGION__CACHEWRITER__HELP) String cacheWriter, @CliOption(key = CliStrings.CREATE_REGION__COLOCATEDWITH, optionContext = ConverterHint.REGION_PATH, help = CliStrings.CREATE_REGION__COLOCATEDWITH__HELP) String prColocatedWith, @CliOption(key = CliStrings.CREATE_REGION__COMPRESSOR, help = CliStrings.CREATE_REGION__COMPRESSOR__HELP) String compressor, @CliOption(key = CliStrings.CREATE_REGION__CONCURRENCYLEVEL, help = CliStrings.CREATE_REGION__CONCURRENCYLEVEL__HELP) Integer concurrencyLevel, @CliOption(key = CliStrings.CREATE_REGION__DISKSTORE, help = CliStrings.CREATE_REGION__DISKSTORE__HELP) String diskStore, @CliOption(key = CliStrings.CREATE_REGION__ENABLEASYNCCONFLATION, help = CliStrings.CREATE_REGION__ENABLEASYNCCONFLATION__HELP) Boolean enableAsyncConflation, @CliOption(key = CliStrings.CREATE_REGION__CLONINGENABLED, help = CliStrings.CREATE_REGION__CLONINGENABLED__HELP) Boolean cloningEnabled, @CliOption(key = CliStrings.CREATE_REGION__CONCURRENCYCHECKSENABLED, help = CliStrings.CREATE_REGION__CONCURRENCYCHECKSENABLED__HELP) Boolean concurrencyChecksEnabled, @CliOption(key = CliStrings.CREATE_REGION__MULTICASTENABLED, help = CliStrings.CREATE_REGION__MULTICASTENABLED__HELP) Boolean mcastEnabled, @CliOption(key = CliStrings.CREATE_REGION__STATISTICSENABLED, help = CliStrings.CREATE_REGION__STATISTICSENABLED__HELP) Boolean statisticsEnabled, @CliOption(key = CliStrings.CREATE_REGION__ENABLESUBSCRIPTIONCONFLATION, help = CliStrings.CREATE_REGION__ENABLESUBSCRIPTIONCONFLATION__HELP) Boolean enableSubscriptionConflation, @CliOption(key = CliStrings.CREATE_REGION__DISKSYNCHRONOUS, help = CliStrings.CREATE_REGION__DISKSYNCHRONOUS__HELP) Boolean diskSynchronous, @CliOption(key = CliStrings.CREATE_REGION__ENTRYEXPIRATIONIDLETIME, help = CliStrings.CREATE_REGION__ENTRYEXPIRATIONIDLETIME__HELP) Integer entryExpirationIdleTime, @CliOption(key = CliStrings.CREATE_REGION__ENTRYEXPIRATIONIDLETIMEACTION, help = CliStrings.CREATE_REGION__ENTRYEXPIRATIONIDLETIMEACTION__HELP) String entryExpirationIdleTimeAction, @CliOption(key = CliStrings.CREATE_REGION__ENTRYEXPIRATIONTIMETOLIVE, help = CliStrings.CREATE_REGION__ENTRYEXPIRATIONTIMETOLIVE__HELP) Integer entryExpirationTTL, @CliOption(key = CliStrings.CREATE_REGION__ENTRYEXPIRATIONTTLACTION, help = CliStrings.CREATE_REGION__ENTRYEXPIRATIONTTLACTION__HELP) String entryExpirationTTLAction, @CliOption(key = CliStrings.CREATE_REGION__GATEWAYSENDERID, help = CliStrings.CREATE_REGION__GATEWAYSENDERID__HELP) String[] gatewaySenderIds, @CliOption(key = CliStrings.CREATE_REGION__KEYCONSTRAINT, help = CliStrings.CREATE_REGION__KEYCONSTRAINT__HELP) String keyConstraint, @CliOption(key = CliStrings.CREATE_REGION__LOCALMAXMEMORY, help = CliStrings.CREATE_REGION__LOCALMAXMEMORY__HELP) Integer prLocalMaxMemory, @CliOption(key = CliStrings.CREATE_REGION__OFF_HEAP, specifiedDefaultValue = "true", help = CliStrings.CREATE_REGION__OFF_HEAP__HELP) Boolean offHeap, @CliOption(key = CliStrings.CREATE_REGION__PARTITION_RESOLVER, help = CliStrings.CREATE_REGION__PARTITION_RESOLVER__HELP) String partitionResolver, @CliOption(key = CliStrings.CREATE_REGION__REGIONEXPIRATIONIDLETIME, help = CliStrings.CREATE_REGION__REGIONEXPIRATIONIDLETIME__HELP) Integer regionExpirationIdleTime, @CliOption(key = CliStrings.CREATE_REGION__REGIONEXPIRATIONIDLETIMEACTION, help = CliStrings.CREATE_REGION__REGIONEXPIRATIONIDLETIMEACTION__HELP) String regionExpirationIdleTimeAction, @CliOption(key = CliStrings.CREATE_REGION__REGIONEXPIRATIONTTL, help = CliStrings.CREATE_REGION__REGIONEXPIRATIONTTL__HELP) Integer regionExpirationTTL, @CliOption(key = CliStrings.CREATE_REGION__REGIONEXPIRATIONTTLACTION, help = CliStrings.CREATE_REGION__REGIONEXPIRATIONTTLACTION__HELP) String regionExpirationTTLAction, @CliOption(key = CliStrings.CREATE_REGION__RECOVERYDELAY, help = CliStrings.CREATE_REGION__RECOVERYDELAY__HELP) Long prRecoveryDelay, @CliOption(key = CliStrings.CREATE_REGION__REDUNDANTCOPIES, help = CliStrings.CREATE_REGION__REDUNDANTCOPIES__HELP) Integer prRedundantCopies, @CliOption(key = CliStrings.CREATE_REGION__STARTUPRECOVERYDDELAY, help = CliStrings.CREATE_REGION__STARTUPRECOVERYDDELAY__HELP) Long prStartupRecoveryDelay, @CliOption(key = CliStrings.CREATE_REGION__TOTALMAXMEMORY, help = CliStrings.CREATE_REGION__TOTALMAXMEMORY__HELP) Long prTotalMaxMemory, @CliOption(key = CliStrings.CREATE_REGION__TOTALNUMBUCKETS, help = CliStrings.CREATE_REGION__TOTALNUMBUCKETS__HELP) Integer prTotalNumBuckets, @CliOption(key = CliStrings.CREATE_REGION__VALUECONSTRAINT, help = CliStrings.CREATE_REGION__VALUECONSTRAINT__HELP) String valueConstraint // NOTICE: keep the region attributes params in alphabetical order ) {// ww w .j a v a 2 s . c o m Result result; AtomicReference<XmlEntity> xmlEntity = new AtomicReference<>(); try { InternalCache cache = getCache(); if (regionShortcut != null && useAttributesFrom != null) { throw new IllegalArgumentException( CliStrings.CREATE_REGION__MSG__ONLY_ONE_OF_REGIONSHORTCUT_AND_USEATTRIBUESFROM_CAN_BE_SPECIFIED); } else if (regionShortcut == null && useAttributesFrom == null) { throw new IllegalArgumentException( CliStrings.CREATE_REGION__MSG__ONE_OF_REGIONSHORTCUT_AND_USEATTRIBUTESFROM_IS_REQUIRED); } validateRegionPathAndParent(cache, regionPath); RegionCommandsUtils.validateGroups(cache, groups); RegionFunctionArgs.ExpirationAttrs entryIdle = null; if (entryExpirationIdleTime != null) { entryIdle = new RegionFunctionArgs.ExpirationAttrs( RegionFunctionArgs.ExpirationAttrs.ExpirationFor.ENTRY_IDLE, entryExpirationIdleTime, entryExpirationIdleTimeAction); } RegionFunctionArgs.ExpirationAttrs entryTTL = null; if (entryExpirationTTL != null) { entryTTL = new RegionFunctionArgs.ExpirationAttrs( RegionFunctionArgs.ExpirationAttrs.ExpirationFor.ENTRY_TTL, entryExpirationTTL, entryExpirationTTLAction); } RegionFunctionArgs.ExpirationAttrs regionIdle = null; if (regionExpirationIdleTime != null) { regionIdle = new RegionFunctionArgs.ExpirationAttrs( RegionFunctionArgs.ExpirationAttrs.ExpirationFor.REGION_IDLE, regionExpirationIdleTime, regionExpirationIdleTimeAction); } RegionFunctionArgs.ExpirationAttrs regionTTL = null; if (regionExpirationTTL != null) { regionTTL = new RegionFunctionArgs.ExpirationAttrs( RegionFunctionArgs.ExpirationAttrs.ExpirationFor.REGION_TTL, regionExpirationTTL, regionExpirationTTLAction); } RegionFunctionArgs regionFunctionArgs; if (useAttributesFrom != null) { if (!regionExists(cache, useAttributesFrom)) { throw new IllegalArgumentException(CliStrings.format( CliStrings.CREATE_REGION__MSG__SPECIFY_VALID_REGION_PATH_FOR_0_REGIONPATH_1_NOT_FOUND, CliStrings.CREATE_REGION__USEATTRIBUTESFROM, useAttributesFrom)); } FetchRegionAttributesFunction.FetchRegionAttributesFunctionResult<Object, Object> regionAttributesResult = getRegionAttributes( cache, useAttributesFrom); RegionAttributes<?, ?> regionAttributes = regionAttributesResult.getRegionAttributes(); // give preference to user specified plugins than the ones retrieved from other region String[] cacheListenerClasses = cacheListener != null && cacheListener.length != 0 ? cacheListener : regionAttributesResult.getCacheListenerClasses(); String cacheLoaderClass = cacheLoader != null ? cacheLoader : regionAttributesResult.getCacheLoaderClass(); String cacheWriterClass = cacheWriter != null ? cacheWriter : regionAttributesResult.getCacheWriterClass(); regionFunctionArgs = new RegionFunctionArgs(regionPath, useAttributesFrom, skipIfExists, keyConstraint, valueConstraint, statisticsEnabled, entryIdle, entryTTL, regionIdle, regionTTL, diskStore, diskSynchronous, enableAsyncConflation, enableSubscriptionConflation, cacheListenerClasses, cacheLoaderClass, cacheWriterClass, asyncEventQueueIds, gatewaySenderIds, concurrencyChecksEnabled, cloningEnabled, concurrencyLevel, prColocatedWith, prLocalMaxMemory, prRecoveryDelay, prRedundantCopies, prStartupRecoveryDelay, prTotalMaxMemory, prTotalNumBuckets, offHeap, mcastEnabled, regionAttributes, partitionResolver); if (regionAttributes.getPartitionAttributes() == null && regionFunctionArgs.hasPartitionAttributes()) { throw new IllegalArgumentException(CliStrings.format( CliStrings.CREATE_REGION__MSG__OPTION_0_CAN_BE_USED_ONLY_FOR_PARTITIONEDREGION, regionFunctionArgs.getPartitionArgs().getUserSpecifiedPartitionAttributes()) + " " + CliStrings.format(CliStrings.CREATE_REGION__MSG__0_IS_NOT_A_PARITIONEDREGION, useAttributesFrom)); } } else { regionFunctionArgs = new RegionFunctionArgs(regionPath, regionShortcut, useAttributesFrom, skipIfExists, keyConstraint, valueConstraint, statisticsEnabled, entryIdle, entryTTL, regionIdle, regionTTL, diskStore, diskSynchronous, enableAsyncConflation, enableSubscriptionConflation, cacheListener, cacheLoader, cacheWriter, asyncEventQueueIds, gatewaySenderIds, concurrencyChecksEnabled, cloningEnabled, concurrencyLevel, prColocatedWith, prLocalMaxMemory, prRecoveryDelay, prRedundantCopies, prStartupRecoveryDelay, prTotalMaxMemory, prTotalNumBuckets, null, compressor, offHeap, mcastEnabled, partitionResolver); if (!regionShortcut.name().startsWith("PARTITION") && regionFunctionArgs.hasPartitionAttributes()) { throw new IllegalArgumentException(CliStrings.format( CliStrings.CREATE_REGION__MSG__OPTION_0_CAN_BE_USED_ONLY_FOR_PARTITIONEDREGION, regionFunctionArgs.getPartitionArgs().getUserSpecifiedPartitionAttributes()) + " " + CliStrings.format(CliStrings.CREATE_REGION__MSG__0_IS_NOT_A_PARITIONEDREGION, useAttributesFrom)); } } // Do we prefer to validate or authorize first? validateRegionFunctionArgs(cache, regionFunctionArgs); if (isPersistentShortcut(regionFunctionArgs.getRegionShortcut()) || isAttributePersistent(regionFunctionArgs.getRegionAttributes())) { getSecurityService().authorize(ResourcePermission.Resource.CLUSTER, ResourcePermission.Operation.WRITE, ResourcePermission.Target.DISK); } Set<DistributedMember> membersToCreateRegionOn; if (groups != null && groups.length != 0) { membersToCreateRegionOn = CliUtil.getDistributedMembersByGroup(cache, groups); // have only normal members from the group membersToCreateRegionOn .removeIf(distributedMember -> ((InternalDistributedMember) distributedMember) .getVmKind() == DistributionManager.LOCATOR_DM_TYPE); } else { membersToCreateRegionOn = CliUtil.getAllNormalMembers(cache); } if (membersToCreateRegionOn.isEmpty()) { return ResultBuilder.createUserErrorResult(CliStrings.NO_CACHING_MEMBERS_FOUND_MESSAGE); } ResultCollector<?, ?> resultCollector = CliUtil.executeFunction(RegionCreateFunction.INSTANCE, regionFunctionArgs, membersToCreateRegionOn); @SuppressWarnings("unchecked") List<CliFunctionResult> regionCreateResults = (List<CliFunctionResult>) resultCollector.getResult(); TabularResultData tabularResultData = ResultBuilder.createTabularResultData(); final String errorPrefix = "ERROR: "; for (CliFunctionResult regionCreateResult : regionCreateResults) { boolean success = regionCreateResult.isSuccessful(); tabularResultData.accumulate("Member", regionCreateResult.getMemberIdOrName()); tabularResultData.accumulate("Status", (success ? "" : errorPrefix) + regionCreateResult.getMessage()); if (success) { xmlEntity.set(regionCreateResult.getXmlEntity()); } } result = ResultBuilder.buildResult(tabularResultData); verifyDistributedRegionMbean(cache, regionPath); } catch (IllegalArgumentException | IllegalStateException e) { LogWrapper.getInstance().info(e.getMessage()); result = ResultBuilder.createUserErrorResult(e.getMessage()); } if (xmlEntity.get() != null) { persistClusterConfiguration(result, () -> getSharedConfiguration().addXmlEntity(xmlEntity.get(), groups)); } return result; }
From source file:org.apache.gobblin.publisher.BaseDataPublisher.java
/** * Merge all of the metadata output from each work-unit and publish the merged record. * @param states States from all tasks/*from ww w. ja v a 2s . c om*/ * @throws IOException If there is an error publishing the file */ @Override public void publishMetadata(Collection<? extends WorkUnitState> states) throws IOException { Set<String> partitions = new HashSet<>(); // There should be one merged metadata file per branch; first merge all of the pieces together mergeMetadataAndCollectPartitionNames(states, partitions); partitions.removeIf(Objects::isNull); // Now, pick an arbitrary WorkUnitState to get config information around metadata such as // the desired output filename. We assume that publisher config settings // are the same across all workunits so it doesn't really matter which workUnit we retrieve this information // from. WorkUnitState anyState = states.iterator().next(); for (int branchId = 0; branchId < numBranches; branchId++) { String mdOutputPath = getMetadataOutputPathFromState(anyState, branchId); String userSpecifiedPath = getUserSpecifiedOutputPathFromState(anyState, branchId); if (partitions.isEmpty() || userSpecifiedPath != null) { publishMetadata(getMergedMetadataForPartitionAndBranch(null, branchId), branchId, getMetadataOutputFileForBranch(anyState, branchId)); } else { String metadataFilename = getMetadataFileNameForBranch(anyState, branchId); if (mdOutputPath == null || metadataFilename == null) { LOG.info("Metadata filename not set for branch " + String.valueOf(branchId) + ": not publishing metadata."); continue; } for (String partition : partitions) { publishMetadata(getMergedMetadataForPartitionAndBranch(partition, branchId), branchId, new Path(new Path(mdOutputPath, partition), metadataFilename)); } } } }
From source file:org.apache.nifi.properties.ProtectedNiFiProperties.java
/** * Retrieves all known property keys./* w ww. j a va 2 s. c o m*/ * * @return all known property keys */ @Override public Set<String> getPropertyKeys() { Set<String> filteredKeys = getPropertyKeysIncludingProtectionSchemes(); filteredKeys.removeIf(p -> p.endsWith(".protected")); return filteredKeys; }
From source file:org.apache.nifi.registry.properties.ProtectedNiFiRegistryProperties.java
/** * Returns the set of property keys in the NiFiRegistryProperties, * excluding any protection keys (i.e. 'x.y.z.protected'). * * @return the set of property keys/* w ww . jav a 2 s .c o m*/ */ Set<String> getPropertyKeysExcludingProtectionSchemes() { Set<String> filteredKeys = getPropertyKeysIncludingProtectionSchemes(); filteredKeys.removeIf(p -> p.endsWith(".protected")); return filteredKeys; }
From source file:org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.java
/** * Removes the brokers which have more bundles assigned to them in the same namespace as the incoming bundle than at * least one other available broker from consideration. * * @param assignedBundleName// w w w .j a v a 2 s . co m * Name of bundle to be assigned. * @param candidates * BrokersBase available for placement. * @param brokerToNamespaceToBundleRange * Map from brokers to namespaces to bundle ranges. */ public static void removeMostServicingBrokersForNamespace(final String assignedBundleName, final Set<String> candidates, final Map<String, Map<String, Set<String>>> brokerToNamespaceToBundleRange) { if (candidates.isEmpty()) { return; } final String namespaceName = getNamespaceNameFromBundleName(assignedBundleName); int leastBundles = Integer.MAX_VALUE; for (final String broker : candidates) { if (brokerToNamespaceToBundleRange.containsKey(broker)) { final Set<String> bundleRanges = brokerToNamespaceToBundleRange.get(broker).get(namespaceName); if (bundleRanges == null) { // Assume that when the namespace is absent, there are no bundles for this namespace assigned to // that broker. leastBundles = 0; break; } leastBundles = Math.min(leastBundles, bundleRanges.size()); } else { // Assume non-present brokers have 0 bundles. leastBundles = 0; break; } } if (leastBundles == 0) { // By assumption, the namespace name will not be present if there are no bundles in the namespace // assigned to the broker. candidates.removeIf(broker -> brokerToNamespaceToBundleRange.containsKey(broker) && brokerToNamespaceToBundleRange.get(broker).containsKey(namespaceName)); } else { final int finalLeastBundles = leastBundles; // We may safely assume that each broker has at least one bundle for this namespace. // Note that this case is far less likely since it implies that there are at least as many bundles for this // namespace as brokers. candidates.removeIf(broker -> brokerToNamespaceToBundleRange.get(broker).get(namespaceName) .size() != finalLeastBundles); } }
From source file:org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.java
/** * It tries to filter out brokers which own namespace with same anti-affinity-group as given namespace. If all the * domains own namespace with same anti-affinity group then it will try to keep brokers with domain that has least * number of namespaces. It also tries to keep brokers which has least number of namespace with in domain. * eg./* w ww . j a v a 2 s .c o m*/ * <pre> * Before: * Domain-count BrokersBase-count * ____________ ____________ * d1-3 b1-2,b2-1 * d2-3 b3-2,b4-1 * d3-4 b5-2,b6-2 * * After filtering: "candidates" brokers * Domain-count BrokersBase-count * ____________ ____________ * d1-3 b2-1 * d2-3 b4-1 * * "candidate" broker to own anti-affinity-namespace = b2 or b4 * * </pre> * * @param pulsar * @param assignedBundleName * @param candidates * @param brokerToNamespaceToBundleRange */ public static void filterAntiAffinityGroupOwnedBrokers(final PulsarService pulsar, final String assignedBundleName, final Set<String> candidates, final Map<String, Map<String, Set<String>>> brokerToNamespaceToBundleRange, Map<String, String> brokerToDomainMap) { if (candidates.isEmpty()) { return; } final String namespaceName = getNamespaceNameFromBundleName(assignedBundleName); try { final Map<String, Integer> brokerToAntiAffinityNamespaceCount = getAntiAffinityNamespaceOwnedBrokers( pulsar, namespaceName, brokerToNamespaceToBundleRange).get(30, TimeUnit.SECONDS); if (brokerToAntiAffinityNamespaceCount == null) { // none of the broker owns anti-affinity-namespace so, none of the broker will be filtered return; } if (pulsar.getConfiguration().isFailureDomainsEnabled()) { // this will remove all the brokers which are part of domains that don't have least number of // anti-affinity-namespaces filterDomainsNotHavingLeastNumberAntiAffinityNamespaces(brokerToAntiAffinityNamespaceCount, candidates, brokerToDomainMap); } // now, "candidates" has list of brokers which are part of domain that can accept this namespace. now, // with in these domains, remove brokers which don't have least number of namespaces. so, brokers with least // number of namespace can be selected int leastNamaespaceCount = Integer.MAX_VALUE; for (final String broker : candidates) { if (brokerToAntiAffinityNamespaceCount.containsKey(broker)) { Integer namespaceCount = brokerToAntiAffinityNamespaceCount.get(broker); if (namespaceCount == null) { // Assume that when the namespace is absent, there are no namespace assigned to // that broker. leastNamaespaceCount = 0; break; } leastNamaespaceCount = Math.min(leastNamaespaceCount, namespaceCount); } else { // Assume non-present brokers have 0 bundles. leastNamaespaceCount = 0; break; } } // filter out broker based on namespace distribution if (leastNamaespaceCount == 0) { candidates.removeIf(broker -> brokerToAntiAffinityNamespaceCount.containsKey(broker) && brokerToAntiAffinityNamespaceCount.get(broker) > 0); } else { final int finalLeastNamespaceCount = leastNamaespaceCount; candidates.removeIf( broker -> brokerToAntiAffinityNamespaceCount.get(broker) != finalLeastNamespaceCount); } } catch (Exception e) { log.error("Failed to filter anti-affinity group namespace {}", e.getMessage()); } }
From source file:org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.java
/** * It computes least number of namespace owned by any of the domain and then it filters out all the domains that own * namespaces more than this count.// ww w.ja v a 2 s .c o m * * @param brokerToAntiAffinityNamespaceCount * @param candidates * @param brokerToDomainMap */ private static void filterDomainsNotHavingLeastNumberAntiAffinityNamespaces( Map<String, Integer> brokerToAntiAffinityNamespaceCount, Set<String> candidates, Map<String, String> brokerToDomainMap) { if (brokerToDomainMap == null || brokerToDomainMap.isEmpty()) { return; } final Map<String, Integer> domainNamespaceCount = Maps.newHashMap(); int leastNamespaceCount = Integer.MAX_VALUE; candidates.forEach(broker -> { final String domain = brokerToDomainMap.getOrDefault(broker, DEFAULT_DOMAIN); final int count = brokerToAntiAffinityNamespaceCount.getOrDefault(broker, 0); domainNamespaceCount.compute(domain, (domainName, nsCount) -> nsCount == null ? count : nsCount + count); }); // find leastNameSpaceCount for (Entry<String, Integer> domainNsCountEntry : domainNamespaceCount.entrySet()) { if (domainNsCountEntry.getValue() < leastNamespaceCount) { leastNamespaceCount = domainNsCountEntry.getValue(); } } final int finalLeastNamespaceCount = leastNamespaceCount; // only keep domain brokers which has leastNamespaceCount candidates.removeIf(broker -> { Integer nsCount = domainNamespaceCount.get(brokerToDomainMap.getOrDefault(broker, DEFAULT_DOMAIN)); return nsCount != null && nsCount != finalLeastNamespaceCount; }); }
From source file:org.artifactory.maven.index.MavenIndexerServiceImpl.java
private Set<? extends RepoDescriptor> calcSpecificReposForIndexing(List<String> repoKeys) { Set<RepoBaseDescriptor> repos = Sets.newHashSet(); repos.addAll(repositoryService.getLocalRepoDescriptors()); repos.addAll(repositoryService.getRemoteRepoDescriptors()); repos.addAll(getAllVirtualReposExceptGlobal()); repos.removeIf(descriptor -> !repoKeys.contains(descriptor.getKey())); return repos; }
From source file:org.cryptomator.frontend.webdav.mount.WindowsWebDavMounter.java
private void addProxyOverrides(URI uri) throws IOException, CommandFailedException { try {/* w w w . ja va 2s. com*/ // get existing value for ProxyOverride key from reqistry: ProcessBuilder query = new ProcessBuilder("reg", "query", "\"HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\"", "/v", "ProxyOverride"); Process queryCmd = query.start(); String queryStdOut = IOUtils.toString(queryCmd.getInputStream(), StandardCharsets.UTF_8); int queryResult = queryCmd.waitFor(); // determine new value for ProxyOverride key: Set<String> overrides = new HashSet<>(); Matcher matcher = REG_QUERY_PROXY_OVERRIDES_PATTERN.matcher(queryStdOut); if (queryResult == 0 && matcher.find()) { String[] existingOverrides = StringUtils.split(matcher.group(1), ';'); overrides.addAll(Arrays.asList(existingOverrides)); } overrides.removeIf(s -> s.startsWith(uri.getHost() + ":")); overrides.add("<local>"); overrides.add(uri.getHost()); overrides.add(uri.getHost() + ":" + uri.getPort()); // set new value: String overridesStr = StringUtils.join(overrides, ';'); ProcessBuilder add = new ProcessBuilder("reg", "add", "\"HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\"", "/v", "ProxyOverride", "/d", "\"" + overridesStr + "\"", "/f"); LOG.debug("Invoking command: " + StringUtils.join(add.command(), ' ')); Process addCmd = add.start(); int addResult = addCmd.waitFor(); if (addResult != 0) { String addStdErr = IOUtils.toString(addCmd.getErrorStream(), StandardCharsets.UTF_8); throw new CommandFailedException(addStdErr); } } catch (IOException | CommandFailedException e) { LOG.info("Failed to add proxy overrides", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); InterruptedIOException ioException = new InterruptedIOException(); ioException.initCause(e); throw ioException; } }