List of usage examples for java.util TreeSet addAll
public boolean addAll(Collection<? extends E> c)
From source file:org.unitime.timetable.solver.TimetableDatabaseLoader.java
private Collection<InstrOfferingConfig> sortedConfigs(InstructionalOffering offering) { if (offering.getInstrOfferingConfigs().size() <= 1) return offering.getInstrOfferingConfigs(); TreeSet<InstrOfferingConfig> configs = new TreeSet<InstrOfferingConfig>(new InstrOfferingConfigComparator( offering.getControllingCourseOffering().getSubjectArea().getUniqueId())); configs.addAll(offering.getInstrOfferingConfigs()); return configs; }
From source file:org.wso2.andes.kernel.slot.SlotManager.java
/** * Get an ordered set of existing, assigned slots that overlap with the input slot range. * * @param queueName name of destination queue * @param startMsgID start message ID of input slot * @param endMsgID end message ID of input slot * @return TreeSet<Slot>//from w w w .j ava 2s. co m */ private TreeSet<Slot> getOverlappedAssignedSlots(String queueName, long startMsgID, long endMsgID) { TreeSet<Slot> overlappedSlots = new TreeSet<Slot>(); // Sweep all assigned slots to find overlaps. // slotAssignmentMap, cos its optimized for node,queue-wise iteration. if (AndesContext.getInstance().isClusteringEnabled()) { // The requirement here is to clear slot associations for the queue on all nodes. List<String> nodeIDs = HazelcastAgent.getInstance().getMembersNodeIDs(); for (String nodeID : nodeIDs) { String lockKey = nodeID + SlotManager.class; TreeSet<Slot> overlappingSlotsOnNode = new TreeSet<Slot>(); synchronized (lockKey.intern()) { HashmapStringTreeSetWrapper wrapper = slotAssignmentMap.get(nodeID); if (!overLappedSlotMap.containsKey(nodeID)) { overLappedSlotMap.put(nodeID, new HashmapStringTreeSetWrapper()); } HashmapStringTreeSetWrapper olWrapper = overLappedSlotMap.get(nodeID); HashMap<String, TreeSet<Slot>> olSlotMap = olWrapper.getStringListHashMap(); if (!olSlotMap.containsKey(queueName)) { olSlotMap.put(queueName, new TreeSet<Slot>()); olWrapper.setStringListHashMap(olSlotMap); overLappedSlotMap.set(nodeID, olWrapper); } if (wrapper != null) { HashMap<String, TreeSet<Slot>> queueToSlotMap = wrapper.getStringListHashMap(); if (queueToSlotMap != null) { TreeSet<Slot> slotListForQueueOnNode = queueToSlotMap.get(queueName); if (null != slotListForQueueOnNode) { for (Slot slot : slotListForQueueOnNode) { if (endMsgID < slot.getStartMessageId()) continue; // skip this one, its below our range if (startMsgID > slot.getEndMessageId()) continue; // skip this one, its above our range slot.setAnOverlappingSlot(true); if (log.isDebugEnabled()) { log.debug("Marked already assigned slot as an overlapping" + " slot. Slot= " + slot); } overlappingSlotsOnNode.add(slot); if (log.isDebugEnabled()) { log.debug("Found an overlapping slot : " + slot); } //Add to global overlappedSlotMap olSlotMap.get(queueName).remove(slot); olSlotMap.get(queueName).add(slot); } } } wrapper.setStringListHashMap(queueToSlotMap); slotAssignmentMap.set(nodeID, wrapper); } // Add all marked slots collected into the olSlot to global overlappedSlotsMap. olWrapper.setStringListHashMap(olSlotMap); overLappedSlotMap.set(nodeID, olWrapper); // Add to return collection overlappedSlots.addAll(overlappingSlotsOnNode); } } } return overlappedSlots; }
From source file:org.lockss.subscription.SubscriptionManager.java
/** * Populates the list of weighted repositories, to be used in a round-robin * fashion for the subsequent AU configurations. * /*from w w w . j ava2 s.c om*/ * @param repositoryMap * A Map<String, PlatformUtil.DF> with the map of all distinct * repositories available. * @return a List<String> with the list of weighted repositories. */ List<String> populateRepositories(Map<String, PlatformUtil.DF> repositoryMap) { final String DEBUG_HEADER = "populateRepositories(): "; if (log.isDebug2()) log.debug2(DEBUG_HEADER + "repositoryMap.size() = " + repositoryMap.size()); // Initialize the list of available repositories. List<String> repos = new ArrayList<String>(); // Handle an empty repository map. if (repositoryMap.size() < 1) { if (log.isDebug2()) log.debug2(DEBUG_HEADER + "repos = " + repos); return repos; } // Get the available repositories sorted by their available space. TreeSet<Entry<String, PlatformUtil.DF>> sortedRepos = new TreeSet<Entry<String, PlatformUtil.DF>>( DF_BY_AVAIL_COMPARATOR); sortedRepos.addAll(repositoryMap.entrySet()); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "sortedRepos.size() = " + sortedRepos.size()); // Handle the case of a single repository. if (sortedRepos.size() == 1) { repos.add(sortedRepos.first().getKey()); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "Added " + sortedRepos.first().getKey()); if (log.isDebug2()) log.debug2(DEBUG_HEADER + "repos = " + repos); return repos; } // Get the repository available space threshold from the configuration. int repoThreshold = ConfigManager.getCurrentConfig().getInt(PARAM_REPOSITORY_AVAIL_SPACE_THRESHOLD, DEFAULT_REPOSITORY_AVAIL_SPACE_THRESHOLD); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "repoThreshold = " + repoThreshold); // Get the available space of the repository with the least amount of // available space. long minAvail = sortedRepos.first().getValue().getAvail(); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "minAvail = " + minAvail); // Remove repositories that don't have a minimum of space, except the last // one. while (minAvail < repoThreshold) { sortedRepos.remove(sortedRepos.first()); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "sortedRepos.size() = " + sortedRepos.size()); // If there is only one repository left, use it. if (sortedRepos.size() == 1) { repos.add(sortedRepos.first().getKey()); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "Added " + sortedRepos.first().getKey()); if (log.isDebug2()) log.debug2(DEBUG_HEADER + "repos = " + repos); return repos; } // Get the available space of the repository with the least amount of // available space. minAvail = sortedRepos.first().getValue().getAvail(); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "minAvail = " + minAvail); } // Count the remaining repositories. int repoCount = sortedRepos.size(); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "repoCount = " + repoCount); // Initialize the array of repositories and the total available space. long totalAvailable = 0l; int i = 0; Entry<String, PlatformUtil.DF>[] repoArray = new Entry[repoCount]; for (Entry<String, PlatformUtil.DF> df : sortedRepos) { totalAvailable += df.getValue().getAvail(); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "totalAvailable = " + totalAvailable); repoArray[i++] = df; } // For each repository, compute the target fraction and initialize the count // of appearances in the final list. i = 0; double[] repoTargetFraction = new double[repoCount]; int[] repoAppearances = new int[repoCount]; for (Entry<String, PlatformUtil.DF> df : repoArray) { repoTargetFraction[i] = df.getValue().getAvail() / (double) totalAvailable; if (log.isDebug3()) log.debug3(DEBUG_HEADER + "i = " + i + ", repoTargetFraction[i] = " + repoTargetFraction[i]); repoAppearances[i++] = 0; } // The first repository in the list is the one with the largest amount of // available space. repos.add(repoArray[repoCount - 1].getKey()); repoAppearances[repoCount - 1]++; // An indication of whether the created list matches the target fractions of // all the repositories. boolean done = false; while (!done) { // If no differences between the target fractions and the fractions of // appearances are found in the process below, the list is complete. done = true; double difference = 0; double maxDifference = 0; int nextRepo = -1; // Loop through all the repositories. for (int j = 0; j < repoCount; j++) { if (log.isDebug3()) log.debug3(DEBUG_HEADER + "j = " + j + ", repoAppearances[j]/(double)repos.size() = " + repoAppearances[j] / (double) repos.size() + ", repoTargetFraction[j] = " + repoTargetFraction[j]); // Find the difference between the target fraction and the fraction of // appearances. difference = repoTargetFraction[j] - repoAppearances[j] / (double) repos.size(); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "difference = " + difference); // Update the largest difference, if necessary. if (maxDifference < difference) { maxDifference = difference; nextRepo = j; } } // Check whether a repository with the largest difference was found. if (nextRepo != -1) { // Yes: Add it to the list. repos.add(repoArray[nextRepo].getKey()); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "Added " + repoArray[nextRepo].getKey()); // Increment its appearance count. repoAppearances[nextRepo]++; // Check whether not all the target fractions have been achieved. for (int k = 0; k < repoCount; k++) { difference = repoAppearances[k] / (double) repos.size() - repoTargetFraction[k]; if (log.isDebug3()) log.debug3(DEBUG_HEADER + "k = " + k + ", difference = " + difference); // Within one per cent is a match. if (Math.abs(difference) > 0.01) { done = false; break; } } } } if (log.isDebug2()) log.debug2(DEBUG_HEADER + "repos = " + repos); return repos; }
From source file:com.joliciel.talismane.parser.TransitionBasedGlobalLearningParser.java
public List<ParseConfiguration> parseSentence(List<PosTagSequence> posTagSequences, FeatureWeightVector weightVector, RankingSolution correctSolution) { MONITOR.startTask("parseSentence"); try {/*from ww w .ja v a 2s . co m*/ long startTime = (new Date()).getTime(); int maxAnalysisTimeMilliseconds = maxAnalysisTimePerSentence * 1000; int minFreeMemoryBytes = minFreeMemory * KILOBYTE; TokenSequence tokenSequence = posTagSequences.get(0).getTokenSequence(); TreeMap<Integer, TreeSet<ParseConfiguration>> heaps = new TreeMap<Integer, TreeSet<ParseConfiguration>>(); TreeSet<ParseConfiguration> heap0 = new TreeSet<ParseConfiguration>(); for (PosTagSequence posTagSequence : posTagSequences) { // add an initial ParseConfiguration for each postag sequence ParseConfiguration initialConfiguration = this.getParserServiceInternal() .getInitialConfiguration(posTagSequence); initialConfiguration.setScoringStrategy(new SimpleRankingScoringStrategy()); initialConfiguration.setRankingScore(0.0); heap0.add(initialConfiguration); if (LOG.isDebugEnabled()) { LOG.debug("Adding initial posTagSequence: " + posTagSequence); } } heaps.put(0, heap0); TreeSet<ParseConfiguration> backupHeap = null; TreeSet<ParseConfiguration> finalHeap = null; while (heaps.size() > 0) { Entry<Integer, TreeSet<ParseConfiguration>> heapEntry = heaps.firstEntry(); TreeSet<ParseConfiguration> currentHeap = heapEntry.getValue(); int currentHeapIndex = heapEntry.getKey(); if (LOG.isTraceEnabled()) { LOG.trace("##### Polling next heap: " + heapEntry.getKey() + ", size: " + heapEntry.getValue().size()); } boolean finished = false; // systematically set the final heap here, just in case we exit "naturally" with no more heaps finalHeap = heapEntry.getValue(); backupHeap = new TreeSet<ParseConfiguration>(); // we jump out when either (a) all tokens have been attached or (b) we go over the max alloted time ParseConfiguration topConf = currentHeap.first(); if (topConf.isTerminal()) { LOG.trace("Exiting with terminal heap: " + heapEntry.getKey() + ", size: " + heapEntry.getValue().size()); finished = true; } // check if we've gone over alloted time for this sentence long analysisTime = (new Date()).getTime() - startTime; if (maxAnalysisTimePerSentence > 0 && analysisTime > maxAnalysisTimeMilliseconds) { LOG.info("Parse tree analysis took too long for sentence: " + tokenSequence.getText()); LOG.info("Breaking out after " + maxAnalysisTimePerSentence + " seconds."); finished = true; } // check if we've enough memory to process this sentence if (minFreeMemory > 0) { long freeMemory = Runtime.getRuntime().freeMemory(); if (freeMemory < minFreeMemoryBytes) { LOG.info("Not enough memory left to parse sentence: " + tokenSequence.getText()); LOG.info("Min free memory (bytes):" + minFreeMemoryBytes); LOG.info("Current free memory (bytes): " + freeMemory); finished = true; } } // check if any of the remaining top-N solutions on any heap can lead to the correct solution if (correctSolution != null) { boolean canReachCorrectSolution = false; for (TreeSet<ParseConfiguration> heap : heaps.values()) { int j = 1; for (ParseConfiguration solution : heap) { if (j > beamWidth) break; if (solution.canReach(correctSolution)) { canReachCorrectSolution = true; break; } j++; } if (canReachCorrectSolution) break; } if (!canReachCorrectSolution) { LOG.debug("None of the solutions on the heap can reach the gold solution. Exiting."); finished = true; } } if (finished) { // combine any remaining heaps for (TreeSet<ParseConfiguration> heap : heaps.values()) { if (finalHeap != heap) { finalHeap.addAll(heap); } } break; } // remove heap from set of heaps heapEntry = heaps.pollFirstEntry(); // limit the breadth to K int maxSolutions = currentHeap.size() > this.beamWidth ? this.beamWidth : currentHeap.size(); int j = 0; while (currentHeap.size() > 0) { ParseConfiguration history = currentHeap.pollFirst(); backupHeap.add(history); if (LOG.isTraceEnabled()) { LOG.trace("### Next configuration on heap " + heapEntry.getKey() + ":"); LOG.trace(history.toString()); LOG.trace("Score: " + df.format(history.getScore())); LOG.trace(history.getPosTagSequence()); } Set<Transition> transitions = new HashSet<Transition>(); // test the positive rules on the current configuration boolean ruleApplied = false; if (parserPositiveRules != null) { MONITOR.startTask("check rules"); try { for (ParserRule rule : parserPositiveRules) { if (LOG.isTraceEnabled()) { LOG.trace("Checking rule: " + rule.getCondition().getName()); } RuntimeEnvironment env = this.featureService.getRuntimeEnvironment(); FeatureResult<Boolean> ruleResult = rule.getCondition().check(history, env); if (ruleResult != null && ruleResult.getOutcome()) { transitions.add(rule.getTransition()); ruleApplied = true; if (LOG.isTraceEnabled()) { LOG.trace("Rule applies. Setting transition to: " + rule.getTransition().getCode()); } if (!rule.getTransition().checkPreconditions(history)) { LOG.error("Cannot apply rule, preconditions not met."); ruleApplied = false; } break; } } } finally { MONITOR.endTask("check rules"); } } if (!ruleApplied) { transitions = parsingConstrainer.getPossibleTransitions(history); Set<Transition> eliminatedTransitions = new HashSet<Transition>(); for (Transition transition : transitions) { if (!transition.checkPreconditions(history)) { eliminatedTransitions.add(transition); } } transitions.removeAll(eliminatedTransitions); // apply the negative rules eliminatedTransitions = new HashSet<Transition>(); if (parserNegativeRules != null) { MONITOR.startTask("check negative rules"); try { for (ParserRule rule : parserNegativeRules) { if (LOG.isTraceEnabled()) { LOG.trace("Checking negative rule: " + rule.getCondition().getName()); } RuntimeEnvironment env = this.featureService.getRuntimeEnvironment(); FeatureResult<Boolean> ruleResult = rule.getCondition().check(history, env); if (ruleResult != null && ruleResult.getOutcome()) { eliminatedTransitions.add(rule.getTransition()); if (LOG.isTraceEnabled()) { LOG.debug("Rule applies. Eliminating transition: " + rule.getTransition().getCode()); } } } if (eliminatedTransitions.size() == transitions.size()) { LOG.debug("All transitions eliminated! Restoring original transitions."); } else { transitions.removeAll(eliminatedTransitions); } } finally { MONITOR.endTask("check negative rules"); } } } // has a positive rule been applied? if (transitions.size() == 0) { // just in case the we run out of both heaps and analyses, we build this backup heap backupHeap.add(history); if (LOG.isTraceEnabled()) LOG.trace( "No transitions could be applied: not counting this solution as part of the beam"); } else { // up the counter, since we will count this solution towards the heap j++; // add solutions to the heap, one per valid transition MONITOR.startTask("heap sort"); try { Map<Transition, Double> deltaScorePerTransition = new HashMap<Transition, Double>(); double absoluteMax = 1; for (Transition transition : transitions) { if (LOG.isTraceEnabled()) { LOG.trace("Applying transition: " + transition.getCode()); } ParseConfiguration configuration = this.parserServiceInternal .getConfiguration(history); transition.apply(configuration); configuration.setRankingScore(history.getRankingScore()); configuration.getIncrementalFeatureResults() .addAll(history.getIncrementalFeatureResults()); // test the features on the new configuration double scoreDelta = 0.0; MONITOR.startTask("feature analyse"); List<FeatureResult<?>> featureResults = new ArrayList<FeatureResult<?>>(); try { for (ParseConfigurationFeature<?> feature : this.parseFeatures) { MONITOR.startTask(feature.getName()); try { RuntimeEnvironment env = this.featureService.getRuntimeEnvironment(); FeatureResult<?> featureResult = feature.check(configuration, env); if (featureResult != null) { featureResults.add(featureResult); double weight = weightVector.getWeight(featureResult); scoreDelta += weight; if (LOG.isTraceEnabled()) { LOG.trace(featureResult.toString() + " = " + weight); } } } finally { MONITOR.endTask(feature.getName()); } } configuration.getIncrementalFeatureResults().add(featureResults); if (LOG.isTraceEnabled()) { LOG.trace("Score = " + configuration.getRankingScore() + " + " + scoreDelta + " = " + (configuration.getRankingScore() + scoreDelta)); } configuration.setRankingScore(configuration.getRankingScore() + scoreDelta); deltaScorePerTransition.put(transition, scoreDelta); if (Math.abs(scoreDelta) > absoluteMax) absoluteMax = Math.abs(scoreDelta); } finally { MONITOR.endTask("feature analyse"); } int nextHeapIndex = parseComparisonStrategy.getComparisonIndex(configuration) * 1000; while (nextHeapIndex <= currentHeapIndex) nextHeapIndex++; TreeSet<ParseConfiguration> nextHeap = heaps.get(nextHeapIndex); if (nextHeap == null) { nextHeap = new TreeSet<ParseConfiguration>(); heaps.put(nextHeapIndex, nextHeap); if (LOG.isTraceEnabled()) LOG.trace("Created heap with index: " + nextHeapIndex); } nextHeap.add(configuration); if (LOG.isTraceEnabled()) { LOG.trace("Added configuration with score " + configuration.getScore() + " to heap: " + nextHeapIndex + ", total size: " + nextHeap.size()); } configuration.clearMemory(); } // next transition // Create a probability distribution of transitions // normalise probabilities for each transition via normalised exponential // e^(x/absmax)/sum(e^(x/absmax)) // where x/absmax is in [-1,1] // e^(x/absmax) is in [1/e,e] double total = 0.0; for (Transition transition : deltaScorePerTransition.keySet()) { double deltaScore = deltaScorePerTransition.get(transition); deltaScore = Math.exp(deltaScore / absoluteMax); deltaScorePerTransition.put(transition, deltaScore); total += deltaScore; } for (Transition transition : deltaScorePerTransition.keySet()) { double probability = deltaScorePerTransition.get(transition); probability /= total; Decision<Transition> decision = machineLearningService.createDecision(transition, probability); transition.setDecision(decision); if (LOG.isTraceEnabled()) { LOG.trace("Transition: " + transition.getCode() + ", Prob: " + probability); } } } finally { MONITOR.endTask("heap sort"); } } // have we any transitions? // beam width test if (j == maxSolutions) break; } // next history } // next atomic index // return the best sequences on the heap List<ParseConfiguration> bestConfigurations = new ArrayList<ParseConfiguration>(); int i = 0; if (finalHeap.isEmpty()) finalHeap = backupHeap; while (!finalHeap.isEmpty()) { bestConfigurations.add(finalHeap.pollFirst()); i++; if (i >= this.getBeamWidth()) break; } if (LOG.isDebugEnabled()) { if (correctSolution != null) { LOG.debug("Gold transitions: " + correctSolution.getIncrementalOutcomes()); } for (ParseConfiguration finalConfiguration : bestConfigurations) { LOG.debug(df.format(finalConfiguration.getScore()) + ": " + finalConfiguration.toString()); LOG.debug("Pos tag sequence: " + finalConfiguration.getPosTagSequence()); LOG.debug("Transitions: " + finalConfiguration.getTransitions()); if (LOG.isTraceEnabled()) { StringBuilder sb = new StringBuilder(); sb.append(" * PosTag sequence score "); sb.append(df.format(finalConfiguration.getPosTagSequence().getScore())); sb.append(" = "); for (PosTaggedToken posTaggedToken : finalConfiguration.getPosTagSequence()) { sb.append(" * "); sb.append(df.format(posTaggedToken.getDecision().getProbability())); } sb.append(" root "); sb.append(finalConfiguration.getPosTagSequence().size()); LOG.trace(sb.toString()); sb = new StringBuilder(); sb.append(" * Token sequence score = "); sb.append(df.format(finalConfiguration.getPosTagSequence().getTokenSequence().getScore())); LOG.trace(sb.toString()); } } } return bestConfigurations; } finally { MONITOR.endTask("parseSentence"); } }
From source file:org.jahia.services.importexport.ImportExportBaseService.java
private void exportNodes(JCRNodeWrapper rootNode, TreeSet<JCRNodeWrapper> sortedNodes, OutputStream outputStream, Set<String> typesToIgnore, Set<String> externalReferences, Map<String, Object> params, boolean logProgress) throws IOException, RepositoryException, SAXException, TransformerException { long startSitesExportTime = System.currentTimeMillis(); ExportContext exportContext = null;/*www. ja v a 2 s . com*/ if (logProgress) { // estimate the number of nodes to exports and logs this information long estimatedNodes = estimateNodesToExport(sortedNodes, rootNode.getSession(), typesToIgnore); logger.info("Approximate number of nodes to export: {}, estimated in: {} seconds", estimatedNodes, getDuration(startSitesExportTime)); exportContext = new ExportContext(estimatedNodes); } final String xsl = (String) params.get(XSL_PATH); final boolean skipBinary = !Boolean.FALSE.equals(params.get(SKIP_BINARY)); final boolean noRecurse = Boolean.TRUE.equals(params.get(NO_RECURSE)); OutputStream tmpOut = outputStream; if (xsl != null) { String filename = Patterns.SPACE.matcher(rootNode.getName()).replaceAll("_"); File tempFile = File.createTempFile("exportTemplates-" + filename, ".xml"); tmpOut = new DeferredFileOutputStream(1024 * 1024 * 10, tempFile); } DataWriter dw = new DataWriter(new OutputStreamWriter(tmpOut, "UTF-8")); if (Boolean.TRUE.equals(params.get(SYSTEM_VIEW))) { SystemViewExporter exporter = new SystemViewExporter(rootNode.getSession(), dw, !noRecurse, !skipBinary); exporter.export(rootNode); } else { DocumentViewExporter exporter = new DocumentViewExporter(rootNode.getSession(), dw, skipBinary, noRecurse); exporter.setExportContext(exportContext); exporter.addObserver(this); if (externalReferences != null) { exporter.setExternalReferences(externalReferences); } typesToIgnore.add("rep:system"); if (params.containsKey(INCLUDE_LIVE_EXPORT)) { List<String> l = new ArrayList<String>(exporter.getPropertiestoIgnore()); l.remove("jcr:uuid"); exporter.setPropertiestoIgnore(l); if (rootNode.getSession().getWorkspace().getName().equals(Constants.EDIT_WORKSPACE)) { exporter.setPublicationStatusSession( jcrStoreService.getSessionFactory().getCurrentUserSession("live")); } } exporter.setTypesToIgnore(typesToIgnore); exporter.export(rootNode, sortedNodes); sortedNodes.addAll(exporter.getNodesList()); } if (exportContext != null) { // Nodes are now exported in the .xml, so we log the time difference for this export logger.info("Exported {} nodes in {} seconds", exportContext.getExportIndex(), getDuration(startSitesExportTime)); } dw.flush(); if (xsl != null) { DeferredFileOutputStream stream = (DeferredFileOutputStream) tmpOut; InputStream inputStream = new BufferedInputStream(new FileInputStream(stream.getFile())); fileCleaningTracker.track(stream.getFile(), inputStream); if (stream.isInMemory()) { inputStream.close(); inputStream = new ByteArrayInputStream(stream.getData()); } Transformer transformer = getTransformer(xsl); long startXmlCleanup = System.currentTimeMillis(); if (logProgress) { // since the xml transformation can be heavy in process depending on the .xml size // we logs some basics data logger.info("Starting cleanup transformation ..."); transformer.transform(new StreamSource(inputStream), new StreamResult(outputStream)); logger.info("Cleanup transformation finished in {} seconds", getDuration(startXmlCleanup)); } else { transformer.transform(new StreamSource(inputStream), new StreamResult(outputStream)); } } }
From source file:com.offbynull.voip.kademlia.model.RouteTreeNode.java
public void dumpAllNodesUnderTreeNode(Id id, TreeSet<Activity> output, int max, boolean includeStale, Set<BitString> skipPrefixes) { Validate.notNull(id);/*from www . j a v a 2 s . c om*/ Validate.notNull(output); // technically shouldn't contain any null elements, but we don't care since we're just adding to this Validate.notNull(skipPrefixes); Validate.noNullElements(skipPrefixes); Validate.isTrue(max >= 0); // why would anyone want 0 here? let thru anwyways // No more room in bucket? just leave right away. if (output.size() >= max) { return; } // Sort branches at this treenode by how close the are to the ID we're searching for... Go through the sorted branches in // order... // // If it's a bucket: dump it. // If it's a branch: recurse in to the branch and repeat // ArrayList<RouteTreeBranch> sortedBranches = new ArrayList<>(branches); Collections.sort(sortedBranches, new PrefixClosenessComparator(id, prefix.getBitLength(), suffixLen)); // What is the point of taking in an ID and sorting the branches in this tree node such that the we access the "closer" prefixes // first? We want to access the branches that are closer to the suffix of the ID first because ... // // // 1. Given the same prefix, we don't end up accessing the exact same set of nodes given. For example... // // 0/\1 // / EMPTY // 0/\1 // / FULL // 0/\1 // ME FULL // // Assume the routing tree above. We want to route to node 111, but bucket 1xx is empty. We then go down the other branch and // start grabbing nodes starting with prefix 0xx. We then use the suffix of 111 (x11) to determine which branches to traverse // down first for our 0xx nodes to return. We do this because we don't want to return the same set of nodes everytime someone // tries to access a 1xx node and we have an empty branch. // // For example... // if someone wanted 111 and 1xx was empty, path to search under 0xx would be 011, then 001, then 000. // if someone wanted 101 and 1xx was empty, path to search under 0xx would be 001, then 000, then 011. // // If we did something like a depth-first search, we'd always target 000 first, then 001, then 011. We don't want to do that // because we don't want to return the same set of nodes everytime. It would end up being an undue burden on those nodes. // // // // 2. Remember our notion of closeness: XOR and normal integer less-than to see which is closer. So for example, lets say we're // looking for ID 111110 and the prefix at this point in the tree is is 110xxx. Even though the prefix 110 doesn't match, we // still want to match as closely to the remaining suffix as possible, because when we XOR those extra 0's at the beginning of // the suffix mean that we're closer. // // For example... // // This tree node has the prefix 110xxx and the ID we're searching for is 111110. There are 2 branches at this tree node: // 1100xx and 1101xx // // 110xxx // /\ // / \ // / \ // 0 / \ 1 // / \ // 1100xx 1101xx // // We know that for ID 111110, the IDs under 1101xx WILL ALWAYS BE CLOSER than the IDs at 1100xx. // // XORing with the 1100xx bucket ... XOR(111110, 1100xx) = 0011xx // // XORing with the 1101xx bucket ... XOR(111110, 1101xx) = 0010xx // // // Remember how < works... go compare each single bit from the beginning until you come across a pair of bits that aren't // equal (one is 0 and the other is 1). The ID with 0 at that position is less-than the other one. // // // The one on the bottom (1101xx) will ALWAYS CONTAIN CLOSER IDs... // // An example ID in top: 110011 ... XOR(111110, 110011) = 001101 = 13 // An exmaple ID in bottom: 110100 ... XOR(111110, 110100) = 001010 = 9 // for (RouteTreeBranch sortedBranch : sortedBranches) { if (skipPrefixes.contains(sortedBranch.getPrefix())) { continue; } if (sortedBranch instanceof RouteTreeNodeBranch) { RouteTreeNode node = sortedBranch.getItem(); node.dumpAllNodesUnderTreeNode(id, output, max, includeStale, emptySet()); // dont propogate skipPrefixes (not relevant) // Bucket's full after dumping nodes in that branch. No point in continued processing. if (output.size() >= max) { return; } } else if (sortedBranch instanceof RouteTreeBucketBranch) { KBucket bucket = sortedBranch.getItem(); // don't bother with locked nodes for now, we're not supporting them output.addAll(bucket.dumpBucket(true, includeStale, false)); // Bucket's full after that add. No point in continued processing. if (output.size() >= max) { // If we have more than max elements from that last add, start evicting farthest away nodes while (output.size() > max) { output.pollLast(); } return; } } else { throw new IllegalStateException(); // should never happen } } }
From source file:com.jhh.hdb.sqlparser.MySemanticAnalyzer.java
protected static boolean distinctExprsExists(QB qb) { QBParseInfo qbp = qb.getParseInfo(); TreeSet<String> ks = new TreeSet<String>(); ks.addAll(qbp.getClauseNames()); for (String dest : ks) { List<ASTNode> list = qbp.getDistinctFuncExprsForClause(dest); if (!list.isEmpty()) { return true; }//from w w w. j a v a2s . co m } return false; }
From source file:com.jhh.hdb.sqlparser.MySemanticAnalyzer.java
private List<List<String>> getCommonGroupByDestGroups(QB qb, Map<String, Operator<? extends OperatorDesc>> inputs) throws SemanticException { QBParseInfo qbp = qb.getParseInfo(); TreeSet<String> ks = new TreeSet<String>(); ks.addAll(qbp.getClauseNames()); List<List<String>> commonGroupByDestGroups = new ArrayList<List<String>>(); // If this is a trivial query block return if (ks.size() <= 1) { List<String> oneList = new ArrayList<String>(1); if (ks.size() == 1) { oneList.add(ks.first());//from w w w . jav a 2 s . c o m } commonGroupByDestGroups.add(oneList); return commonGroupByDestGroups; } List<Operator<? extends OperatorDesc>> inputOperators = new ArrayList<Operator<? extends OperatorDesc>>( ks.size()); List<List<ExprNodeDesc>> sprayKeyLists = new ArrayList<List<ExprNodeDesc>>(ks.size()); List<List<ExprNodeDesc>> distinctKeyLists = new ArrayList<List<ExprNodeDesc>>(ks.size()); // Iterate over each clause for (String dest : ks) { Operator input = inputs.get(dest); RowResolver inputRR = opParseCtx.get(input).getRowResolver(); List<ExprNodeDesc> distinctKeys = getDistinctExprs(qbp, dest, inputRR); List<ExprNodeDesc> sprayKeys = new ArrayList<ExprNodeDesc>(); // Add the group by expressions List<ASTNode> grpByExprs = getGroupByForClause(qbp, dest); for (ASTNode grpByExpr : grpByExprs) { ExprNodeDesc exprDesc = genExprNodeDesc(grpByExpr, inputRR); if (ExprNodeDescUtils.indexOf(exprDesc, sprayKeys) < 0) { sprayKeys.add(exprDesc); } } // Loop through each of the lists of exprs, looking for a match boolean found = false; for (int i = 0; i < sprayKeyLists.size(); i++) { if (!input.equals(inputOperators.get(i))) { continue; } if (distinctKeys.isEmpty()) { // current dest has no distinct keys. List<ExprNodeDesc> combinedList = new ArrayList<ExprNodeDesc>(); combineExprNodeLists(sprayKeyLists.get(i), distinctKeyLists.get(i), combinedList); if (!matchExprLists(combinedList, sprayKeys)) { continue; } // else do the common code at the end. } else { if (distinctKeyLists.get(i).isEmpty()) { List<ExprNodeDesc> combinedList = new ArrayList<ExprNodeDesc>(); combineExprNodeLists(sprayKeys, distinctKeys, combinedList); if (!matchExprLists(combinedList, sprayKeyLists.get(i))) { continue; } else { // we have found a match. insert this distinct clause to head. distinctKeyLists.remove(i); sprayKeyLists.remove(i); distinctKeyLists.add(i, distinctKeys); sprayKeyLists.add(i, sprayKeys); commonGroupByDestGroups.get(i).add(0, dest); found = true; break; } } else { if (!matchExprLists(distinctKeyLists.get(i), distinctKeys)) { continue; } if (!matchExprLists(sprayKeyLists.get(i), sprayKeys)) { continue; } // else do common code } } // common code // A match was found, so add the clause to the corresponding list commonGroupByDestGroups.get(i).add(dest); found = true; break; } // No match was found, so create new entries if (!found) { inputOperators.add(input); sprayKeyLists.add(sprayKeys); distinctKeyLists.add(distinctKeys); List<String> destGroup = new ArrayList<String>(); destGroup.add(dest); commonGroupByDestGroups.add(destGroup); } } return commonGroupByDestGroups; }