List of usage examples for java.util TreeSet first
public E first()
From source file:org.lockss.subscription.SubscriptionManager.java
/** * Populates the list of weighted repositories, to be used in a round-robin * fashion for the subsequent AU configurations. * //from ww w . j a v a2 s. com * @param repositoryMap * A Map<String, PlatformUtil.DF> with the map of all distinct * repositories available. * @return a List<String> with the list of weighted repositories. */ List<String> populateRepositories(Map<String, PlatformUtil.DF> repositoryMap) { final String DEBUG_HEADER = "populateRepositories(): "; if (log.isDebug2()) log.debug2(DEBUG_HEADER + "repositoryMap.size() = " + repositoryMap.size()); // Initialize the list of available repositories. List<String> repos = new ArrayList<String>(); // Handle an empty repository map. if (repositoryMap.size() < 1) { if (log.isDebug2()) log.debug2(DEBUG_HEADER + "repos = " + repos); return repos; } // Get the available repositories sorted by their available space. TreeSet<Entry<String, PlatformUtil.DF>> sortedRepos = new TreeSet<Entry<String, PlatformUtil.DF>>( DF_BY_AVAIL_COMPARATOR); sortedRepos.addAll(repositoryMap.entrySet()); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "sortedRepos.size() = " + sortedRepos.size()); // Handle the case of a single repository. if (sortedRepos.size() == 1) { repos.add(sortedRepos.first().getKey()); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "Added " + sortedRepos.first().getKey()); if (log.isDebug2()) log.debug2(DEBUG_HEADER + "repos = " + repos); return repos; } // Get the repository available space threshold from the configuration. int repoThreshold = ConfigManager.getCurrentConfig().getInt(PARAM_REPOSITORY_AVAIL_SPACE_THRESHOLD, DEFAULT_REPOSITORY_AVAIL_SPACE_THRESHOLD); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "repoThreshold = " + repoThreshold); // Get the available space of the repository with the least amount of // available space. long minAvail = sortedRepos.first().getValue().getAvail(); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "minAvail = " + minAvail); // Remove repositories that don't have a minimum of space, except the last // one. while (minAvail < repoThreshold) { sortedRepos.remove(sortedRepos.first()); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "sortedRepos.size() = " + sortedRepos.size()); // If there is only one repository left, use it. if (sortedRepos.size() == 1) { repos.add(sortedRepos.first().getKey()); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "Added " + sortedRepos.first().getKey()); if (log.isDebug2()) log.debug2(DEBUG_HEADER + "repos = " + repos); return repos; } // Get the available space of the repository with the least amount of // available space. minAvail = sortedRepos.first().getValue().getAvail(); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "minAvail = " + minAvail); } // Count the remaining repositories. int repoCount = sortedRepos.size(); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "repoCount = " + repoCount); // Initialize the array of repositories and the total available space. long totalAvailable = 0l; int i = 0; Entry<String, PlatformUtil.DF>[] repoArray = new Entry[repoCount]; for (Entry<String, PlatformUtil.DF> df : sortedRepos) { totalAvailable += df.getValue().getAvail(); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "totalAvailable = " + totalAvailable); repoArray[i++] = df; } // For each repository, compute the target fraction and initialize the count // of appearances in the final list. i = 0; double[] repoTargetFraction = new double[repoCount]; int[] repoAppearances = new int[repoCount]; for (Entry<String, PlatformUtil.DF> df : repoArray) { repoTargetFraction[i] = df.getValue().getAvail() / (double) totalAvailable; if (log.isDebug3()) log.debug3(DEBUG_HEADER + "i = " + i + ", repoTargetFraction[i] = " + repoTargetFraction[i]); repoAppearances[i++] = 0; } // The first repository in the list is the one with the largest amount of // available space. repos.add(repoArray[repoCount - 1].getKey()); repoAppearances[repoCount - 1]++; // An indication of whether the created list matches the target fractions of // all the repositories. boolean done = false; while (!done) { // If no differences between the target fractions and the fractions of // appearances are found in the process below, the list is complete. done = true; double difference = 0; double maxDifference = 0; int nextRepo = -1; // Loop through all the repositories. for (int j = 0; j < repoCount; j++) { if (log.isDebug3()) log.debug3(DEBUG_HEADER + "j = " + j + ", repoAppearances[j]/(double)repos.size() = " + repoAppearances[j] / (double) repos.size() + ", repoTargetFraction[j] = " + repoTargetFraction[j]); // Find the difference between the target fraction and the fraction of // appearances. difference = repoTargetFraction[j] - repoAppearances[j] / (double) repos.size(); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "difference = " + difference); // Update the largest difference, if necessary. if (maxDifference < difference) { maxDifference = difference; nextRepo = j; } } // Check whether a repository with the largest difference was found. if (nextRepo != -1) { // Yes: Add it to the list. repos.add(repoArray[nextRepo].getKey()); if (log.isDebug3()) log.debug3(DEBUG_HEADER + "Added " + repoArray[nextRepo].getKey()); // Increment its appearance count. repoAppearances[nextRepo]++; // Check whether not all the target fractions have been achieved. for (int k = 0; k < repoCount; k++) { difference = repoAppearances[k] / (double) repos.size() - repoTargetFraction[k]; if (log.isDebug3()) log.debug3(DEBUG_HEADER + "k = " + k + ", difference = " + difference); // Within one per cent is a match. if (Math.abs(difference) > 0.01) { done = false; break; } } } } if (log.isDebug2()) log.debug2(DEBUG_HEADER + "repos = " + repos); return repos; }
From source file:org.apache.lens.cube.parse.StorageCandidate.java
/** * Gets FactPartitions for the given fact using the following logic * * 1. Find the max update interval that will be used for the query. Lets assume time * range is 15 Sep to 15 Dec and the fact has two storage with update periods as MONTHLY,DAILY,HOURLY. * In this case the data for [15 sep - 1 oct)U[1 Dec - 15 Dec) will be answered by DAILY partitions * and [1 oct - 1Dec) will be answered by MONTHLY partitions. The max interavl for this query will be MONTHLY. * * 2.Prune Storgaes that do not fall in the queries time range. * {@link org.apache.lens.cube.metadata.CubeMetastoreClient#isStorageTableCandidateForRange(String, Date, Date)} * * 3. Iterate over max interavl . In out case it will give two months Oct and Nov. Find partitions for * these two months.Check validity of FactPartitions for Oct and Nov * via {@link #updatePartitionStorage(FactPartition)}. * If the partition is missing, try getting partitions for the time range form other update periods (DAILY,HOURLY). * This is achieved by calling getPartitions() recursively but passing only 2 update periods (DAILY,HOURLY) * * 4.If the monthly partitions are found, check for lookahead partitions and call getPartitions recursively for the * remaining time intervals i.e, [15 sep - 1 oct) and [1 Dec - 15 Dec) * * TODO union : Move this into util.//w ww.j a v a 2 s.c om */ private boolean getPartitions(Date fromDate, Date toDate, String partCol, Set<FactPartition> partitions, TreeSet<UpdatePeriod> updatePeriods, boolean addNonExistingParts, boolean failOnPartialData, PartitionRangesForPartitionColumns missingPartitions) throws LensException { if (fromDate.equals(toDate) || fromDate.after(toDate)) { return true; } if (updatePeriods == null || updatePeriods.isEmpty()) { return false; } UpdatePeriod maxInterval = CubeFactTable.maxIntervalInRange(fromDate, toDate, updatePeriods); if (maxInterval == null) { log.info("No max interval for range: {} to {}", fromDate, toDate); return false; } if (maxInterval == UpdatePeriod.CONTINUOUS && cubeQueryContext.getRangeWriter().getClass().equals(BetweenTimeRangeWriter.class)) { FactPartition part = new FactPartition(partCol, fromDate, maxInterval, null, partWhereClauseFormat); partitions.add(part); part.getStorageTables().add(storageTable); part = new FactPartition(partCol, toDate, maxInterval, null, partWhereClauseFormat); partitions.add(part); part.getStorageTables().add(storageTable); this.participatingUpdatePeriods.add(maxInterval); log.info("Added continuous fact partition for storage table {}", storageName); return true; } if (!getCubeMetastoreClient().partColExists(this.getFact(), storageName, partCol)) { log.info("{} does not exist in {}", partCol, name); return false; } Date maxIntervalStorageTblStartDate = getStorageTableStartDate(maxInterval); Date maxIntervalStorageTblEndDate = getStorageTableEndDate(maxInterval); TreeSet<UpdatePeriod> remainingIntervals = new TreeSet<>(updatePeriods); remainingIntervals.remove(maxInterval); if (!isCandidatePartiallyValidForTimeRange(maxIntervalStorageTblStartDate, maxIntervalStorageTblEndDate, fromDate, toDate)) { //Check the time range in remainingIntervals as maxInterval is not useful return getPartitions(fromDate, toDate, partCol, partitions, remainingIntervals, addNonExistingParts, failOnPartialData, missingPartitions); } Date ceilFromDate = DateUtil.getCeilDate( fromDate.after(maxIntervalStorageTblStartDate) ? fromDate : maxIntervalStorageTblStartDate, maxInterval); Date floorToDate = DateUtil.getFloorDate( toDate.before(maxIntervalStorageTblEndDate) ? toDate : maxIntervalStorageTblEndDate, maxInterval); if (ceilFromDate.equals(floorToDate) || floorToDate.before(ceilFromDate)) { return getPartitions(fromDate, toDate, partCol, partitions, remainingIntervals, addNonExistingParts, failOnPartialData, missingPartitions); } int lookAheadNumParts = getConf().getInt(CubeQueryConfUtil.getLookAheadPTPartsKey(maxInterval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_PT_PARTS); TimeRange.Iterable.Iterator iter = TimeRange.iterable(ceilFromDate, floorToDate, maxInterval, 1).iterator(); // add partitions from ceilFrom to floorTo while (iter.hasNext()) { Date dt = iter.next(); Date nextDt = iter.peekNext(); FactPartition part = new FactPartition(partCol, dt, maxInterval, null, partWhereClauseFormat); updatePartitionStorage(part); log.debug("Storage tables containing Partition {} are: {}", part, part.getStorageTables()); if (part.isFound()) { log.debug("Adding existing partition {}", part); partitions.add(part); this.participatingUpdatePeriods.add(maxInterval); log.debug("Looking for look ahead process time partitions for {}", part); if (processTimePartCol == null) { log.debug("processTimePartCol is null"); } else if (partCol.equals(processTimePartCol)) { log.debug("part column is process time col"); } else if (updatePeriods.first().equals(maxInterval)) { log.debug("Update period is the least update period"); } else if ((iter.getNumIters() - iter.getCounter()) > lookAheadNumParts) { // see if this is the part of the last-n look ahead partitions log.debug("Not a look ahead partition"); } else { log.debug("Looking for look ahead process time partitions for {}", part); // check if finer partitions are required // final partitions are required if no partitions from // look-ahead // process time are present TimeRange.Iterable.Iterator processTimeIter = TimeRange .iterable(nextDt, lookAheadNumParts, maxInterval, 1).iterator(); while (processTimeIter.hasNext()) { Date pdt = processTimeIter.next(); Date nextPdt = processTimeIter.peekNext(); FactPartition processTimePartition = new FactPartition(processTimePartCol, pdt, maxInterval, null, partWhereClauseFormat); updatePartitionStorage(processTimePartition); if (processTimePartition.isFound()) { log.debug("Finer parts not required for look-ahead partition :{}", part); } else { log.debug("Looked ahead process time partition {} is not found", processTimePartition); TreeSet<UpdatePeriod> newset = new TreeSet<UpdatePeriod>(); newset.addAll(updatePeriods); newset.remove(maxInterval); log.debug("newset of update periods:{}", newset); if (!newset.isEmpty()) { // Get partitions for look ahead process time log.debug("Looking for process time partitions between {} and {}", pdt, nextPdt); Set<FactPartition> processTimeParts = getPartitions( TimeRange.builder().fromDate(pdt).toDate(nextPdt) .partitionColumn(processTimePartCol).build(), newset, true, failOnPartialData, missingPartitions); log.debug("Look ahead partitions: {}", processTimeParts); TimeRange timeRange = TimeRange.builder().fromDate(dt).toDate(nextDt).build(); for (FactPartition pPart : processTimeParts) { log.debug("Looking for finer partitions in pPart: {}", pPart); for (Date date : timeRange.iterable(pPart.getPeriod(), 1)) { FactPartition innerPart = new FactPartition(partCol, date, pPart.getPeriod(), pPart, partWhereClauseFormat); updatePartitionStorage(innerPart); innerPart.setFound(pPart.isFound()); if (innerPart.isFound()) { partitions.add(innerPart); } } log.debug("added all sub partitions blindly in pPart: {}", pPart); } } } } } } else { log.info("Partition:{} does not exist in any storage table", part); if (!getPartitions(dt, nextDt, partCol, partitions, remainingIntervals, false, failOnPartialData, missingPartitions)) { log.debug("Adding non existing partition {}", part); if (addNonExistingParts) { // Add non existing partitions for all cases of whether we populate all non existing or not. this.participatingUpdatePeriods.add(maxInterval); missingPartitions.add(part); if (!failOnPartialData) { partitions.add(part); part.getStorageTables().add(storageTable); } } else { log.info("No finer granualar partitions exist for {}", part); return false; } } else { log.debug("Finer granualar partitions added for {}", part); } } } return getPartitions(fromDate, ceilFromDate, partCol, partitions, remainingIntervals, addNonExistingParts, failOnPartialData, missingPartitions) && getPartitions(floorToDate, toDate, partCol, partitions, remainingIntervals, addNonExistingParts, failOnPartialData, missingPartitions); }
From source file:org.commoncrawl.service.listcrawler.CrawlHistoryManager.java
private void iterateHDFSCrawlHistoryLog(long listId, long timestamp, TreeSet<URLFP> criteria, ItemUpdater targetList) throws IOException { // ok copy stuff locally if possible ... File localIndexPath = new File(getLocalDataDir(), CRAWL_HISTORY_HDFS_LOGFILE_PREFIX + timestamp + ".index"); File localDataPath = new File(getLocalDataDir(), CRAWL_HISTORY_HDFS_LOGFILE_PREFIX + timestamp + ".data"); File localBloomFilterPath = new File(getLocalDataDir(), CRAWL_HISTORY_HDFS_LOGFILE_PREFIX + timestamp + ".bloom"); SequenceFile.Reader reader = null; Path mapFilePath = new Path(_remoteDataDirectory, CRAWL_HISTORY_HDFS_LOGFILE_PREFIX + timestamp); Path indexFilePath = new Path(mapFilePath, "index"); Path dataFilePath = new Path(mapFilePath, "data"); Path bloomFilePath = new Path(_remoteDataDirectory, CRAWL_HISTORY_HDFS_BLOOMFILTER_PREFIX + timestamp); // ok copy local first if (!localIndexPath.exists()) { LOG.info("LIST:" + listId + " Copying Index File:" + indexFilePath + " to Local:" + localIndexPath.getAbsolutePath()); try {/*from www .ja v a 2s .c o m*/ _remoteFileSystem.copyToLocalFile(indexFilePath, new Path(localIndexPath.getAbsolutePath())); } catch (IOException e) { LOG.error(CCStringUtils.stringifyException(e)); localIndexPath.delete(); throw e; } } if (!localDataPath.exists()) { LOG.info("LIST:" + listId + " Copying Data File:" + dataFilePath + " to Local:" + localDataPath.getAbsolutePath()); try { _remoteFileSystem.copyToLocalFile(dataFilePath, new Path(localDataPath.getAbsolutePath())); } catch (IOException e) { LOG.error(CCStringUtils.stringifyException(e)); localDataPath.delete(); throw e; } } if (!localBloomFilterPath.exists()) { LOG.info("LIST:" + listId + " Copying Bloom File:" + bloomFilePath + " to Local:" + localBloomFilterPath.getAbsolutePath()); try { _remoteFileSystem.copyToLocalFile(bloomFilePath, new Path(localBloomFilterPath.getAbsolutePath())); } catch (IOException e) { LOG.error(CCStringUtils.stringifyException(e)); localBloomFilterPath.delete(); throw e; } } // ok open local FileSystem localFileSystem = FileSystem.getLocal(CrawlEnvironment.getHadoopConfig()); SequenceFile.Reader indexReader = new SequenceFile.Reader(localFileSystem, new Path(localIndexPath.getAbsolutePath()), CrawlEnvironment.getHadoopConfig()); try { URLFP firstIndexKey = null; URLFP lastIndexKey = new URLFP(); LongWritable position = new LongWritable(); while (indexReader.next(lastIndexKey, position)) { if (firstIndexKey == null) { try { firstIndexKey = (URLFP) lastIndexKey.clone(); } catch (CloneNotSupportedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } LOG.info("LIST:" + listId + " ### Index First Domain:" + firstIndexKey.getDomainHash() + " URLHash:" + firstIndexKey.getUrlHash() + " Last Domain:" + lastIndexKey.getDomainHash() + " URLHash:" + lastIndexKey.getUrlHash()); URLFP criteriaFirstKey = criteria.first(); URLFP criteriaLastKey = criteria.last(); if (firstIndexKey.compareTo(criteriaLastKey) > 0 || lastIndexKey.compareTo(criteriaFirstKey) < 0) { LOG.info("LIST:" + listId + " Entire Index is Out of Range. Skipping!"); LOG.info("LIST:" + listId + " ### Criteria First Domain:" + criteriaFirstKey.getDomainHash() + " URLHash:" + criteriaFirstKey.getUrlHash() + " Last Domain:" + criteriaLastKey.getDomainHash() + " URLHash:" + criteriaLastKey.getUrlHash()); return; } } finally { indexReader.close(); } LOG.info("LIST:" + listId + " ### Index:" + timestamp + " Passed Test. Doing Full Scan"); // load bloom filter FSDataInputStream bloomFilterStream = localFileSystem .open(new Path(localBloomFilterPath.getAbsolutePath())); int hitCount = 0; try { URLFPBloomFilter filter = URLFPBloomFilter.load(bloomFilterStream); URLFP fpOut = new URLFP(); ProxyCrawlHistoryItem itemOut = new ProxyCrawlHistoryItem(); DataOutputBuffer valueBytesUncompressed = new DataOutputBuffer(); ValueBytes valueBytes = null; DataInputBuffer valueReader = new DataInputBuffer(); DataOutputBuffer keyBytes = new DataOutputBuffer(); DataInputBuffer keyReader = new DataInputBuffer(); URLFP lastFP = null; outerLoop: // now iterate each item in the criteria for (URLFP targetFP : criteria) { // if fingerprint is present in filter ... if (filter.isPresent(targetFP)) { // check to see if reader is initialzied ... if (reader == null) { LOG.info("LIST:" + listId + " BloomFilter First Hit. Initializing Reader for file at:" + localDataPath.getAbsolutePath()); reader = new SequenceFile.Reader(localFileSystem, new Path(localDataPath.getAbsolutePath()), CrawlEnvironment.getHadoopConfig()); LOG.info("LIST:" + listId + " BloomFilter First Hit. Initialized Reader for file at:" + localDataPath.getAbsolutePath()); valueBytes = reader.createValueBytes(); } // if last read fingerprint was not null ... if (lastFP != null) { // does it match the current item if (lastFP.compareTo(targetFP) == 0) { // decompress value bytes ... valueBytesUncompressed.reset(); valueBytes.writeUncompressedBytes(valueBytesUncompressed); // init valueReader valueReader.reset(valueBytesUncompressed.getData(), valueBytesUncompressed.getLength()); itemOut.readFields(valueReader); LOG.info("LIST:" + listId + " GOT HISTORY ITEM HIT. URL:" + +lastFP.getUrlHash() + " File:" + dataFilePath); // if so, null out last fp lastFP = null; // and update item state ... targetList.updateItemState(targetFP, itemOut); hitCount++; continue; } } // ok at this point .. read the next item in the list ... lastFP = null; while (reader.nextRaw(keyBytes, valueBytes) != -1) { // init reader ... keyReader.reset(keyBytes.getData(), keyBytes.getLength()); // read key fpOut.readFields(keyReader); // reset output buffer keyBytes.reset(); // LOG.info("LIST:" + listId +" nextRaw Returned DH:" + // fpOut.getDomainHash() + " UH:" + fpOut.getUrlHash() + " TDH:" + // targetFP.getDomainHash() + " TUH:" + targetFP.getUrlHash()); // compare it to target ... int result = fpOut.compareTo(targetFP); // ok does it match .. ? if (result == 0) { // decompress value bytes ... valueBytesUncompressed.reset(); valueBytes.writeUncompressedBytes(valueBytesUncompressed); // init valueReader valueReader.reset(valueBytesUncompressed.getData(), valueBytesUncompressed.getLength()); itemOut.readFields(valueReader); LOG.info("LIST:" + listId + " GOT HISTORY ITEM HIT. URL:" + fpOut.getUrlHash() + " File:" + dataFilePath); // update item state ... targetList.updateItemState(targetFP, itemOut); hitCount++; // and break to outer loop continue outerLoop; } else if (result == 1) { // LOG.info("LIST:" + listId + // " FP Comparison Returned 1. Going to OuterLoop"); // update last FP lastFP = fpOut; // continue outer loop continue outerLoop; } else { // otherwise skip } } // ok if we got here .. we are done reading the sequence file and did // not find a trailing match LOG.warn("LIST:" + listId + " ### Reached End Of File Searching for item in MapFile while BloomFilter returned positivie result (DomainHash:" + targetFP.getDomainHash() + "FP:" + targetFP.getUrlHash() + ")"); // break out of outer loop break; } } } finally { bloomFilterStream.close(); if (reader != null) { reader.close(); } LOG.info("LIST:" + listId + " File:" + dataFilePath + " DONE. HitCount:" + hitCount); } }
From source file:org.alfresco.web.forms.xforms.Schema2XForms.java
private Element addElementWithMultipleCompatibleTypes(final Document xformsDocument, Element modelSection, final Element defaultInstanceElement, final Element formSection, final XSModel schema, final XSElementDeclaration elementDecl, final TreeSet<XSTypeDefinition> compatibleTypes, final String pathToRoot, final ResourceBundle resourceBundle, final SchemaUtil.Occurrence occurs) throws FormBuilderException { if (LOGGER.isDebugEnabled()) LOGGER.debug("[addElementWithMultipleCompatibleTypes] adding element " + elementDecl + " at path " + pathToRoot);/*w w w . j ava 2s.c o m*/ // look for compatible types final XSTypeDefinition controlType = elementDecl.getTypeDefinition(); //get possible values final List<XSTypeDefinition> enumValues = new LinkedList<XSTypeDefinition>(); //add the type (if not abstract) if (!((XSComplexTypeDefinition) controlType).getAbstract()) { enumValues.add(controlType); } //add compatible types enumValues.addAll(compatibleTypes); // multiple compatible types for this element exist // in the schema - allow the user to choose from // between compatible non-abstract types boolean isRepeated = isRepeated(occurs, controlType); Element bindElement = this.createBind(xformsDocument, pathToRoot + "/@xsi:type"); String bindId = bindElement.getAttributeNS(null, "id"); modelSection.appendChild(bindElement); this.startBindElement(bindElement, schema, controlType, null, occurs); //add the "element" bind, in addition final Element bindElement2 = this.createBind(xformsDocument, pathToRoot + (isRepeated ? "[position() != last()]" : "")); modelSection.appendChild(bindElement2); this.startBindElement(bindElement2, schema, controlType, null, occurs); // add content to select1 final Map<String, Element> caseTypes = this.addChoicesForSelectSwitchControl(xformsDocument, formSection, enumValues, bindId); //add switch final Element switchElement = xformsDocument.createElementNS(NamespaceConstants.XFORMS_NS, NamespaceConstants.XFORMS_PREFIX + ":switch"); switchElement.setAttributeNS(NamespaceConstants.XFORMS_NS, NamespaceConstants.XFORMS_PREFIX + ":bind", bindId); switchElement.setAttributeNS(NamespaceConstants.XFORMS_NS, NamespaceConstants.XFORMS_PREFIX + ":appearance", "full"); formSection.appendChild(switchElement); if (!((XSComplexTypeDefinition) controlType).getAbstract()) { final Element firstCaseElement = caseTypes.get(controlType.getName()); switchElement.appendChild(firstCaseElement); final Element firstGroupElement = this.addComplexType(xformsDocument, modelSection, defaultInstanceElement, firstCaseElement, schema, (XSComplexTypeDefinition) controlType, elementDecl, pathToRoot, SchemaUtil.getOccurrence(elementDecl), true, false, resourceBundle); firstGroupElement.setAttributeNS(NamespaceConstants.XFORMS_NS, NamespaceConstants.XFORMS_PREFIX + ":appearance", ""); } defaultInstanceElement.setAttributeNS(NamespaceConstants.XMLSCHEMA_INSTANCE_NS, NamespaceConstants.XMLSCHEMA_INSTANCE_PREFIX + ":type", (((XSComplexTypeDefinition) controlType).getAbstract() ? compatibleTypes.first().getName() : controlType.getName())); defaultInstanceElement.setAttributeNS(NamespaceConstants.XMLSCHEMA_INSTANCE_NS, NamespaceConstants.XMLSCHEMA_INSTANCE_PREFIX + ":nil", "true"); /////////////// add sub types ////////////// // add each compatible type within // a case statement for (final XSTypeDefinition type : compatibleTypes) { final String compatibleTypeName = type.getName(); if (LOGGER.isDebugEnabled()) { LOGGER.debug(type == null ? ("[addElementWithMultipleCompatibleTypes] compatible type is null!! type = " + compatibleTypeName + ", targetNamespace = " + this.targetNamespace) : ("[addElementWithMultipleCompatibleTypes] adding compatible type " + type.getName())); } if (type == null || type.getTypeCategory() != XSTypeDefinition.COMPLEX_TYPE) { continue; } final Element caseElement = caseTypes.get(type.getName()); switchElement.appendChild(caseElement); // ALF-9524 fix, add an extra element to the instance for each type that extends the abstract parent Element newDefaultInstanceElement = xformsDocument.createElement(getElementName(type, xformsDocument)); Attr nodesetAttr = modelSection.getAttributeNodeNS(NamespaceConstants.XFORMS_NS, "nodeset"); // construct the nodeset that is used in bind for abstract type String desiredBindNodeset = getElementName(elementDecl, xformsDocument) + (isRepeated ? "[position() != last()]" : ""); // check the current bind if (nodesetAttr == null || !nodesetAttr.getValue().equals(desiredBindNodeset)) { // look for desired bind in children Element newModelSection = DOMUtil.getElementByAttributeValueNS(modelSection, NamespaceConstants.XFORMS_NS, "bind", NamespaceConstants.XFORMS_NS, "nodeset", desiredBindNodeset); if (newModelSection == null) { // look for absolute path desiredBindNodeset = "/" + desiredBindNodeset; newModelSection = DOMUtil.getElementByAttributeValueNS(modelSection, NamespaceConstants.XFORMS_NS, "bind", NamespaceConstants.XFORMS_NS, "nodeset", desiredBindNodeset); } modelSection = newModelSection; } // create the extra bind for each child of abstract type Element bindElement3 = this.createBind(xformsDocument, getElementName(type, xformsDocument)); modelSection.appendChild(bindElement3); bindElement3 = this.startBindElement(bindElement3, schema, controlType, elementDecl, occurs); // add the relevant attribute that checks the value of parent' @xsi:type bindElement3.setAttributeNS(NamespaceConstants.XFORMS_NS, NamespaceConstants.XFORMS_PREFIX + ":relevant", "../@xsi:type='" + type.getName() + "'"); final Element groupElement = this.addComplexType(xformsDocument, modelSection, newDefaultInstanceElement, caseElement, schema, (XSComplexTypeDefinition) type, elementDecl, pathToRoot, SchemaUtil.getOccurrence(elementDecl), true, true, resourceBundle); groupElement.setAttributeNS(NamespaceConstants.XFORMS_NS, NamespaceConstants.XFORMS_PREFIX + ":appearance", ""); defaultInstanceElement.appendChild(newDefaultInstanceElement.cloneNode(true)); // modify bind to add a "relevant" attribute that checks the value of @xsi:type if (LOGGER.isDebugEnabled()) { LOGGER.debug("[addElementWithMultipleCompatibleTypes] Model section =\n" + XMLUtil.toString(bindElement3)); } final NodeList binds = bindElement3.getElementsByTagNameNS(NamespaceConstants.XFORMS_NS, "bind"); for (int i = 0; i < binds.getLength(); i++) { final Element subBind = (Element) binds.item(i); String name = subBind.getAttributeNS(NamespaceConstants.XFORMS_NS, "nodeset"); // ETHREEOH-3308 fix name = repeatableNamePattern.matcher(name).replaceAll(""); if (!subBind.getParentNode().getAttributes().getNamedItem("id").getNodeValue() .equals(bindElement3.getAttribute("id"))) { continue; } if (LOGGER.isDebugEnabled()) { LOGGER.debug("[addElementWithMultipleCompatibleTypes] Testing sub-bind with nodeset " + name); } Pair<String, String> parsed = parseName(name, xformsDocument); if (!SchemaUtil.isElementDeclaredIn(parsed.getFirst(), parsed.getSecond(), (XSComplexTypeDefinition) type, false) && !SchemaUtil.isAttributeDeclaredIn(parsed.getFirst(), parsed.getSecond(), (XSComplexTypeDefinition) type, false)) { continue; } if (LOGGER.isDebugEnabled()) { LOGGER.debug("[addElementWithMultipleCompatibleTypes] Element/Attribute " + name + " declared in type " + type.getName() + ": adding relevant attribute"); } //test sub types of this type //TreeSet subCompatibleTypes = (TreeSet) typeTree.get(type); String newRelevant = "../../@xsi:type='" + type.getName() + "'"; if (this.typeTree.containsKey(type.getName())) { for (XSTypeDefinition otherType : this.typeTree.get(type.getName())) { newRelevant = newRelevant + " or ../../@xsi:type='" + otherType.getName() + "'"; } } //change relevant attribute final String relevant = subBind.getAttributeNS(NamespaceConstants.XFORMS_NS, "relevant"); if (relevant != null && relevant.length() != 0) { newRelevant = ("(" + relevant + ") and " + newRelevant); } subBind.setAttributeNS(NamespaceConstants.XFORMS_NS, NamespaceConstants.XFORMS_PREFIX + ":relevant", newRelevant); } } return switchElement; }
From source file:net.spfbl.spf.SPF.java
/** * Processa a consulta e retorna o resultado. * * @param query a expresso da consulta.// w w w. j a v a2 s . c om * @return o resultado do processamento. */ protected static String processSPF(InetAddress ipAddress, Client client, User user, String query, LinkedList<User> userList) { try { String result = ""; if (query.length() == 0) { return "INVALID QUERY\n"; } else { String origin; if (client == null) { origin = ipAddress.getHostAddress(); } else if (client.hasEmail()) { origin = ipAddress.getHostAddress() + " " + client.getDomain() + " " + client.getEmail(); } else { origin = ipAddress.getHostAddress() + " " + client.getDomain(); } StringTokenizer tokenizer = new StringTokenizer(query, " "); String firstToken = tokenizer.nextToken(); if (firstToken.equals("SPAM") && tokenizer.countTokens() == 1) { String ticket = tokenizer.nextToken(); TreeSet<String> tokenSet = addComplainURLSafe(origin, ticket, null); if (tokenSet == null) { result = "DUPLICATE COMPLAIN\n"; } else { String userEmail; try { userEmail = SPF.getClientURLSafe(ticket); } catch (Exception ex) { userEmail = client == null ? null : client.getEmail(); } user = User.get(userEmail); if (user != null) { userList.add(user); } String recipient; try { recipient = SPF.getRecipientURLSafe(ticket); } catch (ProcessException ex) { recipient = null; } result = "OK " + tokenSet + (recipient == null ? "" : " >" + recipient) + "\n"; } } else if (firstToken.equals("ABUSE") && tokenizer.hasMoreTokens()) { String token = tokenizer.nextToken(); if (token.startsWith("In-Reply-To:") && tokenizer.countTokens() == 1) { token = tokenizer.nextToken(); if (token.startsWith("From:")) { int index = token.indexOf(':') + 1; String recipient = token.substring(index); User recipientUser = User.get(recipient); if (recipientUser == null) { // Se a consulta originar de destinatrio com postmaster cadastrado, // considerar o prprio postmaster como usurio da consulta. index = recipient.indexOf('@'); String postmaster = "postmaster" + recipient.substring(index); User postmasterUser = User.get(postmaster); if (postmasterUser != null) { userList.add(user = postmasterUser); } } else { userList.add(user = recipientUser); } index = query.indexOf(':') + 1; String messageID = query.substring(index); result = "INVALID ID\n"; index = messageID.indexOf('<'); if (index >= 0) { messageID = messageID.substring(index + 1); index = messageID.indexOf('>'); if (index > 0) { messageID = messageID.substring(0, index); result = user.blockByMessageID(messageID) + '\n'; } } } else { result = "INVALID FROM\n"; } } else { result = "INVALID COMMAND\n"; } } else if (firstToken.equals("HOLDING") && tokenizer.countTokens() == 1) { String ticket = tokenizer.nextToken(); result = getHoldStatus(client, ticket, userList) + '\n'; } else if (firstToken.equals("LINK") && tokenizer.hasMoreTokens()) { String ticketSet = tokenizer.nextToken(); TreeSet<String> linkSet = new TreeSet<String>(); while (tokenizer.hasMoreTokens()) { linkSet.add(tokenizer.nextToken()); } StringTokenizer tokenizerTicket = new StringTokenizer(ticketSet, ";"); String unblockURL = null; boolean blocked = false; Action action = null; while (tokenizerTicket.hasMoreTokens()) { String ticket = tokenizerTicket.nextToken(); String userEmail; try { userEmail = SPF.getClientURLSafe(ticket); } catch (Exception ex) { userEmail = client == null ? null : client.getEmail(); } if ((user = User.get(userEmail)) != null) { userList.add(user); long dateTicket = SPF.getDateTicket(ticket); User.Query queryTicket = user.getQuery(dateTicket); if (queryTicket != null) { if (queryTicket.setLinkSet(linkSet)) { SPF.setSpam(dateTicket, queryTicket.getTokenSet()); if (!queryTicket.isWhite() && queryTicket.blockSender(dateTicket)) { Server.logDebug( "new BLOCK '" + queryTicket.getBlockSender() + "' added by LINK."); } action = client == null ? Action.REJECT : client.getActionBLOCK(); unblockURL = queryTicket.getUnblockURL(); blocked = true; } else if (queryTicket.isAnyLinkRED()) { action = client == null ? Action.FLAG : client.getActionRED(); } if (action == Action.HOLD) { queryTicket.setResult("HOLD"); } else if (action == Action.FLAG) { queryTicket.setResult("FLAG"); } else if (action == Action.REJECT) { queryTicket.setResult("REJECT"); } User.storeDB(dateTicket, queryTicket); } } } if (unblockURL != null) { result = "BLOCKED " + unblockURL + "\n"; } else if (blocked) { result = "BLOCKED\n"; } else if (action == Action.HOLD) { result = "HOLD\n"; } else if (action == Action.FLAG) { result = "FLAG\n"; } else if (action == Action.REJECT) { result = "REJECT\n"; } else { result = "CLEAR\n"; } } else if (firstToken.equals("MALWARE") && tokenizer.hasMoreTokens()) { String ticketSet = tokenizer.nextToken(); StringBuilder nameBuilder = new StringBuilder(); while (tokenizer.hasMoreTokens()) { if (nameBuilder.length() > 0) { nameBuilder.append(' '); } nameBuilder.append(tokenizer.nextToken()); } StringBuilder resultBuilder = new StringBuilder(); StringTokenizer ticketTokenizer = new StringTokenizer(ticketSet, ";"); while (ticketTokenizer.hasMoreTokens()) { String ticket = ticketTokenizer.nextToken(); TreeSet<String> tokenSet = addComplainURLSafe(origin, ticket, "MALWARE"); if (tokenSet == null) { resultBuilder.append("DUPLICATE COMPLAIN\n"); } else { // Processar reclamao. String userEmail; try { userEmail = SPF.getClientURLSafe(ticket); } catch (Exception ex) { userEmail = client == null ? null : client.getEmail(); } user = User.get(userEmail); if (user != null) { userList.add(user); long dateTicket = getDateTicket(ticket); User.Query userQuery = user.getQuery(dateTicket); if (userQuery != null && userQuery.setMalware(nameBuilder.toString())) { User.storeDB(dateTicket, userQuery); } } String recipient; try { recipient = SPF.getRecipientURLSafe(ticket); } catch (ProcessException ex) { recipient = null; } // Bloquear automaticamente todos // os tokens com reputao amarela ou vermelha. // Processar reclamao. for (String token : tokenSet) { String block; Status status = SPF.getStatus(token); if (status == Status.RED && (block = Block.add(token)) != null) { Server.logDebug( "new BLOCK '" + block + "' added by '" + recipient + ";MALWARE'."); Peer.sendBlockToAll(block); } if (status != Status.GREEN && !Subnet.isValidIP(token) && (block = Block.addIfNotNull(user, token)) != null) { Server.logDebug( "new BLOCK '" + block + "' added by '" + recipient + ";MALWARE'."); } } resultBuilder.append("OK "); resultBuilder.append(tokenSet); resultBuilder.append(recipient == null ? "" : " >" + recipient); resultBuilder.append("\n"); } } result = resultBuilder.toString(); } else if (firstToken.equals("HEADER") && tokenizer.hasMoreTokens()) { String ticketSet = tokenizer.nextToken(); String key = null; String from = null; String replyto = null; String messageID = null; String unsubscribe = null; String subject = null; while (tokenizer.hasMoreTokens()) { String token = tokenizer.nextToken(); if (token.startsWith("From:")) { key = "From"; int index = token.indexOf(':'); from = token.substring(index + 1); } else if (token.startsWith("ReplyTo:") || token.startsWith("Reply-To:")) { key = "Reply-To"; int index = token.indexOf(':'); replyto = token.substring(index + 1); } else if (token.startsWith("Message-ID:")) { key = "Message-ID"; int index = token.indexOf(':'); messageID = token.substring(index + 1); } else if (token.startsWith("List-Unsubscribe:")) { key = "List-Unsubscribe"; int index = token.indexOf(':'); unsubscribe = token.substring(index + 1); } else if (token.startsWith("Subject:")) { key = "Subject"; int index = token.indexOf(':'); subject = token.substring(index + 1); } else if (key == null) { from = null; replyto = null; unsubscribe = null; subject = null; break; } else if (key.equals("From")) { from += ' ' + token; } else if (key.equals("Reply-To")) { replyto += ' ' + token; } else if (key.equals("Message-ID")) { messageID += ' ' + token; } else if (key.equals("List-Unsubscribe")) { unsubscribe += ' ' + token; } else if (key.equals("Subject")) { subject += ' ' + token; } } if ((from == null || from.length() == 0) && (replyto == null || replyto.length() == 0) && (messageID == null || messageID.length() == 0) && (unsubscribe == null || unsubscribe.length() == 0) && (subject == null || subject.length() == 0)) { result = "INVALID COMMAND\n"; } else { boolean whitelisted = false; boolean blocklisted = false; TreeSet<String> unblockURLSet = new TreeSet<String>(); StringTokenizer ticketRokenizer = new StringTokenizer(ticketSet, ";"); int n = ticketRokenizer.countTokens(); ArrayList<User.Query> queryList = new ArrayList<User.Query>(n); while (ticketRokenizer.hasMoreTokens()) { String ticket = ticketRokenizer.nextToken(); String userEmail; try { userEmail = SPF.getClientURLSafe(ticket); } catch (Exception ex) { userEmail = client == null ? null : client.getEmail(); } if ((user = User.get(userEmail)) != null) { userList.add(user); long dateTicket = SPF.getDateTicket(ticket); User.Query queryTicket = user.getQuery(dateTicket); if (queryTicket != null) { queryList.add(queryTicket); String resultLocal = queryTicket.setHeader(from, replyto, subject, messageID, unsubscribe); if ("WHITE".equals(resultLocal)) { whitelisted = true; } else if ("BLOCK".equals(resultLocal)) { blocklisted = true; String url = queryTicket.getUnblockURL(); if (url != null) { unblockURLSet.add(url); } } User.storeDB(dateTicket, queryTicket); } } } if (whitelisted) { for (User.Query queryTicket : queryList) { queryTicket.setResult("WHITE"); } result = "WHITE\n"; } else if (blocklisted) { for (User.Query queryTicket : queryList) { queryTicket.setResult("BLOCK"); } if (unblockURLSet.size() == 1) { result = "BLOCKED " + unblockURLSet.first() + "\n"; } else { result = "BLOCKED\n"; } } else { result = "CLEAR\n"; } } } else if (firstToken.equals("HAM") && tokenizer.countTokens() == 1) { String ticket = tokenizer.nextToken(); TreeSet<String> tokenSet = deleteComplainURLSafe(origin, ticket); if (tokenSet == null) { result = "ALREADY REMOVED\n"; } else { String recipient; try { recipient = SPF.getRecipientURLSafe(ticket); } catch (ProcessException ex) { recipient = null; } result = "OK " + tokenSet + (recipient == null ? "" : " >" + recipient) + "\n"; } } else if (firstToken.equals("REFRESH") && tokenizer.countTokens() == 1) { String address = tokenizer.nextToken(); try { if (CacheSPF.refresh(address, true)) { result = "UPDATED\n"; } else { result = "NOT LOADED\n"; } } catch (ProcessException ex) { result = ex.getMessage() + "\n"; } } else if ((firstToken.equals("SPF") && tokenizer.countTokens() >= 4) || tokenizer.countTokens() == 2 || tokenizer.countTokens() == 1 || (firstToken.equals("CHECK") && tokenizer.countTokens() == 4) || (firstToken.equals("CHECK") && tokenizer.countTokens() == 3) || (firstToken.equals("CHECK") && tokenizer.countTokens() == 2)) { try { String ip; String sender; String helo; String recipient; String origem; String fluxo; if (firstToken.equals("SPF")) { // Nova formatao de consulta. ip = tokenizer.nextToken(); sender = tokenizer.nextToken(); while (!sender.endsWith("'") && tokenizer.hasMoreTokens()) { sender += " " + tokenizer.nextToken(); } helo = tokenizer.hasMoreTokens() ? tokenizer.nextToken() : "''"; recipient = tokenizer.hasMoreTokens() ? tokenizer.nextToken() : "''"; ip = ip.substring(1, ip.length() - 1); sender = sender.substring(1, sender.length() - 1); helo = helo.substring(1, helo.length() - 1); if (recipient.equals("'")) { recipient = tokenizer.hasMoreTokens() ? tokenizer.nextToken() : ""; if (recipient.endsWith("'")) { recipient = recipient.substring(0, recipient.length() - 1); } } else { recipient = recipient.substring(1, recipient.length() - 1); } if (sender.length() == 0) { sender = null; } else { sender = sender.toLowerCase(); } recipient = recipient.toLowerCase(); recipient = recipient.replace("\"", ""); } else if (firstToken.equals("CHECK") && tokenizer.countTokens() == 4) { ip = tokenizer.nextToken().toLowerCase(); sender = tokenizer.nextToken().toLowerCase(); helo = tokenizer.nextToken(); recipient = tokenizer.nextToken().toLowerCase(); if (ip.startsWith("'") && ip.endsWith("'")) { ip = ip.substring(1, ip.length() - 1); } if (sender.startsWith("'") && sender.endsWith("'")) { sender = sender.substring(1, sender.length() - 1); } if (helo.startsWith("'") && helo.endsWith("'")) { helo = helo.substring(1, helo.length() - 1); } if (recipient.startsWith("'") && recipient.endsWith("'")) { recipient = recipient.substring(1, recipient.length() - 1); } if (ip.length() == 0) { ip = null; } if (sender.length() == 0) { sender = null; } if (!Domain.isHostname(helo)) { helo = null; } if (recipient.length() == 0) { recipient = null; } else { recipient = recipient.toLowerCase(); } } else { // Manter compatibilidade da verso antiga. // Verso obsoleta. if (firstToken.equals("CHECK")) { ip = tokenizer.nextToken(); } else { ip = firstToken; } if (tokenizer.countTokens() == 2) { sender = tokenizer.nextToken().toLowerCase(); helo = tokenizer.nextToken(); } else { sender = null; helo = tokenizer.nextToken(); } recipient = null; if (ip.startsWith("'") && ip.endsWith("'")) { ip = ip.substring(1, ip.length() - 1); } if (sender != null && sender.startsWith("'") && sender.endsWith("'")) { sender = sender.substring(1, sender.length() - 1); if (sender.length() == 0) { sender = null; } } if (helo.startsWith("'") && helo.endsWith("'")) { helo = helo.substring(1, helo.length() - 1); } } if (!Subnet.isValidIP(ip)) { return "INVALID\n"; } else if (sender != null && !Domain.isEmail(sender)) { return "INVALID\n"; } else if (recipient != null && !Domain.isEmail(recipient)) { return "INVALID\n"; } else if (Subnet.isReservedIP(ip)) { // Message from LAN. return "LAN\n"; } else if (client != null && client.containsFull(ip)) { // Message from LAN. return "LAN\n"; } else { TreeSet<String> tokenSet = new TreeSet<String>(); ip = Subnet.normalizeIP(ip); tokenSet.add(ip); if (Domain.isValidEmail(recipient)) { // Se houver um remetente vlido, // Adicionar no ticket para controle. tokenSet.add('>' + recipient); } if (recipient != null) { User recipientUser = User.get(recipient); if (recipientUser == null) { // Se a consulta originar de destinatrio com postmaster cadastrado, // considerar o prprio postmaster como usurio da consulta. int index = recipient.indexOf('@'); String postmaster = "postmaster" + recipient.substring(index); User postmasterUser = User.get(postmaster); if (postmasterUser != null) { user = postmasterUser; } } else { user = recipientUser; } } if (user != null) { userList.add(user); tokenSet.add(user.getEmail() + ':'); } else if (client != null && client.hasEmail()) { tokenSet.add(client.getEmail() + ':'); } // Passar a acompanhar todos os // HELO quando apontados para o IP para // uma nova forma de interpretar dados. String hostname; if (CacheHELO.match(ip, helo, false)) { hostname = Domain.normalizeHostname(helo, true); } else { hostname = Reverse.getHostname(ip); hostname = Domain.normalizeHostname(hostname, true); } if (hostname == null) { Server.logDebug("no rDNS for " + ip + "."); } else if (Domain.isOfficialTLD(hostname)) { return "INVALID\n"; } else { // Verificao de pilha dupla, // para pontuao em ambas pilhas. String ipv4 = CacheHELO.getUniqueIPv4(hostname); String ipv6 = CacheHELO.getUniqueIPv6(hostname); if (ip.equals(ipv6) && CacheHELO.match(ipv4, hostname, false)) { // Equivalncia de pilha dupla se // IPv4 for nico para o hostname. tokenSet.add(ipv4); } else if (ip.equals(ipv4) && CacheHELO.match(ipv6, hostname, false)) { // Equivalncia de pilha dupla se // IPv6 for nico para o hostname. tokenSet.add(ipv6); } } if (Generic.containsGenericSoft(hostname)) { // Quando o reverso for // genrico, no consider-lo. hostname = null; } else if (hostname != null) { tokenSet.add(hostname); } LinkedList<String> logList = null; if (sender != null && firstToken.equals("CHECK")) { int index = sender.lastIndexOf('@'); String domain = sender.substring(index + 1); logList = new LinkedList<String>(); try { CacheSPF.refresh(domain, false); } catch (ProcessException ex) { logList.add("Cannot refresh SPF registry: " + ex.getErrorMessage()); logList.add("Using cached SPF registry."); } } SPF spf; if (sender == null) { spf = null; result = "NONE"; } else if (Domain.isOfficialTLD(sender)) { spf = null; result = "NONE"; } else if (Generic.containsGeneric(sender)) { spf = null; result = "NONE"; } else if ((spf = CacheSPF.get(sender)) == null) { result = "NONE"; } else if (spf.isInexistent()) { result = "NONE"; } else { result = spf.getResult(ip, sender, helo, logList); } String mx = Domain.extractHost(sender, true); if (user != null && user.isLocal()) { // Message from local user. return "LAN\n"; } else if (recipient != null && result.equals("PASS")) { if (recipient.endsWith(mx)) { // Message from same domain. return "LAN\n"; } else if (recipient.equals(Core.getAbuseEmail()) && User.exists(sender, "postmaster" + mx)) { // Message to abuse. return "LAN\n"; } } if (result.equals("PASS") || (sender != null && Provider.containsHELO(ip, hostname))) { // Quando fo PASS, significa que o domnio // autorizou envio pelo IP, portanto o dono dele // responsavel pelas mensagens. if (!Provider.containsExact(mx)) { // No um provedor ento // o MX deve ser listado. tokenSet.add(mx); origem = mx; } else if (Domain.isValidEmail(sender)) { // Listar apenas o remetente se o // hostname for um provedor de e-mail. String userEmail = null; String recipientEmail = null; for (String token : tokenSet) { if (token.endsWith(":")) { userEmail = token; } else if (token.startsWith(">")) { recipientEmail = token; } } tokenSet.clear(); tokenSet.add(sender); if (userEmail != null) { tokenSet.add(userEmail); } if (recipientEmail != null) { tokenSet.add(recipientEmail); } origem = sender; } else { origem = sender; } fluxo = origem + ">" + recipient; } else if (hostname == null) { origem = (sender == null ? "" : sender + '>') + ip; fluxo = origem + ">" + recipient; } else { String dominio = Domain.extractDomain(hostname, true); origem = (sender == null ? "" : sender + '>') + (dominio == null ? hostname : dominio.substring(1)); fluxo = origem + ">" + recipient; } Long recipientTrapTime = Trap.getTimeRecipient(client, user, recipient); if (firstToken.equals("CHECK")) { String results = "\nSPF resolution results:\n"; if (spf != null && spf.isInexistent()) { results += " NXDOMAIN\n"; } else if (logList == null || logList.isEmpty()) { results += " NONE\n"; } else { for (String log : logList) { results += " " + log + "\n"; } } String white; String block; if ((white = White.find(client, user, ip, sender, hostname, result, recipient)) != null) { results += "\nFirst WHITE match: " + white + "\n"; } else if ((block = Block.find(client, user, ip, sender, hostname, result, recipient, false, true, true, false)) != null) { results += "\nFirst BLOCK match: " + block + "\n"; } TreeSet<String> graceSet = new TreeSet<String>(); if (Domain.isGraceTime(sender)) { graceSet.add(Domain.extractDomain(sender, false)); } if (Domain.isGraceTime(hostname)) { graceSet.add(Domain.extractDomain(hostname, false)); } if (!graceSet.isEmpty()) { results += "\n"; results += "Domains in grace time:\n"; for (String grace : graceSet) { results += " " + grace + "\n"; } } results += "\n"; results += "Considered identifiers and status:\n"; tokenSet = expandTokenSet(tokenSet); TreeMap<String, Distribution> distributionMap = CacheDistribution.getMap(tokenSet); int count = 0; for (String token : tokenSet) { if (!token.startsWith(">") && !token.endsWith(":")) { if (!Ignore.contains(token)) { float probability; Status status; if (distributionMap.containsKey(token)) { Distribution distribution = distributionMap.get(token); probability = distribution.getSpamProbability(token); status = distribution.getStatus(token); } else { probability = 0.0f; status = SPF.Status.GREEN; } results += " " + token + " " + status.name() + " " + Core.DECIMAL_FORMAT.format(probability) + "\n"; count++; } } } if (count == 0) { results += " NONE\n"; } results += "\n"; return results; } else if (recipientTrapTime == null && White.contains(client, user, ip, sender, hostname, result, recipient)) { if (White.contains(client, user, ip, sender, hostname, result, null)) { // Limpa da lista BLOCK um possvel falso positivo. Block.clear(client, user, ip, sender, hostname, result, null); } // Calcula frequencia de consultas. String url = Core.getURL(); String ticket = SPF.addQueryHam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "WHITE"); return "WHITE " + (url == null ? ticket : url + ticket) + "\n"; } else if (Block.contains(client, user, ip, sender, hostname, result, recipient, true, true, true, true)) { Action action = client == null ? Action.REJECT : client.getActionBLOCK(); if (action == Action.REJECT) { // Calcula frequencia de consultas. long time = Server.getNewUniqueTime(); User.Query queryLocal = SPF.addQuerySpam(time, client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "BLOCK"); action = client == null ? Action.FLAG : client.getActionRED(); if (action != Action.REJECT && queryLocal != null && queryLocal.needHeader()) { if (action == Action.FLAG) { queryLocal.setResult("FLAG"); String url = Core.getURL(); String ticket = SPF.createTicket(time, tokenSet); return "FLAG " + (url == null ? ticket : url + ticket) + "\n"; } else if (action == Action.HOLD) { queryLocal.setResult("HOLD"); String url = Core.getURL(); String ticket = SPF.createTicket(time, tokenSet); return "HOLD " + (url == null ? ticket : url + ticket) + "\n"; } else { return "ERROR: UNDEFINED ACTION\n"; } } else { String url = Core.getUnblockURL(client, user, ip, sender, hostname, recipient); if (url == null) { return "BLOCKED\n"; } else { return "BLOCKED " + url + "\n"; } } } else if (action == Action.FLAG) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "FLAG"); return "FLAG " + (url == null ? ticket : url + ticket) + "\n"; } else if (action == Action.HOLD) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "HOLD"); return "HOLD " + (url == null ? ticket : url + ticket) + "\n"; } else { return "ERROR: UNDEFINED ACTION\n"; } } else if (Generic.containsDynamicDomain(hostname)) { // Bloquear automaticamente range de IP dinmico. String cidr = Subnet .normalizeCIDR(SubnetIPv4.isValidIPv4(ip) ? ip + "/24" : ip + "/48"); if (Block.tryOverlap(cidr)) { Server.logDebug( "new BLOCK '" + cidr + "' added by '" + hostname + ";DYNAMIC'."); } else if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by '" + hostname + ";DYNAMIC'."); } SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); return "INVALID\n"; } else if (spf != null && spf.isDefinitelyInexistent()) { // Bloquear automaticamente IP com reputao vermelha. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by '" + mx + ";NXDOMAIN'."); } } Analise.processToday(ip); // O domnio foi dado como inexistente inmeras vezes. // Rejeitar e denunciar o host pois h abuso de tentativas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "NXDOMAIN"); return "NXDOMAIN\n"; } else if (spf != null && spf.isInexistent()) { Analise.processToday(ip); SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "NXDOMAIN"); return "NXDOMAIN\n"; } else if (result.equals("FAIL")) { // Bloquear automaticamente IP com reputao vermelha. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by '" + sender + ";FAIL'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "FAIL"); // Retornar FAIL somente se no houver // liberao literal do remetente com FAIL. return "FAIL\n"; } else if (sender != null && !Domain.isEmail(sender)) { // Bloquear automaticamente IP com reputao vermelha. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug( "new BLOCK '" + ip + "' added by '" + sender + ";INVALID'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); return "INVALID\n"; } else if (sender != null && Domain.isOfficialTLD(sender)) { // Bloquear automaticamente IP com reputao vermelha. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug( "new BLOCK '" + ip + "' added by '" + sender + ";RESERVED'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); return "INVALID\n"; } else if (sender == null && !CacheHELO.match(ip, hostname, false)) { // Bloquear automaticamente IP com reputao ruim. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by 'INVALID'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); // HELO invlido sem remetente. return "INVALID\n"; } else if (hostname == null && Core.isReverseRequired()) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by 'NONE'."); } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); // Require a valid HELO or reverse. return "INVALID\n"; } else if (recipient != null && !Domain.isValidEmail(recipient)) { Analise.processToday(ip); Analise.processToday(mx); SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INEXISTENT"); return "INEXISTENT\n"; } else if (recipientTrapTime != null) { if (System.currentTimeMillis() > recipientTrapTime) { // Spamtrap for (String token : tokenSet) { String block; Status status = SPF.getStatus(token); if (status == Status.RED && (block = Block.add(token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + recipient + ";SPAMTRAP'."); Peer.sendBlockToAll(block); } if (status != Status.GREEN && !Subnet.isValidIP(token) && (block = Block.addIfNotNull(user, token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + recipient + ";SPAMTRAP'."); } } Analise.processToday(ip); Analise.processToday(mx); // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "TRAP"); return "SPAMTRAP\n"; } else { // Inexistent for (String token : tokenSet) { String block; Status status = SPF.getStatus(token); if (status == Status.RED && (block = Block.add(token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + recipient + ";INEXISTENT'."); Peer.sendBlockToAll(block); } if (status != Status.GREEN && !Subnet.isValidIP(token) && (block = Block.addIfNotNull(user, token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + recipient + ";INEXISTENT'."); } } Analise.processToday(ip); Analise.processToday(mx); SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INEXISTENT"); return "INEXISTENT\n"; } } else if (Defer.count(fluxo) > Core.getFloodMaxRetry()) { Analise.processToday(ip); Analise.processToday(mx); // A origem atingiu o limite de atraso // para liberao do destinatrio. long time = System.currentTimeMillis(); Defer.end(fluxo); Server.logDefer(time, fluxo, "DEFER FLOOD"); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "REJECT"); return "BLOCKED\n"; } else if (!result.equals("PASS") && !CacheHELO.match(ip, hostname, false)) { // Bloquear automaticamente IP com reputao amarela. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug( "new BLOCK '" + ip + "' added by '" + recipient + ";INVALID'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); return "INVALID\n"; } else if (recipient != null && recipient.startsWith("postmaster@")) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "ACCEPT"); return result + " " + (url == null ? ticket : url + URLEncoder.encode(ticket, "UTF-8")) + "\n"; } else if (result.equals("PASS") && SPF.isGood(Provider.containsExact(mx) ? sender : mx)) { // O remetente vlido e tem excelente reputao, // ainda que o provedor dele esteja com reputao ruim. String url = Core.getURL(); String ticket = SPF.addQueryHam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "ACCEPT"); return "PASS " + (url == null ? ticket : url + URLEncoder.encode(ticket, "UTF-8")) + "\n"; } else if (SPF.hasRed(tokenSet) || Analise.isCusterRED(ip, sender, hostname)) { Analise.processToday(ip); Analise.processToday(mx); Action action = client == null ? Action.REJECT : client.getActionRED(); if (action == Action.REJECT) { // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "REJECT"); return "BLOCKED\n"; } else if (action == Action.DEFER) { if (Defer.defer(fluxo, Core.getDeferTimeRED())) { String url = Core.getReleaseURL(fluxo); SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "LISTED"); if (url == null || Defer.count(fluxo) > 1) { return "LISTED\n"; } else if (result.equals("PASS") && enviarLiberacao(url, sender, recipient)) { // Envio da liberao por e-mail se // houver validao do remetente por PASS. return "LISTED\n"; } else { return "LISTED " + url + "\n"; } } else { // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "REJECT"); return "BLOCKED\n"; } } else if (action == Action.FLAG) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "FLAG"); return "FLAG " + (url == null ? ticket : url + ticket) + "\n"; } else if (action == Action.HOLD) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "HOLD"); return "HOLD " + (url == null ? ticket : url + ticket) + "\n"; } else { return "ERROR: UNDEFINED ACTION\n"; } } else if (Domain.isGraceTime(sender) || Domain.isGraceTime(hostname)) { Server.logTrace("domain in grace time."); for (String token : tokenSet) { String block; Status status = SPF.getStatus(token); if (status == Status.RED && (block = Block.add(token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + status + "'."); Peer.sendBlockToAll(block); } if (status != Status.GREEN && !Subnet.isValidIP(token) && (block = Block.addIfNotNull(user, token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + status + "'."); } } Analise.processToday(ip); Analise.processToday(mx); Action action = client == null ? Action.REJECT : client.getActionGRACE(); if (action == Action.REJECT) { // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "REJECT"); return "BLOCKED\n"; } else if (action == Action.DEFER) { if (Defer.defer(fluxo, Core.getDeferTimeRED())) { String url = Core.getReleaseURL(fluxo); SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "LISTED"); if (url == null || Defer.count(fluxo) > 1) { return "LISTED\n"; } else if (result.equals("PASS") && enviarLiberacao(url, sender, recipient)) { // Envio da liberao por e-mail se // houver validao do remetente por PASS. return "LISTED\n"; } else { return "LISTED " + url + "\n"; } } else { // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "REJECT"); return "BLOCKED\n"; } } else if (action == Action.FLAG) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "FLAG"); return "FLAG " + (url == null ? ticket : url + ticket) + "\n"; } else if (action == Action.HOLD) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "HOLD"); return "HOLD " + (url == null ? ticket : url + ticket) + "\n"; } else { return "ERROR: UNDEFINED ACTION\n"; } } else if (SPF.hasYellow(tokenSet) && Defer.defer(fluxo, Core.getDeferTimeYELLOW())) { Analise.processToday(ip); Analise.processToday(mx); Action action = client == null ? Action.DEFER : client.getActionYELLOW(); if (action == Action.DEFER) { // Pelo menos um identificador do conjunto est em greylisting com atrazo de 10min. SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "GREYLIST"); return "GREYLIST\n"; } else if (action == Action.HOLD) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "HOLD"); return "HOLD " + (url == null ? ticket : url + ticket) + "\n"; } else { return "ERROR: UNDEFINED ACTION\n"; } } else if (SPF.isFlood(tokenSet) && !Provider.containsHELO(ip, hostname) && Defer.defer(origem, Core.getDeferTimeFLOOD())) { Analise.processToday(ip); Analise.processToday(mx); // Pelo menos um identificador est com frequncia superior ao permitido. Server.logDebug("FLOOD " + tokenSet); SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "GREYLIST"); return "GREYLIST\n"; } else if (result.equals("SOFTFAIL") && !Provider.containsHELO(ip, hostname) && Defer.defer(fluxo, Core.getDeferTimeSOFTFAIL())) { Analise.processToday(ip); Analise.processToday(mx); // SOFTFAIL com atrazo de 1min. SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "GREYLIST"); return "GREYLIST\n"; } else { Analise.processToday(ip); Analise.processToday(mx); // Calcula frequencia de consultas. String url = Core.getURL(); String ticket = SPF.addQueryHam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "ACCEPT"); return result + " " + (url == null ? ticket : url + URLEncoder.encode(ticket, "UTF-8")) + "\n"; } } } catch (ProcessException ex) { if (ex.isErrorMessage("HOST NOT FOUND")) { return "NXDOMAIN\n"; } else { throw ex; } } } else { return "INVALID QUERY\n"; } } return result; } catch (ProcessException ex) { Server.logError(ex); return ex.getMessage() + "\n"; } catch (Exception ex) { Server.logError(ex); return "ERROR: FATAL\n"; } }
From source file:com.jhh.hdb.sqlparser.MySemanticAnalyzer.java
private List<List<String>> getCommonGroupByDestGroups(QB qb, Map<String, Operator<? extends OperatorDesc>> inputs) throws SemanticException { QBParseInfo qbp = qb.getParseInfo(); TreeSet<String> ks = new TreeSet<String>(); ks.addAll(qbp.getClauseNames());/*w w w. j a v a2s. com*/ List<List<String>> commonGroupByDestGroups = new ArrayList<List<String>>(); // If this is a trivial query block return if (ks.size() <= 1) { List<String> oneList = new ArrayList<String>(1); if (ks.size() == 1) { oneList.add(ks.first()); } commonGroupByDestGroups.add(oneList); return commonGroupByDestGroups; } List<Operator<? extends OperatorDesc>> inputOperators = new ArrayList<Operator<? extends OperatorDesc>>( ks.size()); List<List<ExprNodeDesc>> sprayKeyLists = new ArrayList<List<ExprNodeDesc>>(ks.size()); List<List<ExprNodeDesc>> distinctKeyLists = new ArrayList<List<ExprNodeDesc>>(ks.size()); // Iterate over each clause for (String dest : ks) { Operator input = inputs.get(dest); RowResolver inputRR = opParseCtx.get(input).getRowResolver(); List<ExprNodeDesc> distinctKeys = getDistinctExprs(qbp, dest, inputRR); List<ExprNodeDesc> sprayKeys = new ArrayList<ExprNodeDesc>(); // Add the group by expressions List<ASTNode> grpByExprs = getGroupByForClause(qbp, dest); for (ASTNode grpByExpr : grpByExprs) { ExprNodeDesc exprDesc = genExprNodeDesc(grpByExpr, inputRR); if (ExprNodeDescUtils.indexOf(exprDesc, sprayKeys) < 0) { sprayKeys.add(exprDesc); } } // Loop through each of the lists of exprs, looking for a match boolean found = false; for (int i = 0; i < sprayKeyLists.size(); i++) { if (!input.equals(inputOperators.get(i))) { continue; } if (distinctKeys.isEmpty()) { // current dest has no distinct keys. List<ExprNodeDesc> combinedList = new ArrayList<ExprNodeDesc>(); combineExprNodeLists(sprayKeyLists.get(i), distinctKeyLists.get(i), combinedList); if (!matchExprLists(combinedList, sprayKeys)) { continue; } // else do the common code at the end. } else { if (distinctKeyLists.get(i).isEmpty()) { List<ExprNodeDesc> combinedList = new ArrayList<ExprNodeDesc>(); combineExprNodeLists(sprayKeys, distinctKeys, combinedList); if (!matchExprLists(combinedList, sprayKeyLists.get(i))) { continue; } else { // we have found a match. insert this distinct clause to head. distinctKeyLists.remove(i); sprayKeyLists.remove(i); distinctKeyLists.add(i, distinctKeys); sprayKeyLists.add(i, sprayKeys); commonGroupByDestGroups.get(i).add(0, dest); found = true; break; } } else { if (!matchExprLists(distinctKeyLists.get(i), distinctKeys)) { continue; } if (!matchExprLists(sprayKeyLists.get(i), sprayKeys)) { continue; } // else do common code } } // common code // A match was found, so add the clause to the corresponding list commonGroupByDestGroups.get(i).add(dest); found = true; break; } // No match was found, so create new entries if (!found) { inputOperators.add(input); sprayKeyLists.add(sprayKeys); distinctKeyLists.add(distinctKeys); List<String> destGroup = new ArrayList<String>(); destGroup.add(dest); commonGroupByDestGroups.add(destGroup); } } return commonGroupByDestGroups; }
From source file:com.joliciel.jochre.graphics.ShapeImpl.java
@Override public Collection<BridgeCandidate> getBridgeCandidates(double maxBridgeWidth) { if (this.bridgeCandidates == null) { TreeSet<VerticalLineSegment> lines = this.getVerticalLineSegments(); // Now, detect "bridges" which could indicate that the shape should be split // First, detect which spaces are "enclosed" and which touch the outer walls // To do this, build up a set of all inverse (white) lines TreeSet<VerticalLineSegment> inverseLines = new TreeSet<VerticalLineSegment>(); int currentX = -1; VerticalLineSegment previousLine = null; for (VerticalLineSegment line : lines) { //LOG.debug("Checking line x = " + line.x + ", top = " + line.yTop + ", bottom = " + line.yBottom); if (line.x != currentX) { // new x-coordinate if (previousLine != null && previousLine.yBottom < this.getHeight() - 1) { VerticalLineSegment inverseLine = new VerticalLineSegment(previousLine.x, previousLine.yBottom + 1); inverseLine.yBottom = this.getHeight() - 1; inverseLines.add(inverseLine); //LOG.debug("Adding inverse line x = " + inverseLine.x + ", top = " + inverseLine.yTop + ", bottom = " + inverseLine.yBottom); }/* www. j a v a 2s. c o m*/ if (line.yTop > 0) { VerticalLineSegment inverseLine = new VerticalLineSegment(line.x, line.yTop - 1); inverseLine.yTop = 0; inverseLines.add(inverseLine); //LOG.debug("Adding inverse line x = " + inverseLine.x + ", top = " + inverseLine.yTop + ", bottom = " + inverseLine.yBottom); } currentX = line.x; } else if (previousLine != null) { VerticalLineSegment inverseLine = new VerticalLineSegment(previousLine.x, previousLine.yBottom + 1); inverseLine.yBottom = line.yTop - 1; inverseLines.add(inverseLine); //LOG.debug("Adding inverse line x = " + inverseLine.x + ", top = " + inverseLine.yTop + ", bottom = " + inverseLine.yBottom); } previousLine = line; } if (previousLine != null && previousLine.yBottom < this.getHeight() - 1) { VerticalLineSegment inverseLine = new VerticalLineSegment(previousLine.x, previousLine.yBottom + 1); inverseLine.yBottom = this.getHeight() - 1; inverseLines.add(inverseLine); //LOG.debug("Adding inverse line x = " + inverseLine.x + ", top = " + inverseLine.yTop + ", bottom = " + inverseLine.yBottom); } LOG.debug("inverseLines size: " + inverseLines.size()); // Calculate neighbours for inverse lines for (VerticalLineSegment inverseLine : inverseLines) { for (VerticalLineSegment otherLine : inverseLines) { if (otherLine.x == inverseLine.x + 1) { if (inverseLine.yTop - 1 <= otherLine.yBottom && otherLine.yTop <= inverseLine.yBottom + 1) { inverseLine.rightSegments.add(otherLine); otherLine.leftSegments.add(inverseLine); } } if (otherLine.x == inverseLine.x - 1) { if (inverseLine.yTop - 1 <= otherLine.yBottom && otherLine.yTop <= inverseLine.yBottom + 1) { inverseLine.leftSegments.add(otherLine); otherLine.rightSegments.add(inverseLine); } } } } // Eliminate any white lines which somehow touch an edge Stack<VerticalLineSegment> lineStack = new Stack<VerticalLineSegment>(); Set<VerticalLineSegment> outerInverseLines = new HashSet<VerticalLineSegment>(); for (VerticalLineSegment inverseLine : inverseLines) { if (inverseLine.yTop == 0 || inverseLine.x == 0 || inverseLine.yBottom == this.getHeight() - 1 || inverseLine.x == this.getWidth() - 1) lineStack.push(inverseLine); } while (!lineStack.isEmpty()) { VerticalLineSegment inverseLine = lineStack.pop(); if (!inverseLine.touched) { inverseLine.touched = true; outerInverseLines.add(inverseLine); //LOG.debug("Outer inverse line x = " + inverseLine.x + ", top = " + inverseLine.yTop + ", bottom = " + inverseLine.yBottom); for (VerticalLineSegment rightLine : inverseLine.rightSegments) lineStack.push(rightLine); for (VerticalLineSegment leftLine : inverseLine.leftSegments) { lineStack.push(leftLine); } } } LOG.debug("outerInverseLines size: " + outerInverseLines.size()); Set<VerticalLineSegment> enclosedInverseLines = new HashSet<VerticalLineSegment>(inverseLines); enclosedInverseLines.removeAll(outerInverseLines); LOG.debug("enclosedInverseLines.size: " + enclosedInverseLines.size()); if (LOG.isDebugEnabled()) { for (VerticalLineSegment inverseLine : enclosedInverseLines) LOG.debug("Enclosed inverse line x = " + inverseLine.x + ", top = " + inverseLine.yTop + ", bottom = " + inverseLine.yBottom); } // Add bridge candidates // based on maximum line length and having exactly one neighbour on each side LOG.debug("Adding bridge candidates"); List<BridgeCandidate> candidateList = new ArrayList<BridgeCandidate>(); for (VerticalLineSegment line : lines) { if (line.rightSegments.size() == 1 && line.leftSegments.size() == 1 && line.length() <= maxBridgeWidth) { // also the bridge width should be considered where two vertical lines touch each other // rather than for the full length of the line BridgeCandidate candidate = null; VerticalLineSegment rightLine = line.rightSegments.iterator().next(); VerticalLineSegment leftLine = line.leftSegments.iterator().next(); int leftTopTouch = (leftLine.yTop > line.yTop ? leftLine.yTop : line.yTop); int leftBottomTouch = (leftLine.yBottom < line.yBottom ? leftLine.yBottom : line.yBottom); int rightTopTouch = (rightLine.yTop > line.yTop ? rightLine.yTop : line.yTop); int rightBottomTouch = (rightLine.yBottom < line.yBottom ? rightLine.yBottom : line.yBottom); int rightLength = rightTopTouch - rightBottomTouch; int leftLength = leftTopTouch - leftBottomTouch; if (line.length() <= maxBridgeWidth || rightLength <= maxBridgeWidth || leftLength <= maxBridgeWidth) { candidate = new BridgeCandidate(this, line); if (rightLength < leftLength && rightLength < line.length()) { candidate.topTouch = rightTopTouch; candidate.bottomTouch = rightBottomTouch; } else if (leftLength < line.length()) { candidate.topTouch = leftTopTouch; candidate.bottomTouch = leftBottomTouch; } LOG.debug("Adding bridge candidate x = " + candidate.x + ", top = " + candidate.yTop + ", bottom = " + candidate.yBottom); candidateList.add(candidate); } } } LOG.debug("Bridge candidate size: " + candidateList.size()); LOG.debug("Eliminating candidates with shorter neighbor"); Set<BridgeCandidate> candidatesToEliminate = null; if (candidateList.size() > 0) { // eliminate any bridge candidates that touch a shorter bridge candidate candidatesToEliminate = new HashSet<BridgeCandidate>(); for (int i = 0; i < candidateList.size() - 1; i++) { BridgeCandidate candidate = candidateList.get(i); for (int j = i + 1; j < candidateList.size(); j++) { BridgeCandidate otherCandidate = candidateList.get(j); if (otherCandidate.x == candidate.x + 1 && candidate.rightSegments.contains(otherCandidate)) { if ((candidate.bridgeWidth()) <= (otherCandidate.bridgeWidth())) { LOG.debug("Eliminating candidate x = " + otherCandidate.x + ", top = " + otherCandidate.yTop + ", bottom = " + otherCandidate.yBottom); candidatesToEliminate.add(otherCandidate); } else { LOG.debug("Eliminating candidate x = " + candidate.x + ", top = " + candidate.yTop + ", bottom = " + candidate.yBottom); candidatesToEliminate.add(candidate); } } } } candidateList.removeAll(candidatesToEliminate); LOG.debug("Bridge candidate size: " + candidateList.size()); // To be a bridge, three additional things have to be true: // (A) intersection between right & left shape = null // (B) weight of right shape & weight of left shape > a certain threshold // (C) little overlap right boundary of left shape, left boundary of right shape LOG.debug("Eliminating candidates touching enclosed space"); // (A) intersection between right & left shape = null // Intersection between right and left shape is non-null // if the line segment X touches an enclosed space immediately above or below candidatesToEliminate = new HashSet<BridgeCandidate>(); for (BridgeCandidate candidate : candidateList) { boolean nullIntersection = true; for (VerticalLineSegment inverseLine : enclosedInverseLines) { if (candidate.x == inverseLine.x) { if (inverseLine.yBottom == candidate.yTop - 1 || inverseLine.yTop == candidate.yBottom + 1) { nullIntersection = false; break; } } } if (!nullIntersection) { LOG.debug("Eliminating candidate x = " + candidate.x + ", top = " + candidate.yTop + ", bottom = " + candidate.yBottom); candidatesToEliminate.add(candidate); } } candidateList.removeAll(candidatesToEliminate); LOG.debug("Remaining bridge candidate size: " + candidateList.size()); // another criterion for avoiding "false splits" is that on both side of the bridge // the shapes pretty rapidly expand in width both up and down LOG.debug("Eliminating candidates without vertical expansion on both sides"); candidatesToEliminate = new HashSet<BridgeCandidate>(); int expansionLimit = (int) Math.ceil(((double) this.getWidth()) / 6.0); for (BridgeCandidate candidate : candidateList) { // take into account the portion touching on the right or left boolean isCandidate = true; Stack<VerticalLineSegment> leftLines = new Stack<VerticalLineSegment>(); Stack<Integer> leftDepths = new Stack<Integer>(); leftLines.push(candidate); leftDepths.push(0); int leftTop = candidate.topTouch; int leftBottom = candidate.bottomTouch; while (!leftLines.isEmpty()) { VerticalLineSegment line = leftLines.pop(); int depth = leftDepths.pop(); if (line.yTop < leftTop) leftTop = line.yTop; if (line.yBottom > leftBottom) leftBottom = line.yBottom; if (depth <= expansionLimit) { for (VerticalLineSegment leftSegment : line.leftSegments) { leftLines.push(leftSegment); leftDepths.push(depth + 1); } } } if (leftTop == candidate.topTouch || leftBottom == candidate.bottomTouch) isCandidate = false; if (isCandidate) { Stack<VerticalLineSegment> rightLines = new Stack<VerticalLineSegment>(); Stack<Integer> rightDepths = new Stack<Integer>(); rightLines.push(candidate); rightDepths.push(0); int rightTop = candidate.topTouch; int rightBottom = candidate.bottomTouch; while (!rightLines.isEmpty()) { VerticalLineSegment line = rightLines.pop(); int depth = rightDepths.pop(); if (line.yTop < rightTop) rightTop = line.yTop; if (line.yBottom > rightBottom) rightBottom = line.yBottom; if (depth <= expansionLimit) { for (VerticalLineSegment rightSegment : line.rightSegments) { rightLines.push(rightSegment); rightDepths.push(depth + 1); } } } if (rightTop == candidate.topTouch || rightBottom == candidate.bottomTouch) isCandidate = false; } if (!isCandidate) { LOG.debug("Eliminating candidate x = " + candidate.x + ", top = " + candidate.yTop + ", bottom = " + candidate.yBottom); candidatesToEliminate.add(candidate); } } candidateList.removeAll(candidatesToEliminate); LOG.debug("Remaining bridge candidate size: " + candidateList.size()); if (LOG.isDebugEnabled()) { for (VerticalLineSegment candidate : candidateList) { LOG.debug("Remaining candidate x = " + candidate.x + ", top = " + candidate.yTop + ", bottom = " + candidate.yBottom); } } } if (candidateList.size() > 0) { // (B) weight of right shape & weight of left shape > a certain threshold // (C) little overlap right boundary of left shape, left boundary of right shape // // We can now divide the shape into n groups, each separated by a candidate // We recursively build a group until we reach a candidate // and indicate whether it's the right or left border of the candidate. // We then keep going from the candidate on to the next one // We keep tab of the size of each group and of its right & left boundaries // at the end we can easily determine the right and left boundaries of each, // as well as the right & left pixel weight List<VerticalLineGroup> groups = new ArrayList<VerticalLineGroup>(); VerticalLineSegment firstLine = lines.first(); lineStack = new Stack<VerticalLineSegment>(); Stack<BridgeCandidate> candidateStack = new Stack<BridgeCandidate>(); Stack<Boolean> fromLeftStack = new Stack<Boolean>(); Stack<Boolean> candidateFromLeftStack = new Stack<Boolean>(); lineStack.push(firstLine); fromLeftStack.push(true); VerticalLineGroup group = new VerticalLineGroup(this); List<BridgeCandidate> touchedCandidates = new ArrayList<BridgeCandidate>(); while (!lineStack.isEmpty()) { while (!lineStack.isEmpty()) { VerticalLineSegment line = lineStack.pop(); boolean fromLeft = fromLeftStack.pop(); if (line.touched) continue; line.touched = true; if (candidateList.contains(line)) { // a candidate! LOG.debug("Touching candidate x = " + line.x + ", top = " + line.yTop + ", bottom = " + line.yBottom); BridgeCandidate candidate = null; for (BridgeCandidate existingCandidate : candidateList) { if (existingCandidate.equals(line)) { candidate = existingCandidate; break; } } boolean foundCandidate = touchedCandidates.contains(candidate); if (!foundCandidate) { touchedCandidates.add(candidate); candidateStack.push(candidate); candidateFromLeftStack.push(fromLeft); if (fromLeft) { // coming from the left group.rightCandidates.add(candidate); candidate.leftGroup = group; } else { group.leftCandidates.add(candidate); candidate.rightGroup = group; } } } else { // not a candidate LOG.debug("Touching line length = " + line.length() + ", x = " + line.x + ", top = " + line.yTop + ", bottom = " + line.yBottom); group.pixelCount += line.length(); if (line.x < group.leftBoundary) group.leftBoundary = line.x; if (line.x > group.rightBoundary) group.rightBoundary = line.x; if (line.yTop < group.topBoundary) group.topBoundary = line.yTop; if (line.yBottom > group.bottomBoundary) group.bottomBoundary = line.yBottom; for (VerticalLineSegment leftLine : line.leftSegments) { lineStack.push(leftLine); fromLeftStack.push(false); } for (VerticalLineSegment rightLine : line.rightSegments) { lineStack.push(rightLine); fromLeftStack.push(true); } } } // no more lines in this group groups.add(group); if (!candidateStack.isEmpty()) { BridgeCandidate candidate = candidateStack.pop(); boolean fromLeft = candidateFromLeftStack.pop(); //lineStack.push(candidate.line); //fromLeftStack.push(fromLeft); LOG.debug("*** New Group ***"); LOG.debug("Next candidate: x = " + candidate.x + ", top = " + candidate.yTop + ", bottom = " + candidate.yBottom); group = new VerticalLineGroup(this); if (fromLeft) { group.leftCandidates.add(candidate); candidate.rightGroup = group; } else { group.rightCandidates.add(candidate); candidate.leftGroup = group; } // add this candidate's neighbours to the lineStack for (VerticalLineSegment leftLine : candidate.leftSegments) { lineStack.push(leftLine); fromLeftStack.push(false); } for (VerticalLineSegment rightLine : candidate.rightSegments) { lineStack.push(rightLine); fromLeftStack.push(true); } } // next candidate on candidate stack } // no more lines to process if (LOG.isDebugEnabled()) { LOG.debug("Found " + groups.size() + " groups"); int i = 1; for (VerticalLineGroup aGroup : groups) { LOG.debug("Group " + i++ + ", pixelCount: " + aGroup.pixelCount + ", leftBoundary: " + aGroup.leftBoundary + ", rightBoundary: " + aGroup.rightBoundary); LOG.debug("Candidates on left: "); for (BridgeCandidate candidate : aGroup.leftCandidates) LOG.debug("Candidate x = " + candidate.x + ", top = " + candidate.yTop + ", bottom = " + candidate.yBottom); LOG.debug("Candidates on right: "); for (BridgeCandidate candidate : aGroup.rightCandidates) LOG.debug("Candidate x = " + candidate.x + ", top = " + candidate.yTop + ", bottom = " + candidate.yBottom); } LOG.debug("Found " + candidateList.size() + " candidates"); for (BridgeCandidate candidate : candidateList) { LOG.debug("Candidate x = " + candidate.x + ", top = " + candidate.yTop + ", bottom = " + candidate.yBottom); LOG.debug("- Left group = pixelCount: " + candidate.leftGroup.pixelCount + ", leftBoundary: " + candidate.leftGroup.leftBoundary + ", rightBoundary: " + candidate.leftGroup.rightBoundary); LOG.debug("- Right group = pixelCount: " + candidate.rightGroup.pixelCount + ", leftBoundary: " + candidate.rightGroup.leftBoundary + ", rightBoundary: " + candidate.rightGroup.rightBoundary); } } // should we log? // calculate each candidate's pixel totals and boundaries for (BridgeCandidate candidate : candidateList) { for (VerticalLineGroup lineGroup : groups) lineGroup.touched = false; Stack<VerticalLineGroup> groupStack = new Stack<VerticalLineGroup>(); groupStack.push(candidate.leftGroup); while (!groupStack.isEmpty()) { VerticalLineGroup lineGroup = groupStack.pop(); if (lineGroup.touched) continue; lineGroup.touched = true; candidate.leftPixels += lineGroup.pixelCount; if (lineGroup.leftBoundary < candidate.leftShapeLeftBoundary) candidate.leftShapeLeftBoundary = lineGroup.leftBoundary; if (lineGroup.rightBoundary > candidate.leftShapeRightBoundary) candidate.leftShapeRightBoundary = lineGroup.rightBoundary; for (BridgeCandidate leftCandidate : lineGroup.leftCandidates) { if (!candidate.equals(leftCandidate)) { candidate.leftPixels += leftCandidate.length(); groupStack.push(leftCandidate.leftGroup); } } for (BridgeCandidate rightCandidate : lineGroup.rightCandidates) { if (!candidate.equals(rightCandidate)) { candidate.leftPixels += rightCandidate.length(); groupStack.push(rightCandidate.rightGroup); } } } // next left group groupStack.push(candidate.rightGroup); while (!groupStack.isEmpty()) { VerticalLineGroup lineGroup = groupStack.pop(); if (lineGroup.touched) continue; lineGroup.touched = true; candidate.rightPixels += lineGroup.pixelCount; if (lineGroup.leftBoundary < candidate.rightShapeLeftBoundary) candidate.rightShapeLeftBoundary = lineGroup.leftBoundary; if (lineGroup.rightBoundary > candidate.rightShapeRightBoundary) candidate.rightShapeRightBoundary = lineGroup.rightBoundary; for (BridgeCandidate leftCandidate : lineGroup.leftCandidates) { if (!candidate.equals(leftCandidate)) { candidate.rightPixels += leftCandidate.length(); groupStack.push(leftCandidate.leftGroup); } } for (BridgeCandidate rightCandidate : lineGroup.rightCandidates) { if (!candidate.equals(rightCandidate)) { candidate.rightPixels += rightCandidate.length(); groupStack.push(rightCandidate.rightGroup); } } } // next right group } // next candidate } // do we have any candidates? this.bridgeCandidates = candidateList; } // lazy load return this.bridgeCandidates; }