List of usage examples for java.util TreeSet clear
public void clear()
From source file:Main.java
public static void main(String[] args) { TreeSet<Integer> tSet = new TreeSet<Integer>(); tSet.add(new Integer("1")); tSet.add(new Integer("2")); tSet.add(new Integer("3")); System.out.println("TreeSet before removal : " + tSet); tSet.clear(); System.out.println("TreeSet after removal : " + tSet); System.out.println("Is TreeSet empty ? " + tSet.isEmpty()); }
From source file:Main.java
public static void main(String[] args) { TreeSet<Integer> treeadd = new TreeSet<Integer>(); treeadd.add(12);/* w ww.j a va 2 s. c om*/ treeadd.add(13); treeadd.add(14); treeadd.add(15); Iterator<Integer> iterator = treeadd.iterator(); while (iterator.hasNext()) { System.out.println(iterator.next()); } treeadd.clear(); System.out.println(treeadd.size()); }
From source file:org.opendatakit.security.server.SecurityServiceUtil.java
/** * Method to enforce an access configuration constraining only registered users, authenticated * users and anonymous access.//from w w w . jav a2 s . co m * * Add additional checks of the incoming parameters and patch things up if the incoming list of * users omits the super-user. * * @param users * @param anonGrants * @param allGroups * @param cc * @throws DatastoreFailureException * @throws AccessDeniedException */ public static final void setStandardSiteAccessConfiguration(ArrayList<UserSecurityInfo> users, ArrayList<GrantedAuthorityName> allGroups, CallingContext cc) throws DatastoreFailureException, AccessDeniedException { // remove anonymousUser from the set of users and collect its // permissions (anonGrantStrings) which will be placed in // the granted authority hierarchy table. List<String> anonGrantStrings = new ArrayList<String>(); { UserSecurityInfo anonUser = null; for (UserSecurityInfo i : users) { if (i.getType() == UserType.ANONYMOUS) { anonUser = i; // clean up grants for anonymousUser -- // ignore anonAuth (the grant under which we will place things) // and forbid Site Admin for (GrantedAuthorityName a : i.getAssignedUserGroups()) { if (anonAuth.getAuthority().equals(a.name())) continue; // avoid circularity... // only allow ROLE_ATTACHMENT_VIEWER and GROUP_ assignments. if (!a.name().startsWith(GrantedAuthorityName.GROUP_PREFIX)) { continue; } // do not allow Site Admin assignments for Anonymous -- // or Tables super-user or Tables Administrator. // those all give access to the full set of users on the system // and giving that information to Anonymous is a security // risk. if (GrantedAuthorityName.GROUP_SITE_ADMINS.equals(a) || GrantedAuthorityName.GROUP_ADMINISTER_TABLES.equals(a) || GrantedAuthorityName.GROUP_SUPER_USER_TABLES.equals(a)) { continue; } anonGrantStrings.add(a.name()); } break; } } if (anonUser != null) { users.remove(anonUser); } } // scan through the users and remove any entries under assigned user groups // that do not begin with GROUP_. // // Additionally, if the user is an e-mail, remove the GROUP_DATA_COLLECTORS // permission since ODK Collect does not support oauth2 authentication. { TreeSet<GrantedAuthorityName> toRemove = new TreeSet<GrantedAuthorityName>(); for (UserSecurityInfo i : users) { // only working with registered users if (i.getType() != UserType.REGISTERED) { continue; } // get the list of assigned groups // -- this is not a copy -- we can directly manipulate this. TreeSet<GrantedAuthorityName> assignedGroups = i.getAssignedUserGroups(); // scan the set of assigned groups and remove any that don't begin with GROUP_ toRemove.clear(); for (GrantedAuthorityName name : assignedGroups) { if (!name.name().startsWith(GrantedAuthorityName.GROUP_PREFIX)) { toRemove.add(name); } } if (!toRemove.isEmpty()) { assignedGroups.removeAll(toRemove); } // for e-mail accounts, remove the Data Collector permission since ODK Collect // does not support an oauth2 authentication mechanism. if (i.getEmail() != null) { assignedGroups.remove(GrantedAuthorityName.GROUP_DATA_COLLECTORS); } } } // find the entry(entries) for the designated super-user(s) String superUserUsername = cc.getUserService().getSuperUserUsername(); int expectedSize = ((superUserUsername != null) ? 1 : 0); ArrayList<UserSecurityInfo> superUsers = new ArrayList<UserSecurityInfo>(); for (UserSecurityInfo i : users) { if (i.getType() == UserType.REGISTERED) { if (i.getUsername() != null && superUserUsername != null && i.getUsername().equals(superUserUsername)) { superUsers.add(i); } } } if (superUsers.size() != expectedSize) { // we are missing one or both super-users. // remove any we have and recreate them from scratch. users.removeAll(superUsers); superUsers.clear(); // Synthesize a UserSecurityInfo object for the super-user(s) // and add it(them) to the list. try { List<RegisteredUsersTable> tList = RegisteredUsersTable.assertSuperUsers(cc); for (RegisteredUsersTable t : tList) { UserSecurityInfo i = new UserSecurityInfo(t.getUsername(), t.getFullName(), t.getEmail(), UserSecurityInfo.UserType.REGISTERED); superUsers.add(i); users.add(i); } } catch (ODKDatastoreException e) { e.printStackTrace(); throw new DatastoreFailureException("Incomplete update"); } } // reset super-user privileges to have (just) site admin privileges // even if caller attempts to change, add, or remove them. for (UserSecurityInfo i : superUsers) { TreeSet<GrantedAuthorityName> grants = new TreeSet<GrantedAuthorityName>(); grants.add(GrantedAuthorityName.GROUP_SITE_ADMINS); grants.add(GrantedAuthorityName.ROLE_SITE_ACCESS_ADMIN); // override whatever the user gave us. i.setAssignedUserGroups(grants); } try { // enforce our fixed set of groups and their inclusion hierarchy. // this is generally a no-op during normal operations. GrantedAuthorityHierarchyTable.assertGrantedAuthorityHierarchy(siteAuth, SecurityServiceUtil.siteAdministratorGrants, cc); GrantedAuthorityHierarchyTable.assertGrantedAuthorityHierarchy(administerTablesAuth, SecurityServiceUtil.administerTablesGrants, cc); GrantedAuthorityHierarchyTable.assertGrantedAuthorityHierarchy(superUserTablesAuth, SecurityServiceUtil.superUserTablesGrants, cc); GrantedAuthorityHierarchyTable.assertGrantedAuthorityHierarchy(synchronizeTablesAuth, SecurityServiceUtil.synchronizeTablesGrants, cc); GrantedAuthorityHierarchyTable.assertGrantedAuthorityHierarchy(dataOwnerAuth, SecurityServiceUtil.dataOwnerGrants, cc); GrantedAuthorityHierarchyTable.assertGrantedAuthorityHierarchy(dataViewerAuth, SecurityServiceUtil.dataViewerGrants, cc); GrantedAuthorityHierarchyTable.assertGrantedAuthorityHierarchy(dataCollectorAuth, SecurityServiceUtil.dataCollectorGrants, cc); // place the anonymous user's permissions in the granted authority table. GrantedAuthorityHierarchyTable.assertGrantedAuthorityHierarchy(anonAuth, anonGrantStrings, cc); // get all granted authority names TreeSet<String> authorities = GrantedAuthorityHierarchyTable .getAllPermissionsAssignableGrantedAuthorities(cc.getDatastore(), cc.getCurrentUser()); // remove the groups that have structure (i.e., those defined above). authorities.remove(siteAuth.getAuthority()); authorities.remove(administerTablesAuth.getAuthority()); authorities.remove(superUserTablesAuth.getAuthority()); authorities.remove(synchronizeTablesAuth.getAuthority()); authorities.remove(dataOwnerAuth.getAuthority()); authorities.remove(dataViewerAuth.getAuthority()); authorities.remove(dataCollectorAuth.getAuthority()); authorities.remove(anonAuth.getAuthority()); // delete all hierarchy structures under anything else. // i.e., if somehow USER_IS_REGISTERED had been granted GROUP_FORM_MANAGER // then this loop would leave USER_IS_REGISTERED without any grants. // (it repairs the database to conform to our privilege hierarchy expectations). List<String> empty = Collections.emptyList(); for (String s : authorities) { GrantedAuthorityHierarchyTable.assertGrantedAuthorityHierarchy(new SimpleGrantedAuthority(s), empty, cc); } // declare all the users (and remove users that are not in this set) Map<UserSecurityInfo, String> pkMap = setUsers(users, cc); // now, for each GROUP_..., update the user granted authority // table with the users that have that GROUP_... assignment. setUsersOfGrantedAuthority(pkMap, siteAuth, cc); setUsersOfGrantedAuthority(pkMap, administerTablesAuth, cc); setUsersOfGrantedAuthority(pkMap, superUserTablesAuth, cc); setUsersOfGrantedAuthority(pkMap, synchronizeTablesAuth, cc); setUsersOfGrantedAuthority(pkMap, dataOwnerAuth, cc); setUsersOfGrantedAuthority(pkMap, dataViewerAuth, cc); setUsersOfGrantedAuthority(pkMap, dataCollectorAuth, cc); // all super-users would already have their site admin role and // we leave that unchanged. The key is to ensure that the // super users are in the users list so they don't get // accidentally removed and that they have siteAuth group // membership. I.e., we don't need to manage ROLE_SITE_ACCESS_ADMIN // here. it is done elsewhere. } catch (ODKDatastoreException e) { e.printStackTrace(); throw new DatastoreFailureException("Incomplete update"); } finally { Datastore ds = cc.getDatastore(); User user = cc.getCurrentUser(); try { SecurityRevisionsTable.setLastRegisteredUsersRevisionDate(ds, user); } catch (ODKDatastoreException e) { // if it fails, use RELOAD_INTERVAL to force reload. e.printStackTrace(); } try { SecurityRevisionsTable.setLastRoleHierarchyRevisionDate(ds, user); } catch (ODKDatastoreException e) { // if it fails, use RELOAD_INTERVAL to force reload. e.printStackTrace(); } } }
From source file:com.samples.platform.service.common.GetServiceStatusOperation.java
/** * @param message// ww w.ja v a 2 s .c om * the {@link JAXBElement} containing a * {@link GetServiceStatusRequestType}. * @return the {@link JAXBElement} with a * {@link GetServiceStatusResponseType}. */ @InsightEndPoint @ServiceActivator public final JAXBElement<GetServiceStatusResponseType> getServiceStatus( final JAXBElement<GetServiceStatusRequestType> message) { this.logger.debug("+getServiceStatus"); GetServiceStatusResponseType response = this.of.createGetServiceStatusResponseType(); try { PropertyType p; ClassLoader cl; URL[] urls; ClassLoader sysCl = ClassLoader.getSystemClassLoader(); response.setStatus("Service is available"); /* System properties */ p = new PropertyType(); p.setName("System Properties"); response.getDetails().add(p); TreeSet<String> propertyNames = new TreeSet<String>(); propertyNames.addAll(System.getProperties().stringPropertyNames()); for (String propertyName : propertyNames) { p.getValue().add(new StringBuffer(64).append(propertyName).append("=") .append(System.getProperty(propertyName)).toString()); } /* Application properties. */ p = new PropertyType(); p.setName("Application loaded properties"); response.getDetails().add(p); propertyNames.clear(); propertyNames.addAll(this.properties.stringPropertyNames()); for (String propertyName : propertyNames) { p.getValue().add(new StringBuffer(64).append(propertyName).append("=") .append(this.properties.getProperty(propertyName)).toString()); } /* Current lass loader */ cl = this.getClass().getClassLoader(); p = new PropertyType(); p.setName("This ClassLoader"); response.getDetails().add(p); p.getValue().add(cl.getClass().getName()); if (URLClassLoader.class.isInstance(cl)) { urls = ((URLClassLoader) cl).getURLs(); p.getValue().add(new StringBuffer("Url: ").append(urls.length).toString()); for (URL url : urls) { p.getValue().add(url.toString()); } } cl = cl.getParent(); while (cl != sysCl) { p = new PropertyType(); p.setName("Parent Classloader"); response.getDetails().add(p); p.getValue().add(cl.getClass().getName()); if (URLClassLoader.class.isInstance(cl)) { urls = ((URLClassLoader) cl).getURLs(); p.getValue().add(new StringBuffer("Url: ").append(urls.length).toString()); for (URL url : urls) { p.getValue().add(url.toString()); } } cl = cl.getParent(); } /* System class loader */ cl = sysCl; p = new PropertyType(); p.setName("SystemClassLoader"); response.getDetails().add(p); p.getValue().add(cl.getClass().getName()); if (URLClassLoader.class.isInstance(cl)) { urls = ((URLClassLoader) cl).getURLs(); p.getValue().add(new StringBuffer("Url: ").append(urls.length).toString()); for (URL url : urls) { p.getValue().add(url.toString()); } } } catch (Throwable e) { this.logger.error(e.getMessage(), e); } finally { this.logger.debug("-getServiceStatus #{}, #f{}", response/* .get() */ != null ? 1 : 0, response.getFailure().size()); } return this.of.createGetServiceStatusResponse(response); }
From source file:de.julielab.jcore.ae.lingpipegazetteer.chunking.ChunkerProviderImpl.java
private TreeSet<String> flushDictionary(TreeSet<String> dictionarySet, AbstractDictionary<String> dict) throws AnalysisEngineProcessException { Iterator<String> it = dictionarySet.iterator(); String[] split;//from w ww.j a v a2 s .c om while (it.hasNext()) { split = it.next().split(SEPARATOR); if (split.length != 2) { LOGGER.error("readDictionary() - wrong split length: " + split.length); throw new AnalysisEngineProcessException(AnalysisEngineProcessException.ANNOTATOR_EXCEPTION, null); } dict.addEntry(new DictionaryEntry<String>(split[0], split[1], CHUNK_SCORE)); } it = null; dictionarySet.clear(); return dictionarySet; }
From source file:crawler.HackerEarthCrawler.java
@Override public void crawl() { int flag = 0; //set of urls which should be crawled TreeSet<String> linksset = new TreeSet<String>(); TreeSet<String> tempset = new TreeSet<String>(); TreeSet<String> tutorialset = new TreeSet<String>(); //final set of problem urls TreeSet<String> problemset = new TreeSet<String>(); //visited for maintaing status of if url is already crawled or not TreeMap<String, Integer> visited = new TreeMap<String, Integer>(); //add base url linksset.add(baseUrl);//w w w .j a v a 2 s . c o m //mark base url as not crawled visited.put(baseUrl, 0); try { while (true) { flag = 0; tempset.clear(); for (String str : linksset) { //check if url is already crawled or not and it has valid domain name if ((visited.get(str) == 0) && (str.startsWith("https://www.hackerearth.com/"))) { System.out.println("crawling " + str); //retriving response of current url as document Document doc = Jsoup.connect(str).timeout(0).userAgent( "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0") .referrer("http://www.google.com").ignoreHttpErrors(true).get(); //retriving all urls from current page Elements links = doc.select("a[href]"); //mark url as crawled visited.put(str, 1); //mark flag as url is crawled flag = 1; //retrive all urls for (Element link : links) { if (link.absUrl("href").endsWith("/tutorial/")) { tutorialset.add(link.absUrl("href")); } //check if url is problem url then add it in problemurlset if (link.absUrl("href").startsWith("https://www.hackerearth.com/") && isProblemUrl(link.absUrl("href"))) { problemset.add(link.absUrl("href")); } //check if url has valid domain and it has problem urls or not if (link.absUrl("href").contains(("https://www.hackerearth.com/")) && isCrawlable(link.absUrl("href"))) { //if link is not visited then mark it as uncrawled if (!visited.containsKey(link.absUrl("href"))) { visited.put(link.absUrl("href"), 0); } //add it in tempsetorary set tempset.add(link.absUrl("href")); //System.out.println("\n base: "+str+" ::: link : " + link.absUrl("href")); } } } } //if nothing is left to crawl break the loop if (flag == 0) { break; } //add all retrieved links to linksset linksset.addAll(tempset); } System.out.println("\n\ntotal problem urls " + problemset.size()); int i = 0; for (String str : problemset) { System.out.println("link " + i + " : " + str); i++; } } catch (IOException ex) { Logger.getLogger(HackerEarthCrawler.class.getName()).log(Level.SEVERE, null, ex); } //scrap and store into database //for every problem url scrap problem page for (String problemUrl : problemset) { System.out.println("problemUrl :" + problemUrl); try { //create problem class to store in database Problem problem = new Problem(); String problemSIOC = "", problemIOC = ""; String problemTitle = "", problemStatement = "", problemInput = "", problemOutput = "", problemConstraints = ""; String sampleInput = "", sampleOutput = ""; String problemExplanation = ""; //set default timelimit to 1 second double problemTimeLimit = 1.0; ArrayList<String> tags = new ArrayList<String>(); //get response for given problem url Response response = Jsoup.connect(problemUrl).execute(); Document doc = response.parse(); //retrieve problem title from page Element elementTitle = doc.getElementsByTag("title").first(); StringTokenizer stTitle = new StringTokenizer(elementTitle.text(), "|"); problemTitle = stTitle.nextToken().trim(); Element content = doc.getElementsByClass("starwars-lab").first(); problemSIOC = content.text(); Elements e = content.children(); //to find problem statement String breakloop[] = { "input", "input:", "input :", "input format:", "input format :", "input format", "Input and output", "constraints :", "constraints:", "constraints", "$$Input :$$" }; flag = 0; for (Element p : e) { String tempStatement = ""; for (Element pp : p.getAllElements()) { for (String strbreak : breakloop) { if (StringUtils.equalsIgnoreCase(pp.ownText(), strbreak)) { //System.out.println("strbreak :"+strbreak); tempStatement = p.text().substring(0, p.text().toLowerCase().indexOf(strbreak.toLowerCase())); // System.out.println("temp "+tempStatement); flag = 1; break; } } } if (flag == 1) { problemStatement += tempStatement; //remove extra space at end if (tempStatement.length() == 0) { problemStatement = problemStatement.substring(0, problemStatement.length() - 1); } break; } problemStatement += p.text() + " "; } System.out.println("problemSIOC :" + problemSIOC); System.out.println("problemStatement :" + problemStatement); if (problemStatement.length() <= problemSIOC.length()) { //remove problem statement from whole text and remove extra spaces at the beginning and the end problemIOC = problemSIOC.substring(problemStatement.length()).trim(); } else { problemIOC = ""; } System.out.println("problemIOC :" + problemIOC); //keywords for identifying input String decideInput[] = { "Input format :", "Input format:", "Input format", "inputformat:", "inputformat :", "inputformat", "input and output", "input :", "input:", "input" }; //keywords for identifying output String decideOutput[] = { "output format :", "output format:", "Output format", "outputformat:", "outputformat :", "outputformat", "output :", "output:", "output" }; //keywords for identifying constraint String decideConstraint[] = { "constraints:", "constraints :", "constraints", "Constraints :", "constraint:", "constraint :", "constraint", "Contraints :" }; int posin = 0, posoutput = 0, poscon = 0, idxin, idxout, idxcon, flaginput = 0, flagoutput = 0, flagcon = 0, inlen = 0, outlen = 0, conlen = 0; //find inputformat position,length of keyword for (idxin = 0; idxin < decideInput.length; idxin++) { if (StringUtils.containsIgnoreCase(problemIOC, decideInput[idxin])) { posin = problemIOC.toLowerCase().indexOf(decideInput[idxin].toLowerCase()); flaginput = 1; inlen = decideInput[idxin].length(); //decide it is keyowrd for actucal input or it is "sample input" if (StringUtils.containsIgnoreCase(problemIOC, "sample input")) { if (posin > problemIOC.toLowerCase().indexOf("sample input")) { flaginput = 0; inlen = 0; } else { break; } } else { break; } } } //find outputformat position,length of keyword for (idxout = 0; idxout < decideOutput.length; idxout++) { if (StringUtils.containsIgnoreCase(problemIOC, decideOutput[idxout])) { posoutput = problemIOC.toLowerCase().indexOf(decideOutput[idxout].toLowerCase()); flagoutput = 1; outlen = decideOutput[idxout].length(); break; } } //find constraint position,length of keyword for (idxcon = 0; idxcon < decideConstraint.length; idxcon++) { if (StringUtils.containsIgnoreCase(problemIOC, decideConstraint[idxcon])) { poscon = problemIOC.toLowerCase().indexOf(decideConstraint[idxcon].toLowerCase()); flagcon = 1; conlen = decideConstraint[idxcon].length(); break; } } System.out.println("input " + flaginput + " " + inlen + " " + posin); System.out.println("output " + flagoutput + " " + outlen + " " + posoutput); System.out.println("constraint " + flagcon + " " + conlen + " " + poscon); //retrieve problem input and output if present in problem page //if input format is present if (flaginput == 1) { //if input keyword is "input and output" and contraint is present in problem page if (idxin == 6 && flagcon == 1) { problemInput = problemIOC.substring(inlen, poscon); } //if input keyword is "input and output" and contraint is not present in problem page else if (idxin == 6 && flagcon == 0) { problemInput = problemIOC.substring(inlen); } //if output format and constraint is present else if (flagoutput == 1 && flagcon == 1) { //if constraint is present before input format if (poscon < posin) { problemInput = problemIOC.substring(posin + inlen, posoutput); problemOutput = problemIOC.substring(posoutput + outlen); } //if constraint is present before sample else if (poscon < posoutput) { problemInput = problemIOC.substring(inlen, poscon); problemOutput = problemIOC.substring(posoutput + outlen); } else { problemInput = problemIOC.substring(inlen, posoutput); problemOutput = problemIOC.substring(posoutput + outlen, poscon); } } //if constraint is not present else if (flagoutput == 1 && flagcon == 0) { problemInput = problemIOC.substring(inlen, posoutput); problemOutput = problemIOC.substring(posoutput + outlen); } else if (flagoutput == 0 && flagcon == 1) { if (poscon < posin) { problemInput = problemIOC.substring(posin + inlen); } else { problemInput = problemIOC.substring(poscon + conlen, posin); } problemOutput = ""; } else { problemInput = problemIOC.substring(inlen); problemOutput = ""; } } //if input format and output format is not present else { problemInput = ""; problemOutput = ""; } //if constraint is present if (flagcon == 1) { //if constraint is present before input format if (poscon < posin) { problemConstraints = problemIOC.substring(0, posin); } //if constraint is present before output format else if (poscon < posoutput) { problemConstraints = problemIOC.substring(poscon + conlen, posoutput); } else { problemConstraints = problemIOC.substring(poscon + conlen); } } System.out.println("problemInput :" + problemInput); System.out.println("problemOutput :" + problemOutput); System.out.println("problemConstraints :" + problemConstraints); //retrieve problem tags from problem page Element elementtag = doc.getElementsByClass("problem-tags").first().child(1); StringTokenizer st = new StringTokenizer(elementtag.text(), ","); while (st.hasMoreTokens()) { tags.add(st.nextToken().trim()); } //retrieve sample input sample output if present Element elementSIO = doc.getElementsByClass("input-output-container").first(); //if sample input output is present if (elementSIO != null) { //find position of sample output int soutpos = elementSIO.text().indexOf("SAMPLE OUTPUT"); sampleInput = elementSIO.text().substring(12, soutpos); sampleOutput = elementSIO.text().substring(soutpos + 13); System.out.println("Sample input :\n" + sampleInput + "\n\n\n"); System.out.println("Sample Output :\n" + sampleOutput); } else { sampleInput = ""; sampleOutput = ""; } //retrieve problem explanation from problem page if present Element elementExplanation = doc.getElementsByClass("standard-margin").first().child(0); if (elementExplanation.text().toLowerCase().contains("explanation")) { problemExplanation = elementExplanation.nextElementSibling().text(); } System.out.println("Explanation :" + problemExplanation); //retrieve timelimit Element elementTL = doc.getElementsByClass("problem-guidelines").first().child(0).child(1); StringTokenizer stTL = new StringTokenizer(elementTL.ownText(), " "); problemTimeLimit = Double.parseDouble(stTL.nextToken()); //System.out.println("problemTimeLimit :"+problemTimeLimit); //set all retrieved information to problem class problem.setProblemUrl(problemUrl); if (problemTitle.length() == 0) { problemTitle = null; } if (problemStatement.length() == 0) { problemStatement = null; } if (problemInput.length() == 0) { problemInput = null; } if (problemOutput.length() == 0) { problemOutput = null; } if (problemExplanation.length() == 0) { problemExplanation = null; } if (problemConstraints.length() == 0) { problemConstraints = null; } problem.setTitle(problemTitle); problem.setProblemUrl(problemUrl); problem.setProblemStatement(problemStatement); problem.setInputFormat(problemInput); problem.setOutputFormat(problemOutput); problem.setTimeLimit(problemTimeLimit); problem.setExplanation(problemExplanation); problem.setConstraints(problemConstraints); //set sample input output to problem class SampleInputOutput sampleInputOutput = new SampleInputOutput(problem, sampleInput, sampleOutput); problem.getSampleInputOutputs().add(sampleInputOutput); //set platform as hackerearth problem.setPlatform(Platform.HackerEarth); for (String strtag : tags) { problem.getTags().add(strtag); } //store in database Session session = null; Transaction transaction = null; try { //start session session = HibernateUtil.getSessionFactory().openSession(); transaction = session.beginTransaction(); //check if problem is already stored in database String hql = "FROM Problem p where p.problemUrl = :problem_url"; Problem oldProblem = (Problem) session.createQuery(hql).setString("problem_url", problemUrl) .uniqueResult(); String task; //if problem is present in database if (oldProblem != null) { //update the old problem task = "updated"; //retrieve id of old problem problem.setId(oldProblem.getId()); session.delete(oldProblem); session.flush(); session.save(problem); } else { task = "saved"; session.save(problem); } transaction.commit(); //log the info to console Logger.getLogger(CodeForcesCrawler.class.getName()).log(Level.INFO, "{0} {1}", new Object[] { task, problem.getProblemUrl() }); } catch (HibernateException ee) { if (transaction != null) { transaction.rollback(); } Logger.getLogger(CodeForcesCrawler.class.getName()).log(Level.SEVERE, "Cannot Insert/Update problem into databse: " + problemUrl, e); } finally { //close the session if (session != null) { session.close(); } } } catch (Exception ee) { System.out.println(ee.toString()); } } System.out.println("\n\n\n\ntutorial urls\n\n"); try { for (String tutorialurl : tutorialset) { //System.out.println(tutorialurl+"\n\n"); Response tutorialres = Jsoup.connect(tutorialurl).execute(); Document doc = tutorialres.parse(); Tutorial tutorial = new Tutorial(); tutorial.setContent(doc.getElementsByClass("tutorial").first().text()); tutorial.setName(baseUrl); tutorialurl = tutorialurl.substring(0, tutorialurl.length() - 10); StringTokenizer tutorialtok = new StringTokenizer(tutorialurl, "/"); String tempstr = ""; while (tutorialtok.hasMoreTokens()) { tempstr = tutorialtok.nextToken(); } Session session = null; Transaction transaction = null; try { //start session session = HibernateUtil.getSessionFactory().openSession(); transaction = session.beginTransaction(); //check if problem is already stored in database String hql = "FROM Tutorial p where p.name = :name"; Tutorial oldProblem = (Tutorial) session.createQuery(hql).setString("name", tempstr) .uniqueResult(); String task; //if problem is present in database if (oldProblem != null) { //update the old problem task = "updated"; //retrieve id of old problem tutorial.setName(oldProblem.getName()); session.delete(oldProblem); session.flush(); session.save(tutorial); } else { task = "saved"; tutorial.setName(tempstr); session.save(tutorial); } transaction.commit(); //log the info to console Logger.getLogger(CodeForcesCrawler.class.getName()).log(Level.INFO, "{0} {1}", new Object[] { task, tutorial.getName() }); } catch (HibernateException ee) { if (transaction != null) { transaction.rollback(); } Logger.getLogger(CodeForcesCrawler.class.getName()).log(Level.SEVERE, "Cannot Insert/Update problem into databse: " + tempstr, ee); } finally { //close the session if (session != null) { session.close(); } } } } catch (Exception e) { System.out.println(e.getMessage()); } }
From source file:org.dasein.cloud.azure.AzureStorageMethod.java
private String calculatedSharedKeyLiteSignature(@Nonnull HttpRequestBase method, @Nonnull Map<String, String> queryParams) throws CloudException, InternalException { fetchKeys();/*from ww w . j a va 2 s . c o m*/ ProviderContext ctx = provider.getContext(); if (ctx == null) { throw new AzureConfigException("No context was specified for this request"); } Header h = method.getFirstHeader("content-type"); String contentType = (h == null ? null : h.getValue()); if (contentType == null) { contentType = ""; } StringBuilder stringToSign = new StringBuilder(); stringToSign.append(method.getMethod().toUpperCase()).append("\n"); stringToSign.append("\n"); // content-md5 stringToSign.append(contentType).append("\n"); stringToSign.append(method.getFirstHeader("date").getValue()).append("\n"); Header[] headers = method.getAllHeaders(); TreeSet<String> keys = new TreeSet<String>(); for (Header header : headers) { if (header.getName().startsWith(Header_Prefix_MS)) { keys.add(header.getName().toLowerCase()); } } for (String key : keys) { Header header = method.getFirstHeader(key); if (header != null) { Header[] all = method.getHeaders(key); stringToSign.append(key.toLowerCase().trim()).append(":"); if (all != null && all.length > 0) { for (Header current : all) { String v = (current.getValue() != null ? current.getValue() : ""); stringToSign.append(v.trim().replaceAll("\n", " ")).append(","); } } stringToSign.deleteCharAt(stringToSign.lastIndexOf(",")); } else { stringToSign.append(key.toLowerCase().trim()).append(":"); } stringToSign.append("\n"); } stringToSign.append("/").append(getStorageAccount()).append(method.getURI().getPath()); keys.clear(); for (String key : queryParams.keySet()) { if (key.equalsIgnoreCase("comp")) { key = key.toLowerCase(); keys.add(key); } } if (!keys.isEmpty()) { stringToSign.append("?"); for (String key : keys) { String value = queryParams.get(key); if (value == null) { value = ""; } stringToSign.append(key).append("=").append(value).append("&"); } stringToSign.deleteCharAt(stringToSign.lastIndexOf("&")); } try { if (logger.isDebugEnabled()) { logger.debug("BEGIN STRING TO SIGN"); logger.debug(stringToSign.toString()); logger.debug("END STRING TO SIGN"); } Mac mac = Mac.getInstance("HmacSHA256"); mac.init(new SecretKeySpec(Base64.decodeBase64(ctx.getStoragePrivate()), "HmacSHA256")); String signature = new String( Base64.encodeBase64(mac.doFinal(stringToSign.toString().getBytes("UTF-8")))); if (logger.isDebugEnabled()) { logger.debug("signature=" + signature); } return signature; } catch (UnsupportedEncodingException e) { logger.error("UTF-8 not supported: " + e.getMessage()); throw new InternalException(e); } catch (NoSuchAlgorithmException e) { logger.error("No such algorithm: " + e.getMessage()); throw new InternalException(e); } catch (InvalidKeyException e) { logger.error("Invalid key: " + e.getMessage()); throw new InternalException(e); } }
From source file:net.spfbl.data.White.java
public static synchronized TreeSet<String> dropAll() { TreeSet<String> set = SET.clear(); set.addAll(CIDR.clear());/* www. j a v a2 s . co m*/ set.addAll(REGEX.clear()); set.addAll(WHOIS.clear()); CHANGED = true; return set; }
From source file:net.spfbl.spf.SPF.java
protected static String processPostfixSPF(InetAddress ipAddress, Client client, User user, String ip, String sender, String helo, String recipient, LinkedList<User> userResult) throws ProcessException { if (sender == null) { sender = null;/* ww w . j a va2s . c om*/ } else if (sender.trim().length() == 0) { sender = null; } else if (Domain.isEmail(sender)) { sender = sender.toLowerCase(); } else { return "action=554 5.7.1 SPFBL " + sender + " " + "is not a valid e-mail address.\n\n"; } if (recipient == null) { recipient = null; } else if (recipient.trim().length() == 0) { recipient = null; } else if (Domain.isEmail(recipient)) { recipient = recipient.toLowerCase(); } else { return "action=554 5.7.1 SPFBL " + recipient + " " + "is not a valid e-mail address.\n\n"; } if (!Domain.isHostname(helo)) { helo = null; } if (!Subnet.isValidIP(ip)) { return "action=554 5.7.1 SPFBL " + ip + " is not a valid public IP.\n\n"; } else if (Subnet.isReservedIP(ip)) { // Message from LAN. return "action=DUNNO\n\n"; } else if (client != null && client.containsFull(ip)) { // Message from LAN. return "action=DUNNO\n\n"; } else { try { TreeSet<String> tokenSet = new TreeSet<String>(); ip = Subnet.normalizeIP(ip); tokenSet.add(Subnet.normalizeIP(ip)); if (Domain.isValidEmail(recipient)) { // Se houver um remetente vlido, // Adicionar no ticket para controle externo. tokenSet.add('>' + recipient); } if (recipient != null) { User recipientUser = User.get(recipient); if (recipientUser == null) { // Se a consulta originar de destinatrio com postmaster cadastrado, // considerar o prprio postmaster como usurio da consulta. int index = recipient.indexOf('@'); String postmaster = "postmaster" + recipient.substring(index); User postmasterUser = User.get(postmaster); if (postmasterUser != null) { user = postmasterUser; } } else { user = recipientUser; } } if (user != null) { userResult.add(user); tokenSet.add(user.getEmail() + ':'); } else if (client != null && client.hasEmail()) { tokenSet.add(client.getEmail() + ':'); } // Passar a acompanhar todos os // HELO quando apontados para o IP para // uma nova forma de interpretar dados. String hostname; if (CacheHELO.match(ip, helo, false)) { hostname = Domain.normalizeHostname(helo, true); } else { hostname = Reverse.getHostname(ip); hostname = Domain.normalizeHostname(hostname, true); } if (hostname == null) { Server.logDebug("no rDNS for " + ip + "."); } else if (Domain.isOfficialTLD(hostname)) { return "action=554 5.7.1 SPFBL " + hostname + " is a reserved domain.\n\n"; } else { // Verificao de pilha dupla, // para pontuao em ambas pilhas. String ipv4 = CacheHELO.getUniqueIPv4(hostname); String ipv6 = CacheHELO.getUniqueIPv6(hostname); if (ip.equals(ipv6) && CacheHELO.match(ipv4, hostname, false)) { // Equivalncia de pilha dupla se // IPv4 for nico para o hostname. tokenSet.add(ipv4); } else if (ip.equals(ipv4) && CacheHELO.match(ipv6, hostname, false)) { // Equivalncia de pilha dupla se // IPv6 for nico para o hostname. tokenSet.add(ipv6); } } if (Generic.containsGenericSoft(hostname)) { // Quando o reverso for // genrico, no consider-lo. hostname = null; } else if (hostname != null) { tokenSet.add(hostname); } String result; LinkedList<String> logList = new LinkedList<String>(); SPF spf; if (sender == null) { spf = null; result = "NONE"; } else if (Domain.isOfficialTLD(sender)) { spf = null; result = "NONE"; } else if (Generic.containsGeneric(sender)) { spf = null; result = "NONE"; } else if ((spf = CacheSPF.get(sender)) == null) { result = "NONE"; } else if (spf.isInexistent()) { result = "NONE"; } else { result = spf.getResult(ip, sender, helo, logList); } String origem; String fluxo; String mx = Domain.extractHost(sender, true); if (user != null && user.isLocal()) { // Message from local user. return "action=DUNNO\n\n"; } else if (recipient != null && result.equals("PASS")) { if (recipient.endsWith(mx)) { // Message from same domain. return "action=DUNNO\n\n"; } else if (recipient.equals(Core.getAbuseEmail()) && User.exists(sender, "postmaster" + mx)) { // Message to abuse. return "action=DUNNO\n\n"; } } if (result.equals("PASS") || (sender != null && Provider.containsHELO(ip, helo))) { // Quando fo PASS, significa que o domnio // autorizou envio pelo IP, portanto o dono dele // responsavel pelas mensagens. if (!Provider.containsExact(mx)) { // No um provedor ento // o MX deve ser listado. tokenSet.add(mx); origem = mx; } else if (Domain.isValidEmail(sender)) { // Listar apenas o remetente se o // hostname for um provedor de e-mail. String userEmail = null; String recipientEmail = null; for (String token : tokenSet) { if (token.endsWith(":")) { userEmail = token; } else if (token.startsWith(">")) { recipientEmail = token; } } tokenSet.clear(); tokenSet.add(sender); if (userEmail != null) { tokenSet.add(userEmail); } if (recipientEmail != null) { tokenSet.add(recipientEmail); } origem = sender; } else { origem = sender; } fluxo = origem + ">" + recipient; } else if (hostname == null) { origem = (sender == null ? "" : sender + '>') + ip; fluxo = origem + ">" + recipient; } else { String dominio = Domain.extractDomain(hostname, true); origem = (sender == null ? "" : sender + '>') + (dominio == null ? hostname : dominio.substring(1)); fluxo = origem + ">" + recipient; } Long recipientTrapTime = Trap.getTimeRecipient(client, user, recipient); if (recipientTrapTime == null && White.contains(client, user, ip, sender, hostname, result, recipient)) { if (White.contains(client, user, ip, sender, hostname, result, null)) { // Limpa da lista BLOCK um possvel falso positivo. Block.clear(client, user, ip, sender, hostname, result, null); } // Calcula frequencia de consultas. String url = Core.getURL(); String ticket = SPF.addQueryHam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "WHITE"); return "action=PREPEND " + "Received-SPFBL: " + result + " " + (url == null ? ticket : url + ticket) + "\n\n"; } else if (Block.contains(client, user, ip, sender, hostname, result, recipient, true, true, true, true)) { Action action = client == null ? Action.REJECT : client.getActionBLOCK(); if (action == Action.REJECT) { // Calcula frequencia de consultas. User.Query queryLocal = SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "BLOCK"); action = client == null ? Action.FLAG : client.getActionRED(); if (action != Action.REJECT && queryLocal != null && queryLocal.needHeader()) { if (action == Action.FLAG) { queryLocal.setResult("FLAG"); return "action=PREPEND X-Spam-Flag: YES\n\n"; } else if (action == Action.HOLD) { queryLocal.setResult("HOLD"); return "action=HOLD blocked.\n\n"; } else { return "action=WARN undefined action.\n\n"; } } else { String url = Core.getUnblockURL(client, user, ip, sender, hostname, recipient); if (url == null) { return "action=554 5.7.1 SPFBL " + "you are permanently blocked in this server.\n\n"; } else { return "action=554 5.7.1 SPFBL " + "BLOCKED " + url + "\n\n"; } } } else if (action == Action.FLAG) { SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "FLAG"); return "action=PREPEND X-Spam-Flag: YES\n\n"; } else if (action == Action.HOLD) { SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "HOLD"); return "action=HOLD blocked.\n\n"; } else { return "action=WARN undefined action.\n\n"; } } else if (Generic.containsDynamicDomain(hostname)) { // Bloquear automaticamente range de IP dinmico. String cidr = Subnet.normalizeCIDR(SubnetIPv4.isValidIPv4(ip) ? ip + "/24" : ip + "/48"); if (Block.tryOverlap(cidr)) { Server.logDebug("new BLOCK '" + cidr + "' added by '" + hostname + ";DYNAMIC'."); } else if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by '" + hostname + ";DYNAMIC'."); } SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); return "action=554 5.7.1 SPFBL dynamic IP.\n\n"; } else if (spf != null && spf.isDefinitelyInexistent()) { // Bloquear automaticamente IP com reputao vermelha. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by '" + mx + ";NXDOMAIN'."); } } Analise.processToday(ip); // O domnio foi dado como inexistente inmeras vezes. // Rejeitar e denunciar o host pois h abuso de tentativas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "NXDOMAIN"); return "action=554 5.7.1 SPFBL " + "sender has non-existent internet domain.\n\n"; } else if (spf != null && spf.isInexistent()) { Analise.processToday(ip); SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "NXDOMAIN"); return "action=554 5.7.1 SPFBL " + "sender has non-existent internet domain.\n\n"; } else if (result.equals("FAIL")) { // Bloquear automaticamente IP com reputao vermelha. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by '" + sender + ";FAIL'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "FAIL"); // Retornar REJECT somente se no houver // liberao literal do remetente com FAIL. return "action=554 5.7.1 SPFBL " + sender + " is not allowed to " + "send mail from " + ip + ".\n\n"; } else if (sender != null && !Domain.isEmail(sender)) { // Bloquear automaticamente IP com reputao vermelha. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by '" + sender + ";INVALID'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); return "action=554 5.7.1 SPFBL " + sender + " is not a valid e-mail address.\n\n"; } else if (sender != null && Domain.isOfficialTLD(sender)) { // Bloquear automaticamente IP com reputao vermelha. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by '" + sender + ";RESERVED'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "RESERVED"); return "action=554 5.7.1 SPFBL " + sender + " has a reserved domain.\n\n"; } else if (sender == null && !CacheHELO.match(ip, hostname, false)) { // Bloquear automaticamente IP com reputao ruim. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by '" + hostname + ";INVALID'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); return "action=554 5.7.1 SPFBL invalid hostname.\n\n"; } else if (hostname == null && Core.isReverseRequired()) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by 'NONE'."); } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); return "action=554 5.7.1 SPFBL " + ip + " has no rDNS.\n\n"; } else if (recipient != null && !Domain.isValidEmail(recipient)) { Analise.processToday(ip); Analise.processToday(mx); SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INEXISTENT"); return "action=550 5.1.1 SPFBL the email account that you tried to reach does not exist.\n\n"; } else if (recipientTrapTime != null) { if (System.currentTimeMillis() > recipientTrapTime) { // Spamtrap. for (String token : tokenSet) { String block; Status status = SPF.getStatus(token); if (status == Status.RED && (block = Block.add(token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + recipient + ";SPAMTRAP'."); Peer.sendBlockToAll(block); } if (status != Status.GREEN && !Subnet.isValidIP(token) && (block = Block.addIfNotNull(user, token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + recipient + ";SPAMTRAP'."); } } Analise.processToday(ip); Analise.processToday(mx); // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "TRAP"); return "action=DISCARD SPFBL discarded by spamtrap.\n\n"; } else { // Inexistent. for (String token : tokenSet) { String block; Status status = SPF.getStatus(token); if (status == Status.RED && (block = Block.add(token)) != null) { Server.logDebug( "new BLOCK '" + block + "' added by '" + recipient + ";INEXISTENT'."); Peer.sendBlockToAll(block); } if (status != Status.GREEN && !Subnet.isValidIP(token) && (block = Block.addIfNotNull(user, token)) != null) { Server.logDebug( "new BLOCK '" + block + "' added by '" + recipient + ";INEXISTENT'."); } } Analise.processToday(ip); Analise.processToday(mx); SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INEXISTENT"); return "action=550 5.1.1 SPFBL the email account that you tried to reach does not exist.\n\n"; } } else if (Defer.count(fluxo) > Core.getFloodMaxRetry()) { Analise.processToday(ip); Analise.processToday(mx); // A origem atingiu o limite de atraso // para liberao do destinatrio. long time = System.currentTimeMillis(); Defer.end(fluxo); Server.logDefer(time, fluxo, "DEFER FLOOD"); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "FLOOD"); return "action=554 5.7.1 SPFBL too many retries.\n\n"; } else if (!result.equals("PASS") && !CacheHELO.match(ip, hostname, false)) { // Bloquear automaticamente IP com reputao amarela. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by '" + recipient + ";INVALID'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); return "action=554 5.7.1 SPFBL invalid hostname.\n\n"; } else if (recipient != null && recipient.startsWith("postmaster@")) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "ACCEPT"); return "action=PREPEND " + "Received-SPFBL: " + result + " " + (url == null ? ticket : url + URLEncoder.encode(ticket, "UTF-8")) + "\n\n"; } else if (result.equals("PASS") && SPF.isGood(Provider.containsExact(mx) ? sender : mx)) { // O remetente vlido e tem excelente reputao, // ainda que o provedor dele esteja com reputao ruim. String url = Core.getURL(); String ticket = SPF.addQueryHam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "ACCEPT"); return "action=PREPEND " + "Received-SPFBL: PASS " + (url == null ? ticket : url + URLEncoder.encode(ticket, "UTF-8")) + "\n\n"; } else if (SPF.hasRed(tokenSet) || Analise.isCusterRED(ip, sender, hostname)) { Analise.processToday(ip); Analise.processToday(mx); Action action = client == null ? Action.REJECT : client.getActionRED(); if (action == Action.REJECT) { // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "REJECT"); return "action=554 5.7.1 SPFBL " + "you are temporarily listed.\n\n"; } else if (action == Action.DEFER) { if (Defer.defer(fluxo, Core.getDeferTimeRED())) { // Pelo menos um identificador est listado e com atrazo programado de um dia. String url = Core.getReleaseURL(fluxo); SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "LISTED"); if (url == null || Defer.count(fluxo) > 1) { return "action=451 4.7.2 SPFBL " + "you are temporarily listed.\n\n"; } else if (result.equals("PASS") && enviarLiberacao(url, sender, recipient)) { // Envio da liberao por e-mail se // houver validao do remetente por PASS. return "action=451 4.7.2 SPFBL " + "you are temporarily listed.\n\n"; } else { return "action=451 4.7.2 SPFBL LISTED " + url + "\n\n"; } } else { // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "REJECT"); return "action=554 5.7.1 SPFBL too many retries.\n\n"; } } else if (action == Action.FLAG) { SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "FLAG"); return "action=PREPEND X-Spam-Flag: YES\n\n"; } else if (action == Action.HOLD) { SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "HOLD"); return "action=HOLD very bad reputation.\n\n"; } else { SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "UNDEFINED"); return "action=WARN undefined action.\n\n"; } } else if (Domain.isGraceTime(sender) || Domain.isGraceTime(hostname)) { Server.logTrace("domain in grace time."); for (String token : tokenSet) { String block; Status status = SPF.getStatus(token); if (status == Status.RED && (block = Block.add(token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + status + "'."); Peer.sendBlockToAll(block); } if (status != Status.GREEN && !Subnet.isValidIP(token) && (block = Block.addIfNotNull(user, token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + status + "'."); } } Analise.processToday(ip); Analise.processToday(mx); Action action = client == null ? Action.REJECT : client.getActionGRACE(); if (action == Action.REJECT) { // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "REJECT"); return "action=554 5.7.1 SPFBL " + "your domain is in grace time.\n\n"; } else if (action == Action.DEFER) { if (Defer.defer(fluxo, Core.getDeferTimeRED())) { // Pelo menos um identificador est listado e com atrazo programado de um dia. String url = Core.getReleaseURL(fluxo); SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "LISTED"); if (url == null || Defer.count(fluxo) > 1) { return "action=451 4.7.2 SPFBL " + "you are temporarily listed.\n\n"; } else if (result.equals("PASS") && enviarLiberacao(url, sender, recipient)) { // Envio da liberao por e-mail se // houver validao do remetente por PASS. return "action=451 4.7.2 SPFBL " + "you are temporarily listed.\n\n"; } else { return "action=451 4.7.2 SPFBL LISTED " + url + "\n\n"; } } else { // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "REJECT"); return "action=554 5.7.1 SPFBL too many retries.\n\n"; } } else if (action == Action.FLAG) { SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "FLAG"); return "action=PREPEND X-Spam-Flag: YES\n\n"; } else if (action == Action.HOLD) { SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "HOLD"); return "action=HOLD domain in grace time.\n\n"; } else { SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "UNDEFINED"); return "action=WARN undefined action.\n\n"; } } else if (SPF.hasYellow(tokenSet) && Defer.defer(fluxo, Core.getDeferTimeYELLOW())) { Analise.processToday(ip); Analise.processToday(mx); Action action = client == null ? Action.DEFER : client.getActionYELLOW(); if (action == Action.DEFER) { // Pelo menos um identificador est em greylisting com atrazo programado de 10min. SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "GREYLISTED"); return "action=451 4.7.1 SPFBL you are greylisted.\n\n"; } else if (action == Action.HOLD) { SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "HOLD"); return "action=HOLD not good reputation.\n\n"; } else { SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "UNDEFINED"); return "action=WARN undefined action.\n\n"; } } else if (SPF.isFlood(tokenSet) && !Provider.containsHELO(ip, hostname) && Defer.defer(origem, Core.getDeferTimeFLOOD())) { Analise.processToday(ip); Analise.processToday(mx); // Pelo menos um identificador est com frequncia superior ao permitido. Server.logDebug("FLOOD " + tokenSet); SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "GREYLISTED"); return "action=451 4.7.1 SPFBL you are greylisted.\n\n"; } else if (result.equals("SOFTFAIL") && !Provider.containsHELO(ip, hostname) && Defer.defer(fluxo, Core.getDeferTimeSOFTFAIL())) { Analise.processToday(ip); Analise.processToday(mx); // SOFTFAIL com atrazo programado de 1min. SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "GREYLISTED"); return "action=451 4.7.1 SPFBL you are greylisted.\n\n"; } else { Analise.processToday(ip); Analise.processToday(mx); // Calcula frequencia de consultas. String url = Core.getURL(); String ticket = SPF.addQueryHam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "ACCEPT"); return "action=PREPEND " + "Received-SPFBL: " + result + " " + (url == null ? ticket : url + URLEncoder.encode(ticket, "UTF-8")) + "\n\n"; } } catch (ProcessException ex) { if (ex.isErrorMessage("SPF PARSE")) { return "action=REJECT [SPF] " + "One or more SPF records from " + sender + " " + "could not be interpreted. " + "Please see http://www.openspf.org/SPF_" + "Record_Syntax for details.\n\n"; } else if (ex.isErrorMessage("RESERVED")) { return "action=REJECT [SPF] " + "The domain of " + sender + " is a reserved TLD.\n\n"; } else if (sender == null) { Server.logError(ex); return "action=DEFER [SPF] " + "A transient error occurred. " + "Try again later.\n\n"; } else { return "action=DEFER [SPF] " + "A transient error occurred when " + "checking SPF record from " + sender + ", " + "preventing a result from being reached. " + "Try again later.\n\n"; } } catch (Exception ex) { Server.logError(ex); return "action=WARN SPFBL fatal error.\n\n"; } } }
From source file:net.spfbl.spf.SPF.java
/** * Processa a consulta e retorna o resultado. * * @param query a expresso da consulta.//from w w w . ja v a2 s .co m * @return o resultado do processamento. */ protected static String processSPF(InetAddress ipAddress, Client client, User user, String query, LinkedList<User> userList) { try { String result = ""; if (query.length() == 0) { return "INVALID QUERY\n"; } else { String origin; if (client == null) { origin = ipAddress.getHostAddress(); } else if (client.hasEmail()) { origin = ipAddress.getHostAddress() + " " + client.getDomain() + " " + client.getEmail(); } else { origin = ipAddress.getHostAddress() + " " + client.getDomain(); } StringTokenizer tokenizer = new StringTokenizer(query, " "); String firstToken = tokenizer.nextToken(); if (firstToken.equals("SPAM") && tokenizer.countTokens() == 1) { String ticket = tokenizer.nextToken(); TreeSet<String> tokenSet = addComplainURLSafe(origin, ticket, null); if (tokenSet == null) { result = "DUPLICATE COMPLAIN\n"; } else { String userEmail; try { userEmail = SPF.getClientURLSafe(ticket); } catch (Exception ex) { userEmail = client == null ? null : client.getEmail(); } user = User.get(userEmail); if (user != null) { userList.add(user); } String recipient; try { recipient = SPF.getRecipientURLSafe(ticket); } catch (ProcessException ex) { recipient = null; } result = "OK " + tokenSet + (recipient == null ? "" : " >" + recipient) + "\n"; } } else if (firstToken.equals("ABUSE") && tokenizer.hasMoreTokens()) { String token = tokenizer.nextToken(); if (token.startsWith("In-Reply-To:") && tokenizer.countTokens() == 1) { token = tokenizer.nextToken(); if (token.startsWith("From:")) { int index = token.indexOf(':') + 1; String recipient = token.substring(index); User recipientUser = User.get(recipient); if (recipientUser == null) { // Se a consulta originar de destinatrio com postmaster cadastrado, // considerar o prprio postmaster como usurio da consulta. index = recipient.indexOf('@'); String postmaster = "postmaster" + recipient.substring(index); User postmasterUser = User.get(postmaster); if (postmasterUser != null) { userList.add(user = postmasterUser); } } else { userList.add(user = recipientUser); } index = query.indexOf(':') + 1; String messageID = query.substring(index); result = "INVALID ID\n"; index = messageID.indexOf('<'); if (index >= 0) { messageID = messageID.substring(index + 1); index = messageID.indexOf('>'); if (index > 0) { messageID = messageID.substring(0, index); result = user.blockByMessageID(messageID) + '\n'; } } } else { result = "INVALID FROM\n"; } } else { result = "INVALID COMMAND\n"; } } else if (firstToken.equals("HOLDING") && tokenizer.countTokens() == 1) { String ticket = tokenizer.nextToken(); result = getHoldStatus(client, ticket, userList) + '\n'; } else if (firstToken.equals("LINK") && tokenizer.hasMoreTokens()) { String ticketSet = tokenizer.nextToken(); TreeSet<String> linkSet = new TreeSet<String>(); while (tokenizer.hasMoreTokens()) { linkSet.add(tokenizer.nextToken()); } StringTokenizer tokenizerTicket = new StringTokenizer(ticketSet, ";"); String unblockURL = null; boolean blocked = false; Action action = null; while (tokenizerTicket.hasMoreTokens()) { String ticket = tokenizerTicket.nextToken(); String userEmail; try { userEmail = SPF.getClientURLSafe(ticket); } catch (Exception ex) { userEmail = client == null ? null : client.getEmail(); } if ((user = User.get(userEmail)) != null) { userList.add(user); long dateTicket = SPF.getDateTicket(ticket); User.Query queryTicket = user.getQuery(dateTicket); if (queryTicket != null) { if (queryTicket.setLinkSet(linkSet)) { SPF.setSpam(dateTicket, queryTicket.getTokenSet()); if (!queryTicket.isWhite() && queryTicket.blockSender(dateTicket)) { Server.logDebug( "new BLOCK '" + queryTicket.getBlockSender() + "' added by LINK."); } action = client == null ? Action.REJECT : client.getActionBLOCK(); unblockURL = queryTicket.getUnblockURL(); blocked = true; } else if (queryTicket.isAnyLinkRED()) { action = client == null ? Action.FLAG : client.getActionRED(); } if (action == Action.HOLD) { queryTicket.setResult("HOLD"); } else if (action == Action.FLAG) { queryTicket.setResult("FLAG"); } else if (action == Action.REJECT) { queryTicket.setResult("REJECT"); } User.storeDB(dateTicket, queryTicket); } } } if (unblockURL != null) { result = "BLOCKED " + unblockURL + "\n"; } else if (blocked) { result = "BLOCKED\n"; } else if (action == Action.HOLD) { result = "HOLD\n"; } else if (action == Action.FLAG) { result = "FLAG\n"; } else if (action == Action.REJECT) { result = "REJECT\n"; } else { result = "CLEAR\n"; } } else if (firstToken.equals("MALWARE") && tokenizer.hasMoreTokens()) { String ticketSet = tokenizer.nextToken(); StringBuilder nameBuilder = new StringBuilder(); while (tokenizer.hasMoreTokens()) { if (nameBuilder.length() > 0) { nameBuilder.append(' '); } nameBuilder.append(tokenizer.nextToken()); } StringBuilder resultBuilder = new StringBuilder(); StringTokenizer ticketTokenizer = new StringTokenizer(ticketSet, ";"); while (ticketTokenizer.hasMoreTokens()) { String ticket = ticketTokenizer.nextToken(); TreeSet<String> tokenSet = addComplainURLSafe(origin, ticket, "MALWARE"); if (tokenSet == null) { resultBuilder.append("DUPLICATE COMPLAIN\n"); } else { // Processar reclamao. String userEmail; try { userEmail = SPF.getClientURLSafe(ticket); } catch (Exception ex) { userEmail = client == null ? null : client.getEmail(); } user = User.get(userEmail); if (user != null) { userList.add(user); long dateTicket = getDateTicket(ticket); User.Query userQuery = user.getQuery(dateTicket); if (userQuery != null && userQuery.setMalware(nameBuilder.toString())) { User.storeDB(dateTicket, userQuery); } } String recipient; try { recipient = SPF.getRecipientURLSafe(ticket); } catch (ProcessException ex) { recipient = null; } // Bloquear automaticamente todos // os tokens com reputao amarela ou vermelha. // Processar reclamao. for (String token : tokenSet) { String block; Status status = SPF.getStatus(token); if (status == Status.RED && (block = Block.add(token)) != null) { Server.logDebug( "new BLOCK '" + block + "' added by '" + recipient + ";MALWARE'."); Peer.sendBlockToAll(block); } if (status != Status.GREEN && !Subnet.isValidIP(token) && (block = Block.addIfNotNull(user, token)) != null) { Server.logDebug( "new BLOCK '" + block + "' added by '" + recipient + ";MALWARE'."); } } resultBuilder.append("OK "); resultBuilder.append(tokenSet); resultBuilder.append(recipient == null ? "" : " >" + recipient); resultBuilder.append("\n"); } } result = resultBuilder.toString(); } else if (firstToken.equals("HEADER") && tokenizer.hasMoreTokens()) { String ticketSet = tokenizer.nextToken(); String key = null; String from = null; String replyto = null; String messageID = null; String unsubscribe = null; String subject = null; while (tokenizer.hasMoreTokens()) { String token = tokenizer.nextToken(); if (token.startsWith("From:")) { key = "From"; int index = token.indexOf(':'); from = token.substring(index + 1); } else if (token.startsWith("ReplyTo:") || token.startsWith("Reply-To:")) { key = "Reply-To"; int index = token.indexOf(':'); replyto = token.substring(index + 1); } else if (token.startsWith("Message-ID:")) { key = "Message-ID"; int index = token.indexOf(':'); messageID = token.substring(index + 1); } else if (token.startsWith("List-Unsubscribe:")) { key = "List-Unsubscribe"; int index = token.indexOf(':'); unsubscribe = token.substring(index + 1); } else if (token.startsWith("Subject:")) { key = "Subject"; int index = token.indexOf(':'); subject = token.substring(index + 1); } else if (key == null) { from = null; replyto = null; unsubscribe = null; subject = null; break; } else if (key.equals("From")) { from += ' ' + token; } else if (key.equals("Reply-To")) { replyto += ' ' + token; } else if (key.equals("Message-ID")) { messageID += ' ' + token; } else if (key.equals("List-Unsubscribe")) { unsubscribe += ' ' + token; } else if (key.equals("Subject")) { subject += ' ' + token; } } if ((from == null || from.length() == 0) && (replyto == null || replyto.length() == 0) && (messageID == null || messageID.length() == 0) && (unsubscribe == null || unsubscribe.length() == 0) && (subject == null || subject.length() == 0)) { result = "INVALID COMMAND\n"; } else { boolean whitelisted = false; boolean blocklisted = false; TreeSet<String> unblockURLSet = new TreeSet<String>(); StringTokenizer ticketRokenizer = new StringTokenizer(ticketSet, ";"); int n = ticketRokenizer.countTokens(); ArrayList<User.Query> queryList = new ArrayList<User.Query>(n); while (ticketRokenizer.hasMoreTokens()) { String ticket = ticketRokenizer.nextToken(); String userEmail; try { userEmail = SPF.getClientURLSafe(ticket); } catch (Exception ex) { userEmail = client == null ? null : client.getEmail(); } if ((user = User.get(userEmail)) != null) { userList.add(user); long dateTicket = SPF.getDateTicket(ticket); User.Query queryTicket = user.getQuery(dateTicket); if (queryTicket != null) { queryList.add(queryTicket); String resultLocal = queryTicket.setHeader(from, replyto, subject, messageID, unsubscribe); if ("WHITE".equals(resultLocal)) { whitelisted = true; } else if ("BLOCK".equals(resultLocal)) { blocklisted = true; String url = queryTicket.getUnblockURL(); if (url != null) { unblockURLSet.add(url); } } User.storeDB(dateTicket, queryTicket); } } } if (whitelisted) { for (User.Query queryTicket : queryList) { queryTicket.setResult("WHITE"); } result = "WHITE\n"; } else if (blocklisted) { for (User.Query queryTicket : queryList) { queryTicket.setResult("BLOCK"); } if (unblockURLSet.size() == 1) { result = "BLOCKED " + unblockURLSet.first() + "\n"; } else { result = "BLOCKED\n"; } } else { result = "CLEAR\n"; } } } else if (firstToken.equals("HAM") && tokenizer.countTokens() == 1) { String ticket = tokenizer.nextToken(); TreeSet<String> tokenSet = deleteComplainURLSafe(origin, ticket); if (tokenSet == null) { result = "ALREADY REMOVED\n"; } else { String recipient; try { recipient = SPF.getRecipientURLSafe(ticket); } catch (ProcessException ex) { recipient = null; } result = "OK " + tokenSet + (recipient == null ? "" : " >" + recipient) + "\n"; } } else if (firstToken.equals("REFRESH") && tokenizer.countTokens() == 1) { String address = tokenizer.nextToken(); try { if (CacheSPF.refresh(address, true)) { result = "UPDATED\n"; } else { result = "NOT LOADED\n"; } } catch (ProcessException ex) { result = ex.getMessage() + "\n"; } } else if ((firstToken.equals("SPF") && tokenizer.countTokens() >= 4) || tokenizer.countTokens() == 2 || tokenizer.countTokens() == 1 || (firstToken.equals("CHECK") && tokenizer.countTokens() == 4) || (firstToken.equals("CHECK") && tokenizer.countTokens() == 3) || (firstToken.equals("CHECK") && tokenizer.countTokens() == 2)) { try { String ip; String sender; String helo; String recipient; String origem; String fluxo; if (firstToken.equals("SPF")) { // Nova formatao de consulta. ip = tokenizer.nextToken(); sender = tokenizer.nextToken(); while (!sender.endsWith("'") && tokenizer.hasMoreTokens()) { sender += " " + tokenizer.nextToken(); } helo = tokenizer.hasMoreTokens() ? tokenizer.nextToken() : "''"; recipient = tokenizer.hasMoreTokens() ? tokenizer.nextToken() : "''"; ip = ip.substring(1, ip.length() - 1); sender = sender.substring(1, sender.length() - 1); helo = helo.substring(1, helo.length() - 1); if (recipient.equals("'")) { recipient = tokenizer.hasMoreTokens() ? tokenizer.nextToken() : ""; if (recipient.endsWith("'")) { recipient = recipient.substring(0, recipient.length() - 1); } } else { recipient = recipient.substring(1, recipient.length() - 1); } if (sender.length() == 0) { sender = null; } else { sender = sender.toLowerCase(); } recipient = recipient.toLowerCase(); recipient = recipient.replace("\"", ""); } else if (firstToken.equals("CHECK") && tokenizer.countTokens() == 4) { ip = tokenizer.nextToken().toLowerCase(); sender = tokenizer.nextToken().toLowerCase(); helo = tokenizer.nextToken(); recipient = tokenizer.nextToken().toLowerCase(); if (ip.startsWith("'") && ip.endsWith("'")) { ip = ip.substring(1, ip.length() - 1); } if (sender.startsWith("'") && sender.endsWith("'")) { sender = sender.substring(1, sender.length() - 1); } if (helo.startsWith("'") && helo.endsWith("'")) { helo = helo.substring(1, helo.length() - 1); } if (recipient.startsWith("'") && recipient.endsWith("'")) { recipient = recipient.substring(1, recipient.length() - 1); } if (ip.length() == 0) { ip = null; } if (sender.length() == 0) { sender = null; } if (!Domain.isHostname(helo)) { helo = null; } if (recipient.length() == 0) { recipient = null; } else { recipient = recipient.toLowerCase(); } } else { // Manter compatibilidade da verso antiga. // Verso obsoleta. if (firstToken.equals("CHECK")) { ip = tokenizer.nextToken(); } else { ip = firstToken; } if (tokenizer.countTokens() == 2) { sender = tokenizer.nextToken().toLowerCase(); helo = tokenizer.nextToken(); } else { sender = null; helo = tokenizer.nextToken(); } recipient = null; if (ip.startsWith("'") && ip.endsWith("'")) { ip = ip.substring(1, ip.length() - 1); } if (sender != null && sender.startsWith("'") && sender.endsWith("'")) { sender = sender.substring(1, sender.length() - 1); if (sender.length() == 0) { sender = null; } } if (helo.startsWith("'") && helo.endsWith("'")) { helo = helo.substring(1, helo.length() - 1); } } if (!Subnet.isValidIP(ip)) { return "INVALID\n"; } else if (sender != null && !Domain.isEmail(sender)) { return "INVALID\n"; } else if (recipient != null && !Domain.isEmail(recipient)) { return "INVALID\n"; } else if (Subnet.isReservedIP(ip)) { // Message from LAN. return "LAN\n"; } else if (client != null && client.containsFull(ip)) { // Message from LAN. return "LAN\n"; } else { TreeSet<String> tokenSet = new TreeSet<String>(); ip = Subnet.normalizeIP(ip); tokenSet.add(ip); if (Domain.isValidEmail(recipient)) { // Se houver um remetente vlido, // Adicionar no ticket para controle. tokenSet.add('>' + recipient); } if (recipient != null) { User recipientUser = User.get(recipient); if (recipientUser == null) { // Se a consulta originar de destinatrio com postmaster cadastrado, // considerar o prprio postmaster como usurio da consulta. int index = recipient.indexOf('@'); String postmaster = "postmaster" + recipient.substring(index); User postmasterUser = User.get(postmaster); if (postmasterUser != null) { user = postmasterUser; } } else { user = recipientUser; } } if (user != null) { userList.add(user); tokenSet.add(user.getEmail() + ':'); } else if (client != null && client.hasEmail()) { tokenSet.add(client.getEmail() + ':'); } // Passar a acompanhar todos os // HELO quando apontados para o IP para // uma nova forma de interpretar dados. String hostname; if (CacheHELO.match(ip, helo, false)) { hostname = Domain.normalizeHostname(helo, true); } else { hostname = Reverse.getHostname(ip); hostname = Domain.normalizeHostname(hostname, true); } if (hostname == null) { Server.logDebug("no rDNS for " + ip + "."); } else if (Domain.isOfficialTLD(hostname)) { return "INVALID\n"; } else { // Verificao de pilha dupla, // para pontuao em ambas pilhas. String ipv4 = CacheHELO.getUniqueIPv4(hostname); String ipv6 = CacheHELO.getUniqueIPv6(hostname); if (ip.equals(ipv6) && CacheHELO.match(ipv4, hostname, false)) { // Equivalncia de pilha dupla se // IPv4 for nico para o hostname. tokenSet.add(ipv4); } else if (ip.equals(ipv4) && CacheHELO.match(ipv6, hostname, false)) { // Equivalncia de pilha dupla se // IPv6 for nico para o hostname. tokenSet.add(ipv6); } } if (Generic.containsGenericSoft(hostname)) { // Quando o reverso for // genrico, no consider-lo. hostname = null; } else if (hostname != null) { tokenSet.add(hostname); } LinkedList<String> logList = null; if (sender != null && firstToken.equals("CHECK")) { int index = sender.lastIndexOf('@'); String domain = sender.substring(index + 1); logList = new LinkedList<String>(); try { CacheSPF.refresh(domain, false); } catch (ProcessException ex) { logList.add("Cannot refresh SPF registry: " + ex.getErrorMessage()); logList.add("Using cached SPF registry."); } } SPF spf; if (sender == null) { spf = null; result = "NONE"; } else if (Domain.isOfficialTLD(sender)) { spf = null; result = "NONE"; } else if (Generic.containsGeneric(sender)) { spf = null; result = "NONE"; } else if ((spf = CacheSPF.get(sender)) == null) { result = "NONE"; } else if (spf.isInexistent()) { result = "NONE"; } else { result = spf.getResult(ip, sender, helo, logList); } String mx = Domain.extractHost(sender, true); if (user != null && user.isLocal()) { // Message from local user. return "LAN\n"; } else if (recipient != null && result.equals("PASS")) { if (recipient.endsWith(mx)) { // Message from same domain. return "LAN\n"; } else if (recipient.equals(Core.getAbuseEmail()) && User.exists(sender, "postmaster" + mx)) { // Message to abuse. return "LAN\n"; } } if (result.equals("PASS") || (sender != null && Provider.containsHELO(ip, hostname))) { // Quando fo PASS, significa que o domnio // autorizou envio pelo IP, portanto o dono dele // responsavel pelas mensagens. if (!Provider.containsExact(mx)) { // No um provedor ento // o MX deve ser listado. tokenSet.add(mx); origem = mx; } else if (Domain.isValidEmail(sender)) { // Listar apenas o remetente se o // hostname for um provedor de e-mail. String userEmail = null; String recipientEmail = null; for (String token : tokenSet) { if (token.endsWith(":")) { userEmail = token; } else if (token.startsWith(">")) { recipientEmail = token; } } tokenSet.clear(); tokenSet.add(sender); if (userEmail != null) { tokenSet.add(userEmail); } if (recipientEmail != null) { tokenSet.add(recipientEmail); } origem = sender; } else { origem = sender; } fluxo = origem + ">" + recipient; } else if (hostname == null) { origem = (sender == null ? "" : sender + '>') + ip; fluxo = origem + ">" + recipient; } else { String dominio = Domain.extractDomain(hostname, true); origem = (sender == null ? "" : sender + '>') + (dominio == null ? hostname : dominio.substring(1)); fluxo = origem + ">" + recipient; } Long recipientTrapTime = Trap.getTimeRecipient(client, user, recipient); if (firstToken.equals("CHECK")) { String results = "\nSPF resolution results:\n"; if (spf != null && spf.isInexistent()) { results += " NXDOMAIN\n"; } else if (logList == null || logList.isEmpty()) { results += " NONE\n"; } else { for (String log : logList) { results += " " + log + "\n"; } } String white; String block; if ((white = White.find(client, user, ip, sender, hostname, result, recipient)) != null) { results += "\nFirst WHITE match: " + white + "\n"; } else if ((block = Block.find(client, user, ip, sender, hostname, result, recipient, false, true, true, false)) != null) { results += "\nFirst BLOCK match: " + block + "\n"; } TreeSet<String> graceSet = new TreeSet<String>(); if (Domain.isGraceTime(sender)) { graceSet.add(Domain.extractDomain(sender, false)); } if (Domain.isGraceTime(hostname)) { graceSet.add(Domain.extractDomain(hostname, false)); } if (!graceSet.isEmpty()) { results += "\n"; results += "Domains in grace time:\n"; for (String grace : graceSet) { results += " " + grace + "\n"; } } results += "\n"; results += "Considered identifiers and status:\n"; tokenSet = expandTokenSet(tokenSet); TreeMap<String, Distribution> distributionMap = CacheDistribution.getMap(tokenSet); int count = 0; for (String token : tokenSet) { if (!token.startsWith(">") && !token.endsWith(":")) { if (!Ignore.contains(token)) { float probability; Status status; if (distributionMap.containsKey(token)) { Distribution distribution = distributionMap.get(token); probability = distribution.getSpamProbability(token); status = distribution.getStatus(token); } else { probability = 0.0f; status = SPF.Status.GREEN; } results += " " + token + " " + status.name() + " " + Core.DECIMAL_FORMAT.format(probability) + "\n"; count++; } } } if (count == 0) { results += " NONE\n"; } results += "\n"; return results; } else if (recipientTrapTime == null && White.contains(client, user, ip, sender, hostname, result, recipient)) { if (White.contains(client, user, ip, sender, hostname, result, null)) { // Limpa da lista BLOCK um possvel falso positivo. Block.clear(client, user, ip, sender, hostname, result, null); } // Calcula frequencia de consultas. String url = Core.getURL(); String ticket = SPF.addQueryHam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "WHITE"); return "WHITE " + (url == null ? ticket : url + ticket) + "\n"; } else if (Block.contains(client, user, ip, sender, hostname, result, recipient, true, true, true, true)) { Action action = client == null ? Action.REJECT : client.getActionBLOCK(); if (action == Action.REJECT) { // Calcula frequencia de consultas. long time = Server.getNewUniqueTime(); User.Query queryLocal = SPF.addQuerySpam(time, client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "BLOCK"); action = client == null ? Action.FLAG : client.getActionRED(); if (action != Action.REJECT && queryLocal != null && queryLocal.needHeader()) { if (action == Action.FLAG) { queryLocal.setResult("FLAG"); String url = Core.getURL(); String ticket = SPF.createTicket(time, tokenSet); return "FLAG " + (url == null ? ticket : url + ticket) + "\n"; } else if (action == Action.HOLD) { queryLocal.setResult("HOLD"); String url = Core.getURL(); String ticket = SPF.createTicket(time, tokenSet); return "HOLD " + (url == null ? ticket : url + ticket) + "\n"; } else { return "ERROR: UNDEFINED ACTION\n"; } } else { String url = Core.getUnblockURL(client, user, ip, sender, hostname, recipient); if (url == null) { return "BLOCKED\n"; } else { return "BLOCKED " + url + "\n"; } } } else if (action == Action.FLAG) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "FLAG"); return "FLAG " + (url == null ? ticket : url + ticket) + "\n"; } else if (action == Action.HOLD) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "HOLD"); return "HOLD " + (url == null ? ticket : url + ticket) + "\n"; } else { return "ERROR: UNDEFINED ACTION\n"; } } else if (Generic.containsDynamicDomain(hostname)) { // Bloquear automaticamente range de IP dinmico. String cidr = Subnet .normalizeCIDR(SubnetIPv4.isValidIPv4(ip) ? ip + "/24" : ip + "/48"); if (Block.tryOverlap(cidr)) { Server.logDebug( "new BLOCK '" + cidr + "' added by '" + hostname + ";DYNAMIC'."); } else if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by '" + hostname + ";DYNAMIC'."); } SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); return "INVALID\n"; } else if (spf != null && spf.isDefinitelyInexistent()) { // Bloquear automaticamente IP com reputao vermelha. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by '" + mx + ";NXDOMAIN'."); } } Analise.processToday(ip); // O domnio foi dado como inexistente inmeras vezes. // Rejeitar e denunciar o host pois h abuso de tentativas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "NXDOMAIN"); return "NXDOMAIN\n"; } else if (spf != null && spf.isInexistent()) { Analise.processToday(ip); SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "NXDOMAIN"); return "NXDOMAIN\n"; } else if (result.equals("FAIL")) { // Bloquear automaticamente IP com reputao vermelha. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by '" + sender + ";FAIL'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "FAIL"); // Retornar FAIL somente se no houver // liberao literal do remetente com FAIL. return "FAIL\n"; } else if (sender != null && !Domain.isEmail(sender)) { // Bloquear automaticamente IP com reputao vermelha. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug( "new BLOCK '" + ip + "' added by '" + sender + ";INVALID'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); return "INVALID\n"; } else if (sender != null && Domain.isOfficialTLD(sender)) { // Bloquear automaticamente IP com reputao vermelha. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug( "new BLOCK '" + ip + "' added by '" + sender + ";RESERVED'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); return "INVALID\n"; } else if (sender == null && !CacheHELO.match(ip, hostname, false)) { // Bloquear automaticamente IP com reputao ruim. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by 'INVALID'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); // HELO invlido sem remetente. return "INVALID\n"; } else if (hostname == null && Core.isReverseRequired()) { if (Block.tryAdd(ip)) { Server.logDebug("new BLOCK '" + ip + "' added by 'NONE'."); } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); // Require a valid HELO or reverse. return "INVALID\n"; } else if (recipient != null && !Domain.isValidEmail(recipient)) { Analise.processToday(ip); Analise.processToday(mx); SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INEXISTENT"); return "INEXISTENT\n"; } else if (recipientTrapTime != null) { if (System.currentTimeMillis() > recipientTrapTime) { // Spamtrap for (String token : tokenSet) { String block; Status status = SPF.getStatus(token); if (status == Status.RED && (block = Block.add(token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + recipient + ";SPAMTRAP'."); Peer.sendBlockToAll(block); } if (status != Status.GREEN && !Subnet.isValidIP(token) && (block = Block.addIfNotNull(user, token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + recipient + ";SPAMTRAP'."); } } Analise.processToday(ip); Analise.processToday(mx); // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "TRAP"); return "SPAMTRAP\n"; } else { // Inexistent for (String token : tokenSet) { String block; Status status = SPF.getStatus(token); if (status == Status.RED && (block = Block.add(token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + recipient + ";INEXISTENT'."); Peer.sendBlockToAll(block); } if (status != Status.GREEN && !Subnet.isValidIP(token) && (block = Block.addIfNotNull(user, token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + recipient + ";INEXISTENT'."); } } Analise.processToday(ip); Analise.processToday(mx); SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INEXISTENT"); return "INEXISTENT\n"; } } else if (Defer.count(fluxo) > Core.getFloodMaxRetry()) { Analise.processToday(ip); Analise.processToday(mx); // A origem atingiu o limite de atraso // para liberao do destinatrio. long time = System.currentTimeMillis(); Defer.end(fluxo); Server.logDefer(time, fluxo, "DEFER FLOOD"); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "REJECT"); return "BLOCKED\n"; } else if (!result.equals("PASS") && !CacheHELO.match(ip, hostname, false)) { // Bloquear automaticamente IP com reputao amarela. if (SPF.isRed(ip)) { if (Block.tryAdd(ip)) { Server.logDebug( "new BLOCK '" + ip + "' added by '" + recipient + ";INVALID'."); } } Analise.processToday(ip); SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "INVALID"); return "INVALID\n"; } else if (recipient != null && recipient.startsWith("postmaster@")) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "ACCEPT"); return result + " " + (url == null ? ticket : url + URLEncoder.encode(ticket, "UTF-8")) + "\n"; } else if (result.equals("PASS") && SPF.isGood(Provider.containsExact(mx) ? sender : mx)) { // O remetente vlido e tem excelente reputao, // ainda que o provedor dele esteja com reputao ruim. String url = Core.getURL(); String ticket = SPF.addQueryHam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "ACCEPT"); return "PASS " + (url == null ? ticket : url + URLEncoder.encode(ticket, "UTF-8")) + "\n"; } else if (SPF.hasRed(tokenSet) || Analise.isCusterRED(ip, sender, hostname)) { Analise.processToday(ip); Analise.processToday(mx); Action action = client == null ? Action.REJECT : client.getActionRED(); if (action == Action.REJECT) { // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "REJECT"); return "BLOCKED\n"; } else if (action == Action.DEFER) { if (Defer.defer(fluxo, Core.getDeferTimeRED())) { String url = Core.getReleaseURL(fluxo); SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "LISTED"); if (url == null || Defer.count(fluxo) > 1) { return "LISTED\n"; } else if (result.equals("PASS") && enviarLiberacao(url, sender, recipient)) { // Envio da liberao por e-mail se // houver validao do remetente por PASS. return "LISTED\n"; } else { return "LISTED " + url + "\n"; } } else { // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "REJECT"); return "BLOCKED\n"; } } else if (action == Action.FLAG) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "FLAG"); return "FLAG " + (url == null ? ticket : url + ticket) + "\n"; } else if (action == Action.HOLD) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "HOLD"); return "HOLD " + (url == null ? ticket : url + ticket) + "\n"; } else { return "ERROR: UNDEFINED ACTION\n"; } } else if (Domain.isGraceTime(sender) || Domain.isGraceTime(hostname)) { Server.logTrace("domain in grace time."); for (String token : tokenSet) { String block; Status status = SPF.getStatus(token); if (status == Status.RED && (block = Block.add(token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + status + "'."); Peer.sendBlockToAll(block); } if (status != Status.GREEN && !Subnet.isValidIP(token) && (block = Block.addIfNotNull(user, token)) != null) { Server.logDebug("new BLOCK '" + block + "' added by '" + status + "'."); } } Analise.processToday(ip); Analise.processToday(mx); Action action = client == null ? Action.REJECT : client.getActionGRACE(); if (action == Action.REJECT) { // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "REJECT"); return "BLOCKED\n"; } else if (action == Action.DEFER) { if (Defer.defer(fluxo, Core.getDeferTimeRED())) { String url = Core.getReleaseURL(fluxo); SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "LISTED"); if (url == null || Defer.count(fluxo) > 1) { return "LISTED\n"; } else if (result.equals("PASS") && enviarLiberacao(url, sender, recipient)) { // Envio da liberao por e-mail se // houver validao do remetente por PASS. return "LISTED\n"; } else { return "LISTED " + url + "\n"; } } else { // Calcula frequencia de consultas. SPF.addQuerySpam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "REJECT"); return "BLOCKED\n"; } } else if (action == Action.FLAG) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "FLAG"); return "FLAG " + (url == null ? ticket : url + ticket) + "\n"; } else if (action == Action.HOLD) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "HOLD"); return "HOLD " + (url == null ? ticket : url + ticket) + "\n"; } else { return "ERROR: UNDEFINED ACTION\n"; } } else if (SPF.hasYellow(tokenSet) && Defer.defer(fluxo, Core.getDeferTimeYELLOW())) { Analise.processToday(ip); Analise.processToday(mx); Action action = client == null ? Action.DEFER : client.getActionYELLOW(); if (action == Action.DEFER) { // Pelo menos um identificador do conjunto est em greylisting com atrazo de 10min. SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "GREYLIST"); return "GREYLIST\n"; } else if (action == Action.HOLD) { String url = Core.getURL(); String ticket = SPF.getTicket(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "HOLD"); return "HOLD " + (url == null ? ticket : url + ticket) + "\n"; } else { return "ERROR: UNDEFINED ACTION\n"; } } else if (SPF.isFlood(tokenSet) && !Provider.containsHELO(ip, hostname) && Defer.defer(origem, Core.getDeferTimeFLOOD())) { Analise.processToday(ip); Analise.processToday(mx); // Pelo menos um identificador est com frequncia superior ao permitido. Server.logDebug("FLOOD " + tokenSet); SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "GREYLIST"); return "GREYLIST\n"; } else if (result.equals("SOFTFAIL") && !Provider.containsHELO(ip, hostname) && Defer.defer(fluxo, Core.getDeferTimeSOFTFAIL())) { Analise.processToday(ip); Analise.processToday(mx); // SOFTFAIL com atrazo de 1min. SPF.addQuery(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "GREYLIST"); return "GREYLIST\n"; } else { Analise.processToday(ip); Analise.processToday(mx); // Calcula frequencia de consultas. String url = Core.getURL(); String ticket = SPF.addQueryHam(client, user, ip, helo, hostname, sender, result, recipient, tokenSet, "ACCEPT"); return result + " " + (url == null ? ticket : url + URLEncoder.encode(ticket, "UTF-8")) + "\n"; } } } catch (ProcessException ex) { if (ex.isErrorMessage("HOST NOT FOUND")) { return "NXDOMAIN\n"; } else { throw ex; } } } else { return "INVALID QUERY\n"; } } return result; } catch (ProcessException ex) { Server.logError(ex); return ex.getMessage() + "\n"; } catch (Exception ex) { Server.logError(ex); return "ERROR: FATAL\n"; } }