List of usage examples for java.util Iterator next
E next();
From source file:executables.Align.java
@SuppressWarnings("static-access") public static void main(String[] args) throws IOException { Options options = new Options() .addOption(OptionBuilder.withArgName("f1").withDescription("Fasta file 1").hasArg().create("f1")) .addOption(OptionBuilder.withArgName("f2").withDescription("Fasta file 2").hasArg().create("f2")) .addOption(OptionBuilder.withArgName("s1").withDescription("sequence 1").hasArg().create("s1")) .addOption(OptionBuilder.withArgName("s2").withDescription("sequence 2").hasArg().create("s2")) .addOption(OptionBuilder.withArgName("gap-linear").withDescription("Linear gap cost").hasArg() .create("gl")) .addOption(OptionBuilder.withArgName("gap-open").withDescription("Affine gap open cost").hasArg() .create("go")) .addOption(OptionBuilder.withArgName("gap-extend").withDescription("Affine gap extend cost") .hasArg().create("ge")) .addOption(OptionBuilder.withArgName("gap-function").withDescription("Gap function file").hasArg() .create("gf")) .addOption(//from w w w . j av a 2 s . c o m OptionBuilder.withArgName("gapless").withDescription("Gapless alignment").create("gapless")) .addOption(OptionBuilder.withArgName("mode") .withDescription("Alignment mode: global,local,freeshift (Default: freeshift)").hasArg() .create('m')) .addOption(OptionBuilder.withArgName("match").withDescription("Match score").hasArg().create("ma")) .addOption(OptionBuilder.withArgName("mismatch").withDescription("Mismatch score").hasArg() .create("mi")) .addOption(OptionBuilder.withDescription("Do not append unaligned flanking sequences") .create("noflank")) .addOption(OptionBuilder.withArgName("check").withDescription("Calculate checkscore").create('c')) .addOption(OptionBuilder.withArgName("format").withDescription( "Output format, see String.format, parameters are: id1,id2,score,alignment (alignment only, if -f is specified); (default: '%s %s %.4f' w/o -f and '%s %s %.4f\n%s' w/ -f)") .hasArg().create("format")) .addOption(OptionBuilder.withArgName("matrix") .withDescription("Output dynamic programming matrix as well").create("matrix")) .addOption(OptionBuilder.withArgName("quasar-format") .withDescription("Scoring matrix in quasar format").hasArg().create('q')) .addOption( OptionBuilder.withArgName("pairs").withDescription("Pairs file").hasArg().create("pairs")) .addOption(OptionBuilder.withArgName("output").withDescription("Output").hasArg().create('o')) .addOption(OptionBuilder.withArgName("seqlib").withDescription("Seqlib file").hasArg() .create("seqlib")) .addOption(OptionBuilder.withArgName("full").withDescription("Full output").create('f')); CommandLineParser parser = new PosixParser(); try { CommandLine cmd = parser.parse(options, args); LongScoring<CharSequence> scoring = createScoring(cmd); AlignmentMode mode = createMode(cmd); if (mode == null) throw new ParseException("Mode unknown: " + cmd.getOptionValue('m')); Iterator<MutablePair<String, String>> idIterator = createSequences(scoring, cmd); GapCostFunction gap = createGapFunction(cmd); String format = getFormat(cmd); LongAligner<CharSequence> aligner; if (gap instanceof AffineGapCostFunction) aligner = new LongAligner<CharSequence>(scoring, ((AffineGapCostFunction) gap).getGapOpen(), ((AffineGapCostFunction) gap).getGapExtend(), mode); else if (gap instanceof LinearGapCostFunction) aligner = new LongAligner<CharSequence>(scoring, ((LinearGapCostFunction) gap).getGap(), mode); else if (gap instanceof InfiniteGapCostFunction) aligner = new LongAligner<CharSequence>(scoring, mode); else throw new RuntimeException("Gap cost function " + gap.toString() + " currently not supported!"); SimpleAlignmentFormatter formatter = cmd.hasOption('f') ? new SimpleAlignmentFormatter().setAppendUnaligned(!cmd.hasOption("noflank")) : null; CheckScore checkscore = cmd.hasOption('c') ? new CheckScore() : null; Alignment alignment = checkscore != null || formatter != null ? new Alignment() : null; float score; String ali; LineOrientedFile out = new LineOrientedFile( cmd.hasOption('o') ? cmd.getOptionValue('o') : LineOrientedFile.STDOUT); Writer wr = out.startWriting(); while (idIterator.hasNext()) { MutablePair<String, String> ids = idIterator.next(); score = alignment == null ? aligner.alignCache(ids.Item1, ids.Item2) : aligner.alignCache(ids.Item1, ids.Item2, alignment); ali = formatter != null ? formatter.format(alignment, scoring, gap, mode, scoring.getCachedSubject(ids.Item1), scoring.getCachedSubject(ids.Item2)) : ""; out.writeLine(String.format(Locale.US, format, ids.Item1, ids.Item2, score, ali)); if (cmd.hasOption("matrix")) { aligner.writeMatrix(wr, aligner.getScoring().getCachedSubject(ids.Item1).toString().toCharArray(), aligner.getScoring().getCachedSubject(ids.Item2).toString().toCharArray()); } if (checkscore != null) checkscore.checkScore(aligner, scoring.getCachedSubject(ids.Item1).length(), scoring.getCachedSubject(ids.Item2).length(), alignment, score); } out.finishWriting(); } catch (ParseException e) { e.printStackTrace(); HelpFormatter f = new HelpFormatter(); f.printHelp("Align", options); } }
From source file:com.cloud.test.stress.TestClientWithAPI.java
public static void main(String[] args) { String host = "http://localhost"; String port = "8092"; String devPort = "8080"; String apiUrl = "/client/api"; try {/*ww w . j a va2 s. c om*/ // Parameters List<String> argsList = Arrays.asList(args); Iterator<String> iter = argsList.iterator(); while (iter.hasNext()) { String arg = iter.next(); // host if (arg.equals("-h")) { host = "http://" + iter.next(); } if (arg.equals("-p")) { port = iter.next(); } if (arg.equals("-dp")) { devPort = iter.next(); } if (arg.equals("-t")) { numThreads = Integer.parseInt(iter.next()); } if (arg.equals("-s")) { sleepTime = Long.parseLong(iter.next()); } if (arg.equals("-a")) { accountName = iter.next(); } if (arg.equals("-c")) { cleanUp = Boolean.parseBoolean(iter.next()); if (!cleanUp) sleepTime = 0L; // no need to wait if we don't ever // cleanup } if (arg.equals("-r")) { repeat = Boolean.parseBoolean(iter.next()); } if (arg.equals("-u")) { numOfUsers = Integer.parseInt(iter.next()); } if (arg.equals("-i")) { internet = Boolean.parseBoolean(iter.next()); } if (arg.equals("-w")) { wait = Integer.parseInt(iter.next()); } if (arg.equals("-z")) { zoneId = iter.next(); } if (arg.equals("-snapshot")) { snapshot_test = "yes"; } if (arg.equals("-so")) { serviceOfferingId = iter.next(); } if (arg.equals("-do")) { diskOfferingId = iter.next(); } if (arg.equals("-no")) { networkOfferingId = iter.next(); } if (arg.equals("-pass")) { vmPassword = iter.next(); } if (arg.equals("-url")) { downloadUrl = iter.next(); } } final String server = host + ":" + port + "/"; final String developerServer = host + ":" + devPort + apiUrl; s_logger.info("Starting test against server: " + server + " with " + numThreads + " thread(s)"); if (cleanUp) s_logger.info("Clean up is enabled, each test will wait " + sleepTime + " ms before cleaning up"); if (numOfUsers > 0) { s_logger.info("Pre-generating users for test of size : " + numOfUsers); users = new String[numOfUsers]; Random ran = new Random(); for (int i = 0; i < numOfUsers; i++) { users[i] = Math.abs(ran.nextInt()) + "-user"; } } for (int i = 0; i < numThreads; i++) { new Thread(new Runnable() { @Override public void run() { do { String username = null; try { long now = System.currentTimeMillis(); Random ran = new Random(); if (users != null) { username = users[Math.abs(ran.nextInt()) % numOfUsers]; } else { username = Math.abs(ran.nextInt()) + "-user"; } NDC.push(username); s_logger.info("Starting test for the user " + username); int response = executeDeployment(server, developerServer, username, snapshot_test); boolean success = false; String reason = null; if (response == 200) { success = true; if (internet) { s_logger.info("Deploy successful...waiting 5 minute before SSH tests"); Thread.sleep(300000L); // Wait 60 // seconds so // the windows VM // can boot up and do a sys prep. if (accountName == null) { s_logger.info("Begin Linux SSH test for account " + _account.get()); reason = sshTest(_linuxIP.get(), _linuxPassword.get(), snapshot_test); } if (reason == null) { s_logger.info( "Linux SSH test successful for account " + _account.get()); s_logger.info("Begin WindowsSSH test for account " + _account.get()); reason = sshTest(_linuxIP.get(), _linuxPassword.get(), snapshot_test); // reason = sshWinTest(_windowsIP.get()); } // release the linux IP now... _linuxIP.set(null); // release the Windows IP now _windowsIP.set(null); } // sleep for 3 min before getting the latest network stat // s_logger.info("Sleeping for 5 min before getting the lates network stat for the account"); // Thread.sleep(300000); // verify that network stat is correct for the user; if it's not - stop all the resources // for the user // if ((reason == null) && (getNetworkStat(server) == false) ) { // s_logger.error("Stopping all the resources for the account " + _account.get() + // " as network stat is incorrect"); // int stopResponseCode = executeStop( // server, developerServer, // username, false); // s_logger // .info("stop command finished with response code: " // + stopResponseCode); // success = false; // since the SSH test // // } else if (reason == null) { if (internet) { s_logger.info( "Windows SSH test successful for account " + _account.get()); } else { s_logger.info("deploy test successful....now cleaning up"); if (cleanUp) { s_logger.info( "Waiting " + sleepTime + " ms before cleaning up vms"); Thread.sleep(sleepTime); } else { success = true; } } if (usageIterator >= numThreads) { int eventsAndBillingResponseCode = executeEventsAndBilling(server, developerServer); s_logger.info( "events and usage records command finished with response code: " + eventsAndBillingResponseCode); usageIterator = 1; } else { s_logger.info( "Skipping events and usage records for this user: usageIterator " + usageIterator + " and number of Threads " + numThreads); usageIterator++; } if ((users == null) && (accountName == null)) { s_logger.info("Sending cleanup command"); int cleanupResponseCode = executeCleanup(server, developerServer, username); s_logger.info("cleanup command finished with response code: " + cleanupResponseCode); success = (cleanupResponseCode == 200); } else { s_logger.info("Sending stop DomR / destroy VM command"); int stopResponseCode = executeStop(server, developerServer, username, true); s_logger.info("stop(destroy) command finished with response code: " + stopResponseCode); success = (stopResponseCode == 200); } } else { // Just stop but don't destroy the // VMs/Routers s_logger.info("SSH test failed for account " + _account.get() + "with reason '" + reason + "', stopping VMs"); int stopResponseCode = executeStop(server, developerServer, username, false); s_logger.info( "stop command finished with response code: " + stopResponseCode); success = false; // since the SSH test // failed, mark the // whole test as // failure } } else { // Just stop but don't destroy the // VMs/Routers s_logger.info("Deploy test failed with reason '" + reason + "', stopping VMs"); int stopResponseCode = executeStop(server, developerServer, username, true); s_logger.info("stop command finished with response code: " + stopResponseCode); success = false; // since the deploy test // failed, mark the // whole test as failure } if (success) { s_logger.info("***** Completed test for user : " + username + " in " + ((System.currentTimeMillis() - now) / 1000L) + " seconds"); } else { s_logger.info("##### FAILED test for user : " + username + " in " + ((System.currentTimeMillis() - now) / 1000L) + " seconds with reason : " + reason); } s_logger.info("Sleeping for " + wait + " seconds before starting next iteration"); Thread.sleep(wait); } catch (Exception e) { s_logger.warn("Error in thread", e); try { int stopResponseCode = executeStop(server, developerServer, username, true); s_logger.info("stop response code: " + stopResponseCode); } catch (Exception e1) { } } finally { NDC.clear(); } } while (repeat); } }).start(); } } catch (Exception e) { s_logger.error(e); } }
From source file:net.sf.extjwnl.cli.ewn.java
public static void main(String[] args) throws IOException, JWNLException { if (args.length < 1) { System.out.println(USAGE); System.exit(0);/* w ww .j a v a 2 s. com*/ } //find dictionary Dictionary d = null; File config = new File(defaultConfig); if (!config.exists()) { if (System.getenv().containsKey("WNHOME")) { String wnHomePath = System.getenv().get("WNHOME"); File wnHome = new File(wnHomePath); if (wnHome.exists()) { d = Dictionary.getFileBackedInstance(wnHomePath); } else { log.error("Cannot find dictionary. Make sure " + defaultConfig + " is available or WNHOME variable is set."); } } } else { d = Dictionary.getInstance(new FileInputStream(config)); } if (null != d) { //parse and execute command line if ((-1 < args[0].indexOf('%') && -1 < args[0].indexOf(':')) || "-script".equals(args[0]) || (-1 < args[0].indexOf('#'))) { d.edit(); //edit if ("-script".equals(args[0])) { if (args.length < 2) { log.error("Filename missing for -script command"); System.exit(1); } else { File script = new File(args[1]); if (script.exists()) { //load into args ArrayList<String> newArgs = new ArrayList<String>(); BufferedReader in = new BufferedReader( new InputStreamReader(new FileInputStream(script), "UTF-8")); try { String str; while ((str = in.readLine()) != null) { String[] bits = str.split(" "); StringBuilder tempArg = null; for (String bit : bits) { int quoteCnt = 0; for (int j = 0; j < bit.length(); j++) { if ('"' == bit.charAt(j)) { quoteCnt++; } } if (null != tempArg) { if (0 == quoteCnt) { tempArg.append(" ").append(bit); } else { tempArg.append(" ").append(bit.replaceAll("\"\"", "\"")); if (1 == (quoteCnt % 2)) { newArgs.add( tempArg.toString().substring(1, tempArg.length() - 1)); tempArg = null; } } } else { if (0 == quoteCnt) { newArgs.add(bit); } else { if (1 == (quoteCnt % 2)) { tempArg = new StringBuilder(bit.replaceAll("\"\"", "\"")); } else { newArgs.add(bit.replaceAll("\"\"", "\"")); } } } } if (null != tempArg) { newArgs.add(tempArg.toString()); } } } finally { try { in.close(); } catch (IOException e) { //nop } } args = newArgs.toArray(args); } } } Word workWord = null; String key = null; String lemma = null; int lexFileNum = -1; int lexId = -1; // String headLemma = null; // int headLexId = -1; POS pos = null; String derivation = null; for (int i = 0; i < args.length; i++) { if (null == key && '-' != args[i].charAt(0) && ((-1 < args[i].indexOf('%') && -1 < args[i].indexOf(':')))) { key = args[i]; log.info("Searching " + key + "..."); if (null != key) { workWord = d.getWordBySenseKey(key); } if (null == workWord) { //parse sensekey lemma = key.substring(0, key.indexOf('%')).replace('_', ' '); String posId = key.substring(key.indexOf('%') + 1, key.indexOf(':')); if ("1".equals(posId) || "2".equals(posId) || "3".equals(posId) || "4".equals(posId) || "5".equals(posId)) { pos = POS.getPOSForId(Integer.parseInt(posId)); String lexFileString = key.substring(key.indexOf(':') + 1); if (-1 < lexFileString.indexOf(':')) { lexFileNum = Integer .parseInt(lexFileString.substring(0, lexFileString.indexOf(':'))); if (lexFileString.indexOf(':') + 1 < lexFileString.length()) { String lexIdString = lexFileString .substring(lexFileString.indexOf(':') + 1); if (-1 < lexIdString.indexOf(':')) { lexId = Integer .parseInt(lexIdString.substring(0, lexIdString.indexOf(':'))); // if (lexIdString.indexOf(':') + 1 < lexIdString.length()) { // headLemma = lexIdString.substring(lexIdString.indexOf(':') + 1); // if (-1 < headLemma.indexOf(':')) { // headLemma = headLemma.substring(0, headLemma.indexOf(':')); // if (null != headLemma && !"".equals(headLemma) && lexIdString.lastIndexOf(':') + 1 < lexIdString.length()) { // headLexId = Integer.parseInt(lexIdString.substring(lexIdString.lastIndexOf(':') + 1)); // } // } else { // log.error("Malformed sensekey " + key); // System.exit(1); // } // } } else { log.error("Malformed sensekey " + key); System.exit(1); } } else { log.error("Malformed sensekey " + key); System.exit(1); } } else { log.error("Malformed sensekey " + key); System.exit(1); } } else { log.error("Malformed sensekey " + key); System.exit(1); } } } else if (-1 < args[i].indexOf('#')) { if (2 < args[i].length()) { derivation = args[i].substring(2); if (null == derivation) { log.error("Missing derivation"); System.exit(1); } else { pos = POS.getPOSForKey(args[i].substring(0, 1)); if (null == pos) { log.error("POS " + args[i] + " is not recognized for derivation " + derivation); System.exit(1); } } } } if ("-add".equals(args[i])) { if (null == key) { log.error("Missing sensekey"); System.exit(1); } if (null != workWord) { log.error("Duplicate sensekey " + workWord.getSenseKey()); System.exit(1); } log.info("Creating " + pos.getLabel() + " synset..."); Synset tempSynset = d.createSynset(pos); log.info("Creating word " + lemma + "..."); workWord = new Word(d, tempSynset, 1, lemma); workWord.setLexId(lexId); tempSynset.getWords().add(workWord); tempSynset.setLexFileNum(lexFileNum); key = null; } if ("-remove".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { d.removeSynset(workWord.getSynset()); workWord = null; key = null; } } if ("-addword".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length && '-' != args[i].charAt(0)) { Word tempWord = new Word(d, workWord.getSynset(), workWord.getSynset().getWords().size() + 1, args[i]); workWord.getSynset().getWords().add(tempWord); key = null; } else { log.error( "Missing word for addword command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-removeword".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { workWord.getSynset().getWords().remove(workWord); key = null; } } if ("-setgloss".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length && '-' != args[i].charAt(0)) { workWord.getSynset().setGloss(args[i]); key = null; } else { log.error("Missing gloss for setgloss command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-setadjclus".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length && '-' != args[i].charAt(0)) { workWord.getSynset().setIsAdjectiveCluster(Boolean.parseBoolean(args[i])); key = null; } else { log.error("Missing flag for setadjclus command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-setverbframe".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length) { if (workWord instanceof Verb) { Verb verb = (Verb) workWord; if ('-' == args[i].charAt(0)) { verb.getVerbFrameFlags().clear(Integer.parseInt(args[i].substring(1))); } else { verb.getVerbFrameFlags().set(Integer.parseInt(args[i])); } } else { log.error("Word at " + workWord.getSenseKey() + " should be verb"); System.exit(1); } key = null; } else { log.error("Missing index for setverbframe command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-setverbframeall".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length) { if (workWord.getSynset() instanceof VerbSynset) { if ('-' == args[i].charAt(0)) { workWord.getSynset().getVerbFrameFlags() .clear(Integer.parseInt(args[i].substring(1))); } else { workWord.getSynset().getVerbFrameFlags().set(Integer.parseInt(args[i])); } } else { log.error("Synset at " + workWord.getSenseKey() + " should be verb"); System.exit(1); } key = null; } else { log.error("Missing index for setverbframeall command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-setlexfile".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length && '-' != args[i].charAt(0)) { if (-1 < args[i].indexOf('.')) { workWord.getSynset() .setLexFileNum(LexFileNameLexFileIdMap.getMap().get(args[i])); } else { workWord.getSynset().setLexFileNum(Integer.parseInt(args[i])); } } else { log.error("Missing file number or name for setlexfile command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-addptr".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length) { Word targetWord = d.getWordBySenseKey(args[i]); if (null != targetWord) { i++; if (i < args.length) { PointerType pt = PointerType.getPointerTypeForKey(args[i]); if (null != pt) { Pointer p; if (pt.isLexical()) { p = new Pointer(pt, workWord, targetWord); } else { p = new Pointer(pt, workWord.getSynset(), targetWord.getSynset()); } if (!workWord.getSynset().getPointers().contains(p)) { workWord.getSynset().getPointers().add(p); } else { log.error("Duplicate pointer of type " + pt + " to " + targetWord.getSenseKey() + " in addptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } else { log.error("Invalid pointer type at " + args[i] + " in addptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } else { log.error("Missing pointer type at " + args[i] + " in addptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } else { log.error("Missing target at " + args[i] + " in addptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } key = null; } else { log.error("Missing sensekey for addptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-removeptr".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length) { Word targetWord = d.getWordBySenseKey(args[i]); if (null != targetWord) { i++; if (i < args.length) { PointerType pt = PointerType.getPointerTypeForKey(args[i]); if (null != pt) { Pointer p; if (pt.isLexical()) { p = new Pointer(pt, workWord, targetWord); } else { p = new Pointer(pt, workWord.getSynset(), targetWord.getSynset()); } if (workWord.getSynset().getPointers().contains(p)) { workWord.getSynset().getPointers().remove(p); } else { log.error("Missing pointer of type " + pt + " to " + targetWord.getSenseKey() + " in removeptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } else { log.error("Invalid pointer type at " + args[i] + " in removeptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } else { log.error("Missing pointer type at " + args[i] + " in removeptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } else { log.error("Missing target at " + args[i] + " in removeptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } key = null; } else { log.error("Missing sensekey for removeptr command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-setlexid".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length && '-' != args[i].charAt(0)) { workWord.setLexId(Integer.parseInt(args[i])); key = null; } else { log.error("Missing lexid for setlexid command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-setusecount".equals(args[i])) { if (null == workWord) { log.error("Missing sensekey"); System.exit(1); } else { i++; if (i < args.length && '-' != args[i].charAt(0)) { workWord.setUseCount(Integer.parseInt(args[i])); key = null; } else { log.error("Missing count for setusecount command for sensekey " + workWord.getSenseKey()); System.exit(1); } } } if ("-addexc".equals(args[i])) { i++; if (i < args.length && '-' != args[i].charAt(0)) { String baseform = args[i]; Exc e = d.getException(pos, derivation); if (null != e) { if (null != e.getExceptions()) { if (!e.getExceptions().contains(baseform)) { e.getExceptions().add(baseform); } } } else { ArrayList<String> list = new ArrayList<String>(1); list.add(baseform); d.createException(pos, derivation, list); } derivation = null; } else { log.error("Missing baseform for addexc command for derivation " + derivation); System.exit(1); } } if ("-removeexc".equals(args[i])) { Exc e = d.getException(pos, derivation); if (null != e) { i++; if (i < args.length && '-' != args[i].charAt(0)) { String baseform = args[i]; if (null != e.getExceptions()) { if (e.getExceptions().contains(baseform)) { e.getExceptions().remove(baseform); } if (0 == e.getExceptions().size()) { d.removeException(e); } } } else { d.removeException(e); } } else { log.error("Missing derivation " + derivation); System.exit(1); } derivation = null; } } d.save(); } else { //browse String key = args[0]; if (1 == args.length) { for (POS pos : POS.getAllPOS()) { IndexWord iw = d.getIndexWord(pos, key); if (null == iw) { System.out.println("\nNo information available for " + pos.getLabel() + " " + key); } else { System.out.println( "\nInformation available for " + iw.getPOS().getLabel() + " " + iw.getLemma()); printAvailableInfo(iw); } if (null != d.getMorphologicalProcessor()) { List<String> forms = d.getMorphologicalProcessor().lookupAllBaseForms(pos, key); if (null != forms) { for (String form : forms) { if (!key.equals(form)) { iw = d.getIndexWord(pos, form); if (null != iw) { System.out.println("\nInformation available for " + iw.getPOS().getLabel() + " " + iw.getLemma()); printAvailableInfo(iw); } } } } } } } else { boolean needHelp = false; boolean needGloss = false; boolean needLex = false; boolean needOffset = false; boolean needSenseNum = false; boolean needSenseKeys = false; int needSense = 0; for (String arg : args) { if ("-h".equals(arg)) { needHelp = true; } if ("-g".equals(arg)) { needGloss = true; } if ("-a".equals(arg)) { needLex = true; } if ("-o".equals(arg)) { needOffset = true; } if ("-s".equals(arg)) { needSenseNum = true; } if ("-k".equals(arg)) { needSenseKeys = true; } if (arg.startsWith("-n") && 2 < arg.length()) { needSense = Integer.parseInt(arg.substring(2)); } } for (String arg : args) { if (arg.startsWith("-ants") && 6 == arg.length()) { if (needHelp) { System.out.println( "Display synsets containing direct antonyms of the search string.\n" + "\n" + "Direct antonyms are a pair of words between which there is an\n" + "associative bond built up by co-occurrences.\n" + "\n" + "Antonym synsets are preceded by \"=>\"."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nAntonyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.ANTONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //ants if (arg.startsWith("-hype") && 6 == arg.length()) { if (needHelp) { System.out.println( "Recursively display hypernym (superordinate) tree for the search\n" + "string.\n" + "\n" + "Hypernym is the generic term used to designate a whole class of\n" + "specific instances. Y is a hypernym of X if X is a (kind of) Y.\n" + "\n" + "Hypernym synsets are preceded by \"=>\", and are indented from\n" + "the left according to their level in the hierarchy."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nHypernyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.HYPERNYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //hype if (arg.startsWith("-hypo") && 6 == arg.length()) { if (needHelp) { System.out.println( "Display immediate hyponyms (subordinates) for the search string.\n" + "\n" + "Hyponym is the generic term used to designate a member of a class.\n" + "X is a hyponym of Y if X is a (kind of) Y.\n" + "\n" + "Hyponym synsets are preceded by \"=>\"."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nHyponyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.HYPONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //hypo if (arg.startsWith("-tree") && 6 == arg.length()) { if (needHelp) { System.out.println( "Display hyponym (subordinate) tree for the search string. This is\n" + "a recursive search that finds the hyponyms of each hyponym. \n" + "\n" + "Hyponym is the generic term used to designate a member of a class.\n" + "X is a hyponym of Y if X is a (kind of) Y. \n" + "\n" + "Hyponym synsets are preceded by \"=>\", and are indented from the left\n" + "according to their level in the hierarchy."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nHyponyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.HYPONYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //tree if (arg.startsWith("-enta") && 6 == arg.length()) { if (needHelp) { System.out.println( "Recursively display entailment relations of the search string.\n" + "\n" + "The action represented by the verb X entails Y if X cannot be done\n" + "unless Y is, or has been, done.\n" + "\n" + "Entailment synsets are preceded by \"=>\", and are indented from the left\n" + "according to their level in the hierarchy."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nEntailment of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.ENTAILMENT, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //enta if (arg.startsWith("-syns") && 6 == arg.length()) { POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nSynonyms of " + p.getLabel() + " " + iw.getLemma()); if (POS.ADJECTIVE == p) { if (needHelp) { System.out.println( "Display synonyms and synsets related to synsets containing\n" + "the search string. If the search string is in a head synset\n" + "the 'cluster's' satellite synsets are displayed. If the search\n" + "string is in a satellite synset, its head synset is displayed.\n" + "If the search string is a pertainym the word or synset that it\n" + "pertains to is displayed.\n" + "\n" + "A cluster is a group of adjective synsets that are organized around\n" + "antonymous pairs or triplets. An adjective cluster contains two or more\n" + "head synsets that contan antonyms. Each head synset has one or more\n" + "satellite synsets.\n" + "\n" + "A head synset contains at least one word that has a direct antonym\n" + "in another head synset of the same cluster.\n" + "\n" + "A satellite synset represents a concept that is similar in meaning to\n" + "the concept represented by its head synset.\n" + "\n" + "Direct antonyms are a pair of words between which there is an\n" + "associative bond built up by co-occurrences.\n" + "\n" + "Direct antonyms are printed in parentheses following the adjective.\n" + "The position of an adjective in relation to the noun may be restricted\n" + "to the prenominal, postnominal or predicative position. Where present\n" + "these restrictions are noted in parentheses.\n" + "\n" + "A pertainym is a relational adjective, usually defined by such phrases\n" + "as \"of or pertaining to\" and that does not have an antonym. It pertains\n" + "to a noun or another pertainym.\n" + "\n" + "Senses contained in head synsets are displayed above the satellites,\n" + "which are indented and preceded by \"=>\". Senses contained in\n" + "satellite synsets are displayed with the head synset below. The head\n" + "synset is preceded by \"=>\".\n" + "\n" + "Pertainym senses display the word or synsets that the search string\n" + "pertains to."); } tracePointers(iw, PointerType.SIMILAR_TO, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.PARTICIPLE_OF, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } if (POS.ADVERB == p) { if (needHelp) { System.out.println( "Display synonyms and synsets related to synsets containing\n" + "the search string. If the search string is a pertainym the word\n" + "or synset that it pertains to is displayed.\n" + "\n" + "A pertainym is a relational adverb that is derived from an adjective.\n" + "\n" + "Pertainym senses display the word that the search string is derived from\n" + "and the adjective synset that contains the word. If the adjective synset\n" + "is a satellite synset, its head synset is also displayed."); } tracePointers(iw, PointerType.PERTAINYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } if (POS.NOUN == p || POS.VERB == p) { if (needHelp) { System.out.println( "Recursively display hypernym (superordinate) tree for the search\n" + "string.\n" + "\n" + "Hypernym is the generic term used to designate a whole class of\n" + "specific instances. Y is a hypernym of X if X is a (kind of) Y.\n" + "\n" + "Hypernym synsets are preceded by \"=>\", and are indented from\n" + "the left according to their level in the hierarchy."); } tracePointers(iw, PointerType.HYPERNYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } } //syns if (arg.startsWith("-smem") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all holonyms of the search string.\n" + "\n" + "A holonym is the name of the whole of which the 'meronym' names a part.\n" + "Y is a holonym of X if X is a part of Y.\n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nMember Holonyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.MEMBER_HOLONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //smem if (arg.startsWith("-ssub") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all holonyms of the search string.\n" + "\n" + "A holonym is the name of the whole of which the 'meronym' names a part.\n" + "Y is a holonym of X if X is a part of Y.\n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nSubstance Holonyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.SUBSTANCE_HOLONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //ssub if (arg.startsWith("-sprt") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all holonyms of the search string.\n" + "\n" + "A holonym is the name of the whole of which the 'meronym' names a part.\n" + "Y is a holonym of X if X is a part of Y.\n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nPart Holonyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.PART_HOLONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //sprt if (arg.startsWith("-memb") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all meronyms of the search string. \n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y.\n" + "\n" + "A holonym is the name of the whole of which the meronym names a part.\n" + "Y is a holonym of X if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nMember Meronyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.MEMBER_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //memb if (arg.startsWith("-subs") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all meronyms of the search string. \n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y.\n" + "\n" + "A holonym is the name of the whole of which the meronym names a part.\n" + "Y is a holonym of X if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nSubstance Meronyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.SUBSTANCE_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //subs if (arg.startsWith("-part") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all meronyms of the search string. \n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y.\n" + "\n" + "A holonym is the name of the whole of which the meronym names a part.\n" + "Y is a holonym of X if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nPart Meronyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.PART_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //part if (arg.startsWith("-mero") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all meronyms of the search string. \n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y.\n" + "\n" + "A holonym is the name of the whole of which the meronym names a part.\n" + "Y is a holonym of X if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nMeronyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.MEMBER_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.SUBSTANCE_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.PART_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //mero if (arg.startsWith("-holo") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all holonyms of the search string.\n" + "\n" + "A holonym is the name of the whole of which the 'meronym' names a part.\n" + "Y is a holonym of X if X is a part of Y.\n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nHolonyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.MEMBER_HOLONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.SUBSTANCE_HOLONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.PART_HOLONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //holo if (arg.startsWith("-caus") && 6 == arg.length()) { if (needHelp) { System.out.println("Recursively display CAUSE TO relations of the search string.\n" + "\n" + "The action represented by the verb X causes the action represented by\n" + "the verb Y.\n" + "\n" + "CAUSE TO synsets are preceded by \"=>\", and are indented from the left\n" + "according to their level in the hierarchy."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\n'Cause to' of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.CAUSE, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //caus if (arg.startsWith("-pert") && 6 == arg.length()) { POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nPertainyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.PERTAINYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //pert if (arg.startsWith("-attr") && 6 == arg.length()) { POS p = POS.getPOSForKey(arg.substring(5)); if (needHelp) { if (POS.NOUN == p) { System.out .println("Display adjectives for which search string is an attribute."); } if (POS.ADJECTIVE == p) { System.out.println("Display nouns that are attributes of search string."); } } IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nAttributes of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.ATTRIBUTE, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //attr if (arg.startsWith("-deri") && 6 == arg.length()) { if (needHelp) { System.out.println( "Display derived forms - nouns and verbs that are related morphologically.\n" + "Each related synset is preceeded by its part of speech. Each word in the\n" + "synset is followed by its sense number."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nDerived forms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.NOMINALIZATION, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //deri if (arg.startsWith("-domn") && 6 == arg.length()) { if (needHelp) { System.out.println("Display domain to which this synset belongs."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nDomain of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.CATEGORY, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.USAGE, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.REGION, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //domn if (arg.startsWith("-domt") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all synsets belonging to the domain."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nDomain of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.CATEGORY_MEMBER, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.USAGE_MEMBER, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.REGION_MEMBER, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //domt if (arg.startsWith("-faml") && 6 == arg.length()) { if (needHelp) { System.out.println( "Display familiarity and polysemy information for the search string.\n" + "The polysemy count is the number of senses in WordNet."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { String[] freqs = { "extremely rare", "very rare", "rare", "uncommon", "common", "familiar", "very familiar", "extremely familiar" }; String[] pos = { "a noun", "a verb", "an adjective", "an adverb" }; int cnt = iw.getSenses().size(); int familiar = 0; if (cnt == 0) { familiar = 0; } if (cnt == 1) { familiar = 1; } if (cnt == 2) { familiar = 2; } if (cnt >= 3 && cnt <= 4) { familiar = 3; } if (cnt >= 5 && cnt <= 8) { familiar = 4; } if (cnt >= 9 && cnt <= 16) { familiar = 5; } if (cnt >= 17 && cnt <= 32) { familiar = 6; } if (cnt > 32) { familiar = 7; } System.out.println("\n" + iw.getLemma() + " used as " + pos[p.getId() - 1] + " is " + freqs[familiar] + " (polysemy count = " + cnt + ")"); } } //faml if (arg.startsWith("-fram") && 6 == arg.length()) { if (needHelp) { System.out.println( "Display applicable verb sentence frames for the search string.\n" + "\n" + "A frame is a sentence template illustrating the usage of a verb.\n" + "\n" + "Verb sentence frames are preceded with the string \"*>\" if a sentence\n" + "frame is acceptable for all of the words in the synset, and with \"=>\"\n" + "if a sentence frame is acceptable for the search string only."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nVerb frames of " + p.getLabel() + " " + iw.getLemma()); for (int i = 0; i < iw.getSenses().size(); i++) { Synset synset = iw.getSenses().get(i); for (String vf : synset.getVerbFrames()) { System.out.println("\t*> " + vf); } for (Word word : synset.getWords()) { if (iw.getLemma().equalsIgnoreCase(word.getLemma())) { if (word instanceof Verb) { Verb verb = (Verb) word; for (String vf : verb.getVerbFrames()) { System.out.println("\t=> " + vf); } } } } } } } //fram if (arg.startsWith("-hmer") && 6 == arg.length()) { if (needHelp) { System.out.println( "Display meronyms for search string tree. This is a recursive search\n" + "the prints all the meronyms of the search string and all of its\n" + "hypernyms. \n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nMeronyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.MEMBER_MERONYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.SUBSTANCE_MERONYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.PART_MERONYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //hmer if (arg.startsWith("-hhol") && 6 == arg.length()) { if (needHelp) { System.out.println( "\"Display holonyms for search string tree. This is a recursive search\n" + "that prints all the holonyms of the search string and all of the\n" + "holonym's holonyms.\n" + "\n" + "A holonym is the name of the whole of which the meronym names a part.\n" + "Y is a holonym of X if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nHolonyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.MEMBER_HOLONYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.SUBSTANCE_HOLONYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.PART_HOLONYM, PointerUtils.INFINITY, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //hhol if (arg.startsWith("-mero") && 6 == arg.length()) { if (needHelp) { System.out.println("Display all meronyms of the search string. \n" + "\n" + "A meronym is the name of a constituent part, the substance of, or a\n" + "member of something. X is a meronym of Y if X is a part of Y.\n" + "\n" + "A holonym is the name of the whole of which the meronym names a part.\n" + "Y is a holonym of X if X is a part of Y."); } POS p = POS.getPOSForKey(arg.substring(5)); IndexWord iw = d.lookupIndexWord(p, key); if (null != iw) { System.out.println("\nMeronyms of " + p.getLabel() + " " + iw.getLemma()); tracePointers(iw, PointerType.MEMBER_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.SUBSTANCE_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); tracePointers(iw, PointerType.PART_MERONYM, 1, needSense, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } //mero if (arg.startsWith("-grep") && 6 == arg.length()) { if (needHelp) { System.out .println("Print all strings in the database containing the search string\n" + "as an individual word, or as the first or last string in a word or\n" + "collocation."); } POS p = POS.getPOSForKey(arg.substring(5)); System.out.println("\nGrep of " + p.getLabel() + " " + key); Iterator<IndexWord> ii = d.getIndexWordIterator(p, key); while (ii.hasNext()) { System.out.println(ii.next().getLemma()); } } //grep if ("-over".equals(arg)) { for (POS pos : POS.getAllPOS()) { if (null != d.getMorphologicalProcessor()) { IndexWord iw = d.getIndexWord(pos, key); //for plurals like species, glasses if (null != iw && key.equals(iw.getLemma())) { printOverview(pos, iw, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } List<String> forms = d.getMorphologicalProcessor().lookupAllBaseForms(pos, key); if (null != forms) { for (String form : forms) { if (!form.equals(key)) { iw = d.getIndexWord(pos, form); if (null != iw) { printOverview(pos, iw, needGloss, needLex, needOffset, needSenseNum, needSenseKeys); } } } } } } } //over } } } } }
From source file:com.oneapm.base.SparkAggregation.java
public static void main(String[] args) throws IOException { String configfile = "alert.cnf"; final FlowConstant flowConstant = new FlowConstant(); Properties config = null;// ww w .j a v a 2s .c o m try { config = getConfig(configfile); } catch (IOException e) { LOG.error(configfile + " doesnt exist in /etc/analysis/... exit" + e); System.exit(-1); } final int window = Integer.parseInt(config.getProperty("time.window", "60")); final List<String> percent = Arrays.asList("tryConnectPer", "cBitpacketAccount", "abortConntionCount", "unidirectionalTrafficPer"); final List<String> jsonList = new ArrayList<>(); final Map<String, Tuple2<String, Integer>> stats = new HashMap<String, Tuple2<String, Integer>>(); final Map<ClientKey, BaseData> broadClient = new HashMap<ClientKey, BaseData>(); final Map<HostKey, BaseData> broadHost = new HashMap<HostKey, BaseData>(); final Map<ServiceKey, BaseData> broadService = new HashMap<ServiceKey, BaseData>(); final Map<SubnetKey, BaseData> broadSubnet = new HashMap<SubnetKey, BaseData>(); final Map<VlanKey, BaseData> broadVlan = new HashMap<VlanKey, BaseData>(); final String URL = config.getProperty("alert.url"); final String HEART_BEAT = config.getProperty("alert.heartbeat"); final String RECOVER = config.getProperty("alert.recover"); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/site", new SiteConvertImpl(flowConstant)); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/net", new NetConvertImpl(flowConstant)); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/vlan_sflow", new LinkConvertImpl(flowConstant)); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/subnet", new SubnetConvertImpl(flowConstant)); zookeeperClient.connectZookeeper(config.getProperty("kafaka.zoo")); esGet.setNodeClient(); startZookeeperService(flowConstant, zookeeperClient); zookeeperClient.setRules("/ni/caution"); JavaPairReceiverInputDStream<String, byte[]> rawStream = setupRawStreamFromKafka(config, config.getProperty("group.id", "oneapm-alert")); LOG.info("alert config:" + config.toString()); final Broadcast<IPDataInfo> ipDataBroadcastInfo = getJsc().sparkContext().broadcast(new IPDataInfo()); final Broadcast<ProtocalTypeInfo> protocalTypeBroadcastInfo = getJsc().sparkContext() .broadcast(new ProtocalTypeInfo()); JavaPairDStream<TimeAgg, BeforeAgg> aggJavaPairDStream = rawStream .mapToPair(new PairFunction<Tuple2<String, byte[]>, TimeAgg, BeforeAgg>() { private static final long serialVersionUID = -2751318332921803477L; @Override public Tuple2<TimeAgg, BeforeAgg> call(Tuple2<String, byte[]> stringTuple2) throws Exception { String[] s = TAB.split(new String(stringTuple2._2)); String clientIP = s[1]; String serverIP = s[2]; TimeAgg timeAgg = new TimeAgg(); BeforeAgg beforeAgg = new BeforeAgg(); timeAgg.setServer_ip(serverIP); if (s.length >= 60) { //setProtocal_type if (!"-".equals(s[11]) && !"".equals(s[11])) { timeAgg.setProtocal_type(s[11]); } else { if ("TCP".equals(s[12])) { ProtocalTypeInfo protocalTypeInfo = protocalTypeBroadcastInfo.value(); String protocalType = protocalTypeInfo.config.getProperty(s[4]); if (protocalType != null) { timeAgg.setProtocal_type(protocalType); } else { timeAgg.setProtocal_type("TCP_" + s[4]); } } else { timeAgg.setProtocal_type(s[12]); } } timeAgg.setType("tcp"); //setVlan_id String vlanLinkAlias = flowConstant.VLAN_LINK.get(s[13]); String portLinkAlias = flowConstant.PROBE_POTR_MAP.get(s[0] + "_" + s[15]); if (flowConstant.PROBE_HOST_ENABLE) { if (portLinkAlias != null) { timeAgg.setVlan_id(portLinkAlias); } else { timeAgg.setVlan_id(s[0] + "-nic" + s[15]); } } else if (flowConstant.VLAN_LINK_ENABLE) { if (vlanLinkAlias != null) { timeAgg.setVlan_id(vlanLinkAlias); } else { timeAgg.setVlan_id(s[13]); } } else { timeAgg.setVlan_id("Vlan" + s[13]); } if (!"-".equals(s[6])) { timeAgg.setTimestamp(format.format((long) Double.parseDouble(s[6]) * 1000)); } else { timeAgg.setTimestamp(s[6]); } if (!"-".equals(s[7])) { timeAgg.setTimeEnd(format.format((long) Double.parseDouble(s[7]) * 1000)); } else { timeAgg.setTimeEnd(s[7]); } beforeAgg.setPacket_size(Double.parseDouble(s[55]) + Double.parseDouble(s[56])); beforeAgg.setC_packet_size(Double.parseDouble(s[55])); beforeAgg.setS_packet_size(Double.parseDouble(s[56])); beforeAgg.setPacket_count(Double.parseDouble(s[59]) + Double.parseDouble(s[60])); beforeAgg.setLosspacket_count(Double.parseDouble(s[33]) + Double.parseDouble(s[39]) + Double.parseDouble(s[35]) + Double.parseDouble(s[41])); double cRto = (Double.valueOf(s[19]) + Double.valueOf(s[21]) + Double.valueOf(s[23])) * 1000; double sRto = (Double.valueOf(s[20]) + Double.valueOf(s[22]) + Double.valueOf(s[24])) * 1000; beforeAgg.setTotal_rto(cRto + sRto); // ?/TCP????/TCP????/TCP?MTU?MTU/TCP? beforeAgg.setInt_ZWIN_COUNT(Double.parseDouble(s[43])); beforeAgg.setInt_OOR_COUNT(Double.parseDouble(s[46])); beforeAgg.setInt_CONGEST_COUNT(Double.parseDouble(s[47])); beforeAgg.setMTU(Double.parseDouble(s[61]) + Double.parseDouble(s[62])); Boolean hasSynFlag = Boolean.valueOf(s[5]); timeAgg.setHasSyn(hasSynFlag); timeAgg.setBool_FIN(Boolean.valueOf(s[25])); timeAgg.setBool_RST(Boolean.valueOf(s[26])); if (hasSynFlag && "-".equals(s[9])) { beforeAgg.setAbort(1); } else { beforeAgg.setAbort(0); } beforeAgg.setTcpTurns(Integer.parseInt(s[30])); //int_TURN_COUNT beforeAgg.setConnrequest_count(Double.parseDouble(s[48])); beforeAgg.setsAbortConnCount(Integer.valueOf(s[50])); Long cPayLoad = Long.valueOf(s[53]); Long sPayLoad = Long.valueOf(s[54]); long payLoad = cPayLoad + sPayLoad; double sessionJlRtt = 0; if (!"-".equals(s[9])) { sessionJlRtt = Double.valueOf(s[9]) * 1000; beforeAgg.setRtt(sessionJlRtt); beforeAgg.setServerJlRttCount(1); beforeAgg.setClientJlRttCount(1); } if (hasSynFlag && !"-".equals(s[9])) { beforeAgg.setCount(1); if ("true".equals(s[26]) && payLoad == 0) { beforeAgg.setCount(0); } } if (!"-".equals(s[10])) { double clientJlRtt = Double.valueOf(s[10]) * 1000; double serverJlRtt = sessionJlRtt - clientJlRtt; if (clientJlRtt < sessionJlRtt) { beforeAgg.setServer_rtt(clientJlRtt); beforeAgg.setClient_rtt(serverJlRtt); } else { beforeAgg.setServer_rtt(sessionJlRtt / 2); beforeAgg.setClient_rtt(sessionJlRtt / 2); } } if (beforeAgg.tcpTurns > 0) { beforeAgg.setSessionCount(1); if (Double.parseDouble(s[18]) > 0) { beforeAgg.setServer_reponsetime( Double.parseDouble(s[18]) / beforeAgg.tcpTurns * 1000); } if (Double.parseDouble(s[16]) > 0) { beforeAgg.setResponseTransmissionTime( Double.parseDouble(s[16]) / beforeAgg.tcpTurns * 1000); } if (Double.parseDouble(s[17]) > 0) { beforeAgg.setRequestTransmissionTime( Double.parseDouble(s[17]) / beforeAgg.tcpTurns * 1000); } if (beforeAgg.total_rto > 0) { beforeAgg.setTotal_rto(beforeAgg.getTotal_rto() / beforeAgg.tcpTurns); } beforeAgg.setUserResponseTime( beforeAgg.getRtt() + beforeAgg.getRequestTransmissionTime() + beforeAgg.getResponseTransmissionTime() + beforeAgg.getServer_reponsetime() + beforeAgg.total_rto); } else { beforeAgg.setServer_reponsetime(0); beforeAgg.setResponseTransmissionTime(0); beforeAgg.setRequestTransmissionTime(0); beforeAgg.setTotal_rto(0); beforeAgg.setUserResponseTime(0); } beforeAgg.setC_bitpacket_account(Double.parseDouble(s[57]) + Double.parseDouble(s[58])); } else if (s.length <= 28) { if (!"-".equals(s[8]) && !"".equals(s[8])) { timeAgg.setProtocal_type(s[8]); } else { if ("UDP".equals(s[9])) { ProtocalTypeInfo protocalTypeInfo = protocalTypeBroadcastInfo.value(); String protocalType = protocalTypeInfo.config.getProperty(s[4]); if (protocalType != null) { timeAgg.setProtocal_type(protocalType); } else { timeAgg.setProtocal_type("UDP"); } } else { timeAgg.setProtocal_type(s[9]); } } beforeAgg.setCount(0); timeAgg.setType("udp"); timeAgg.setHasSyn(false); timeAgg.setBool_FIN(false); timeAgg.setBool_RST(false); //setVlan_id String vlanLinkAlias = flowConstant.VLAN_LINK.get(s[12]); String portLinkAlias = flowConstant.PROBE_POTR_MAP.get(s[0] + "_" + s[10]); if (flowConstant.PROBE_HOST_ENABLE) { if (portLinkAlias != null) { timeAgg.setVlan_id(portLinkAlias); } else { timeAgg.setVlan_id(s[0] + "-nic" + s[10]); } } else if (flowConstant.VLAN_LINK_ENABLE) { if (vlanLinkAlias != null) { timeAgg.setVlan_id(vlanLinkAlias); } else { timeAgg.setVlan_id(s[12]); } } else { timeAgg.setVlan_id("Vlan" + s[12]); } if (!"-".equals(s[5])) { timeAgg.setTimestamp(format.format((long) Double.parseDouble(s[5]) * 1000)); } else { timeAgg.setTimestamp(s[5]); } if (!"-".equals(s[6])) { timeAgg.setTimeEnd(format.format((long) Double.parseDouble(s[6]) * 1000)); } else { timeAgg.setTimeEnd(s[6]); } beforeAgg.setPacket_size(Double.parseDouble(s[20]) + Double.parseDouble(s[21])); beforeAgg.setPacket_count(Double.parseDouble(s[24]) + Double.parseDouble(s[25])); beforeAgg.setC_packet_size(Double.parseDouble(s[20])); beforeAgg.setS_packet_size(Double.parseDouble(s[21])); beforeAgg.setInt_ZWIN_COUNT(0); beforeAgg.setInt_OOR_COUNT(0); beforeAgg.setInt_CONGEST_COUNT(0); beforeAgg.setMTU(0); beforeAgg.setC_bitpacket_account(Double.parseDouble(s[22]) + Double.parseDouble(s[23])); beforeAgg.setAbort(0); beforeAgg.setClient_rtt(0); beforeAgg.setServer_rtt(0); beforeAgg.setRtt(0); beforeAgg.setServerJlRttCount(0); beforeAgg.setClientJlRttCount(0); beforeAgg.setLosspacket_count(0); beforeAgg.setServer_reponsetime(0); beforeAgg.setTcpTurns(0); beforeAgg.setConnrequest_count(0); beforeAgg.setTotal_rto(0); beforeAgg.setUserResponseTime(0); beforeAgg.setResponseTransmissionTime(0); beforeAgg.setRequestTransmissionTime(0); beforeAgg.setServer_reponsetime(0); beforeAgg.setsAbortConnCount(0); beforeAgg.setSessionCount(0); } String sInOutFlag = IPUtils.isInHomeNet(serverIP, flowConstant.HOMENET); String cInOutFlag = IPUtils.isInHomeNet(clientIP, flowConstant.HOMENET); //setSubnet if ("IN".equals(sInOutFlag)) { String sSubNet = IPUtils.getSubNet(serverIP, flowConstant.NETMASK); timeAgg.setSubnet(sSubNet + "/" + flowConstant.MASKBITS); } else { timeAgg.setSubnet("-"); } if ("255.255.255.255".equals(clientIP)) { timeAgg.setClient_site("-"); } else { String clientSiteInfo = IPUtils.getSiteInfo(clientIP, flowConstant.SITEINFO_MAP); IPDataInfo ipDataInfo = ipDataBroadcastInfo.getValue(); if (clientSiteInfo != null) { String[] clientSiteInfos = clientSiteInfo.split("_", 3); timeAgg.setClient_site(clientSiteInfos[2]); } else { if ("IN".equals(cInOutFlag)) { timeAgg.setClient_site(""); } else { if (ipDataInfo != null) { String[] ipinfo = ipDataInfo.find(clientIP); // if (ipinfo.length < 3) { timeAgg.setClient_site(""); } else { if ("".equals(ipinfo[0])) { timeAgg.setClient_site(""); } else { //,areasite if ("".equals(ipinfo[1])) { ipinfo[1] = ipinfo[0]; } if ("".equals(ipinfo[2])) { ipinfo[2] = ipinfo[1]; } timeAgg.setClient_site(ipinfo[2]); } } } else { timeAgg.setClient_site("?"); } } } } return new Tuple2<>(timeAgg, beforeAgg); } }).filter(new Function<Tuple2<TimeAgg, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { return !v1._1.getTimestamp().equals("-") && !v1._1.getTimestamp().equals(""); } }); // aggJavaPairDStream.foreachRDD(new Function<JavaPairRDD<TimeAgg, BeforeAgg>, Void>() { // @Override // public Void call(JavaPairRDD<TimeAgg, BeforeAgg> v1) throws Exception { // if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es")) && v1 != null) { // JavaRDD<Map<String, ?>> es = v1.map(new Function<Tuple2<TimeAgg, BeforeAgg>, Map<String, // ?>>() { // // @Override // public Map<String, ?> call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { // ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder(); // // TimeAgg a = v1._1; // BeforeAgg b = v1._2; // String todayStr = sdf.format(format.parse(a.getTimestamp())); // builder.put("server_ip", a.server_ip); // builder.put("protocal_type", a.protocal_type); // builder.put("client_site", a.client_site); // builder.put("vlan_id", a.vlan_id); // builder.put("subnet", a.subnet); // builder.put("timestamp", format.parse(a.timestamp)); // if (b.packet_size > 0) { // builder.put("packet_size", b.packet_size); // } // if (b.c_packet_size > 0) { // builder.put("c_packet_size", b.c_packet_size); // } // // if (b.packet_count > 0) { // builder.put("packet_count", b.packet_count); // } // // if (b.losspacket_count > 0) { // builder.put("losspacket_count", b.losspacket_count); // } // if (b.total_rto > 0) { // builder.put("total_rto", b.total_rto); // builder.put("rtoCount", b.rtoCount); // } // // // if (b.tcpTurns > 0) { // builder.put("tcpTurns", b.tcpTurns); // } // if (b.connrequest_count > 0) { // builder.put("connrequest_count", b.connrequest_count); // } // if (b.abort > 0) { // builder.put("abort", b.abort); // } // if (b.client_rtt > 0) { // builder.put("client_rtt", b.client_rtt); // builder.put("clientJlRttCount", b.clientJlRttCount); // } // if (b.server_rtt > 0) { // builder.put("server_rtt", b.server_rtt); // builder.put("serverJlRttCount", b.serverJlRttCount); // } // // if (b.server_reponsetime > 0) { // builder.put("server_reponsetime", b.server_reponsetime); // builder.put("server_reponsetime_count", b.server_reponsetime_count); // } // // if (b.responseTransmissionTime > 0) { // builder.put("responseTransmissionTime", b.responseTransmissionTime); // builder.put("responseTransmissionTimeCount", b.responseTransmissionTimeCount); // } // if (b.requestTransmissionTime > 0) { // builder.put("requestTransmissionTime", b.requestTransmissionTime); // builder.put("requestTransmissionTimeCount", b.requestTransmissionTimeCount); // // } // // if (b.sAbortConnCount > 0) { // builder.put("sAbortConnCount", b.sAbortConnCount); // } // // if (b.userResponseTime > 0) { // builder.put("userResponseTime", b.userResponseTime); // builder.put("userResponseTimeCount", b.userResponseTimeCount); // } // if (b.c_bitpacket_account > 0) { // builder.put("c_bitpacket_account", b.c_bitpacket_account); // } // builder.put("index_name", todayStr); // // return builder.build(); // } // }).cache(); // if (es != null) { // JavaEsSpark.saveToEs(es, "ni-alert-session-{index_name}/alert", ImmutableMap.of // (ConfigurationOptions.ES_MAPPING_EXCLUDE, "index_name")); // } // } // return null; // } // }); JavaPairDStream<TimeAgg, BeforeAgg> reduceByWindow = aggJavaPairDStream .reduceByKeyAndWindow(new Function2<BeforeAgg, BeforeAgg, BeforeAgg>() { @Override public BeforeAgg call(BeforeAgg v1, BeforeAgg v2) throws Exception { BeforeAgg sum = new BeforeAgg(); sum.setPacket_size(v1.getPacket_size() + v2.getPacket_size()); sum.setC_packet_size(v1.getC_packet_size() + v2.getC_packet_size()); sum.setS_packet_size(v1.getS_packet_size() + v2.getS_packet_size()); sum.setPacket_count(v1.getPacket_count() + v2.getPacket_count()); sum.setC_packet_count(v1.getC_packet_count() + v2.getC_packet_count()); sum.setS_packet_count(v1.getS_packet_count() + v2.getS_packet_count()); sum.setLosspacket_count(v1.getLosspacket_count() + v2.getLosspacket_count()); sum.setTotal_rto(v1.getTotal_rto() + v2.getTotal_rto()); sum.setAbort(v1.getAbort() + v2.getAbort()); sum.setRequestTransmissionTime( v1.getRequestTransmissionTime() + v2.getRequestTransmissionTime()); sum.setResponseTransmissionTime( v1.getResponseTransmissionTime() + v2.getResponseTransmissionTime()); sum.setTcpTurns(v1.getTcpTurns() + v2.getTcpTurns()); sum.setConnrequest_count(v1.getConnrequest_count() + v2.getConnrequest_count()); sum.setRtt(v1.getRtt() + v2.getRtt()); sum.setClient_rtt(v1.getClient_rtt() + v2.getClient_rtt()); sum.setServer_rtt(v1.getServer_rtt() + v2.getServer_rtt()); sum.setServer_reponsetime(v1.getServer_reponsetime() + v2.getServer_reponsetime()); sum.setC_bitpacket_account(v1.getC_bitpacket_account() + v2.getC_bitpacket_account()); sum.setClientJlRttCount(v1.getClientJlRttCount() + v2.getClientJlRttCount()); sum.setServerJlRttCount(v1.getServerJlRttCount() + v2.getServerJlRttCount()); sum.setUserResponseTime(v1.getUserResponseTime() + v2.getUserResponseTime()); sum.setSessionCount(v1.sessionCount + v2.sessionCount); sum.setsAbortConnCount(v1.sAbortConnCount + v2.sAbortConnCount); sum.setCount(v1.getCount() + v2.getCount()); sum.setInt_CONGEST_COUNT(v1.int_CONGEST_COUNT + v2.int_CONGEST_COUNT); sum.setInt_OOR_COUNT(v1.int_OOR_COUNT + v2.int_OOR_COUNT); sum.setInt_ZWIN_COUNT(v1.int_ZWIN_COUNT + v2.int_ZWIN_COUNT); sum.setMTU(v1.MTU + v2.MTU); return sum; } }, Durations.seconds(300), Durations.seconds(60)).cache(); reduceByWindow.foreachRDD(new Function<JavaPairRDD<TimeAgg, BeforeAgg>, Void>() { private static final long serialVersionUID = -4144342491397135515L; @Override public Void call(JavaPairRDD<TimeAgg, BeforeAgg> v1) throws Exception { if (v1.count() > 0) { /** * getStartTime */ List<Long> timeList = v1.map(new Function<Tuple2<TimeAgg, BeforeAgg>, Long>() { @Override public Long call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { return format.parse(v1._1.getTimestamp()).getTime(); } }).distinct().collect(); Collections.sort(timeList, new MyComparator()); long a = timeList.get(3); final String time = format.format(new Date(a)); long b = timeList.get(1); final String endTime = format.format(new Date(b)); if (b > 0) { JavaRDD<Map<String, ?>> active = v1 .filter(new Function<Tuple2<TimeAgg, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { String sInOutFlag = IPUtils.isInHomeNet(v1._1.getServer_ip(), flowConstant.HOMENET); return v1._1.getType().equals("tcp") && "IN".equals(sInOutFlag); } }).mapToPair( new PairFunction<Tuple2<TimeAgg, BeforeAgg>, Tuple2<String, String>, ConnectStatus>() { @Override public Tuple2<Tuple2<String, String>, ConnectStatus> call( Tuple2<TimeAgg, BeforeAgg> timeAggBeforeAggTuple2) throws Exception { ConnectStatus connectStatus = new ConnectStatus(); String serverIp = timeAggBeforeAggTuple2._1.getServer_ip(); String protoType = timeAggBeforeAggTuple2._1.getProtocal_type(); TimeAgg a = timeAggBeforeAggTuple2._1; BeforeAgg b = timeAggBeforeAggTuple2._2; // if (format.parse(a.timestamp).getTime() == format.parse(endTime) .getTime() && a.hasSyn) { connectStatus.setNewCreate(b.getCount()); } else { connectStatus.setNewCreate(0); } //?breakreset?break if (format.parse(a.timeEnd).getTime() == format.parse(endTime) .getTime() && (a.bool_FIN || a.bool_RST)) { connectStatus.setCloseConn(b.getCount()); } else { connectStatus.setCloseConn(0); } if (format.parse(a.timestamp).getTime() <= format.parse(endTime) .getTime() && format.parse(a.timeEnd).getTime() > format.parse(endTime) .getTime() && a.hasSyn) { connectStatus.setActiveConn(b.getCount()); } else if (format.parse(a.timestamp).getTime() == format .parse(endTime).getTime() && format.parse(a.timeEnd).getTime() == format .parse(endTime).getTime() && a.hasSyn) { connectStatus.setActiveConn(b.getCount()); } return new Tuple2<>(new Tuple2<>(serverIp, protoType), connectStatus); } }) .reduceByKey(new Function2<ConnectStatus, ConnectStatus, ConnectStatus>() { @Override public ConnectStatus call(ConnectStatus v1, ConnectStatus v2) throws Exception { ConnectStatus connectStatus = new ConnectStatus(); connectStatus.setNewCreate(v1.newCreate + v2.newCreate); connectStatus.setActiveConn(v1.activeConn + v2.activeConn); connectStatus.setCloseConn(v1.closeConn + v2.closeConn); return connectStatus; } }) .map(new Function<Tuple2<Tuple2<String, String>, ConnectStatus>, Map<String, ?>>() { @Override public Map<String, ?> call(Tuple2<Tuple2<String, String>, ConnectStatus> v1) throws Exception { ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder(); String todayStr = sdf.format(format.parse(endTime)); builder.put("server_ip", v1._1._1); builder.put("protocal_type", v1._1._2); builder.put("newCreateConn", v1._2.getNewCreate()); builder.put("closeConn", v1._2.getCloseConn()); builder.put("activeConn", v1._2.getActiveConn()); builder.put("index_name", todayStr); builder.put("timestamp", format.parse(endTime)); return builder.build(); } }).cache(); if (active != null) { JavaEsSpark.saveToEs(active, "ni-active-conn-{index_name}/active", ImmutableMap.of(ConfigurationOptions.ES_MAPPING_EXCLUDE, "index_name")); } } JavaPairRDD<TimeAgg, BeforeAgg> before = v1 .filter(new Function<Tuple2<TimeAgg, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { return v1._1.getTimestamp().equals(time); } }); if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es")) && before != null) { JavaRDD<Map<String, ?>> es = before .map(new Function<Tuple2<TimeAgg, BeforeAgg>, Map<String, ?>>() { @Override public Map<String, ?> call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder(); TimeAgg a = v1._1; BeforeAgg b = v1._2; String todayStr = sdf.format(format.parse(a.getTimestamp())); builder.put("server_ip", a.server_ip); builder.put("protocal_type", a.protocal_type); builder.put("client_site", a.client_site); builder.put("vlan_id", a.vlan_id); builder.put("subnet", a.subnet); builder.put("timestamp", format.parse(a.timestamp)); if (b.packet_size > 0) { builder.put("packet_size", b.packet_size); } if (b.c_packet_size > 0) { builder.put("c_packet_size", b.c_packet_size); } builder.put("count", b.count); if (b.packet_count > 0) { builder.put("packet_count", b.packet_count); } if (b.losspacket_count > 0) { builder.put("losspacket_count", b.losspacket_count); } if (b.total_rto > 0) { builder.put("total_rto", b.total_rto); } if (b.tcpTurns > 0) { builder.put("tcpTurns", b.tcpTurns); builder.put("sessionCount", b.sessionCount); } if (b.connrequest_count > 0) { builder.put("connrequest_count", b.connrequest_count); } if (b.abort > 0) { builder.put("abort", b.abort); } if (b.client_rtt > 0) { builder.put("client_rtt", b.client_rtt); builder.put("clientJlRttCount", b.clientJlRttCount); } if (b.server_rtt > 0) { builder.put("server_rtt", b.server_rtt); builder.put("serverJlRttCount", b.serverJlRttCount); } if (b.server_reponsetime > 0) { builder.put("server_reponsetime", b.server_reponsetime); } if (b.responseTransmissionTime > 0) { builder.put("responseTransmissionTime", b.responseTransmissionTime); } if (b.requestTransmissionTime > 0) { builder.put("requestTransmissionTime", b.requestTransmissionTime); } if (b.sAbortConnCount > 0) { builder.put("sAbortConnCount", b.sAbortConnCount); } if (b.userResponseTime > 0) { builder.put("userResponseTime", b.userResponseTime); } if (b.c_bitpacket_account > 0) { builder.put("c_bitpacket_account", b.c_bitpacket_account); } builder.put("index_name", todayStr); return builder.build(); } }).cache(); if (es != null) { JavaEsSpark.saveToEs(es, "ni-alert-session-{index_name}/alert", ImmutableMap.of(ConfigurationOptions.ES_MAPPING_EXCLUDE, "index_name")); } } UrlPostMethod.urlPostMethod(HEART_BEAT, "frequency=" + window); rules = zookeeperClient.getRules(); if (rules != null) { ArrayList<String> flagRules = new ArrayList<String>(); //?ruleType for (final RuleRecover ruleRecover : rules) { final Rule rule = ruleRecover.getRule(); if (rule.isEnable() && !flagRules.contains(rule.getType())) { //ruleType? flagRules.add(rule.getType()); //ruleType? JavaPairRDD<String, AggResult> alert = before.mapToPair( new PairFunction<Tuple2<TimeAgg, BeforeAgg>, String, BeforeAgg>() { @Override public Tuple2<String, BeforeAgg> call( Tuple2<TimeAgg, BeforeAgg> timeAggBeforeAggTuple2) throws Exception { Field field1 = timeAggBeforeAggTuple2._1.getClass() .getDeclaredField(rule.getType()); field1.setAccessible(true); String result1 = (String) field1.get(timeAggBeforeAggTuple2._1); if (rule.getType().equals("server_ip")) { String sInOutFlag = IPUtils.isInHomeNet( timeAggBeforeAggTuple2._1.getServer_ip(), flowConstant.HOMENET); if ("IN".equals(sInOutFlag)) { return new Tuple2<>(result1, timeAggBeforeAggTuple2._2); } else { return new Tuple2<>(result1, null); } } else { return new Tuple2<>(result1, timeAggBeforeAggTuple2._2); } } }).filter(new Function<Tuple2<String, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<String, BeforeAgg> v1) throws Exception { return v1._2 != null && !v1._1.equals("") && !v1._1.equals("-"); } }).reduceByKey(new Function2<BeforeAgg, BeforeAgg, BeforeAgg>() { @Override public BeforeAgg call(BeforeAgg v1, BeforeAgg v2) throws Exception { BeforeAgg sum = new BeforeAgg(); sum.setPacket_size(v1.getPacket_size() + v2.getPacket_size()); sum.setC_packet_size(v1.getC_packet_size() + v2.getC_packet_size()); sum.setS_packet_size(v1.getS_packet_size() + v2.getS_packet_size()); sum.setPacket_count(v1.getPacket_count() + v2.getPacket_count()); sum.setC_packet_count( v1.getC_packet_count() + v2.getC_packet_count()); sum.setS_packet_count( v1.getS_packet_count() + v2.getS_packet_count()); sum.setLosspacket_count( v1.getLosspacket_count() + v2.getLosspacket_count()); sum.setTotal_rto(v1.getTotal_rto() + v2.getTotal_rto()); sum.setAbort(v1.getAbort() + v2.getAbort()); sum.setRequestTransmissionTime(v1.getRequestTransmissionTime() + v2.getRequestTransmissionTime()); sum.setResponseTransmissionTime(v1.getResponseTransmissionTime() + v2.getResponseTransmissionTime()); sum.setTcpTurns(v1.getTcpTurns() + v2.getTcpTurns()); sum.setConnrequest_count( v1.getConnrequest_count() + v2.getConnrequest_count()); sum.setRtt(v1.getRtt() + v2.getRtt()); sum.setClient_rtt(v1.getClient_rtt() + v2.getClient_rtt()); sum.setServer_rtt(v1.getServer_rtt() + v2.getServer_rtt()); sum.setServer_reponsetime( v1.getServer_reponsetime() + v2.getServer_reponsetime()); sum.setC_bitpacket_account( v1.getC_bitpacket_account() + v2.getC_bitpacket_account()); sum.setClientJlRttCount( v1.getClientJlRttCount() + v2.getClientJlRttCount()); sum.setServerJlRttCount( v1.getServerJlRttCount() + v2.getServerJlRttCount()); sum.setUserResponseTime( v1.getUserResponseTime() + v2.getUserResponseTime()); sum.setSessionCount(v1.sessionCount + v2.sessionCount); sum.setsAbortConnCount(v1.sAbortConnCount + v2.sAbortConnCount); return sum; } }).mapToPair( new PairFunction<Tuple2<String, BeforeAgg>, String, AggResult>() { @Override public Tuple2<String, AggResult> call( Tuple2<String, BeforeAgg> stringBeforeAggTuple2) throws Exception { BeforeAgg before = stringBeforeAggTuple2._2; AggResult result = new AggResult(); result.setTimestamp(time); result.setThroughput(before.packet_size * 8 / window); result.setS_throughput(before.s_packet_size * 8 / window); result.setC_throughput( before.c_bitpacket_account * 8 / window); result.setPacketThroughput(before.packet_count / window); result.setLossRate(before.losspacket_count / before.packet_count * 100); if (before.sessionCount > 0) { result.setRetransferTime( before.total_rto / before.sessionCount); } else { result.setRetransferTime(0); } if (before.clientJlRttCount > 0) { result.setClientRoundTripTime( before.client_rtt / before.clientJlRttCount); result.setRtt(before.rtt / before.clientJlRttCount); } else { result.setClientRoundTripTime(0); result.setRtt(0); } if (before.serverJlRttCount > 0) { result.setServerRoundTripTime( before.server_rtt / before.serverJlRttCount); } else { result.setServerRoundTripTime(0); } if (before.sessionCount > 0) { result.setUserRespTime(before.getUserResponseTime() / before.sessionCount); result.setTransmissionTime( (before.requestTransmissionTime + before.responseTransmissionTime) / before.sessionCount); } else { result.setUserRespTime(0); result.setTransmissionTime(0); } if (before.sessionCount > 0) { result.setServerRespTime(before.server_reponsetime / before.sessionCount); } else { result.setServerRespTime(0); } result.setConnectFailedRate(before.abort / window); //@Deprecates result.setTryConnectPer(0); result.setCongest_pre(before.getInt_CONGEST_COUNT() / before.getCount() * 100); result.setOutoforder_pre(before.getInt_OOR_COUNT() / before.getCount() * 100); result.setZerowindow_pre(before.getInt_ZWIN_COUNT() / before.getCount() * 100); result.setMTU_pre( before.getMTU() / before.getCount() * 100); if (before.packet_count > 0) { result.setcBitpacketAccount(before.c_bitpacket_account / before.packet_count * 100); } else { result.setcBitpacketAccount(0); } if (before.connrequest_count - before.abort > 0) { result.setAbortConntionCount(before.sAbortConnCount / (before.connrequest_count - before.abort) * 100); } else { result.setAbortConntionCount(0); } return new Tuple2<>(stringBeforeAggTuple2._1, result); } }) .cache(); if (alert.count() > 0) { List<String> alertList = new ArrayList<>(); for (final RuleRecover newRule : rules) { final Rule sameRule = newRule.getRule(); if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es"))) { System.out.println( "rule:" + sameRule.toString() + "--------------"); } final int recover = newRule.getRecover(); if (sameRule.isEnable() && sameRule.getType().equals(flagRules.get(flagRules.size() - 1))) { if (!sameRule.getThresholdErrType().equals("absolute")) { if (esGet.getClient() == null) { esGet.setNodeClient(); } final Calendar now = Calendar.getInstance(); now.setTime(format.parse(time)); int minute = now.get(Calendar.MINUTE); Key key = null; switch (sameRule.getType()) { case "server_ip": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadHost.isEmpty()) { Map<HostKey, BaseData> service = esGet .setHost(now.getTime(), "ni-base-hostname"); broadHost.putAll(service); } } else { Map<HostKey, BaseData> service = esGet .setHost(now.getTime(), "ni-base-hostname"); broadHost.clear(); broadHost.putAll(service); } key = new HostKey(); // baseData = broadHost.get(key); break; case "protocal_type": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadService.isEmpty()) { Map<ServiceKey, BaseData> service = esGet .setService(now.getTime(), "ni-base-service"); broadService.putAll(service); } } else { Map<ServiceKey, BaseData> service = esGet .setService(now.getTime(), "ni-base-service"); broadService.clear(); broadService.putAll(service); } key = new ServiceKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadService.get(key); break; case "client_site": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadClient.isEmpty()) { Map<ClientKey, BaseData> service = esGet .setClient(now.getTime(), "ni-base-clientsite"); broadClient.putAll(service); } } else { Map<ClientKey, BaseData> service = esGet .setClient(now.getTime(), "ni-base-clientsite"); broadClient.clear(); broadClient.putAll(service); } key = new ClientKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadClient.get(key); break; case "vlan_id": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadVlan.isEmpty()) { Map<VlanKey, BaseData> service = esGet .setVlan(now.getTime(), "ni-base-link"); broadVlan.putAll(service); } } else { Map<VlanKey, BaseData> service = esGet .setVlan(now.getTime(), "ni-base-link"); broadVlan.clear(); broadVlan.putAll(service); } key = new VlanKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadVlan.get(key); break; case "subnet": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadSubnet.isEmpty()) { Map<SubnetKey, BaseData> service = esGet .setSubnet(now.getTime(), "ni-base-subnet"); broadSubnet.putAll(service); } } else { Map<SubnetKey, BaseData> service = esGet .setSubnet(now.getTime(), "ni-base-subnet"); broadSubnet.clear(); broadSubnet.putAll(service); } key = new SubnetKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadSubnet.get(key); break; } final Key finalKey = key; alertList = alert .filter(new Function<Tuple2<String, AggResult>, Boolean>() { @Override public Boolean call(Tuple2<String, AggResult> v1) throws Exception { Field field2 = v1._2.getClass() .getDeclaredField(sameRule.getValue()); field2.setAccessible(true); double result2 = (double) field2.get(v1._2); if (result2 == 0) { return false; } String contain = sameRule.getContain(); if (contain.equals("") && !v1._1.equals("")) { return true; } if (v1._1 == null || v1._1.equals("")) { return false; } return v1._1.contains(sameRule.getContain()); } }).mapToPair( new PairFunction<Tuple2<String, AggResult>, String, String>() { @Override public Tuple2<String, String> call( Tuple2<String, AggResult> stringAggResultTuple2) throws Exception { Field field2 = stringAggResultTuple2._2 .getClass().getDeclaredField( sameRule.getValue()); field2.setAccessible(true); double alertCursor = (double) field2 .get(stringAggResultTuple2._2); JSONObject json = new JSONObject(); BaseData baseData = new BaseData(); finalKey.setKeyWord( stringAggResultTuple2._1); finalKey.setStart_timestamp(now.getTime()); baseData = broadService.get(finalKey); if (baseData != null) { Field field = baseData.getClass() .getDeclaredField( sameRule.getValue()); field.setAccessible(true); double result = (double) field .get(baseData); AlertLevel alertLevel = new AlertLevel(); if (sameRule.getThresholdErrOp() .equals("ge")) { if (alertCursor - result >= sameRule .getMax_cardinality()) { alertLevel.setWarningLevel( "levelBad"); } else if (alertCursor - result >= sameRule .getMin_cardinality()) { alertLevel.setWarningLevel( "levelWarn"); } else { alertLevel.setWarningLevel( "levelNormal"); } } if (sameRule.getThresholdErrOp() .equals("le")) { if (result - alertCursor <= sameRule .getMax_cardinality()) { alertLevel.setWarningLevel( "levelBad"); } else if (result - alertCursor <= sameRule .getMin_cardinality()) { alertLevel.setWarningLevel( "levelWarn"); } else { alertLevel.setWarningLevel( "levelNormal"); } } alertLevel.setResourceName( stringAggResultTuple2._1); if (sameRule.getType() .equals("server_ip")) { alertLevel.setIpAddress( stringAggResultTuple2._1); } else { alertLevel.setIpAddress(""); } alertLevel.setOccureTime( esFormat.format(format.parse( stringAggResultTuple2._2 .getTimestamp()))); alertLevel.setResourceType( sameRule.getType()); alertLevel.setMetricId( sameRule.getValue()); alertLevel.setResourceInstanceId( stringAggResultTuple2._1); // top2 d?>10%? Map<String, Double> top2 = new HashMap<String, Double>(); top2.put("MTU?", stringAggResultTuple2._2 .getMTU_pre()); top2.put("?", stringAggResultTuple2._2 .getCongest_pre()); top2.put("??", stringAggResultTuple2._2 .getOutoforder_pre()); top2.put("??", stringAggResultTuple2._2 .getZerowindow_pre()); List<Map.Entry<String, Double>> list = SortHashMap .sortHashMap(top2); if ("lossRate" .equals(sameRule.getValue())) { if (list.get(0).getValue() > 10 && list.get(1) .getValue() > 10) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + list.get(0) .getKey() + "" + list.get(0) .getValue() + "%25," + list.get(1) .getKey() + "" + list.get(1) .getValue() + "%25." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else { if ("userRespTime".equals( sameRule.getValue())) { if (list.get(0).getValue() > 10 && list.get(1) .getValue() > 10) { alertLevel .setWarningContent( alertLevel .getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "ms," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + list.get( 0) .getKey() + "" + list.get( 0) .getValue() + "%25," + list.get( 1) .getKey() + "" + list.get( 1) .getValue() + "%25." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel .setWarningContent( alertLevel .getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "ms," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else if ("rtt".equals( sameRule.getValue())) { alertLevel.setWarningContent( "RTT" + millToSec( (long) alertCursor) + ",RTT" + millToSec( (long) stringAggResultTuple2._2 .getClientRoundTripTime()) + "ms,?RTT" + millToSec( (long) stringAggResultTuple2._2 .getServerRoundTripTime()) + "ms." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("throughput".equals( sameRule.getType())) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + convertUnit( (long) alertCursor) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getS_throughput()) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getC_throughput()) + "." + convertUnit( (long) result) + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("packetThroughput" .equals(sameRule .getValue())) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + ",????" + stringAggResultTuple2._2 .getS_packetThroughput() + ",????" + stringAggResultTuple2._2 .getC_packetThroughput() + "." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if (percent.contains( sameRule.getValue())) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + sameRule .getMin_cardinality() + "." + result + "." + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } alertLevel.setRuleId( sameRule.getRuleName()); alertLevel.setUniqueMark(sameRule .getRuleName() + "-" + stringAggResultTuple2._1 + "-" + sameRule.getValue()); if (stats.containsKey( alertLevel.getUniqueMark())) { String preLevel = stats .get(alertLevel .getUniqueMark())._1; int num = stats.get(alertLevel .getUniqueMark())._2; boolean preWarning = preLevel .equals("levelWarn") || preLevel .equals("levelBad"); boolean newWarning = alertLevel .getWarningLevel() .equals("levelWarn") || alertLevel .getWarningLevel() .equals("levelBad"); if (preWarning && !newWarning) { num = 1 - num; stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else if (!preWarning && newWarning) { stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); } else if (!preWarning && !preWarning) { num = 1 - num; stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else { num = 0 - num; stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } } else { if (alertLevel.getWarningLevel() .equals("levelWarn") || alertLevel .getWarningLevel() .equals("levelBad")) { stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); json = (JSONObject) JSON .toJSON(alertLevel); return new Tuple2<>( stringAggResultTuple2._1, json.toString()); } } } return new Tuple2<>( stringAggResultTuple2._1, ""); } }) .filter(new Function<Tuple2<String, String>, Boolean>() { @Override public Boolean call(Tuple2<String, String> v1) throws Exception { return !v1._2.equals("") && v1._2 != null; } }).map(new Function<Tuple2<String, String>, String>() { @Override public String call(Tuple2<String, String> v1) throws Exception { return v1._2; } }).collect(); } else { alertList = alert .filter(new Function<Tuple2<String, AggResult>, Boolean>() { @Override public Boolean call(Tuple2<String, AggResult> v1) throws Exception { Field field2 = v1._2.getClass() .getDeclaredField(sameRule.getValue()); field2.setAccessible(true); double result2 = (double) field2.get(v1._2); if (result2 == 0.0) { return false; } String contain = sameRule.getContain(); if (contain.equals("") && !v1._1.equals("")) { return true; } if (v1._1 == null || v1._1.equals("")) { return false; } return v1._1.contains(sameRule.getContain()); } }).mapToPair( new PairFunction<Tuple2<String, AggResult>, String, String>() { @Override public Tuple2<String, String> call( Tuple2<String, AggResult> stringAggResultTuple2) throws Exception { Field field2 = stringAggResultTuple2._2 .getClass().getDeclaredField( sameRule.getValue()); field2.setAccessible(true); double alertCursor = (double) field2 .get(stringAggResultTuple2._2); JSONObject json = new JSONObject(); AlertLevel alertLevel = new AlertLevel(); if (alertCursor >= sameRule .getMax_cardinality()) { alertLevel.setWarningLevel("levelBad"); } else if (alertCursor >= sameRule .getMin_cardinality()) { alertLevel.setWarningLevel("levelWarn"); } else { alertLevel .setWarningLevel("levelNormal"); } alertLevel.setResourceName( stringAggResultTuple2._1); if (sameRule.getType() .equals("server_ip")) { alertLevel.setIpAddress( stringAggResultTuple2._1); } else { alertLevel.setIpAddress(""); } alertLevel.setResourceType( sameRule.getType()); alertLevel.setOccureTime( esFormat.format(format.parse( stringAggResultTuple2._2 .getTimestamp()))); alertLevel.setMetricId(sameRule.getValue()); alertLevel.setResourceInstanceId( stringAggResultTuple2._1); // top2 d?>10%? Map<String, Double> top2 = new HashMap<String, Double>(); top2.put("MTU?", stringAggResultTuple2._2 .getMTU_pre()); top2.put("?", stringAggResultTuple2._2 .getCongest_pre()); top2.put("??", stringAggResultTuple2._2 .getOutoforder_pre()); top2.put("??", stringAggResultTuple2._2 .getZerowindow_pre()); List<Map.Entry<String, Double>> list = SortHashMap .sortHashMap(top2); if ("lossRate" .equals(sameRule.getValue())) { if (list.get(0).getValue() > 10 && list .get(1).getValue() > 10) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + list.get(0) .getKey() + "" + list.get(0) .getValue() + "%25," + list.get(1) .getKey() + "" + list.get(1) .getValue() + "%25." + "" + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + "," + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else if ("userRespTime" .equals(sameRule.getValue())) { if (list.get(0).getValue() > 10 && list .get(1).getValue() > 10) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + ",?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + list.get(0) .getKey() + "" + list.get(0) .getValue() + "%25," + list.get(1) .getKey() + "" + list.get(1) .getValue() + "%25." + "" + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + ",?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else if ("rtt" .equals(sameRule.getValue())) { alertLevel.setWarningContent("RTT" + millToSec((long) alertCursor) + ",RTT" + millToSec( (long) stringAggResultTuple2._2 .getClientRoundTripTime()) + ",?RTT" + millToSec( (long) stringAggResultTuple2._2 .getServerRoundTripTime()) + "." + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("throughput" .equals(sameRule.getType())) { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + convertUnit( (long) alertCursor) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getS_throughput()) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getC_throughput()) + "." + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("packetThroughput" .equals(sameRule.getValue())) { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + df.format(alertCursor) + ",????" + stringAggResultTuple2._2 .getS_packetThroughput() + ",????" + stringAggResultTuple2._2 .getC_packetThroughput() + "." + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if (percent .contains(sameRule.getValue())) { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + df.format(alertCursor) + "%25," + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + df.format(alertCursor) + "," + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } alertLevel .setRuleId(sameRule.getRuleName()); alertLevel.setUniqueMark( sameRule.getRuleName() + "-" + stringAggResultTuple2._1 + "-" + sameRule.getValue()); if (stats.containsKey( alertLevel.getUniqueMark())) { String preLevel = stats.get( alertLevel.getUniqueMark())._1; int num = stats.get( alertLevel.getUniqueMark())._2; boolean preWarning = preLevel .equals("levelWarn") || preLevel.equals("levelBad"); boolean newWarning = alertLevel .getWarningLevel() .equals("levelWarn") || alertLevel.getWarningLevel() .equals("levelBad"); if (preWarning && !newWarning) { num = 1 - num; stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else if (!preWarning && newWarning) { stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); } else if (!preWarning && !preWarning) { num = 1 - num; stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else { num = 0 - num; stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } } else { if (alertLevel.getWarningLevel() .equals("levelWarn") || alertLevel.getWarningLevel() .equals("levelBad")) { stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); json = (JSONObject) JSON .toJSON(alertLevel); return new Tuple2<>( stringAggResultTuple2._1, json.toString()); } } return new Tuple2<>( stringAggResultTuple2._1, ""); } }) .filter(new Function<Tuple2<String, String>, Boolean>() { private static final long serialVersionUID = 662946729452638751L; @Override public Boolean call(Tuple2<String, String> v1) throws Exception { return !v1._2.equals("") && v1._2 != null; } }).map(new Function<Tuple2<String, String>, String>() { @Override public String call(Tuple2<String, String> v1) throws Exception { return v1._2; } }).collect(); } } jsonList.addAll(alertList); alertList.clear(); } } } } flagRules.clear(); } Iterator<Map.Entry<String, Tuple2<String, Integer>>> iterator = stats.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry<String, Tuple2<String, Integer>> entry = iterator.next(); int num = entry.getValue()._2; if (num == 0) { UrlPostMethod.urlPostMethod(RECOVER, entry.getValue()._1); iterator.remove(); } else if (num < 0) { num = 0 - num; entry.setValue(new Tuple2<String, Integer>(entry.getValue()._1, num)); } else { num = 1 - num; if (num == 0) { UrlPostMethod.urlPostMethod(RECOVER, entry.getValue()._1); iterator.remove(); } else { entry.setValue(new Tuple2<String, Integer>(entry.getValue()._1, num)); } } } if (stats.size() > 200000) { stats.clear(); } if (jsonList.size() > 0) { if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es"))) { System.out.println( "-------------------" + jsonList.toString() + "-----------------------"); } for (int i = 0; i <= jsonList.size() / 2000; i++) { UrlPostMethod.urlPostMethod(URL, "warnings=" + Arrays.asList(Arrays.copyOfRange( jsonList.toArray(), i * 2000, (i + 1) * 2000 - 1 > jsonList.size() ? jsonList.size() : (i + 1) * 2000 - 1)) .toString()); } jsonList.clear(); } } return null; } }); rawStream.context().start(); rawStream.context().awaitTermination(); }
From source file:Main.java
/** * * @param i an iterator from which we want to retrieve the next element. * @return next element if iterator has more elements, null otherwise. * *//*from www . ja v a 2 s .c o m*/ public static Object nextOrNull(Iterator i) { return i.hasNext() ? i.next() : null; }
From source file:Main.java
private static <T extends Comparable<T>> T getExtremum(Collection<T> vals, int comparator) { if (vals.size() == 0) return null; Iterator<T> it = vals.iterator(); T ret = it.next(); while (it.hasNext()) { T v = it.next();/*from w w w .java2s .co m*/ if (v.compareTo(ret) * comparator > 0) ret = v; } return ret; }
From source file:Main.java
public static void addAll(Collection c, Iterator it) { while (it.hasNext()) { c.add(it.next()); } }
From source file:Main.java
@SuppressWarnings({ "rawtypes", "unchecked" })
public static <E> E maximumValue(Collection<E> c) {
Iterator<E> iterator = c.iterator();
E e = iterator.next();
while (iterator.hasNext()) {
E current = iterator.next();/*w w w . jav a 2s. co m*/
if (((Comparable) e).compareTo((Comparable) current) > 0) {
e = current;
}
}
return e;
}
From source file:Main.java
public static Object firstOrElse(Collection l, Object alt) { Iterator i = l.iterator(); if (i.hasNext()) return i.next(); return alt;/*from w w w. j av a 2 s. c o m*/ }
From source file:Main.java
/** Returns true if all elements in the collection are the same */ public static <T> boolean allEqual(Collection<T> elements) { if (elements.isEmpty()) return false; Iterator<T> it = elements.iterator(); T first = it.next(); while (it.hasNext()) { T el = it.next();/*from ww w . j a va 2s . c om*/ if (!el.equals(first)) return false; } return true; }