List of usage examples for java.util Iterator remove
default void remove()
From source file:LinkedListTest.java
public static void main(String[] args) { List<String> a = new LinkedList<String>(); a.add("Amy"); a.add("Carl"); a.add("Erica"); List<String> b = new LinkedList<String>(); b.add("Bob"); b.add("Doug"); b.add("Frances"); b.add("Gloria"); // merge the words from b into a ListIterator<String> aIter = a.listIterator(); Iterator<String> bIter = b.iterator(); while (bIter.hasNext()) { if (aIter.hasNext()) aIter.next();// w w w . ja v a 2s . com aIter.add(bIter.next()); } System.out.println(a); // remove every second word from b bIter = b.iterator(); while (bIter.hasNext()) { bIter.next(); // skip one element if (bIter.hasNext()) { bIter.next(); // skip next element bIter.remove(); // remove that element } } System.out.println(b); // bulk operation: remove all words in b from a a.removeAll(b); System.out.println(a); }
From source file:cosmos.example.BuildingPermitsExample.java
public static void main(String[] args) throws Exception { BuildingPermitsExample example = new BuildingPermitsExample(); new JCommander(example, args); File inputFile = new File(example.fileName); Preconditions.checkArgument(inputFile.exists() && inputFile.isFile() && inputFile.canRead(), "Expected " + example.fileName + " to be a readable file"); String zookeepers;//from w w w . j a v a2 s. c om String instanceName; Connector connector; MiniAccumuloCluster mac = null; File macDir = null; // Use the MiniAccumuloCluster is requested if (example.useMiniAccumuloCluster) { macDir = Files.createTempDir(); String password = "password"; MiniAccumuloConfig config = new MiniAccumuloConfig(macDir, password); config.setNumTservers(1); mac = new MiniAccumuloCluster(config); mac.start(); zookeepers = mac.getZooKeepers(); instanceName = mac.getInstanceName(); ZooKeeperInstance instance = new ZooKeeperInstance(instanceName, zookeepers); connector = instance.getConnector("root", new PasswordToken(password)); } else { // Otherwise connect to a running instance zookeepers = example.zookeepers; instanceName = example.instanceName; ZooKeeperInstance instance = new ZooKeeperInstance(instanceName, zookeepers); connector = instance.getConnector(example.username, new PasswordToken(example.password)); } // Instantiate an instance of Cosmos Cosmos cosmos = new CosmosImpl(zookeepers); // Create a definition for the data we want to load Store id = Store.create(connector, new Authorizations(), AscendingIndexIdentitySet.create()); // Register the definition with Cosmos so it can track its progress. cosmos.register(id); // Load all of the data from our inputFile LoadBuildingPermits loader = new LoadBuildingPermits(cosmos, id, inputFile); loader.run(); // Finalize the SortableResult which will prevent future writes to the data set cosmos.finalize(id); // Flush the ingest traces to the backend so we can see the results; id.sendTraces(); // Get back the Set of Columns that we've ingested. Set<Column> schema = Sets.newHashSet(cosmos.columns(id)); log.debug("\nColumns: " + schema); Iterator<Column> iter = schema.iterator(); while (iter.hasNext()) { Column c = iter.next(); // Remove the internal ID field and columns that begin with CONTRACTOR_ if (c.equals(LoadBuildingPermits.ID) || c.name().startsWith("CONTRACTOR_")) { iter.remove(); } } Iterable<Index> indices = Iterables.transform(schema, new Function<Column, Index>() { @Override public Index apply(Column col) { return Index.define(col); } }); // Ensure that we have locality groups set as we expect log.info("Ensure locality groups are set"); id.optimizeIndices(indices); // Compact down the data for this SortableResult log.info("Issuing compaction for relevant data"); id.consolidate(); final int numTopValues = 10; // Walk through each column in the result set for (Column c : schema) { Stopwatch sw = new Stopwatch(); sw.start(); // Get the number of times we've seen each value in a given column CloseableIterable<Entry<RecordValue<?>, Long>> groupingsInColumn = cosmos.groupResults(id, c); log.info(c.name() + ":"); // Iterate over the counts, collecting the top N values in each column TreeMap<Long, RecordValue<?>> topValues = Maps.newTreeMap(); for (Entry<RecordValue<?>, Long> entry : groupingsInColumn) { if (topValues.size() == numTopValues) { Entry<Long, RecordValue<?>> least = topValues.pollFirstEntry(); if (least.getKey() < entry.getValue()) { topValues.put(entry.getValue(), entry.getKey()); } else { topValues.put(least.getKey(), least.getValue()); } } else if (topValues.size() < numTopValues) { topValues.put(entry.getValue(), entry.getKey()); } } for (Long key : topValues.descendingKeySet()) { log.info(topValues.get(key).value() + " occurred " + key + " times"); } sw.stop(); log.info("Took " + sw.toString() + " to run query.\n"); } log.info("Deleting records"); // Delete the records we've ingested if (!example.useMiniAccumuloCluster) { // Because I'm lazy and don't want to wait around to run the BatchDeleter when we're just going // to rm -rf the directory in a few secs. cosmos.delete(id); } // And shut down Cosmos cosmos.close(); log.info("Cosmos stopped"); // If we were using MAC, also stop that if (example.useMiniAccumuloCluster && null != mac) { mac.stop(); if (null != macDir) { FileUtils.deleteDirectory(macDir); } } }
From source file:at.tlphotography.jAbuseReport.Reporter.java
/** * The main method./* w w w.j a v a 2s .c o m*/ * * @param args * the arguments */ public static void main(String[] args) { parseArguments(args); File[] directory = new File(logDir).listFiles(); // get the files in the dir for (File file : directory) // iterate over the file { if (!file.isDirectory() && file.getName().contains(logNames)) // if the file is not a dir and the name contains the logName string { if (file.getName().endsWith(".gz")) // is it zipped? { content.putAll(readGZFile(file)); } else { content.putAll(readLogFile(file)); } } } // save the mails to the log lines HashMap<String, ArrayList<LogObject>> finalContent = new HashMap<>(); Iterator<Entry<String, String>> it = content.entrySet().iterator(); while (it.hasNext()) { Map.Entry<String, String> pair = it.next(); String mail = whoIsLookUp(pair.getKey()); if (finalContent.containsKey(mail)) { finalContent.get(mail).add(new LogObject(pair.getValue())); } else { ArrayList<LogObject> temp = new ArrayList<LogObject>(); temp.add(new LogObject(pair.getValue())); finalContent.put(mail, temp); } it.remove(); } // sort them Iterator<Entry<String, ArrayList<LogObject>>> it2 = finalContent.entrySet().iterator(); while (it2.hasNext()) { Entry<String, ArrayList<LogObject>> pair = it2.next(); Collections.sort(pair.getValue()); println(pair.getKey() + " ="); for (LogObject obj : pair.getValue()) { println(obj.logContent); } println("\n"); it2.remove(); } }
From source file:edu.oregonstate.eecs.mcplan.abstraction.EvaluateSimilarityFunction.java
/** * @param args/* w w w . j a v a2 s .com*/ * @throws IOException * @throws FileNotFoundException */ public static void main(final String[] args) throws FileNotFoundException, IOException { final String experiment_file = args[0]; final File root_directory; if (args.length > 1) { root_directory = new File(args[1]); } else { root_directory = new File("."); } final CsvConfigurationParser csv_config = new CsvConfigurationParser(new FileReader(experiment_file)); final String experiment_name = FilenameUtils.getBaseName(experiment_file); final File expr_directory = new File(root_directory, experiment_name); expr_directory.mkdirs(); final Csv.Writer csv = new Csv.Writer( new PrintStream(new FileOutputStream(new File(expr_directory, "results.csv")))); final String[] parameter_headers = new String[] { "kpca.kernel", "kpca.rbf.sigma", "kpca.random_forest.Ntrees", "kpca.random_forest.max_depth", "kpca.Nbases", "multiclass.classifier", "multiclass.random_forest.Ntrees", "multiclass.random_forest.max_depth", "pairwise_classifier.max_branching", "training.label_noise" }; csv.cell("domain").cell("abstraction"); for (final String p : parameter_headers) { csv.cell(p); } csv.cell("Ntrain").cell("Ntest").cell("ami.mean").cell("ami.variance").cell("ami.confidence").newline(); for (int expr = 0; expr < csv_config.size(); ++expr) { try { final KeyValueStore expr_config = csv_config.get(expr); final Configuration config = new Configuration(root_directory.getPath(), expr_directory.getName(), expr_config); System.out.println("[Loading '" + config.training_data_single + "']"); final Instances single = WekaUtil .readLabeledDataset(new File(root_directory, config.training_data_single + ".arff")); final Instances train = new Instances(single, 0); final int[] idx = Fn.range(0, single.size()); int instance_counter = 0; Fn.shuffle(config.rng, idx); final int Ntrain = config.getInt("Ntrain_games"); // TODO: Rename? final double label_noise = config.getDouble("training.label_noise"); final int Nlabels = train.classAttribute().numValues(); assert (Nlabels > 0); for (int i = 0; i < Ntrain; ++i) { final Instance inst = single.get(idx[instance_counter++]); if (label_noise > 0 && config.rng.nextDouble() < label_noise) { int noisy_label = 0; do { noisy_label = config.rng.nextInt(Nlabels); } while (noisy_label == (int) inst.classValue()); System.out.println("Noisy label (" + inst.classValue() + " -> " + noisy_label + ")"); inst.setClassValue(noisy_label); } train.add(inst); inst.setDataset(train); } final Fn.Function2<Boolean, Instance, Instance> plausible_p = createPlausiblePredicate(config); final int Ntest = config.Ntest_games; int Ntest_added = 0; final ArrayList<Instances> tests = new ArrayList<Instances>(); while (instance_counter < single.size() && Ntest_added < Ntest) { final Instance inst = single.get(idx[instance_counter++]); boolean found = false; for (final Instances test : tests) { // Note that 'plausible_p' should be transitive if (plausible_p.apply(inst, test.get(0))) { WekaUtil.addInstance(test, inst); if (test.size() == 30) { Ntest_added += test.size(); } else if (test.size() > 30) { Ntest_added += 1; } found = true; break; } } if (!found) { final Instances test = new Instances(single, 0); WekaUtil.addInstance(test, inst); tests.add(test); } } final Iterator<Instances> test_itr = tests.iterator(); while (test_itr.hasNext()) { if (test_itr.next().size() < 30) { test_itr.remove(); } } System.out.println("=== tests.size() = " + tests.size()); System.out.println("=== Ntest_added = " + Ntest_added); System.out.println("[Training]"); final Evaluator evaluator = createEvaluator(config, train); // final Instances transformed_test = evaluator.prepareInstances( test ); System.out.println("[Evaluating]"); final int Nxval = evaluator.isSensitiveToOrdering() ? 10 : 1; final MeanVarianceAccumulator ami = new MeanVarianceAccumulator(); final MeanVarianceAccumulator errors = new MeanVarianceAccumulator(); final MeanVarianceAccumulator relative_error = new MeanVarianceAccumulator(); int c = 0; for (int xval = 0; xval < Nxval; ++xval) { for (final Instances test : tests) { // TODO: Debugging WekaUtil.writeDataset(new File(config.root_directory), "test_" + (c++), test); // transformed_test.randomize( new RandomAdaptor( config.rng ) ); // final ClusterContingencyTable ct = evaluator.evaluate( transformed_test ); test.randomize(new RandomAdaptor(config.rng)); final ClusterContingencyTable ct = evaluator.evaluate(test); System.out.println(ct); int Nerrors = 0; final MeanVarianceAccumulator mv = new MeanVarianceAccumulator(); for (int i = 0; i < ct.R; ++i) { final int max = Fn.max(ct.n[i]); Nerrors += (ct.a[i] - max); mv.add(((double) ct.a[i]) / ct.N * Nerrors / ct.a[i]); } errors.add(Nerrors); relative_error.add(mv.mean()); System.out.println("exemplar: " + test.get(0)); System.out.println("Nerrors = " + Nerrors); final PrintStream ct_out = new PrintStream( new FileOutputStream(new File(expr_directory, "ct_" + expr + "_" + xval + ".csv"))); ct.writeCsv(ct_out); ct_out.close(); final double ct_ami = ct.adjustedMutualInformation_max(); if (Double.isNaN(ct_ami)) { System.out.println("! ct_ami = NaN"); } else { ami.add(ct_ami); } System.out.println(); } } System.out.println("errors = " + errors.mean() + " (" + errors.confidence() + ")"); System.out.println( "relative_error = " + relative_error.mean() + " (" + relative_error.confidence() + ")"); System.out.println("AMI_max = " + ami.mean() + " (" + ami.confidence() + ")"); csv.cell(config.domain).cell(config.get("abstraction.discovery")); for (final String p : parameter_headers) { csv.cell(config.get(p)); } csv.cell(Ntrain).cell(Ntest).cell(ami.mean()).cell(ami.variance()).cell(ami.confidence()).newline(); } catch (final Exception ex) { ex.printStackTrace(); } } }
From source file:com.zimbra.cs.lmtpserver.utils.LmtpInject.java
public static void main(String[] args) { CliUtil.toolSetup();//from w w w . j ava 2 s. co m CommandLine cl = parseArgs(args); if (cl.hasOption("h")) { usage(null); } boolean quietMode = cl.hasOption("q"); int threads = 1; if (cl.hasOption("t")) { threads = Integer.valueOf(cl.getOptionValue("t")).intValue(); } String host = null; if (cl.hasOption("a")) { host = cl.getOptionValue("a"); } else { host = "localhost"; } int port; Protocol proto = null; if (cl.hasOption("smtp")) { proto = Protocol.SMTP; port = 25; } else port = 7025; if (cl.hasOption("p")) port = Integer.valueOf(cl.getOptionValue("p")).intValue(); String[] recipients = cl.getOptionValues("r"); String sender = cl.getOptionValue("s"); boolean tracingEnabled = cl.hasOption("T"); int everyN; if (cl.hasOption("N")) { everyN = Integer.valueOf(cl.getOptionValue("N")).intValue(); } else { everyN = 100; } // Process files from the -d option. List<File> files = new ArrayList<File>(); if (cl.hasOption("d")) { File dir = new File(cl.getOptionValue("d")); if (!dir.isDirectory()) { System.err.format("%s is not a directory.\n", dir.getPath()); System.exit(1); } File[] fileArray = dir.listFiles(); if (fileArray == null || fileArray.length == 0) { System.err.format("No files found in directory %s.\n", dir.getPath()); } Collections.addAll(files, fileArray); } // Process files specified as arguments. for (String arg : cl.getArgs()) { files.add(new File(arg)); } // Validate file content. if (!cl.hasOption("noValidation")) { Iterator<File> i = files.iterator(); while (i.hasNext()) { InputStream in = null; File file = i.next(); boolean valid = false; try { in = new FileInputStream(file); if (FileUtil.isGzipped(file)) { in = new GZIPInputStream(in); } in = new BufferedInputStream(in); // Required for RFC 822 check if (!EmailUtil.isRfc822Message(in)) { System.err.format("%s does not contain a valid RFC 822 message.\n", file.getPath()); } else { valid = true; } } catch (IOException e) { System.err.format("Unable to validate %s: %s.\n", file.getPath(), e.toString()); } finally { ByteUtil.closeStream(in); } if (!valid) { i.remove(); } } } if (files.size() == 0) { System.err.println("No files to inject."); System.exit(1); } if (!quietMode) { System.out.format( "Injecting %d message(s) to %d recipient(s). Server %s, port %d, using %d thread(s).\n", files.size(), recipients.length, host, port, threads); } int totalFailed = 0; int totalSucceeded = 0; long startTime = System.currentTimeMillis(); boolean verbose = cl.hasOption("v"); boolean skipTLSCertValidation = cl.hasOption("skipTLSCertValidation"); LmtpInject injector = null; try { injector = new LmtpInject(threads, sender, recipients, files, host, port, proto, quietMode, tracingEnabled, verbose, skipTLSCertValidation); } catch (Exception e) { mLog.error("Unable to initialize LmtpInject", e); System.exit(1); } injector.setReportEvery(everyN); injector.markStartTime(); try { injector.run(); } catch (IOException e) { e.printStackTrace(); System.exit(1); } int succeeded = injector.getSuccessCount(); int failedThisTime = injector.getFailureCount(); long elapsedMS = System.currentTimeMillis() - startTime; double elapsed = elapsedMS / 1000.0; double msPerMsg = 0.0; double msgSizeKB = 0.0; if (succeeded > 0) { msPerMsg = elapsedMS; msPerMsg /= succeeded; msgSizeKB = injector.mFileSizeTotal / 1024.0; msgSizeKB /= succeeded; } double msgPerSec = ((double) succeeded / (double) elapsedMS) * 1000; if (!quietMode) { System.out.println(); System.out.printf( "LmtpInject Finished\n" + "submitted=%d failed=%d\n" + "%.2fs, %.2fms/msg, %.2fmsg/s\n" + "average message size = %.2fKB\n", succeeded, failedThisTime, elapsed, msPerMsg, msgPerSec, msgSizeKB); } totalFailed += failedThisTime; totalSucceeded += succeeded; if (totalFailed != 0) System.exit(1); }
From source file:com.oneapm.base.SparkAggregation.java
public static void main(String[] args) throws IOException { String configfile = "alert.cnf"; final FlowConstant flowConstant = new FlowConstant(); Properties config = null;// w w w . j a v a2 s.co m try { config = getConfig(configfile); } catch (IOException e) { LOG.error(configfile + " doesnt exist in /etc/analysis/... exit" + e); System.exit(-1); } final int window = Integer.parseInt(config.getProperty("time.window", "60")); final List<String> percent = Arrays.asList("tryConnectPer", "cBitpacketAccount", "abortConntionCount", "unidirectionalTrafficPer"); final List<String> jsonList = new ArrayList<>(); final Map<String, Tuple2<String, Integer>> stats = new HashMap<String, Tuple2<String, Integer>>(); final Map<ClientKey, BaseData> broadClient = new HashMap<ClientKey, BaseData>(); final Map<HostKey, BaseData> broadHost = new HashMap<HostKey, BaseData>(); final Map<ServiceKey, BaseData> broadService = new HashMap<ServiceKey, BaseData>(); final Map<SubnetKey, BaseData> broadSubnet = new HashMap<SubnetKey, BaseData>(); final Map<VlanKey, BaseData> broadVlan = new HashMap<VlanKey, BaseData>(); final String URL = config.getProperty("alert.url"); final String HEART_BEAT = config.getProperty("alert.heartbeat"); final String RECOVER = config.getProperty("alert.recover"); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/site", new SiteConvertImpl(flowConstant)); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/net", new NetConvertImpl(flowConstant)); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/vlan_sflow", new LinkConvertImpl(flowConstant)); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/subnet", new SubnetConvertImpl(flowConstant)); zookeeperClient.connectZookeeper(config.getProperty("kafaka.zoo")); esGet.setNodeClient(); startZookeeperService(flowConstant, zookeeperClient); zookeeperClient.setRules("/ni/caution"); JavaPairReceiverInputDStream<String, byte[]> rawStream = setupRawStreamFromKafka(config, config.getProperty("group.id", "oneapm-alert")); LOG.info("alert config:" + config.toString()); final Broadcast<IPDataInfo> ipDataBroadcastInfo = getJsc().sparkContext().broadcast(new IPDataInfo()); final Broadcast<ProtocalTypeInfo> protocalTypeBroadcastInfo = getJsc().sparkContext() .broadcast(new ProtocalTypeInfo()); JavaPairDStream<TimeAgg, BeforeAgg> aggJavaPairDStream = rawStream .mapToPair(new PairFunction<Tuple2<String, byte[]>, TimeAgg, BeforeAgg>() { private static final long serialVersionUID = -2751318332921803477L; @Override public Tuple2<TimeAgg, BeforeAgg> call(Tuple2<String, byte[]> stringTuple2) throws Exception { String[] s = TAB.split(new String(stringTuple2._2)); String clientIP = s[1]; String serverIP = s[2]; TimeAgg timeAgg = new TimeAgg(); BeforeAgg beforeAgg = new BeforeAgg(); timeAgg.setServer_ip(serverIP); if (s.length >= 60) { //setProtocal_type if (!"-".equals(s[11]) && !"".equals(s[11])) { timeAgg.setProtocal_type(s[11]); } else { if ("TCP".equals(s[12])) { ProtocalTypeInfo protocalTypeInfo = protocalTypeBroadcastInfo.value(); String protocalType = protocalTypeInfo.config.getProperty(s[4]); if (protocalType != null) { timeAgg.setProtocal_type(protocalType); } else { timeAgg.setProtocal_type("TCP_" + s[4]); } } else { timeAgg.setProtocal_type(s[12]); } } timeAgg.setType("tcp"); //setVlan_id String vlanLinkAlias = flowConstant.VLAN_LINK.get(s[13]); String portLinkAlias = flowConstant.PROBE_POTR_MAP.get(s[0] + "_" + s[15]); if (flowConstant.PROBE_HOST_ENABLE) { if (portLinkAlias != null) { timeAgg.setVlan_id(portLinkAlias); } else { timeAgg.setVlan_id(s[0] + "-nic" + s[15]); } } else if (flowConstant.VLAN_LINK_ENABLE) { if (vlanLinkAlias != null) { timeAgg.setVlan_id(vlanLinkAlias); } else { timeAgg.setVlan_id(s[13]); } } else { timeAgg.setVlan_id("Vlan" + s[13]); } if (!"-".equals(s[6])) { timeAgg.setTimestamp(format.format((long) Double.parseDouble(s[6]) * 1000)); } else { timeAgg.setTimestamp(s[6]); } if (!"-".equals(s[7])) { timeAgg.setTimeEnd(format.format((long) Double.parseDouble(s[7]) * 1000)); } else { timeAgg.setTimeEnd(s[7]); } beforeAgg.setPacket_size(Double.parseDouble(s[55]) + Double.parseDouble(s[56])); beforeAgg.setC_packet_size(Double.parseDouble(s[55])); beforeAgg.setS_packet_size(Double.parseDouble(s[56])); beforeAgg.setPacket_count(Double.parseDouble(s[59]) + Double.parseDouble(s[60])); beforeAgg.setLosspacket_count(Double.parseDouble(s[33]) + Double.parseDouble(s[39]) + Double.parseDouble(s[35]) + Double.parseDouble(s[41])); double cRto = (Double.valueOf(s[19]) + Double.valueOf(s[21]) + Double.valueOf(s[23])) * 1000; double sRto = (Double.valueOf(s[20]) + Double.valueOf(s[22]) + Double.valueOf(s[24])) * 1000; beforeAgg.setTotal_rto(cRto + sRto); // ?/TCP????/TCP????/TCP?MTU?MTU/TCP? beforeAgg.setInt_ZWIN_COUNT(Double.parseDouble(s[43])); beforeAgg.setInt_OOR_COUNT(Double.parseDouble(s[46])); beforeAgg.setInt_CONGEST_COUNT(Double.parseDouble(s[47])); beforeAgg.setMTU(Double.parseDouble(s[61]) + Double.parseDouble(s[62])); Boolean hasSynFlag = Boolean.valueOf(s[5]); timeAgg.setHasSyn(hasSynFlag); timeAgg.setBool_FIN(Boolean.valueOf(s[25])); timeAgg.setBool_RST(Boolean.valueOf(s[26])); if (hasSynFlag && "-".equals(s[9])) { beforeAgg.setAbort(1); } else { beforeAgg.setAbort(0); } beforeAgg.setTcpTurns(Integer.parseInt(s[30])); //int_TURN_COUNT beforeAgg.setConnrequest_count(Double.parseDouble(s[48])); beforeAgg.setsAbortConnCount(Integer.valueOf(s[50])); Long cPayLoad = Long.valueOf(s[53]); Long sPayLoad = Long.valueOf(s[54]); long payLoad = cPayLoad + sPayLoad; double sessionJlRtt = 0; if (!"-".equals(s[9])) { sessionJlRtt = Double.valueOf(s[9]) * 1000; beforeAgg.setRtt(sessionJlRtt); beforeAgg.setServerJlRttCount(1); beforeAgg.setClientJlRttCount(1); } if (hasSynFlag && !"-".equals(s[9])) { beforeAgg.setCount(1); if ("true".equals(s[26]) && payLoad == 0) { beforeAgg.setCount(0); } } if (!"-".equals(s[10])) { double clientJlRtt = Double.valueOf(s[10]) * 1000; double serverJlRtt = sessionJlRtt - clientJlRtt; if (clientJlRtt < sessionJlRtt) { beforeAgg.setServer_rtt(clientJlRtt); beforeAgg.setClient_rtt(serverJlRtt); } else { beforeAgg.setServer_rtt(sessionJlRtt / 2); beforeAgg.setClient_rtt(sessionJlRtt / 2); } } if (beforeAgg.tcpTurns > 0) { beforeAgg.setSessionCount(1); if (Double.parseDouble(s[18]) > 0) { beforeAgg.setServer_reponsetime( Double.parseDouble(s[18]) / beforeAgg.tcpTurns * 1000); } if (Double.parseDouble(s[16]) > 0) { beforeAgg.setResponseTransmissionTime( Double.parseDouble(s[16]) / beforeAgg.tcpTurns * 1000); } if (Double.parseDouble(s[17]) > 0) { beforeAgg.setRequestTransmissionTime( Double.parseDouble(s[17]) / beforeAgg.tcpTurns * 1000); } if (beforeAgg.total_rto > 0) { beforeAgg.setTotal_rto(beforeAgg.getTotal_rto() / beforeAgg.tcpTurns); } beforeAgg.setUserResponseTime( beforeAgg.getRtt() + beforeAgg.getRequestTransmissionTime() + beforeAgg.getResponseTransmissionTime() + beforeAgg.getServer_reponsetime() + beforeAgg.total_rto); } else { beforeAgg.setServer_reponsetime(0); beforeAgg.setResponseTransmissionTime(0); beforeAgg.setRequestTransmissionTime(0); beforeAgg.setTotal_rto(0); beforeAgg.setUserResponseTime(0); } beforeAgg.setC_bitpacket_account(Double.parseDouble(s[57]) + Double.parseDouble(s[58])); } else if (s.length <= 28) { if (!"-".equals(s[8]) && !"".equals(s[8])) { timeAgg.setProtocal_type(s[8]); } else { if ("UDP".equals(s[9])) { ProtocalTypeInfo protocalTypeInfo = protocalTypeBroadcastInfo.value(); String protocalType = protocalTypeInfo.config.getProperty(s[4]); if (protocalType != null) { timeAgg.setProtocal_type(protocalType); } else { timeAgg.setProtocal_type("UDP"); } } else { timeAgg.setProtocal_type(s[9]); } } beforeAgg.setCount(0); timeAgg.setType("udp"); timeAgg.setHasSyn(false); timeAgg.setBool_FIN(false); timeAgg.setBool_RST(false); //setVlan_id String vlanLinkAlias = flowConstant.VLAN_LINK.get(s[12]); String portLinkAlias = flowConstant.PROBE_POTR_MAP.get(s[0] + "_" + s[10]); if (flowConstant.PROBE_HOST_ENABLE) { if (portLinkAlias != null) { timeAgg.setVlan_id(portLinkAlias); } else { timeAgg.setVlan_id(s[0] + "-nic" + s[10]); } } else if (flowConstant.VLAN_LINK_ENABLE) { if (vlanLinkAlias != null) { timeAgg.setVlan_id(vlanLinkAlias); } else { timeAgg.setVlan_id(s[12]); } } else { timeAgg.setVlan_id("Vlan" + s[12]); } if (!"-".equals(s[5])) { timeAgg.setTimestamp(format.format((long) Double.parseDouble(s[5]) * 1000)); } else { timeAgg.setTimestamp(s[5]); } if (!"-".equals(s[6])) { timeAgg.setTimeEnd(format.format((long) Double.parseDouble(s[6]) * 1000)); } else { timeAgg.setTimeEnd(s[6]); } beforeAgg.setPacket_size(Double.parseDouble(s[20]) + Double.parseDouble(s[21])); beforeAgg.setPacket_count(Double.parseDouble(s[24]) + Double.parseDouble(s[25])); beforeAgg.setC_packet_size(Double.parseDouble(s[20])); beforeAgg.setS_packet_size(Double.parseDouble(s[21])); beforeAgg.setInt_ZWIN_COUNT(0); beforeAgg.setInt_OOR_COUNT(0); beforeAgg.setInt_CONGEST_COUNT(0); beforeAgg.setMTU(0); beforeAgg.setC_bitpacket_account(Double.parseDouble(s[22]) + Double.parseDouble(s[23])); beforeAgg.setAbort(0); beforeAgg.setClient_rtt(0); beforeAgg.setServer_rtt(0); beforeAgg.setRtt(0); beforeAgg.setServerJlRttCount(0); beforeAgg.setClientJlRttCount(0); beforeAgg.setLosspacket_count(0); beforeAgg.setServer_reponsetime(0); beforeAgg.setTcpTurns(0); beforeAgg.setConnrequest_count(0); beforeAgg.setTotal_rto(0); beforeAgg.setUserResponseTime(0); beforeAgg.setResponseTransmissionTime(0); beforeAgg.setRequestTransmissionTime(0); beforeAgg.setServer_reponsetime(0); beforeAgg.setsAbortConnCount(0); beforeAgg.setSessionCount(0); } String sInOutFlag = IPUtils.isInHomeNet(serverIP, flowConstant.HOMENET); String cInOutFlag = IPUtils.isInHomeNet(clientIP, flowConstant.HOMENET); //setSubnet if ("IN".equals(sInOutFlag)) { String sSubNet = IPUtils.getSubNet(serverIP, flowConstant.NETMASK); timeAgg.setSubnet(sSubNet + "/" + flowConstant.MASKBITS); } else { timeAgg.setSubnet("-"); } if ("255.255.255.255".equals(clientIP)) { timeAgg.setClient_site("-"); } else { String clientSiteInfo = IPUtils.getSiteInfo(clientIP, flowConstant.SITEINFO_MAP); IPDataInfo ipDataInfo = ipDataBroadcastInfo.getValue(); if (clientSiteInfo != null) { String[] clientSiteInfos = clientSiteInfo.split("_", 3); timeAgg.setClient_site(clientSiteInfos[2]); } else { if ("IN".equals(cInOutFlag)) { timeAgg.setClient_site(""); } else { if (ipDataInfo != null) { String[] ipinfo = ipDataInfo.find(clientIP); // if (ipinfo.length < 3) { timeAgg.setClient_site(""); } else { if ("".equals(ipinfo[0])) { timeAgg.setClient_site(""); } else { //,areasite if ("".equals(ipinfo[1])) { ipinfo[1] = ipinfo[0]; } if ("".equals(ipinfo[2])) { ipinfo[2] = ipinfo[1]; } timeAgg.setClient_site(ipinfo[2]); } } } else { timeAgg.setClient_site("?"); } } } } return new Tuple2<>(timeAgg, beforeAgg); } }).filter(new Function<Tuple2<TimeAgg, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { return !v1._1.getTimestamp().equals("-") && !v1._1.getTimestamp().equals(""); } }); // aggJavaPairDStream.foreachRDD(new Function<JavaPairRDD<TimeAgg, BeforeAgg>, Void>() { // @Override // public Void call(JavaPairRDD<TimeAgg, BeforeAgg> v1) throws Exception { // if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es")) && v1 != null) { // JavaRDD<Map<String, ?>> es = v1.map(new Function<Tuple2<TimeAgg, BeforeAgg>, Map<String, // ?>>() { // // @Override // public Map<String, ?> call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { // ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder(); // // TimeAgg a = v1._1; // BeforeAgg b = v1._2; // String todayStr = sdf.format(format.parse(a.getTimestamp())); // builder.put("server_ip", a.server_ip); // builder.put("protocal_type", a.protocal_type); // builder.put("client_site", a.client_site); // builder.put("vlan_id", a.vlan_id); // builder.put("subnet", a.subnet); // builder.put("timestamp", format.parse(a.timestamp)); // if (b.packet_size > 0) { // builder.put("packet_size", b.packet_size); // } // if (b.c_packet_size > 0) { // builder.put("c_packet_size", b.c_packet_size); // } // // if (b.packet_count > 0) { // builder.put("packet_count", b.packet_count); // } // // if (b.losspacket_count > 0) { // builder.put("losspacket_count", b.losspacket_count); // } // if (b.total_rto > 0) { // builder.put("total_rto", b.total_rto); // builder.put("rtoCount", b.rtoCount); // } // // // if (b.tcpTurns > 0) { // builder.put("tcpTurns", b.tcpTurns); // } // if (b.connrequest_count > 0) { // builder.put("connrequest_count", b.connrequest_count); // } // if (b.abort > 0) { // builder.put("abort", b.abort); // } // if (b.client_rtt > 0) { // builder.put("client_rtt", b.client_rtt); // builder.put("clientJlRttCount", b.clientJlRttCount); // } // if (b.server_rtt > 0) { // builder.put("server_rtt", b.server_rtt); // builder.put("serverJlRttCount", b.serverJlRttCount); // } // // if (b.server_reponsetime > 0) { // builder.put("server_reponsetime", b.server_reponsetime); // builder.put("server_reponsetime_count", b.server_reponsetime_count); // } // // if (b.responseTransmissionTime > 0) { // builder.put("responseTransmissionTime", b.responseTransmissionTime); // builder.put("responseTransmissionTimeCount", b.responseTransmissionTimeCount); // } // if (b.requestTransmissionTime > 0) { // builder.put("requestTransmissionTime", b.requestTransmissionTime); // builder.put("requestTransmissionTimeCount", b.requestTransmissionTimeCount); // // } // // if (b.sAbortConnCount > 0) { // builder.put("sAbortConnCount", b.sAbortConnCount); // } // // if (b.userResponseTime > 0) { // builder.put("userResponseTime", b.userResponseTime); // builder.put("userResponseTimeCount", b.userResponseTimeCount); // } // if (b.c_bitpacket_account > 0) { // builder.put("c_bitpacket_account", b.c_bitpacket_account); // } // builder.put("index_name", todayStr); // // return builder.build(); // } // }).cache(); // if (es != null) { // JavaEsSpark.saveToEs(es, "ni-alert-session-{index_name}/alert", ImmutableMap.of // (ConfigurationOptions.ES_MAPPING_EXCLUDE, "index_name")); // } // } // return null; // } // }); JavaPairDStream<TimeAgg, BeforeAgg> reduceByWindow = aggJavaPairDStream .reduceByKeyAndWindow(new Function2<BeforeAgg, BeforeAgg, BeforeAgg>() { @Override public BeforeAgg call(BeforeAgg v1, BeforeAgg v2) throws Exception { BeforeAgg sum = new BeforeAgg(); sum.setPacket_size(v1.getPacket_size() + v2.getPacket_size()); sum.setC_packet_size(v1.getC_packet_size() + v2.getC_packet_size()); sum.setS_packet_size(v1.getS_packet_size() + v2.getS_packet_size()); sum.setPacket_count(v1.getPacket_count() + v2.getPacket_count()); sum.setC_packet_count(v1.getC_packet_count() + v2.getC_packet_count()); sum.setS_packet_count(v1.getS_packet_count() + v2.getS_packet_count()); sum.setLosspacket_count(v1.getLosspacket_count() + v2.getLosspacket_count()); sum.setTotal_rto(v1.getTotal_rto() + v2.getTotal_rto()); sum.setAbort(v1.getAbort() + v2.getAbort()); sum.setRequestTransmissionTime( v1.getRequestTransmissionTime() + v2.getRequestTransmissionTime()); sum.setResponseTransmissionTime( v1.getResponseTransmissionTime() + v2.getResponseTransmissionTime()); sum.setTcpTurns(v1.getTcpTurns() + v2.getTcpTurns()); sum.setConnrequest_count(v1.getConnrequest_count() + v2.getConnrequest_count()); sum.setRtt(v1.getRtt() + v2.getRtt()); sum.setClient_rtt(v1.getClient_rtt() + v2.getClient_rtt()); sum.setServer_rtt(v1.getServer_rtt() + v2.getServer_rtt()); sum.setServer_reponsetime(v1.getServer_reponsetime() + v2.getServer_reponsetime()); sum.setC_bitpacket_account(v1.getC_bitpacket_account() + v2.getC_bitpacket_account()); sum.setClientJlRttCount(v1.getClientJlRttCount() + v2.getClientJlRttCount()); sum.setServerJlRttCount(v1.getServerJlRttCount() + v2.getServerJlRttCount()); sum.setUserResponseTime(v1.getUserResponseTime() + v2.getUserResponseTime()); sum.setSessionCount(v1.sessionCount + v2.sessionCount); sum.setsAbortConnCount(v1.sAbortConnCount + v2.sAbortConnCount); sum.setCount(v1.getCount() + v2.getCount()); sum.setInt_CONGEST_COUNT(v1.int_CONGEST_COUNT + v2.int_CONGEST_COUNT); sum.setInt_OOR_COUNT(v1.int_OOR_COUNT + v2.int_OOR_COUNT); sum.setInt_ZWIN_COUNT(v1.int_ZWIN_COUNT + v2.int_ZWIN_COUNT); sum.setMTU(v1.MTU + v2.MTU); return sum; } }, Durations.seconds(300), Durations.seconds(60)).cache(); reduceByWindow.foreachRDD(new Function<JavaPairRDD<TimeAgg, BeforeAgg>, Void>() { private static final long serialVersionUID = -4144342491397135515L; @Override public Void call(JavaPairRDD<TimeAgg, BeforeAgg> v1) throws Exception { if (v1.count() > 0) { /** * getStartTime */ List<Long> timeList = v1.map(new Function<Tuple2<TimeAgg, BeforeAgg>, Long>() { @Override public Long call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { return format.parse(v1._1.getTimestamp()).getTime(); } }).distinct().collect(); Collections.sort(timeList, new MyComparator()); long a = timeList.get(3); final String time = format.format(new Date(a)); long b = timeList.get(1); final String endTime = format.format(new Date(b)); if (b > 0) { JavaRDD<Map<String, ?>> active = v1 .filter(new Function<Tuple2<TimeAgg, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { String sInOutFlag = IPUtils.isInHomeNet(v1._1.getServer_ip(), flowConstant.HOMENET); return v1._1.getType().equals("tcp") && "IN".equals(sInOutFlag); } }).mapToPair( new PairFunction<Tuple2<TimeAgg, BeforeAgg>, Tuple2<String, String>, ConnectStatus>() { @Override public Tuple2<Tuple2<String, String>, ConnectStatus> call( Tuple2<TimeAgg, BeforeAgg> timeAggBeforeAggTuple2) throws Exception { ConnectStatus connectStatus = new ConnectStatus(); String serverIp = timeAggBeforeAggTuple2._1.getServer_ip(); String protoType = timeAggBeforeAggTuple2._1.getProtocal_type(); TimeAgg a = timeAggBeforeAggTuple2._1; BeforeAgg b = timeAggBeforeAggTuple2._2; // if (format.parse(a.timestamp).getTime() == format.parse(endTime) .getTime() && a.hasSyn) { connectStatus.setNewCreate(b.getCount()); } else { connectStatus.setNewCreate(0); } //?breakreset?break if (format.parse(a.timeEnd).getTime() == format.parse(endTime) .getTime() && (a.bool_FIN || a.bool_RST)) { connectStatus.setCloseConn(b.getCount()); } else { connectStatus.setCloseConn(0); } if (format.parse(a.timestamp).getTime() <= format.parse(endTime) .getTime() && format.parse(a.timeEnd).getTime() > format.parse(endTime) .getTime() && a.hasSyn) { connectStatus.setActiveConn(b.getCount()); } else if (format.parse(a.timestamp).getTime() == format .parse(endTime).getTime() && format.parse(a.timeEnd).getTime() == format .parse(endTime).getTime() && a.hasSyn) { connectStatus.setActiveConn(b.getCount()); } return new Tuple2<>(new Tuple2<>(serverIp, protoType), connectStatus); } }) .reduceByKey(new Function2<ConnectStatus, ConnectStatus, ConnectStatus>() { @Override public ConnectStatus call(ConnectStatus v1, ConnectStatus v2) throws Exception { ConnectStatus connectStatus = new ConnectStatus(); connectStatus.setNewCreate(v1.newCreate + v2.newCreate); connectStatus.setActiveConn(v1.activeConn + v2.activeConn); connectStatus.setCloseConn(v1.closeConn + v2.closeConn); return connectStatus; } }) .map(new Function<Tuple2<Tuple2<String, String>, ConnectStatus>, Map<String, ?>>() { @Override public Map<String, ?> call(Tuple2<Tuple2<String, String>, ConnectStatus> v1) throws Exception { ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder(); String todayStr = sdf.format(format.parse(endTime)); builder.put("server_ip", v1._1._1); builder.put("protocal_type", v1._1._2); builder.put("newCreateConn", v1._2.getNewCreate()); builder.put("closeConn", v1._2.getCloseConn()); builder.put("activeConn", v1._2.getActiveConn()); builder.put("index_name", todayStr); builder.put("timestamp", format.parse(endTime)); return builder.build(); } }).cache(); if (active != null) { JavaEsSpark.saveToEs(active, "ni-active-conn-{index_name}/active", ImmutableMap.of(ConfigurationOptions.ES_MAPPING_EXCLUDE, "index_name")); } } JavaPairRDD<TimeAgg, BeforeAgg> before = v1 .filter(new Function<Tuple2<TimeAgg, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { return v1._1.getTimestamp().equals(time); } }); if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es")) && before != null) { JavaRDD<Map<String, ?>> es = before .map(new Function<Tuple2<TimeAgg, BeforeAgg>, Map<String, ?>>() { @Override public Map<String, ?> call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder(); TimeAgg a = v1._1; BeforeAgg b = v1._2; String todayStr = sdf.format(format.parse(a.getTimestamp())); builder.put("server_ip", a.server_ip); builder.put("protocal_type", a.protocal_type); builder.put("client_site", a.client_site); builder.put("vlan_id", a.vlan_id); builder.put("subnet", a.subnet); builder.put("timestamp", format.parse(a.timestamp)); if (b.packet_size > 0) { builder.put("packet_size", b.packet_size); } if (b.c_packet_size > 0) { builder.put("c_packet_size", b.c_packet_size); } builder.put("count", b.count); if (b.packet_count > 0) { builder.put("packet_count", b.packet_count); } if (b.losspacket_count > 0) { builder.put("losspacket_count", b.losspacket_count); } if (b.total_rto > 0) { builder.put("total_rto", b.total_rto); } if (b.tcpTurns > 0) { builder.put("tcpTurns", b.tcpTurns); builder.put("sessionCount", b.sessionCount); } if (b.connrequest_count > 0) { builder.put("connrequest_count", b.connrequest_count); } if (b.abort > 0) { builder.put("abort", b.abort); } if (b.client_rtt > 0) { builder.put("client_rtt", b.client_rtt); builder.put("clientJlRttCount", b.clientJlRttCount); } if (b.server_rtt > 0) { builder.put("server_rtt", b.server_rtt); builder.put("serverJlRttCount", b.serverJlRttCount); } if (b.server_reponsetime > 0) { builder.put("server_reponsetime", b.server_reponsetime); } if (b.responseTransmissionTime > 0) { builder.put("responseTransmissionTime", b.responseTransmissionTime); } if (b.requestTransmissionTime > 0) { builder.put("requestTransmissionTime", b.requestTransmissionTime); } if (b.sAbortConnCount > 0) { builder.put("sAbortConnCount", b.sAbortConnCount); } if (b.userResponseTime > 0) { builder.put("userResponseTime", b.userResponseTime); } if (b.c_bitpacket_account > 0) { builder.put("c_bitpacket_account", b.c_bitpacket_account); } builder.put("index_name", todayStr); return builder.build(); } }).cache(); if (es != null) { JavaEsSpark.saveToEs(es, "ni-alert-session-{index_name}/alert", ImmutableMap.of(ConfigurationOptions.ES_MAPPING_EXCLUDE, "index_name")); } } UrlPostMethod.urlPostMethod(HEART_BEAT, "frequency=" + window); rules = zookeeperClient.getRules(); if (rules != null) { ArrayList<String> flagRules = new ArrayList<String>(); //?ruleType for (final RuleRecover ruleRecover : rules) { final Rule rule = ruleRecover.getRule(); if (rule.isEnable() && !flagRules.contains(rule.getType())) { //ruleType? flagRules.add(rule.getType()); //ruleType? JavaPairRDD<String, AggResult> alert = before.mapToPair( new PairFunction<Tuple2<TimeAgg, BeforeAgg>, String, BeforeAgg>() { @Override public Tuple2<String, BeforeAgg> call( Tuple2<TimeAgg, BeforeAgg> timeAggBeforeAggTuple2) throws Exception { Field field1 = timeAggBeforeAggTuple2._1.getClass() .getDeclaredField(rule.getType()); field1.setAccessible(true); String result1 = (String) field1.get(timeAggBeforeAggTuple2._1); if (rule.getType().equals("server_ip")) { String sInOutFlag = IPUtils.isInHomeNet( timeAggBeforeAggTuple2._1.getServer_ip(), flowConstant.HOMENET); if ("IN".equals(sInOutFlag)) { return new Tuple2<>(result1, timeAggBeforeAggTuple2._2); } else { return new Tuple2<>(result1, null); } } else { return new Tuple2<>(result1, timeAggBeforeAggTuple2._2); } } }).filter(new Function<Tuple2<String, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<String, BeforeAgg> v1) throws Exception { return v1._2 != null && !v1._1.equals("") && !v1._1.equals("-"); } }).reduceByKey(new Function2<BeforeAgg, BeforeAgg, BeforeAgg>() { @Override public BeforeAgg call(BeforeAgg v1, BeforeAgg v2) throws Exception { BeforeAgg sum = new BeforeAgg(); sum.setPacket_size(v1.getPacket_size() + v2.getPacket_size()); sum.setC_packet_size(v1.getC_packet_size() + v2.getC_packet_size()); sum.setS_packet_size(v1.getS_packet_size() + v2.getS_packet_size()); sum.setPacket_count(v1.getPacket_count() + v2.getPacket_count()); sum.setC_packet_count( v1.getC_packet_count() + v2.getC_packet_count()); sum.setS_packet_count( v1.getS_packet_count() + v2.getS_packet_count()); sum.setLosspacket_count( v1.getLosspacket_count() + v2.getLosspacket_count()); sum.setTotal_rto(v1.getTotal_rto() + v2.getTotal_rto()); sum.setAbort(v1.getAbort() + v2.getAbort()); sum.setRequestTransmissionTime(v1.getRequestTransmissionTime() + v2.getRequestTransmissionTime()); sum.setResponseTransmissionTime(v1.getResponseTransmissionTime() + v2.getResponseTransmissionTime()); sum.setTcpTurns(v1.getTcpTurns() + v2.getTcpTurns()); sum.setConnrequest_count( v1.getConnrequest_count() + v2.getConnrequest_count()); sum.setRtt(v1.getRtt() + v2.getRtt()); sum.setClient_rtt(v1.getClient_rtt() + v2.getClient_rtt()); sum.setServer_rtt(v1.getServer_rtt() + v2.getServer_rtt()); sum.setServer_reponsetime( v1.getServer_reponsetime() + v2.getServer_reponsetime()); sum.setC_bitpacket_account( v1.getC_bitpacket_account() + v2.getC_bitpacket_account()); sum.setClientJlRttCount( v1.getClientJlRttCount() + v2.getClientJlRttCount()); sum.setServerJlRttCount( v1.getServerJlRttCount() + v2.getServerJlRttCount()); sum.setUserResponseTime( v1.getUserResponseTime() + v2.getUserResponseTime()); sum.setSessionCount(v1.sessionCount + v2.sessionCount); sum.setsAbortConnCount(v1.sAbortConnCount + v2.sAbortConnCount); return sum; } }).mapToPair( new PairFunction<Tuple2<String, BeforeAgg>, String, AggResult>() { @Override public Tuple2<String, AggResult> call( Tuple2<String, BeforeAgg> stringBeforeAggTuple2) throws Exception { BeforeAgg before = stringBeforeAggTuple2._2; AggResult result = new AggResult(); result.setTimestamp(time); result.setThroughput(before.packet_size * 8 / window); result.setS_throughput(before.s_packet_size * 8 / window); result.setC_throughput( before.c_bitpacket_account * 8 / window); result.setPacketThroughput(before.packet_count / window); result.setLossRate(before.losspacket_count / before.packet_count * 100); if (before.sessionCount > 0) { result.setRetransferTime( before.total_rto / before.sessionCount); } else { result.setRetransferTime(0); } if (before.clientJlRttCount > 0) { result.setClientRoundTripTime( before.client_rtt / before.clientJlRttCount); result.setRtt(before.rtt / before.clientJlRttCount); } else { result.setClientRoundTripTime(0); result.setRtt(0); } if (before.serverJlRttCount > 0) { result.setServerRoundTripTime( before.server_rtt / before.serverJlRttCount); } else { result.setServerRoundTripTime(0); } if (before.sessionCount > 0) { result.setUserRespTime(before.getUserResponseTime() / before.sessionCount); result.setTransmissionTime( (before.requestTransmissionTime + before.responseTransmissionTime) / before.sessionCount); } else { result.setUserRespTime(0); result.setTransmissionTime(0); } if (before.sessionCount > 0) { result.setServerRespTime(before.server_reponsetime / before.sessionCount); } else { result.setServerRespTime(0); } result.setConnectFailedRate(before.abort / window); //@Deprecates result.setTryConnectPer(0); result.setCongest_pre(before.getInt_CONGEST_COUNT() / before.getCount() * 100); result.setOutoforder_pre(before.getInt_OOR_COUNT() / before.getCount() * 100); result.setZerowindow_pre(before.getInt_ZWIN_COUNT() / before.getCount() * 100); result.setMTU_pre( before.getMTU() / before.getCount() * 100); if (before.packet_count > 0) { result.setcBitpacketAccount(before.c_bitpacket_account / before.packet_count * 100); } else { result.setcBitpacketAccount(0); } if (before.connrequest_count - before.abort > 0) { result.setAbortConntionCount(before.sAbortConnCount / (before.connrequest_count - before.abort) * 100); } else { result.setAbortConntionCount(0); } return new Tuple2<>(stringBeforeAggTuple2._1, result); } }) .cache(); if (alert.count() > 0) { List<String> alertList = new ArrayList<>(); for (final RuleRecover newRule : rules) { final Rule sameRule = newRule.getRule(); if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es"))) { System.out.println( "rule:" + sameRule.toString() + "--------------"); } final int recover = newRule.getRecover(); if (sameRule.isEnable() && sameRule.getType().equals(flagRules.get(flagRules.size() - 1))) { if (!sameRule.getThresholdErrType().equals("absolute")) { if (esGet.getClient() == null) { esGet.setNodeClient(); } final Calendar now = Calendar.getInstance(); now.setTime(format.parse(time)); int minute = now.get(Calendar.MINUTE); Key key = null; switch (sameRule.getType()) { case "server_ip": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadHost.isEmpty()) { Map<HostKey, BaseData> service = esGet .setHost(now.getTime(), "ni-base-hostname"); broadHost.putAll(service); } } else { Map<HostKey, BaseData> service = esGet .setHost(now.getTime(), "ni-base-hostname"); broadHost.clear(); broadHost.putAll(service); } key = new HostKey(); // baseData = broadHost.get(key); break; case "protocal_type": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadService.isEmpty()) { Map<ServiceKey, BaseData> service = esGet .setService(now.getTime(), "ni-base-service"); broadService.putAll(service); } } else { Map<ServiceKey, BaseData> service = esGet .setService(now.getTime(), "ni-base-service"); broadService.clear(); broadService.putAll(service); } key = new ServiceKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadService.get(key); break; case "client_site": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadClient.isEmpty()) { Map<ClientKey, BaseData> service = esGet .setClient(now.getTime(), "ni-base-clientsite"); broadClient.putAll(service); } } else { Map<ClientKey, BaseData> service = esGet .setClient(now.getTime(), "ni-base-clientsite"); broadClient.clear(); broadClient.putAll(service); } key = new ClientKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadClient.get(key); break; case "vlan_id": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadVlan.isEmpty()) { Map<VlanKey, BaseData> service = esGet .setVlan(now.getTime(), "ni-base-link"); broadVlan.putAll(service); } } else { Map<VlanKey, BaseData> service = esGet .setVlan(now.getTime(), "ni-base-link"); broadVlan.clear(); broadVlan.putAll(service); } key = new VlanKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadVlan.get(key); break; case "subnet": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadSubnet.isEmpty()) { Map<SubnetKey, BaseData> service = esGet .setSubnet(now.getTime(), "ni-base-subnet"); broadSubnet.putAll(service); } } else { Map<SubnetKey, BaseData> service = esGet .setSubnet(now.getTime(), "ni-base-subnet"); broadSubnet.clear(); broadSubnet.putAll(service); } key = new SubnetKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadSubnet.get(key); break; } final Key finalKey = key; alertList = alert .filter(new Function<Tuple2<String, AggResult>, Boolean>() { @Override public Boolean call(Tuple2<String, AggResult> v1) throws Exception { Field field2 = v1._2.getClass() .getDeclaredField(sameRule.getValue()); field2.setAccessible(true); double result2 = (double) field2.get(v1._2); if (result2 == 0) { return false; } String contain = sameRule.getContain(); if (contain.equals("") && !v1._1.equals("")) { return true; } if (v1._1 == null || v1._1.equals("")) { return false; } return v1._1.contains(sameRule.getContain()); } }).mapToPair( new PairFunction<Tuple2<String, AggResult>, String, String>() { @Override public Tuple2<String, String> call( Tuple2<String, AggResult> stringAggResultTuple2) throws Exception { Field field2 = stringAggResultTuple2._2 .getClass().getDeclaredField( sameRule.getValue()); field2.setAccessible(true); double alertCursor = (double) field2 .get(stringAggResultTuple2._2); JSONObject json = new JSONObject(); BaseData baseData = new BaseData(); finalKey.setKeyWord( stringAggResultTuple2._1); finalKey.setStart_timestamp(now.getTime()); baseData = broadService.get(finalKey); if (baseData != null) { Field field = baseData.getClass() .getDeclaredField( sameRule.getValue()); field.setAccessible(true); double result = (double) field .get(baseData); AlertLevel alertLevel = new AlertLevel(); if (sameRule.getThresholdErrOp() .equals("ge")) { if (alertCursor - result >= sameRule .getMax_cardinality()) { alertLevel.setWarningLevel( "levelBad"); } else if (alertCursor - result >= sameRule .getMin_cardinality()) { alertLevel.setWarningLevel( "levelWarn"); } else { alertLevel.setWarningLevel( "levelNormal"); } } if (sameRule.getThresholdErrOp() .equals("le")) { if (result - alertCursor <= sameRule .getMax_cardinality()) { alertLevel.setWarningLevel( "levelBad"); } else if (result - alertCursor <= sameRule .getMin_cardinality()) { alertLevel.setWarningLevel( "levelWarn"); } else { alertLevel.setWarningLevel( "levelNormal"); } } alertLevel.setResourceName( stringAggResultTuple2._1); if (sameRule.getType() .equals("server_ip")) { alertLevel.setIpAddress( stringAggResultTuple2._1); } else { alertLevel.setIpAddress(""); } alertLevel.setOccureTime( esFormat.format(format.parse( stringAggResultTuple2._2 .getTimestamp()))); alertLevel.setResourceType( sameRule.getType()); alertLevel.setMetricId( sameRule.getValue()); alertLevel.setResourceInstanceId( stringAggResultTuple2._1); // top2 d?>10%? Map<String, Double> top2 = new HashMap<String, Double>(); top2.put("MTU?", stringAggResultTuple2._2 .getMTU_pre()); top2.put("?", stringAggResultTuple2._2 .getCongest_pre()); top2.put("??", stringAggResultTuple2._2 .getOutoforder_pre()); top2.put("??", stringAggResultTuple2._2 .getZerowindow_pre()); List<Map.Entry<String, Double>> list = SortHashMap .sortHashMap(top2); if ("lossRate" .equals(sameRule.getValue())) { if (list.get(0).getValue() > 10 && list.get(1) .getValue() > 10) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + list.get(0) .getKey() + "" + list.get(0) .getValue() + "%25," + list.get(1) .getKey() + "" + list.get(1) .getValue() + "%25." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else { if ("userRespTime".equals( sameRule.getValue())) { if (list.get(0).getValue() > 10 && list.get(1) .getValue() > 10) { alertLevel .setWarningContent( alertLevel .getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "ms," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + list.get( 0) .getKey() + "" + list.get( 0) .getValue() + "%25," + list.get( 1) .getKey() + "" + list.get( 1) .getValue() + "%25." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel .setWarningContent( alertLevel .getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "ms," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else if ("rtt".equals( sameRule.getValue())) { alertLevel.setWarningContent( "RTT" + millToSec( (long) alertCursor) + ",RTT" + millToSec( (long) stringAggResultTuple2._2 .getClientRoundTripTime()) + "ms,?RTT" + millToSec( (long) stringAggResultTuple2._2 .getServerRoundTripTime()) + "ms." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("throughput".equals( sameRule.getType())) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + convertUnit( (long) alertCursor) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getS_throughput()) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getC_throughput()) + "." + convertUnit( (long) result) + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("packetThroughput" .equals(sameRule .getValue())) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + ",????" + stringAggResultTuple2._2 .getS_packetThroughput() + ",????" + stringAggResultTuple2._2 .getC_packetThroughput() + "." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if (percent.contains( sameRule.getValue())) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + sameRule .getMin_cardinality() + "." + result + "." + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } alertLevel.setRuleId( sameRule.getRuleName()); alertLevel.setUniqueMark(sameRule .getRuleName() + "-" + stringAggResultTuple2._1 + "-" + sameRule.getValue()); if (stats.containsKey( alertLevel.getUniqueMark())) { String preLevel = stats .get(alertLevel .getUniqueMark())._1; int num = stats.get(alertLevel .getUniqueMark())._2; boolean preWarning = preLevel .equals("levelWarn") || preLevel .equals("levelBad"); boolean newWarning = alertLevel .getWarningLevel() .equals("levelWarn") || alertLevel .getWarningLevel() .equals("levelBad"); if (preWarning && !newWarning) { num = 1 - num; stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else if (!preWarning && newWarning) { stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); } else if (!preWarning && !preWarning) { num = 1 - num; stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else { num = 0 - num; stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } } else { if (alertLevel.getWarningLevel() .equals("levelWarn") || alertLevel .getWarningLevel() .equals("levelBad")) { stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); json = (JSONObject) JSON .toJSON(alertLevel); return new Tuple2<>( stringAggResultTuple2._1, json.toString()); } } } return new Tuple2<>( stringAggResultTuple2._1, ""); } }) .filter(new Function<Tuple2<String, String>, Boolean>() { @Override public Boolean call(Tuple2<String, String> v1) throws Exception { return !v1._2.equals("") && v1._2 != null; } }).map(new Function<Tuple2<String, String>, String>() { @Override public String call(Tuple2<String, String> v1) throws Exception { return v1._2; } }).collect(); } else { alertList = alert .filter(new Function<Tuple2<String, AggResult>, Boolean>() { @Override public Boolean call(Tuple2<String, AggResult> v1) throws Exception { Field field2 = v1._2.getClass() .getDeclaredField(sameRule.getValue()); field2.setAccessible(true); double result2 = (double) field2.get(v1._2); if (result2 == 0.0) { return false; } String contain = sameRule.getContain(); if (contain.equals("") && !v1._1.equals("")) { return true; } if (v1._1 == null || v1._1.equals("")) { return false; } return v1._1.contains(sameRule.getContain()); } }).mapToPair( new PairFunction<Tuple2<String, AggResult>, String, String>() { @Override public Tuple2<String, String> call( Tuple2<String, AggResult> stringAggResultTuple2) throws Exception { Field field2 = stringAggResultTuple2._2 .getClass().getDeclaredField( sameRule.getValue()); field2.setAccessible(true); double alertCursor = (double) field2 .get(stringAggResultTuple2._2); JSONObject json = new JSONObject(); AlertLevel alertLevel = new AlertLevel(); if (alertCursor >= sameRule .getMax_cardinality()) { alertLevel.setWarningLevel("levelBad"); } else if (alertCursor >= sameRule .getMin_cardinality()) { alertLevel.setWarningLevel("levelWarn"); } else { alertLevel .setWarningLevel("levelNormal"); } alertLevel.setResourceName( stringAggResultTuple2._1); if (sameRule.getType() .equals("server_ip")) { alertLevel.setIpAddress( stringAggResultTuple2._1); } else { alertLevel.setIpAddress(""); } alertLevel.setResourceType( sameRule.getType()); alertLevel.setOccureTime( esFormat.format(format.parse( stringAggResultTuple2._2 .getTimestamp()))); alertLevel.setMetricId(sameRule.getValue()); alertLevel.setResourceInstanceId( stringAggResultTuple2._1); // top2 d?>10%? Map<String, Double> top2 = new HashMap<String, Double>(); top2.put("MTU?", stringAggResultTuple2._2 .getMTU_pre()); top2.put("?", stringAggResultTuple2._2 .getCongest_pre()); top2.put("??", stringAggResultTuple2._2 .getOutoforder_pre()); top2.put("??", stringAggResultTuple2._2 .getZerowindow_pre()); List<Map.Entry<String, Double>> list = SortHashMap .sortHashMap(top2); if ("lossRate" .equals(sameRule.getValue())) { if (list.get(0).getValue() > 10 && list .get(1).getValue() > 10) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + list.get(0) .getKey() + "" + list.get(0) .getValue() + "%25," + list.get(1) .getKey() + "" + list.get(1) .getValue() + "%25." + "" + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + "," + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else if ("userRespTime" .equals(sameRule.getValue())) { if (list.get(0).getValue() > 10 && list .get(1).getValue() > 10) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + ",?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + list.get(0) .getKey() + "" + list.get(0) .getValue() + "%25," + list.get(1) .getKey() + "" + list.get(1) .getValue() + "%25." + "" + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + ",?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else if ("rtt" .equals(sameRule.getValue())) { alertLevel.setWarningContent("RTT" + millToSec((long) alertCursor) + ",RTT" + millToSec( (long) stringAggResultTuple2._2 .getClientRoundTripTime()) + ",?RTT" + millToSec( (long) stringAggResultTuple2._2 .getServerRoundTripTime()) + "." + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("throughput" .equals(sameRule.getType())) { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + convertUnit( (long) alertCursor) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getS_throughput()) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getC_throughput()) + "." + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("packetThroughput" .equals(sameRule.getValue())) { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + df.format(alertCursor) + ",????" + stringAggResultTuple2._2 .getS_packetThroughput() + ",????" + stringAggResultTuple2._2 .getC_packetThroughput() + "." + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if (percent .contains(sameRule.getValue())) { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + df.format(alertCursor) + "%25," + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + df.format(alertCursor) + "," + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } alertLevel .setRuleId(sameRule.getRuleName()); alertLevel.setUniqueMark( sameRule.getRuleName() + "-" + stringAggResultTuple2._1 + "-" + sameRule.getValue()); if (stats.containsKey( alertLevel.getUniqueMark())) { String preLevel = stats.get( alertLevel.getUniqueMark())._1; int num = stats.get( alertLevel.getUniqueMark())._2; boolean preWarning = preLevel .equals("levelWarn") || preLevel.equals("levelBad"); boolean newWarning = alertLevel .getWarningLevel() .equals("levelWarn") || alertLevel.getWarningLevel() .equals("levelBad"); if (preWarning && !newWarning) { num = 1 - num; stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else if (!preWarning && newWarning) { stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); } else if (!preWarning && !preWarning) { num = 1 - num; stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else { num = 0 - num; stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } } else { if (alertLevel.getWarningLevel() .equals("levelWarn") || alertLevel.getWarningLevel() .equals("levelBad")) { stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); json = (JSONObject) JSON .toJSON(alertLevel); return new Tuple2<>( stringAggResultTuple2._1, json.toString()); } } return new Tuple2<>( stringAggResultTuple2._1, ""); } }) .filter(new Function<Tuple2<String, String>, Boolean>() { private static final long serialVersionUID = 662946729452638751L; @Override public Boolean call(Tuple2<String, String> v1) throws Exception { return !v1._2.equals("") && v1._2 != null; } }).map(new Function<Tuple2<String, String>, String>() { @Override public String call(Tuple2<String, String> v1) throws Exception { return v1._2; } }).collect(); } } jsonList.addAll(alertList); alertList.clear(); } } } } flagRules.clear(); } Iterator<Map.Entry<String, Tuple2<String, Integer>>> iterator = stats.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry<String, Tuple2<String, Integer>> entry = iterator.next(); int num = entry.getValue()._2; if (num == 0) { UrlPostMethod.urlPostMethod(RECOVER, entry.getValue()._1); iterator.remove(); } else if (num < 0) { num = 0 - num; entry.setValue(new Tuple2<String, Integer>(entry.getValue()._1, num)); } else { num = 1 - num; if (num == 0) { UrlPostMethod.urlPostMethod(RECOVER, entry.getValue()._1); iterator.remove(); } else { entry.setValue(new Tuple2<String, Integer>(entry.getValue()._1, num)); } } } if (stats.size() > 200000) { stats.clear(); } if (jsonList.size() > 0) { if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es"))) { System.out.println( "-------------------" + jsonList.toString() + "-----------------------"); } for (int i = 0; i <= jsonList.size() / 2000; i++) { UrlPostMethod.urlPostMethod(URL, "warnings=" + Arrays.asList(Arrays.copyOfRange( jsonList.toArray(), i * 2000, (i + 1) * 2000 - 1 > jsonList.size() ? jsonList.size() : (i + 1) * 2000 - 1)) .toString()); } jsonList.clear(); } } return null; } }); rawStream.context().start(); rawStream.context().awaitTermination(); }
From source file:com.apextom.util.CollectionUtil.java
/** * ? null .//from w w w . ja v a 2 s. c om * * @param list */ public static void removeNull(Collection<Object> list) { if (list == null || list.size() == 0) { return; } Iterator<Object> iter = list.iterator(); while (iter.hasNext()) { if (iter.next() == null) { iter.remove(); } } }
From source file:Main.java
public static List<String> removeIgnoreCase(List<String> l, String s) { Iterator<String> iter = l.iterator(); while (iter.hasNext()) { if (iter.next().equalsIgnoreCase(s)) { iter.remove(); break; }//from ww w. j a v a2 s. c o m } return l; }
From source file:Main.java
/** * Removes nulls from provided collection. * /*from w w w.j a v a 2 s .c o m*/ * @param c * to remove nulls from * @return number of removed elements */ @SuppressWarnings("rawtypes") public static int removeNulls(Collection c) { int removed = 0; Iterator iterator = c.iterator(); while (iterator.hasNext()) { if (iterator.next() == null) { iterator.remove(); removed++; } } return removed; }
From source file:Main.java
public static final <E, F extends Collection<E>> F removeNulls(F collection) { Iterator<E> iterator = collection.iterator(); while (iterator.hasNext()) { if (iterator.next() == null) { iterator.remove(); }/* w w w . j ava2 s.c o m*/ } return collection; }