List of usage examples for java.util List get
E get(int index);
From source file:edu.upenn.cis.FastAlign.java
/** * Prints alignments for options specified by command line arguments. * @param argv parameters to be used by FastAlign. *//*from w w w . java2 s . com*/ public static void main(String[] argv) { FastAlign align = FastAlign.initCommandLine(argv); if (align == null) { System.err.println("Usage: java " + FastAlign.class.getCanonicalName() + " -i file.fr-en\n" + " Standard options ([USE] = strongly recommended):\n" + " -i: [REQ] Input parallel corpus\n" + " -v: [USE] Use Dirichlet prior on lexical translation distributions\n" + " -d: [USE] Favor alignment points close to the monotonic diagonoal\n" + " -o: [USE] Optimize how close to the diagonal alignment points should be\n" + " -r: Run alignment in reverse (condition on target and predict source)\n" + " -c: Output conditional probability table\n" + " -e: Start with existing conditional probability table\n" + " Advanced options:\n" + " -I: number of iterations in EM training (default = 5)\n" + " -p: p_null parameter (default = 0.08)\n" + " -N: No null word\n" + " -a: alpha parameter for optional Dirichlet prior (default = 0.01)\n" + " -T: starting lambda for diagonal distance parameter (default = 4)\n"); System.exit(1); } boolean use_null = !align.no_null_word; if (align.variational_bayes && align.alpha <= 0.0) { System.err.println("--alpha must be > 0\n"); System.exit(1); } double prob_align_not_null = 1.0 - align.prob_align_null; final int kNULL = align.d.Convert("<eps>"); TTable s2t = new TTable(); if (!align.existing_probability_filename.isEmpty()) { boolean success = s2t.ImportFromFile(align.existing_probability_filename, '\t', align.d); if (!success) { System.err.println("Can't read table " + align.existing_probability_filename); System.exit(1); } } Map<Pair, Integer> size_counts = new HashMap<Pair, Integer>(); double tot_len_ratio = 0; double mean_srclen_multiplier = 0; List<Double> probs = new ArrayList<Double>(); ; // E-M Iterations Loop TODO move this into a method? for (int iter = 0; iter < align.iterations || (iter == 0 && align.iterations == 0); ++iter) { final boolean final_iteration = (iter >= (align.iterations - 1)); System.err.println("ITERATION " + (iter + 1) + (final_iteration ? " (FINAL)" : "")); Scanner in = null; try { in = new Scanner(new File(align.input)); if (!in.hasNextLine()) { System.err.println("Can't read " + align.input); System.exit(1); } } catch (FileNotFoundException e) { e.printStackTrace(); System.err.println("Can't read " + align.input); System.exit(1); } double likelihood = 0; double denom = 0.0; int lc = 0; boolean flag = false; String line; // String ssrc, strg; ArrayList<Integer> src = new ArrayList<Integer>(); ArrayList<Integer> trg = new ArrayList<Integer>(); double c0 = 0; double emp_feat = 0; double toks = 0; // Iterate over each line of the input file while (in.hasNextLine()) { line = in.nextLine(); ++lc; if (lc % 1000 == 0) { System.err.print('.'); flag = true; } if (lc % 50000 == 0) { System.err.println(" [" + lc + "]\n"); System.err.flush(); flag = false; } src.clear(); trg.clear(); // TODO this is redundant; src and tgt cleared in ParseLine // Integerize and split source and target lines. align.ParseLine(line, src, trg); if (align.is_reverse) { ArrayList<Integer> tmp = src; src = trg; trg = tmp; } // TODO Empty lines break the parser. Should this be true? if (src.size() == 0 || trg.size() == 0) { System.err.println("Error in line " + lc + "\n" + line); System.exit(1); } if (iter == 0) { tot_len_ratio += ((double) trg.size()) / ((double) src.size()); } denom += trg.size(); probs.clear(); // Add to pair length counts only if first iteration. if (iter == 0) { Pair pair = new Pair(trg.size(), src.size()); Integer value = size_counts.get(pair); if (value == null) value = 0; size_counts.put(pair, value + 1); } boolean first_al = true; // used when printing alignments toks += trg.size(); // Iterate through the English tokens for (int j = 0; j < trg.size(); ++j) { final int f_j = trg.get(j); double sum = 0; double prob_a_i = 1.0 / (src.size() + (use_null ? 1 : 0)); // uniform (model 1) if (use_null) { if (align.favor_diagonal) { prob_a_i = align.prob_align_null; } probs.add(0, s2t.prob(kNULL, f_j) * prob_a_i); sum += probs.get(0); } double az = 0; if (align.favor_diagonal) az = DiagonalAlignment.computeZ(j + 1, trg.size(), src.size(), align.diagonal_tension) / prob_align_not_null; for (int i = 1; i <= src.size(); ++i) { if (align.favor_diagonal) prob_a_i = DiagonalAlignment.unnormalizedProb(j + 1, i, trg.size(), src.size(), align.diagonal_tension) / az; probs.add(i, s2t.prob(src.get(i - 1), f_j) * prob_a_i); sum += probs.get(i); } if (final_iteration) { double max_p = -1; int max_index = -1; if (use_null) { max_index = 0; max_p = probs.get(0); } for (int i = 1; i <= src.size(); ++i) { if (probs.get(i) > max_p) { max_index = i; max_p = probs.get(i); } } if (max_index > 0) { if (first_al) first_al = false; else System.out.print(' '); if (align.is_reverse) System.out.print("" + j + '-' + (max_index - 1)); else System.out.print("" + (max_index - 1) + '-' + j); } } else { if (use_null) { double count = probs.get(0) / sum; c0 += count; s2t.Increment(kNULL, f_j, count); } for (int i = 1; i <= src.size(); ++i) { final double p = probs.get(i) / sum; s2t.Increment(src.get(i - 1), f_j, p); emp_feat += DiagonalAlignment.feature(j, i, trg.size(), src.size()) * p; } } likelihood += Math.log(sum); } if (final_iteration) System.out.println(); } // log(e) = 1.0 double base2_likelihood = likelihood / Math.log(2); if (flag) { System.err.println(); } if (iter == 0) { mean_srclen_multiplier = tot_len_ratio / lc; System.err.println("expected target length = source length * " + mean_srclen_multiplier); } emp_feat /= toks; System.err.println(" log_e likelihood: " + likelihood); System.err.println(" log_2 likelihood: " + base2_likelihood); System.err.println(" cross entropy: " + (-base2_likelihood / denom)); System.err.println(" perplexity: " + Math.pow(2.0, -base2_likelihood / denom)); System.err.println(" posterior p0: " + c0 / toks); System.err.println(" posterior al-feat: " + emp_feat); //System.err.println(" model tension: " + mod_feat / toks ); System.err.println(" size counts: " + size_counts.size()); if (!final_iteration) { if (align.favor_diagonal && align.optimize_tension && iter > 0) { for (int ii = 0; ii < 8; ++ii) { double mod_feat = 0; Iterator<Map.Entry<Pair, Integer>> it = size_counts.entrySet().iterator(); for (; it.hasNext();) { Map.Entry<Pair, Integer> entry = it.next(); final Pair p = entry.getKey(); for (int j = 1; j <= p.first; ++j) mod_feat += entry.getValue() * DiagonalAlignment.computeDLogZ(j, p.first, p.second, align.diagonal_tension); } mod_feat /= toks; System.err.println(" " + ii + 1 + " model al-feat: " + mod_feat + " (tension=" + align.diagonal_tension + ")"); align.diagonal_tension += (emp_feat - mod_feat) * 20.0; if (align.diagonal_tension <= 0.1) align.diagonal_tension = 0.1; if (align.diagonal_tension > 14) align.diagonal_tension = 14; } System.err.println(" final tension: " + align.diagonal_tension); } if (align.variational_bayes) s2t.NormalizeVB(align.alpha); else s2t.Normalize(); //prob_align_null *= 0.8; // XXX //prob_align_null += (c0 / toks) * 0.2; prob_align_not_null = 1.0 - align.prob_align_null; } } if (!align.conditional_probability_filename.isEmpty()) { System.err.println("conditional probabilities: " + align.conditional_probability_filename); s2t.ExportToFile(align.conditional_probability_filename, align.d); } System.exit(0); }
From source file:com.servoy.extensions.plugins.http.HttpProvider.java
public static void main(String[] args) throws Exception { if (args.length != 3) { System.out.println("Use: ContentFetcher mainurl contenturl destdir"); //$NON-NLS-1$ System.out.println(/*from w w w . j a v a 2 s . c o m*/ "Example: ContentFetcher http://site.com http://site.com/dir[0-2]/image_A[001-040].jpg c:/temp"); //$NON-NLS-1$ System.out.println( "Result: accessing http://site.com for cookie, reading http://site.com/dir1/image_A004.jpg writing c:/temp/dir_1_image_A004.jpg"); //$NON-NLS-1$ } else { String url = args[1]; String destdir = args[2]; List parts = new ArrayList(); int dir_from = 0; int dir_to = 0; int dir_fill = 0; int from = 0; int to = 0; int fill = 0; StringTokenizer tk = new StringTokenizer(url, "[]", true); //$NON-NLS-1$ boolean hasDir = (tk.countTokens() > 5); boolean inDir = hasDir; System.out.println("hasDir " + hasDir); //$NON-NLS-1$ boolean inTag = false; while (tk.hasMoreTokens()) { String token = tk.nextToken(); if (token.equals("[")) //$NON-NLS-1$ { inTag = true; continue; } if (token.equals("]")) //$NON-NLS-1$ { inTag = false; if (inDir) inDir = false; continue; } if (inTag) { int idx = token.indexOf('-'); String s_from = token.substring(0, idx); int a_from = new Integer(s_from).intValue(); int a_fill = s_from.length(); int a_to = new Integer(token.substring(idx + 1)).intValue(); if (inDir) { dir_from = a_from; dir_to = a_to; dir_fill = a_fill; } else { from = a_from; to = a_to; fill = a_fill; } } else { parts.add(token); } } DefaultHttpClient client = new DefaultHttpClient(); client.getParams().setParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS, true); HttpGet main = new HttpGet(args[0]); HttpResponse res = client.execute(main); ; int main_rs = res.getStatusLine().getStatusCode(); if (main_rs != 200) { System.out.println("main page retrieval failed " + main_rs); //$NON-NLS-1$ return; } for (int d = dir_from; d <= dir_to; d++) { String dir_number = "" + d; //$NON-NLS-1$ if (dir_fill > 1) { dir_number = "000000" + d; //$NON-NLS-1$ int dir_digits = (int) (Math.log(fill) / Math.log(10)); System.out.println("dir_digits " + dir_digits); //$NON-NLS-1$ dir_number = dir_number.substring(dir_number.length() - (dir_fill - dir_digits), dir_number.length()); } for (int i = from; i <= to; i++) { try { String number = "" + i; //$NON-NLS-1$ if (fill > 1) { number = "000000" + i; //$NON-NLS-1$ int digits = (int) (Math.log(fill) / Math.log(10)); System.out.println("digits " + digits); //$NON-NLS-1$ number = number.substring(number.length() - (fill - digits), number.length()); } int part = 0; StringBuffer surl = new StringBuffer((String) parts.get(part++)); if (hasDir) { surl.append(dir_number); surl.append(parts.get(part++)); } surl.append(number); surl.append(parts.get(part++)); System.out.println("reading url " + surl); //$NON-NLS-1$ int indx = surl.toString().lastIndexOf('/'); StringBuffer sfile = new StringBuffer(destdir); sfile.append("\\"); //$NON-NLS-1$ if (hasDir) { sfile.append("dir_"); //$NON-NLS-1$ sfile.append(dir_number); sfile.append("_"); //$NON-NLS-1$ } sfile.append(surl.toString().substring(indx + 1)); File file = new File(sfile.toString()); if (file.exists()) { file = new File("" + System.currentTimeMillis() + sfile.toString()); } System.out.println("write file " + file.getAbsolutePath()); //$NON-NLS-1$ // URL iurl = createURLFromString(surl.toString()); HttpGet get = new HttpGet(surl.toString()); HttpResponse response = client.execute(get); int result = response.getStatusLine().getStatusCode(); System.out.println("page http result " + result); //$NON-NLS-1$ if (result == 200) { InputStream is = response.getEntity().getContent(); FileOutputStream fos = new FileOutputStream(file); Utils.streamCopy(is, fos); fos.close(); } } catch (Exception e) { System.err.println(e); } } } } }
From source file:com.oneapm.base.SparkAggregation.java
public static void main(String[] args) throws IOException { String configfile = "alert.cnf"; final FlowConstant flowConstant = new FlowConstant(); Properties config = null;/* ww w . j av a 2 s .co m*/ try { config = getConfig(configfile); } catch (IOException e) { LOG.error(configfile + " doesnt exist in /etc/analysis/... exit" + e); System.exit(-1); } final int window = Integer.parseInt(config.getProperty("time.window", "60")); final List<String> percent = Arrays.asList("tryConnectPer", "cBitpacketAccount", "abortConntionCount", "unidirectionalTrafficPer"); final List<String> jsonList = new ArrayList<>(); final Map<String, Tuple2<String, Integer>> stats = new HashMap<String, Tuple2<String, Integer>>(); final Map<ClientKey, BaseData> broadClient = new HashMap<ClientKey, BaseData>(); final Map<HostKey, BaseData> broadHost = new HashMap<HostKey, BaseData>(); final Map<ServiceKey, BaseData> broadService = new HashMap<ServiceKey, BaseData>(); final Map<SubnetKey, BaseData> broadSubnet = new HashMap<SubnetKey, BaseData>(); final Map<VlanKey, BaseData> broadVlan = new HashMap<VlanKey, BaseData>(); final String URL = config.getProperty("alert.url"); final String HEART_BEAT = config.getProperty("alert.heartbeat"); final String RECOVER = config.getProperty("alert.recover"); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/site", new SiteConvertImpl(flowConstant)); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/net", new NetConvertImpl(flowConstant)); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/vlan_sflow", new LinkConvertImpl(flowConstant)); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/subnet", new SubnetConvertImpl(flowConstant)); zookeeperClient.connectZookeeper(config.getProperty("kafaka.zoo")); esGet.setNodeClient(); startZookeeperService(flowConstant, zookeeperClient); zookeeperClient.setRules("/ni/caution"); JavaPairReceiverInputDStream<String, byte[]> rawStream = setupRawStreamFromKafka(config, config.getProperty("group.id", "oneapm-alert")); LOG.info("alert config:" + config.toString()); final Broadcast<IPDataInfo> ipDataBroadcastInfo = getJsc().sparkContext().broadcast(new IPDataInfo()); final Broadcast<ProtocalTypeInfo> protocalTypeBroadcastInfo = getJsc().sparkContext() .broadcast(new ProtocalTypeInfo()); JavaPairDStream<TimeAgg, BeforeAgg> aggJavaPairDStream = rawStream .mapToPair(new PairFunction<Tuple2<String, byte[]>, TimeAgg, BeforeAgg>() { private static final long serialVersionUID = -2751318332921803477L; @Override public Tuple2<TimeAgg, BeforeAgg> call(Tuple2<String, byte[]> stringTuple2) throws Exception { String[] s = TAB.split(new String(stringTuple2._2)); String clientIP = s[1]; String serverIP = s[2]; TimeAgg timeAgg = new TimeAgg(); BeforeAgg beforeAgg = new BeforeAgg(); timeAgg.setServer_ip(serverIP); if (s.length >= 60) { //setProtocal_type if (!"-".equals(s[11]) && !"".equals(s[11])) { timeAgg.setProtocal_type(s[11]); } else { if ("TCP".equals(s[12])) { ProtocalTypeInfo protocalTypeInfo = protocalTypeBroadcastInfo.value(); String protocalType = protocalTypeInfo.config.getProperty(s[4]); if (protocalType != null) { timeAgg.setProtocal_type(protocalType); } else { timeAgg.setProtocal_type("TCP_" + s[4]); } } else { timeAgg.setProtocal_type(s[12]); } } timeAgg.setType("tcp"); //setVlan_id String vlanLinkAlias = flowConstant.VLAN_LINK.get(s[13]); String portLinkAlias = flowConstant.PROBE_POTR_MAP.get(s[0] + "_" + s[15]); if (flowConstant.PROBE_HOST_ENABLE) { if (portLinkAlias != null) { timeAgg.setVlan_id(portLinkAlias); } else { timeAgg.setVlan_id(s[0] + "-nic" + s[15]); } } else if (flowConstant.VLAN_LINK_ENABLE) { if (vlanLinkAlias != null) { timeAgg.setVlan_id(vlanLinkAlias); } else { timeAgg.setVlan_id(s[13]); } } else { timeAgg.setVlan_id("Vlan" + s[13]); } if (!"-".equals(s[6])) { timeAgg.setTimestamp(format.format((long) Double.parseDouble(s[6]) * 1000)); } else { timeAgg.setTimestamp(s[6]); } if (!"-".equals(s[7])) { timeAgg.setTimeEnd(format.format((long) Double.parseDouble(s[7]) * 1000)); } else { timeAgg.setTimeEnd(s[7]); } beforeAgg.setPacket_size(Double.parseDouble(s[55]) + Double.parseDouble(s[56])); beforeAgg.setC_packet_size(Double.parseDouble(s[55])); beforeAgg.setS_packet_size(Double.parseDouble(s[56])); beforeAgg.setPacket_count(Double.parseDouble(s[59]) + Double.parseDouble(s[60])); beforeAgg.setLosspacket_count(Double.parseDouble(s[33]) + Double.parseDouble(s[39]) + Double.parseDouble(s[35]) + Double.parseDouble(s[41])); double cRto = (Double.valueOf(s[19]) + Double.valueOf(s[21]) + Double.valueOf(s[23])) * 1000; double sRto = (Double.valueOf(s[20]) + Double.valueOf(s[22]) + Double.valueOf(s[24])) * 1000; beforeAgg.setTotal_rto(cRto + sRto); // ?/TCP????/TCP????/TCP?MTU?MTU/TCP? beforeAgg.setInt_ZWIN_COUNT(Double.parseDouble(s[43])); beforeAgg.setInt_OOR_COUNT(Double.parseDouble(s[46])); beforeAgg.setInt_CONGEST_COUNT(Double.parseDouble(s[47])); beforeAgg.setMTU(Double.parseDouble(s[61]) + Double.parseDouble(s[62])); Boolean hasSynFlag = Boolean.valueOf(s[5]); timeAgg.setHasSyn(hasSynFlag); timeAgg.setBool_FIN(Boolean.valueOf(s[25])); timeAgg.setBool_RST(Boolean.valueOf(s[26])); if (hasSynFlag && "-".equals(s[9])) { beforeAgg.setAbort(1); } else { beforeAgg.setAbort(0); } beforeAgg.setTcpTurns(Integer.parseInt(s[30])); //int_TURN_COUNT beforeAgg.setConnrequest_count(Double.parseDouble(s[48])); beforeAgg.setsAbortConnCount(Integer.valueOf(s[50])); Long cPayLoad = Long.valueOf(s[53]); Long sPayLoad = Long.valueOf(s[54]); long payLoad = cPayLoad + sPayLoad; double sessionJlRtt = 0; if (!"-".equals(s[9])) { sessionJlRtt = Double.valueOf(s[9]) * 1000; beforeAgg.setRtt(sessionJlRtt); beforeAgg.setServerJlRttCount(1); beforeAgg.setClientJlRttCount(1); } if (hasSynFlag && !"-".equals(s[9])) { beforeAgg.setCount(1); if ("true".equals(s[26]) && payLoad == 0) { beforeAgg.setCount(0); } } if (!"-".equals(s[10])) { double clientJlRtt = Double.valueOf(s[10]) * 1000; double serverJlRtt = sessionJlRtt - clientJlRtt; if (clientJlRtt < sessionJlRtt) { beforeAgg.setServer_rtt(clientJlRtt); beforeAgg.setClient_rtt(serverJlRtt); } else { beforeAgg.setServer_rtt(sessionJlRtt / 2); beforeAgg.setClient_rtt(sessionJlRtt / 2); } } if (beforeAgg.tcpTurns > 0) { beforeAgg.setSessionCount(1); if (Double.parseDouble(s[18]) > 0) { beforeAgg.setServer_reponsetime( Double.parseDouble(s[18]) / beforeAgg.tcpTurns * 1000); } if (Double.parseDouble(s[16]) > 0) { beforeAgg.setResponseTransmissionTime( Double.parseDouble(s[16]) / beforeAgg.tcpTurns * 1000); } if (Double.parseDouble(s[17]) > 0) { beforeAgg.setRequestTransmissionTime( Double.parseDouble(s[17]) / beforeAgg.tcpTurns * 1000); } if (beforeAgg.total_rto > 0) { beforeAgg.setTotal_rto(beforeAgg.getTotal_rto() / beforeAgg.tcpTurns); } beforeAgg.setUserResponseTime( beforeAgg.getRtt() + beforeAgg.getRequestTransmissionTime() + beforeAgg.getResponseTransmissionTime() + beforeAgg.getServer_reponsetime() + beforeAgg.total_rto); } else { beforeAgg.setServer_reponsetime(0); beforeAgg.setResponseTransmissionTime(0); beforeAgg.setRequestTransmissionTime(0); beforeAgg.setTotal_rto(0); beforeAgg.setUserResponseTime(0); } beforeAgg.setC_bitpacket_account(Double.parseDouble(s[57]) + Double.parseDouble(s[58])); } else if (s.length <= 28) { if (!"-".equals(s[8]) && !"".equals(s[8])) { timeAgg.setProtocal_type(s[8]); } else { if ("UDP".equals(s[9])) { ProtocalTypeInfo protocalTypeInfo = protocalTypeBroadcastInfo.value(); String protocalType = protocalTypeInfo.config.getProperty(s[4]); if (protocalType != null) { timeAgg.setProtocal_type(protocalType); } else { timeAgg.setProtocal_type("UDP"); } } else { timeAgg.setProtocal_type(s[9]); } } beforeAgg.setCount(0); timeAgg.setType("udp"); timeAgg.setHasSyn(false); timeAgg.setBool_FIN(false); timeAgg.setBool_RST(false); //setVlan_id String vlanLinkAlias = flowConstant.VLAN_LINK.get(s[12]); String portLinkAlias = flowConstant.PROBE_POTR_MAP.get(s[0] + "_" + s[10]); if (flowConstant.PROBE_HOST_ENABLE) { if (portLinkAlias != null) { timeAgg.setVlan_id(portLinkAlias); } else { timeAgg.setVlan_id(s[0] + "-nic" + s[10]); } } else if (flowConstant.VLAN_LINK_ENABLE) { if (vlanLinkAlias != null) { timeAgg.setVlan_id(vlanLinkAlias); } else { timeAgg.setVlan_id(s[12]); } } else { timeAgg.setVlan_id("Vlan" + s[12]); } if (!"-".equals(s[5])) { timeAgg.setTimestamp(format.format((long) Double.parseDouble(s[5]) * 1000)); } else { timeAgg.setTimestamp(s[5]); } if (!"-".equals(s[6])) { timeAgg.setTimeEnd(format.format((long) Double.parseDouble(s[6]) * 1000)); } else { timeAgg.setTimeEnd(s[6]); } beforeAgg.setPacket_size(Double.parseDouble(s[20]) + Double.parseDouble(s[21])); beforeAgg.setPacket_count(Double.parseDouble(s[24]) + Double.parseDouble(s[25])); beforeAgg.setC_packet_size(Double.parseDouble(s[20])); beforeAgg.setS_packet_size(Double.parseDouble(s[21])); beforeAgg.setInt_ZWIN_COUNT(0); beforeAgg.setInt_OOR_COUNT(0); beforeAgg.setInt_CONGEST_COUNT(0); beforeAgg.setMTU(0); beforeAgg.setC_bitpacket_account(Double.parseDouble(s[22]) + Double.parseDouble(s[23])); beforeAgg.setAbort(0); beforeAgg.setClient_rtt(0); beforeAgg.setServer_rtt(0); beforeAgg.setRtt(0); beforeAgg.setServerJlRttCount(0); beforeAgg.setClientJlRttCount(0); beforeAgg.setLosspacket_count(0); beforeAgg.setServer_reponsetime(0); beforeAgg.setTcpTurns(0); beforeAgg.setConnrequest_count(0); beforeAgg.setTotal_rto(0); beforeAgg.setUserResponseTime(0); beforeAgg.setResponseTransmissionTime(0); beforeAgg.setRequestTransmissionTime(0); beforeAgg.setServer_reponsetime(0); beforeAgg.setsAbortConnCount(0); beforeAgg.setSessionCount(0); } String sInOutFlag = IPUtils.isInHomeNet(serverIP, flowConstant.HOMENET); String cInOutFlag = IPUtils.isInHomeNet(clientIP, flowConstant.HOMENET); //setSubnet if ("IN".equals(sInOutFlag)) { String sSubNet = IPUtils.getSubNet(serverIP, flowConstant.NETMASK); timeAgg.setSubnet(sSubNet + "/" + flowConstant.MASKBITS); } else { timeAgg.setSubnet("-"); } if ("255.255.255.255".equals(clientIP)) { timeAgg.setClient_site("-"); } else { String clientSiteInfo = IPUtils.getSiteInfo(clientIP, flowConstant.SITEINFO_MAP); IPDataInfo ipDataInfo = ipDataBroadcastInfo.getValue(); if (clientSiteInfo != null) { String[] clientSiteInfos = clientSiteInfo.split("_", 3); timeAgg.setClient_site(clientSiteInfos[2]); } else { if ("IN".equals(cInOutFlag)) { timeAgg.setClient_site(""); } else { if (ipDataInfo != null) { String[] ipinfo = ipDataInfo.find(clientIP); // if (ipinfo.length < 3) { timeAgg.setClient_site(""); } else { if ("".equals(ipinfo[0])) { timeAgg.setClient_site(""); } else { //,areasite if ("".equals(ipinfo[1])) { ipinfo[1] = ipinfo[0]; } if ("".equals(ipinfo[2])) { ipinfo[2] = ipinfo[1]; } timeAgg.setClient_site(ipinfo[2]); } } } else { timeAgg.setClient_site("?"); } } } } return new Tuple2<>(timeAgg, beforeAgg); } }).filter(new Function<Tuple2<TimeAgg, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { return !v1._1.getTimestamp().equals("-") && !v1._1.getTimestamp().equals(""); } }); // aggJavaPairDStream.foreachRDD(new Function<JavaPairRDD<TimeAgg, BeforeAgg>, Void>() { // @Override // public Void call(JavaPairRDD<TimeAgg, BeforeAgg> v1) throws Exception { // if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es")) && v1 != null) { // JavaRDD<Map<String, ?>> es = v1.map(new Function<Tuple2<TimeAgg, BeforeAgg>, Map<String, // ?>>() { // // @Override // public Map<String, ?> call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { // ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder(); // // TimeAgg a = v1._1; // BeforeAgg b = v1._2; // String todayStr = sdf.format(format.parse(a.getTimestamp())); // builder.put("server_ip", a.server_ip); // builder.put("protocal_type", a.protocal_type); // builder.put("client_site", a.client_site); // builder.put("vlan_id", a.vlan_id); // builder.put("subnet", a.subnet); // builder.put("timestamp", format.parse(a.timestamp)); // if (b.packet_size > 0) { // builder.put("packet_size", b.packet_size); // } // if (b.c_packet_size > 0) { // builder.put("c_packet_size", b.c_packet_size); // } // // if (b.packet_count > 0) { // builder.put("packet_count", b.packet_count); // } // // if (b.losspacket_count > 0) { // builder.put("losspacket_count", b.losspacket_count); // } // if (b.total_rto > 0) { // builder.put("total_rto", b.total_rto); // builder.put("rtoCount", b.rtoCount); // } // // // if (b.tcpTurns > 0) { // builder.put("tcpTurns", b.tcpTurns); // } // if (b.connrequest_count > 0) { // builder.put("connrequest_count", b.connrequest_count); // } // if (b.abort > 0) { // builder.put("abort", b.abort); // } // if (b.client_rtt > 0) { // builder.put("client_rtt", b.client_rtt); // builder.put("clientJlRttCount", b.clientJlRttCount); // } // if (b.server_rtt > 0) { // builder.put("server_rtt", b.server_rtt); // builder.put("serverJlRttCount", b.serverJlRttCount); // } // // if (b.server_reponsetime > 0) { // builder.put("server_reponsetime", b.server_reponsetime); // builder.put("server_reponsetime_count", b.server_reponsetime_count); // } // // if (b.responseTransmissionTime > 0) { // builder.put("responseTransmissionTime", b.responseTransmissionTime); // builder.put("responseTransmissionTimeCount", b.responseTransmissionTimeCount); // } // if (b.requestTransmissionTime > 0) { // builder.put("requestTransmissionTime", b.requestTransmissionTime); // builder.put("requestTransmissionTimeCount", b.requestTransmissionTimeCount); // // } // // if (b.sAbortConnCount > 0) { // builder.put("sAbortConnCount", b.sAbortConnCount); // } // // if (b.userResponseTime > 0) { // builder.put("userResponseTime", b.userResponseTime); // builder.put("userResponseTimeCount", b.userResponseTimeCount); // } // if (b.c_bitpacket_account > 0) { // builder.put("c_bitpacket_account", b.c_bitpacket_account); // } // builder.put("index_name", todayStr); // // return builder.build(); // } // }).cache(); // if (es != null) { // JavaEsSpark.saveToEs(es, "ni-alert-session-{index_name}/alert", ImmutableMap.of // (ConfigurationOptions.ES_MAPPING_EXCLUDE, "index_name")); // } // } // return null; // } // }); JavaPairDStream<TimeAgg, BeforeAgg> reduceByWindow = aggJavaPairDStream .reduceByKeyAndWindow(new Function2<BeforeAgg, BeforeAgg, BeforeAgg>() { @Override public BeforeAgg call(BeforeAgg v1, BeforeAgg v2) throws Exception { BeforeAgg sum = new BeforeAgg(); sum.setPacket_size(v1.getPacket_size() + v2.getPacket_size()); sum.setC_packet_size(v1.getC_packet_size() + v2.getC_packet_size()); sum.setS_packet_size(v1.getS_packet_size() + v2.getS_packet_size()); sum.setPacket_count(v1.getPacket_count() + v2.getPacket_count()); sum.setC_packet_count(v1.getC_packet_count() + v2.getC_packet_count()); sum.setS_packet_count(v1.getS_packet_count() + v2.getS_packet_count()); sum.setLosspacket_count(v1.getLosspacket_count() + v2.getLosspacket_count()); sum.setTotal_rto(v1.getTotal_rto() + v2.getTotal_rto()); sum.setAbort(v1.getAbort() + v2.getAbort()); sum.setRequestTransmissionTime( v1.getRequestTransmissionTime() + v2.getRequestTransmissionTime()); sum.setResponseTransmissionTime( v1.getResponseTransmissionTime() + v2.getResponseTransmissionTime()); sum.setTcpTurns(v1.getTcpTurns() + v2.getTcpTurns()); sum.setConnrequest_count(v1.getConnrequest_count() + v2.getConnrequest_count()); sum.setRtt(v1.getRtt() + v2.getRtt()); sum.setClient_rtt(v1.getClient_rtt() + v2.getClient_rtt()); sum.setServer_rtt(v1.getServer_rtt() + v2.getServer_rtt()); sum.setServer_reponsetime(v1.getServer_reponsetime() + v2.getServer_reponsetime()); sum.setC_bitpacket_account(v1.getC_bitpacket_account() + v2.getC_bitpacket_account()); sum.setClientJlRttCount(v1.getClientJlRttCount() + v2.getClientJlRttCount()); sum.setServerJlRttCount(v1.getServerJlRttCount() + v2.getServerJlRttCount()); sum.setUserResponseTime(v1.getUserResponseTime() + v2.getUserResponseTime()); sum.setSessionCount(v1.sessionCount + v2.sessionCount); sum.setsAbortConnCount(v1.sAbortConnCount + v2.sAbortConnCount); sum.setCount(v1.getCount() + v2.getCount()); sum.setInt_CONGEST_COUNT(v1.int_CONGEST_COUNT + v2.int_CONGEST_COUNT); sum.setInt_OOR_COUNT(v1.int_OOR_COUNT + v2.int_OOR_COUNT); sum.setInt_ZWIN_COUNT(v1.int_ZWIN_COUNT + v2.int_ZWIN_COUNT); sum.setMTU(v1.MTU + v2.MTU); return sum; } }, Durations.seconds(300), Durations.seconds(60)).cache(); reduceByWindow.foreachRDD(new Function<JavaPairRDD<TimeAgg, BeforeAgg>, Void>() { private static final long serialVersionUID = -4144342491397135515L; @Override public Void call(JavaPairRDD<TimeAgg, BeforeAgg> v1) throws Exception { if (v1.count() > 0) { /** * getStartTime */ List<Long> timeList = v1.map(new Function<Tuple2<TimeAgg, BeforeAgg>, Long>() { @Override public Long call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { return format.parse(v1._1.getTimestamp()).getTime(); } }).distinct().collect(); Collections.sort(timeList, new MyComparator()); long a = timeList.get(3); final String time = format.format(new Date(a)); long b = timeList.get(1); final String endTime = format.format(new Date(b)); if (b > 0) { JavaRDD<Map<String, ?>> active = v1 .filter(new Function<Tuple2<TimeAgg, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { String sInOutFlag = IPUtils.isInHomeNet(v1._1.getServer_ip(), flowConstant.HOMENET); return v1._1.getType().equals("tcp") && "IN".equals(sInOutFlag); } }).mapToPair( new PairFunction<Tuple2<TimeAgg, BeforeAgg>, Tuple2<String, String>, ConnectStatus>() { @Override public Tuple2<Tuple2<String, String>, ConnectStatus> call( Tuple2<TimeAgg, BeforeAgg> timeAggBeforeAggTuple2) throws Exception { ConnectStatus connectStatus = new ConnectStatus(); String serverIp = timeAggBeforeAggTuple2._1.getServer_ip(); String protoType = timeAggBeforeAggTuple2._1.getProtocal_type(); TimeAgg a = timeAggBeforeAggTuple2._1; BeforeAgg b = timeAggBeforeAggTuple2._2; // if (format.parse(a.timestamp).getTime() == format.parse(endTime) .getTime() && a.hasSyn) { connectStatus.setNewCreate(b.getCount()); } else { connectStatus.setNewCreate(0); } //?breakreset?break if (format.parse(a.timeEnd).getTime() == format.parse(endTime) .getTime() && (a.bool_FIN || a.bool_RST)) { connectStatus.setCloseConn(b.getCount()); } else { connectStatus.setCloseConn(0); } if (format.parse(a.timestamp).getTime() <= format.parse(endTime) .getTime() && format.parse(a.timeEnd).getTime() > format.parse(endTime) .getTime() && a.hasSyn) { connectStatus.setActiveConn(b.getCount()); } else if (format.parse(a.timestamp).getTime() == format .parse(endTime).getTime() && format.parse(a.timeEnd).getTime() == format .parse(endTime).getTime() && a.hasSyn) { connectStatus.setActiveConn(b.getCount()); } return new Tuple2<>(new Tuple2<>(serverIp, protoType), connectStatus); } }) .reduceByKey(new Function2<ConnectStatus, ConnectStatus, ConnectStatus>() { @Override public ConnectStatus call(ConnectStatus v1, ConnectStatus v2) throws Exception { ConnectStatus connectStatus = new ConnectStatus(); connectStatus.setNewCreate(v1.newCreate + v2.newCreate); connectStatus.setActiveConn(v1.activeConn + v2.activeConn); connectStatus.setCloseConn(v1.closeConn + v2.closeConn); return connectStatus; } }) .map(new Function<Tuple2<Tuple2<String, String>, ConnectStatus>, Map<String, ?>>() { @Override public Map<String, ?> call(Tuple2<Tuple2<String, String>, ConnectStatus> v1) throws Exception { ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder(); String todayStr = sdf.format(format.parse(endTime)); builder.put("server_ip", v1._1._1); builder.put("protocal_type", v1._1._2); builder.put("newCreateConn", v1._2.getNewCreate()); builder.put("closeConn", v1._2.getCloseConn()); builder.put("activeConn", v1._2.getActiveConn()); builder.put("index_name", todayStr); builder.put("timestamp", format.parse(endTime)); return builder.build(); } }).cache(); if (active != null) { JavaEsSpark.saveToEs(active, "ni-active-conn-{index_name}/active", ImmutableMap.of(ConfigurationOptions.ES_MAPPING_EXCLUDE, "index_name")); } } JavaPairRDD<TimeAgg, BeforeAgg> before = v1 .filter(new Function<Tuple2<TimeAgg, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { return v1._1.getTimestamp().equals(time); } }); if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es")) && before != null) { JavaRDD<Map<String, ?>> es = before .map(new Function<Tuple2<TimeAgg, BeforeAgg>, Map<String, ?>>() { @Override public Map<String, ?> call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder(); TimeAgg a = v1._1; BeforeAgg b = v1._2; String todayStr = sdf.format(format.parse(a.getTimestamp())); builder.put("server_ip", a.server_ip); builder.put("protocal_type", a.protocal_type); builder.put("client_site", a.client_site); builder.put("vlan_id", a.vlan_id); builder.put("subnet", a.subnet); builder.put("timestamp", format.parse(a.timestamp)); if (b.packet_size > 0) { builder.put("packet_size", b.packet_size); } if (b.c_packet_size > 0) { builder.put("c_packet_size", b.c_packet_size); } builder.put("count", b.count); if (b.packet_count > 0) { builder.put("packet_count", b.packet_count); } if (b.losspacket_count > 0) { builder.put("losspacket_count", b.losspacket_count); } if (b.total_rto > 0) { builder.put("total_rto", b.total_rto); } if (b.tcpTurns > 0) { builder.put("tcpTurns", b.tcpTurns); builder.put("sessionCount", b.sessionCount); } if (b.connrequest_count > 0) { builder.put("connrequest_count", b.connrequest_count); } if (b.abort > 0) { builder.put("abort", b.abort); } if (b.client_rtt > 0) { builder.put("client_rtt", b.client_rtt); builder.put("clientJlRttCount", b.clientJlRttCount); } if (b.server_rtt > 0) { builder.put("server_rtt", b.server_rtt); builder.put("serverJlRttCount", b.serverJlRttCount); } if (b.server_reponsetime > 0) { builder.put("server_reponsetime", b.server_reponsetime); } if (b.responseTransmissionTime > 0) { builder.put("responseTransmissionTime", b.responseTransmissionTime); } if (b.requestTransmissionTime > 0) { builder.put("requestTransmissionTime", b.requestTransmissionTime); } if (b.sAbortConnCount > 0) { builder.put("sAbortConnCount", b.sAbortConnCount); } if (b.userResponseTime > 0) { builder.put("userResponseTime", b.userResponseTime); } if (b.c_bitpacket_account > 0) { builder.put("c_bitpacket_account", b.c_bitpacket_account); } builder.put("index_name", todayStr); return builder.build(); } }).cache(); if (es != null) { JavaEsSpark.saveToEs(es, "ni-alert-session-{index_name}/alert", ImmutableMap.of(ConfigurationOptions.ES_MAPPING_EXCLUDE, "index_name")); } } UrlPostMethod.urlPostMethod(HEART_BEAT, "frequency=" + window); rules = zookeeperClient.getRules(); if (rules != null) { ArrayList<String> flagRules = new ArrayList<String>(); //?ruleType for (final RuleRecover ruleRecover : rules) { final Rule rule = ruleRecover.getRule(); if (rule.isEnable() && !flagRules.contains(rule.getType())) { //ruleType? flagRules.add(rule.getType()); //ruleType? JavaPairRDD<String, AggResult> alert = before.mapToPair( new PairFunction<Tuple2<TimeAgg, BeforeAgg>, String, BeforeAgg>() { @Override public Tuple2<String, BeforeAgg> call( Tuple2<TimeAgg, BeforeAgg> timeAggBeforeAggTuple2) throws Exception { Field field1 = timeAggBeforeAggTuple2._1.getClass() .getDeclaredField(rule.getType()); field1.setAccessible(true); String result1 = (String) field1.get(timeAggBeforeAggTuple2._1); if (rule.getType().equals("server_ip")) { String sInOutFlag = IPUtils.isInHomeNet( timeAggBeforeAggTuple2._1.getServer_ip(), flowConstant.HOMENET); if ("IN".equals(sInOutFlag)) { return new Tuple2<>(result1, timeAggBeforeAggTuple2._2); } else { return new Tuple2<>(result1, null); } } else { return new Tuple2<>(result1, timeAggBeforeAggTuple2._2); } } }).filter(new Function<Tuple2<String, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<String, BeforeAgg> v1) throws Exception { return v1._2 != null && !v1._1.equals("") && !v1._1.equals("-"); } }).reduceByKey(new Function2<BeforeAgg, BeforeAgg, BeforeAgg>() { @Override public BeforeAgg call(BeforeAgg v1, BeforeAgg v2) throws Exception { BeforeAgg sum = new BeforeAgg(); sum.setPacket_size(v1.getPacket_size() + v2.getPacket_size()); sum.setC_packet_size(v1.getC_packet_size() + v2.getC_packet_size()); sum.setS_packet_size(v1.getS_packet_size() + v2.getS_packet_size()); sum.setPacket_count(v1.getPacket_count() + v2.getPacket_count()); sum.setC_packet_count( v1.getC_packet_count() + v2.getC_packet_count()); sum.setS_packet_count( v1.getS_packet_count() + v2.getS_packet_count()); sum.setLosspacket_count( v1.getLosspacket_count() + v2.getLosspacket_count()); sum.setTotal_rto(v1.getTotal_rto() + v2.getTotal_rto()); sum.setAbort(v1.getAbort() + v2.getAbort()); sum.setRequestTransmissionTime(v1.getRequestTransmissionTime() + v2.getRequestTransmissionTime()); sum.setResponseTransmissionTime(v1.getResponseTransmissionTime() + v2.getResponseTransmissionTime()); sum.setTcpTurns(v1.getTcpTurns() + v2.getTcpTurns()); sum.setConnrequest_count( v1.getConnrequest_count() + v2.getConnrequest_count()); sum.setRtt(v1.getRtt() + v2.getRtt()); sum.setClient_rtt(v1.getClient_rtt() + v2.getClient_rtt()); sum.setServer_rtt(v1.getServer_rtt() + v2.getServer_rtt()); sum.setServer_reponsetime( v1.getServer_reponsetime() + v2.getServer_reponsetime()); sum.setC_bitpacket_account( v1.getC_bitpacket_account() + v2.getC_bitpacket_account()); sum.setClientJlRttCount( v1.getClientJlRttCount() + v2.getClientJlRttCount()); sum.setServerJlRttCount( v1.getServerJlRttCount() + v2.getServerJlRttCount()); sum.setUserResponseTime( v1.getUserResponseTime() + v2.getUserResponseTime()); sum.setSessionCount(v1.sessionCount + v2.sessionCount); sum.setsAbortConnCount(v1.sAbortConnCount + v2.sAbortConnCount); return sum; } }).mapToPair( new PairFunction<Tuple2<String, BeforeAgg>, String, AggResult>() { @Override public Tuple2<String, AggResult> call( Tuple2<String, BeforeAgg> stringBeforeAggTuple2) throws Exception { BeforeAgg before = stringBeforeAggTuple2._2; AggResult result = new AggResult(); result.setTimestamp(time); result.setThroughput(before.packet_size * 8 / window); result.setS_throughput(before.s_packet_size * 8 / window); result.setC_throughput( before.c_bitpacket_account * 8 / window); result.setPacketThroughput(before.packet_count / window); result.setLossRate(before.losspacket_count / before.packet_count * 100); if (before.sessionCount > 0) { result.setRetransferTime( before.total_rto / before.sessionCount); } else { result.setRetransferTime(0); } if (before.clientJlRttCount > 0) { result.setClientRoundTripTime( before.client_rtt / before.clientJlRttCount); result.setRtt(before.rtt / before.clientJlRttCount); } else { result.setClientRoundTripTime(0); result.setRtt(0); } if (before.serverJlRttCount > 0) { result.setServerRoundTripTime( before.server_rtt / before.serverJlRttCount); } else { result.setServerRoundTripTime(0); } if (before.sessionCount > 0) { result.setUserRespTime(before.getUserResponseTime() / before.sessionCount); result.setTransmissionTime( (before.requestTransmissionTime + before.responseTransmissionTime) / before.sessionCount); } else { result.setUserRespTime(0); result.setTransmissionTime(0); } if (before.sessionCount > 0) { result.setServerRespTime(before.server_reponsetime / before.sessionCount); } else { result.setServerRespTime(0); } result.setConnectFailedRate(before.abort / window); //@Deprecates result.setTryConnectPer(0); result.setCongest_pre(before.getInt_CONGEST_COUNT() / before.getCount() * 100); result.setOutoforder_pre(before.getInt_OOR_COUNT() / before.getCount() * 100); result.setZerowindow_pre(before.getInt_ZWIN_COUNT() / before.getCount() * 100); result.setMTU_pre( before.getMTU() / before.getCount() * 100); if (before.packet_count > 0) { result.setcBitpacketAccount(before.c_bitpacket_account / before.packet_count * 100); } else { result.setcBitpacketAccount(0); } if (before.connrequest_count - before.abort > 0) { result.setAbortConntionCount(before.sAbortConnCount / (before.connrequest_count - before.abort) * 100); } else { result.setAbortConntionCount(0); } return new Tuple2<>(stringBeforeAggTuple2._1, result); } }) .cache(); if (alert.count() > 0) { List<String> alertList = new ArrayList<>(); for (final RuleRecover newRule : rules) { final Rule sameRule = newRule.getRule(); if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es"))) { System.out.println( "rule:" + sameRule.toString() + "--------------"); } final int recover = newRule.getRecover(); if (sameRule.isEnable() && sameRule.getType().equals(flagRules.get(flagRules.size() - 1))) { if (!sameRule.getThresholdErrType().equals("absolute")) { if (esGet.getClient() == null) { esGet.setNodeClient(); } final Calendar now = Calendar.getInstance(); now.setTime(format.parse(time)); int minute = now.get(Calendar.MINUTE); Key key = null; switch (sameRule.getType()) { case "server_ip": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadHost.isEmpty()) { Map<HostKey, BaseData> service = esGet .setHost(now.getTime(), "ni-base-hostname"); broadHost.putAll(service); } } else { Map<HostKey, BaseData> service = esGet .setHost(now.getTime(), "ni-base-hostname"); broadHost.clear(); broadHost.putAll(service); } key = new HostKey(); // baseData = broadHost.get(key); break; case "protocal_type": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadService.isEmpty()) { Map<ServiceKey, BaseData> service = esGet .setService(now.getTime(), "ni-base-service"); broadService.putAll(service); } } else { Map<ServiceKey, BaseData> service = esGet .setService(now.getTime(), "ni-base-service"); broadService.clear(); broadService.putAll(service); } key = new ServiceKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadService.get(key); break; case "client_site": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadClient.isEmpty()) { Map<ClientKey, BaseData> service = esGet .setClient(now.getTime(), "ni-base-clientsite"); broadClient.putAll(service); } } else { Map<ClientKey, BaseData> service = esGet .setClient(now.getTime(), "ni-base-clientsite"); broadClient.clear(); broadClient.putAll(service); } key = new ClientKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadClient.get(key); break; case "vlan_id": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadVlan.isEmpty()) { Map<VlanKey, BaseData> service = esGet .setVlan(now.getTime(), "ni-base-link"); broadVlan.putAll(service); } } else { Map<VlanKey, BaseData> service = esGet .setVlan(now.getTime(), "ni-base-link"); broadVlan.clear(); broadVlan.putAll(service); } key = new VlanKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadVlan.get(key); break; case "subnet": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadSubnet.isEmpty()) { Map<SubnetKey, BaseData> service = esGet .setSubnet(now.getTime(), "ni-base-subnet"); broadSubnet.putAll(service); } } else { Map<SubnetKey, BaseData> service = esGet .setSubnet(now.getTime(), "ni-base-subnet"); broadSubnet.clear(); broadSubnet.putAll(service); } key = new SubnetKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadSubnet.get(key); break; } final Key finalKey = key; alertList = alert .filter(new Function<Tuple2<String, AggResult>, Boolean>() { @Override public Boolean call(Tuple2<String, AggResult> v1) throws Exception { Field field2 = v1._2.getClass() .getDeclaredField(sameRule.getValue()); field2.setAccessible(true); double result2 = (double) field2.get(v1._2); if (result2 == 0) { return false; } String contain = sameRule.getContain(); if (contain.equals("") && !v1._1.equals("")) { return true; } if (v1._1 == null || v1._1.equals("")) { return false; } return v1._1.contains(sameRule.getContain()); } }).mapToPair( new PairFunction<Tuple2<String, AggResult>, String, String>() { @Override public Tuple2<String, String> call( Tuple2<String, AggResult> stringAggResultTuple2) throws Exception { Field field2 = stringAggResultTuple2._2 .getClass().getDeclaredField( sameRule.getValue()); field2.setAccessible(true); double alertCursor = (double) field2 .get(stringAggResultTuple2._2); JSONObject json = new JSONObject(); BaseData baseData = new BaseData(); finalKey.setKeyWord( stringAggResultTuple2._1); finalKey.setStart_timestamp(now.getTime()); baseData = broadService.get(finalKey); if (baseData != null) { Field field = baseData.getClass() .getDeclaredField( sameRule.getValue()); field.setAccessible(true); double result = (double) field .get(baseData); AlertLevel alertLevel = new AlertLevel(); if (sameRule.getThresholdErrOp() .equals("ge")) { if (alertCursor - result >= sameRule .getMax_cardinality()) { alertLevel.setWarningLevel( "levelBad"); } else if (alertCursor - result >= sameRule .getMin_cardinality()) { alertLevel.setWarningLevel( "levelWarn"); } else { alertLevel.setWarningLevel( "levelNormal"); } } if (sameRule.getThresholdErrOp() .equals("le")) { if (result - alertCursor <= sameRule .getMax_cardinality()) { alertLevel.setWarningLevel( "levelBad"); } else if (result - alertCursor <= sameRule .getMin_cardinality()) { alertLevel.setWarningLevel( "levelWarn"); } else { alertLevel.setWarningLevel( "levelNormal"); } } alertLevel.setResourceName( stringAggResultTuple2._1); if (sameRule.getType() .equals("server_ip")) { alertLevel.setIpAddress( stringAggResultTuple2._1); } else { alertLevel.setIpAddress(""); } alertLevel.setOccureTime( esFormat.format(format.parse( stringAggResultTuple2._2 .getTimestamp()))); alertLevel.setResourceType( sameRule.getType()); alertLevel.setMetricId( sameRule.getValue()); alertLevel.setResourceInstanceId( stringAggResultTuple2._1); // top2 d?>10%? Map<String, Double> top2 = new HashMap<String, Double>(); top2.put("MTU?", stringAggResultTuple2._2 .getMTU_pre()); top2.put("?", stringAggResultTuple2._2 .getCongest_pre()); top2.put("??", stringAggResultTuple2._2 .getOutoforder_pre()); top2.put("??", stringAggResultTuple2._2 .getZerowindow_pre()); List<Map.Entry<String, Double>> list = SortHashMap .sortHashMap(top2); if ("lossRate" .equals(sameRule.getValue())) { if (list.get(0).getValue() > 10 && list.get(1) .getValue() > 10) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + list.get(0) .getKey() + "" + list.get(0) .getValue() + "%25," + list.get(1) .getKey() + "" + list.get(1) .getValue() + "%25." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else { if ("userRespTime".equals( sameRule.getValue())) { if (list.get(0).getValue() > 10 && list.get(1) .getValue() > 10) { alertLevel .setWarningContent( alertLevel .getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "ms," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + list.get( 0) .getKey() + "" + list.get( 0) .getValue() + "%25," + list.get( 1) .getKey() + "" + list.get( 1) .getValue() + "%25." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel .setWarningContent( alertLevel .getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "ms," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else if ("rtt".equals( sameRule.getValue())) { alertLevel.setWarningContent( "RTT" + millToSec( (long) alertCursor) + ",RTT" + millToSec( (long) stringAggResultTuple2._2 .getClientRoundTripTime()) + "ms,?RTT" + millToSec( (long) stringAggResultTuple2._2 .getServerRoundTripTime()) + "ms." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("throughput".equals( sameRule.getType())) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + convertUnit( (long) alertCursor) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getS_throughput()) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getC_throughput()) + "." + convertUnit( (long) result) + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("packetThroughput" .equals(sameRule .getValue())) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + ",????" + stringAggResultTuple2._2 .getS_packetThroughput() + ",????" + stringAggResultTuple2._2 .getC_packetThroughput() + "." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if (percent.contains( sameRule.getValue())) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + sameRule .getMin_cardinality() + "." + result + "." + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } alertLevel.setRuleId( sameRule.getRuleName()); alertLevel.setUniqueMark(sameRule .getRuleName() + "-" + stringAggResultTuple2._1 + "-" + sameRule.getValue()); if (stats.containsKey( alertLevel.getUniqueMark())) { String preLevel = stats .get(alertLevel .getUniqueMark())._1; int num = stats.get(alertLevel .getUniqueMark())._2; boolean preWarning = preLevel .equals("levelWarn") || preLevel .equals("levelBad"); boolean newWarning = alertLevel .getWarningLevel() .equals("levelWarn") || alertLevel .getWarningLevel() .equals("levelBad"); if (preWarning && !newWarning) { num = 1 - num; stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else if (!preWarning && newWarning) { stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); } else if (!preWarning && !preWarning) { num = 1 - num; stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else { num = 0 - num; stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } } else { if (alertLevel.getWarningLevel() .equals("levelWarn") || alertLevel .getWarningLevel() .equals("levelBad")) { stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); json = (JSONObject) JSON .toJSON(alertLevel); return new Tuple2<>( stringAggResultTuple2._1, json.toString()); } } } return new Tuple2<>( stringAggResultTuple2._1, ""); } }) .filter(new Function<Tuple2<String, String>, Boolean>() { @Override public Boolean call(Tuple2<String, String> v1) throws Exception { return !v1._2.equals("") && v1._2 != null; } }).map(new Function<Tuple2<String, String>, String>() { @Override public String call(Tuple2<String, String> v1) throws Exception { return v1._2; } }).collect(); } else { alertList = alert .filter(new Function<Tuple2<String, AggResult>, Boolean>() { @Override public Boolean call(Tuple2<String, AggResult> v1) throws Exception { Field field2 = v1._2.getClass() .getDeclaredField(sameRule.getValue()); field2.setAccessible(true); double result2 = (double) field2.get(v1._2); if (result2 == 0.0) { return false; } String contain = sameRule.getContain(); if (contain.equals("") && !v1._1.equals("")) { return true; } if (v1._1 == null || v1._1.equals("")) { return false; } return v1._1.contains(sameRule.getContain()); } }).mapToPair( new PairFunction<Tuple2<String, AggResult>, String, String>() { @Override public Tuple2<String, String> call( Tuple2<String, AggResult> stringAggResultTuple2) throws Exception { Field field2 = stringAggResultTuple2._2 .getClass().getDeclaredField( sameRule.getValue()); field2.setAccessible(true); double alertCursor = (double) field2 .get(stringAggResultTuple2._2); JSONObject json = new JSONObject(); AlertLevel alertLevel = new AlertLevel(); if (alertCursor >= sameRule .getMax_cardinality()) { alertLevel.setWarningLevel("levelBad"); } else if (alertCursor >= sameRule .getMin_cardinality()) { alertLevel.setWarningLevel("levelWarn"); } else { alertLevel .setWarningLevel("levelNormal"); } alertLevel.setResourceName( stringAggResultTuple2._1); if (sameRule.getType() .equals("server_ip")) { alertLevel.setIpAddress( stringAggResultTuple2._1); } else { alertLevel.setIpAddress(""); } alertLevel.setResourceType( sameRule.getType()); alertLevel.setOccureTime( esFormat.format(format.parse( stringAggResultTuple2._2 .getTimestamp()))); alertLevel.setMetricId(sameRule.getValue()); alertLevel.setResourceInstanceId( stringAggResultTuple2._1); // top2 d?>10%? Map<String, Double> top2 = new HashMap<String, Double>(); top2.put("MTU?", stringAggResultTuple2._2 .getMTU_pre()); top2.put("?", stringAggResultTuple2._2 .getCongest_pre()); top2.put("??", stringAggResultTuple2._2 .getOutoforder_pre()); top2.put("??", stringAggResultTuple2._2 .getZerowindow_pre()); List<Map.Entry<String, Double>> list = SortHashMap .sortHashMap(top2); if ("lossRate" .equals(sameRule.getValue())) { if (list.get(0).getValue() > 10 && list .get(1).getValue() > 10) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + list.get(0) .getKey() + "" + list.get(0) .getValue() + "%25," + list.get(1) .getKey() + "" + list.get(1) .getValue() + "%25." + "" + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + "," + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else if ("userRespTime" .equals(sameRule.getValue())) { if (list.get(0).getValue() > 10 && list .get(1).getValue() > 10) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + ",?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + list.get(0) .getKey() + "" + list.get(0) .getValue() + "%25," + list.get(1) .getKey() + "" + list.get(1) .getValue() + "%25." + "" + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + ",?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else if ("rtt" .equals(sameRule.getValue())) { alertLevel.setWarningContent("RTT" + millToSec((long) alertCursor) + ",RTT" + millToSec( (long) stringAggResultTuple2._2 .getClientRoundTripTime()) + ",?RTT" + millToSec( (long) stringAggResultTuple2._2 .getServerRoundTripTime()) + "." + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("throughput" .equals(sameRule.getType())) { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + convertUnit( (long) alertCursor) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getS_throughput()) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getC_throughput()) + "." + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("packetThroughput" .equals(sameRule.getValue())) { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + df.format(alertCursor) + ",????" + stringAggResultTuple2._2 .getS_packetThroughput() + ",????" + stringAggResultTuple2._2 .getC_packetThroughput() + "." + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if (percent .contains(sameRule.getValue())) { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + df.format(alertCursor) + "%25," + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + df.format(alertCursor) + "," + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } alertLevel .setRuleId(sameRule.getRuleName()); alertLevel.setUniqueMark( sameRule.getRuleName() + "-" + stringAggResultTuple2._1 + "-" + sameRule.getValue()); if (stats.containsKey( alertLevel.getUniqueMark())) { String preLevel = stats.get( alertLevel.getUniqueMark())._1; int num = stats.get( alertLevel.getUniqueMark())._2; boolean preWarning = preLevel .equals("levelWarn") || preLevel.equals("levelBad"); boolean newWarning = alertLevel .getWarningLevel() .equals("levelWarn") || alertLevel.getWarningLevel() .equals("levelBad"); if (preWarning && !newWarning) { num = 1 - num; stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else if (!preWarning && newWarning) { stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); } else if (!preWarning && !preWarning) { num = 1 - num; stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else { num = 0 - num; stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } } else { if (alertLevel.getWarningLevel() .equals("levelWarn") || alertLevel.getWarningLevel() .equals("levelBad")) { stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); json = (JSONObject) JSON .toJSON(alertLevel); return new Tuple2<>( stringAggResultTuple2._1, json.toString()); } } return new Tuple2<>( stringAggResultTuple2._1, ""); } }) .filter(new Function<Tuple2<String, String>, Boolean>() { private static final long serialVersionUID = 662946729452638751L; @Override public Boolean call(Tuple2<String, String> v1) throws Exception { return !v1._2.equals("") && v1._2 != null; } }).map(new Function<Tuple2<String, String>, String>() { @Override public String call(Tuple2<String, String> v1) throws Exception { return v1._2; } }).collect(); } } jsonList.addAll(alertList); alertList.clear(); } } } } flagRules.clear(); } Iterator<Map.Entry<String, Tuple2<String, Integer>>> iterator = stats.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry<String, Tuple2<String, Integer>> entry = iterator.next(); int num = entry.getValue()._2; if (num == 0) { UrlPostMethod.urlPostMethod(RECOVER, entry.getValue()._1); iterator.remove(); } else if (num < 0) { num = 0 - num; entry.setValue(new Tuple2<String, Integer>(entry.getValue()._1, num)); } else { num = 1 - num; if (num == 0) { UrlPostMethod.urlPostMethod(RECOVER, entry.getValue()._1); iterator.remove(); } else { entry.setValue(new Tuple2<String, Integer>(entry.getValue()._1, num)); } } } if (stats.size() > 200000) { stats.clear(); } if (jsonList.size() > 0) { if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es"))) { System.out.println( "-------------------" + jsonList.toString() + "-----------------------"); } for (int i = 0; i <= jsonList.size() / 2000; i++) { UrlPostMethod.urlPostMethod(URL, "warnings=" + Arrays.asList(Arrays.copyOfRange( jsonList.toArray(), i * 2000, (i + 1) * 2000 - 1 > jsonList.size() ? jsonList.size() : (i + 1) * 2000 - 1)) .toString()); } jsonList.clear(); } } return null; } }); rawStream.context().start(); rawStream.context().awaitTermination(); }
From source file:fmiquerytest.Coordinates.java
public static void main(String[] args) { df_short.setTimeZone(tz); df_iso.setTimeZone(tz);/*from ww w . j a va2s. c om*/ df_daycode.setTimeZone(tz); DecimalFormatSymbols otherSymbols = new DecimalFormatSymbols(); otherSymbols.setDecimalSeparator('.'); df_fiveDecimal.setDecimalFormatSymbols(otherSymbols); String startTime = df_short.format(new Date(startTimeMillis)); System.out.println("startTime: " + startTime); //Clean up old weather data //********************************************************************** FileSystemTools.cleanupOldWeatherData(daysToStoreWeatherData); //Google query //********************************************************************** if (gShare.equals("")) { Scanner input = new Scanner(System.in); System.out.println("Paste Google Directions Share:"); gShare = input.nextLine(); } String gQuery = Parser.getQueryFromShare(gShare); System.out.println("Google query URL: " + gQuery); //Check if we already have this route //Valid only if the route option is 0 (default) //Because otherwise we cannot be sure we already have the optional route List<routeStep> gSteps = new ArrayList<>(); if (FileSystemTools.isSavedRoute(gQuery) && gRouteOption == 0) { System.out.println("Route found from saved list. Loading."); gSteps = FileSystemTools.loadSavedRoute(gQuery); } else { gSteps = Parser.getSteps(gQuery); if (gRouteOption == 0) { System.out.println("Saving new route to list."); FileSystemTools.saveRoute(gQuery, gSteps); } } //Compile route table with current settings //********************************************************************** List<routeStep> routeData = RouteTools.compileRoute(gSteps, refreshInterval); String endTime = df_short.format(new Date(startTimeMillis + routeDur * 1000)); System.out.println("endTime: " + endTime); //Forecast from FMI is only for 48h - warning if we are going over //Or is it 54h? http://ilmatieteenlaitos.fi/avoin-data-saaennustedata-hirlam if (((startTimeMillis + routeDur * 1000) - System.currentTimeMillis()) / (1000 * 60 * 60) > 48) { System.out.println("**************************************************" + newLine + "WARNING:" + newLine + "Weather forecast available only for 48 hours" + newLine + "**************************************************"); } //Prepare time and file variables //********************************************************************** String nowAsISO = df_iso.format(new Date()); System.out.println("Start ISO time: " + nowAsISO); double timeMarginal = routeDur * 1.2 + 3600; String endTimeForFmi = df_iso.format(new Date(startTimeMillis + (intValue(timeMarginal)) * 1000)); String endTimeForFile = df_iso.format(new Date(startTimeMillis + (intValue(routeDur + 3600)) * 1000)); System.out.println("End ISO time: " + endTimeForFmi); String fmiParam = new StringBuilder("&starttime=").append(nowAsISO).append("&endtime=") .append(endTimeForFmi).toString(); File weatherDataFileNameFirst = new File("weather" + nowAsISO.replaceAll("[^A-Za-z0-9 ]", "") + ".txt"); File weatherDataFileNameLast = new File("weather" + endTimeForFmi.replaceAll("[^A-Za-z0-9 ]", "") + ".txt"); File weatherDataFileNameStart = new File( "weather" + (df_iso.format(new Date(startTimeMillis))).replaceAll("[^A-Za-z0-9 ]", "") + ".txt"); File weatherDataFileNameEnd = new File("weather" + endTimeForFile.replaceAll("[^A-Za-z0-9 ]", "") + ".txt"); List<stationData> allStations = new ArrayList<>(); List<stationData> fmiData = new ArrayList<>(); List<String> savedFileTimes = new ArrayList<>(); //********************************************************************** //Check if we already have the weather data //********************************************************************** if (!weatherDataFileNameStart.exists() || !weatherDataFileNameEnd.exists()) { //FMI query //********************************************************************** String fmiCities = new StringBuilder(fmiBase).append(fmiKey).append(fmiMiddle).append(fmiQueryCities) .append(fmiParam).toString(); String fmiObsStations = new StringBuilder(fmiBase).append(fmiKey).append(fmiMiddle) .append(fmiQueryObsStations).append(fmiParam).toString(); //System.out.println("FMI cities URL: "+fmiCities); //System.out.println("FMI obsstations URL: "+fmiObsStations); //Collect weather data from FMI //********************************************************************** System.out.print("FMI data:" + newLine + fmiCities + newLine + "Loading and processing..."); fmiData.addAll(Parser.getStations(fmiCities)); System.out.println("SUCCESS."); System.out.print("FMI data:" + newLine + fmiObsStations + newLine + "Loading and processing..."); fmiData.addAll(Parser.getStations(fmiObsStations)); System.out.println("SUCCESS."); //Get unique stations //********************************************************************** List<stationData> uniqueStations = ToolBox.getUniqueStations(fmiData); System.out.println("Parsed stations count: " + uniqueStations.size()); //Save or load stations //********************************************************************** List<stationData> savedStations = new ArrayList<>(); if (!stationFileName.exists()) { //Save current parsed stations to file FileSystemTools.saveObjectToFile(uniqueStations, stationFileName); } else { //Or if the stations were already saved, load them System.out.println("Station information file found: " + stationFileName); System.out.print("Loading..."); savedStations = FileSystemTools.loadStationsFromFile(stationFileName); System.out.println("DONE."); System.out.println("Loaded stations count: " + savedStations.size()); } //Merge station information //********************************************************************** System.out.println("Merging station information."); savedStations.addAll(uniqueStations); allStations = ToolBox.getUniqueStations(savedStations); System.out.println("Merged stations count: " + allStations.size()); //Find names for stations //********************************************************************** String gMapsGeoCode = "https://maps.googleapis.com/maps/api/geocode/xml?latlng="; //for (stationData station : allStations){ for (int i = 0; i < allStations.size(); i++) { if (allStations.get(i).stationName.equals("")) { gQuery = new StringBuilder(gMapsGeoCode).append(allStations.get(i).stationLocation.Lat) .append(",").append(allStations.get(i).stationLocation.Lon).append("&key=").append(gKey) .toString(); System.out.println("Google query URL: " + gQuery); allStations.get(i).stationName = Parser.getStationName(gQuery); } } //System.out.println("Station names parsed."); Collections.sort(allStations); //Print stations and separate them for saving //********************************************************************** List<stationData> onlyStations = new ArrayList<>(); //int indeksi = 0; List<weatherData> weatherPoint = new ArrayList<>(); weatherPoint.add(0, new weatherData("", "", "")); for (stationData station : allStations) { //System.out.format("%-4s%-30s%-10s%-10s%n", // indeksi,station.stationName,station.stationLocation.Lat,station.stationLocation.Lon); //++indeksi; onlyStations.add(new stationData(station.stationLocation, station.stationName, weatherPoint)); } //Save station names //********************************************************************** System.out.println("Saving station names."); FileSystemTools.saveObjectToFile(onlyStations, stationFileName); //Save weather dataset //********************************************************************** //Compute file names between start and end System.out.println("Saving weather data..."); long currentTimeAsDouble = System.currentTimeMillis(); int hoursPassed = intValue(Math.floor(currentTimeAsDouble - startTimeMillis) / 1000 / 60 / 60); File weatherDataFileNameTemp = weatherDataFileNameFirst; while (!weatherDataFileNameTemp.equals(weatherDataFileNameLast)) { String savedFileTime = df_iso.format(new Date(startTimeMillis + ((hoursPassed * 3600) * 1000))); savedFileTimes.add(savedFileTime); weatherDataFileNameTemp = new File( "weather" + savedFileTime.replaceAll("[^A-Za-z0-9 ]", "") + ".txt"); //System.out.println("Weather data file: "+weatherDataFileNameTemp); //This if we don't actually maybe want //if (!weatherDataFileNameTemp.exists()){ List<stationData> thisHourWeather = FileSystemTools.extractHourOfWeatherData(savedFileTime, fmiData); //System.out.println("Saving: "+weatherDataFileNameTemp); FileSystemTools.saveObjectToFile(thisHourWeather, weatherDataFileNameTemp); //} ++hoursPassed; } } //If we have weather data saved, definitely we have the stations also //********************************************************************** else { System.out.println("Loading weather data..."); File weatherDataFileNameTemp = weatherDataFileNameStart; int hoursPassed = 0; while (!weatherDataFileNameTemp.equals(weatherDataFileNameEnd)) { String savedFileTime = df_iso.format(new Date(startTimeMillis + ((hoursPassed * 3600) * 1000))); savedFileTimes.add(savedFileTime); weatherDataFileNameTemp = new File( "weather" + savedFileTime.replaceAll("[^A-Za-z0-9 ]", "") + ".txt"); System.out.println("Weather data file: " + weatherDataFileNameTemp); if (weatherDataFileNameTemp.exists()) { fmiData.addAll(FileSystemTools.loadStationsFromFile(weatherDataFileNameTemp)); } ++hoursPassed; } allStations = FileSystemTools.loadStationsFromFile(stationFileName); System.out.println("DONE."); } //Find closest weather stations in route points and extract their data //********************************************************************** System.out.println("Calculating nearest stations in route points:"); List<Integer> neededStations = new ArrayList<>(); for (routeStep step : routeData) { distance[] stationDistances = RouteTools.calculateStationDistances(step.StartLocation, allStations); System.out.format("%-6s%.5f, %.5f ", "Step: ", step.StartLocation.Lat, step.StartLocation.Lon); for (int i = 0; i < 1; i++) { System.out.format("%-9s%-5s%-20s%.5f%n", "Station: ", stationDistances[i].stationNum, allStations.get(stationDistances[i].stationNum).stationName, stationDistances[i].stationDistance); } neededStations.add(stationDistances[0].stationNum); } System.out.println("Needed stations: " + neededStations.toString().trim()); //Remove duplicates from needed stations list Set<Integer> uniqueEntries = new HashSet<Integer>(neededStations); //Extract weather data from needed stations Map routeWeather = Collections.synchronizedMap(new HashMap()); routeWeather = WeatherTools.extractNeededStations(uniqueEntries, fmiData, allStations); //Find what fields we have List<String> allParameters = new ArrayList<>(); for (int i = 0; i < fmiData.size(); ++i) { allParameters.add(fmiData.get(i).weatherData.get(0).parameterName); } Set<String> uniqueParameters = new HashSet<String>(allParameters); for (String par : uniqueParameters) { for (Integer num : uniqueEntries) { for (String time : savedFileTimes) { //System.out.format("%-5s%-25s%-35s%s%n",num,time,par,routeWeather.get(num+"-"+time+"-"+par)); } } } // Build the final data table //********************************************************************** List<stepWeather> stepDataBase = new ArrayList<>(); stepDataBase = RouteTools.combineRouteDatabase(routeData, neededStations, allStations); //Find sunrise and sunset times during the route //********************************************************************** List<String> sunEvents = DayLightTime.calculateSunEvents(stepDataBase); for (String s : sunEvents) { System.out.println(s.replaceAll(",", ".")); } //Make a webpage to show the weather data //********************************************************************** WeatherTools.makeResultHtml(stepDataBase, allStations, routeWeather, sunEvents); }
From source file:com.github.s4ke.moar.cli.Main.java
public static void main(String[] args) throws ParseException, IOException { // create Options object Options options = new Options(); options.addOption("rf", true, "file containing the regexes to test against (multiple regexes are separated by one empty line)"); options.addOption("r", true, "regex to test against"); options.addOption("mf", true, "file/folder to read the MOA from"); options.addOption("mo", true, "folder to export the MOAs to (overwrites if existent)"); options.addOption("sf", true, "file to read the input string(s) from"); options.addOption("s", true, "string to test the MOA/Regex against"); options.addOption("m", false, "multiline matching mode (search in string for regex)"); options.addOption("ls", false, "treat every line of the input string file as one string"); options.addOption("t", false, "trim lines if -ls is set"); options.addOption("d", false, "only do determinism check"); options.addOption("help", false, "prints this dialog"); CommandLineParser parser = new DefaultParser(); CommandLine cmd = parser.parse(options, args); if (args.length == 0 || cmd.hasOption("help")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("moar-cli", options); return;//from ww w . j a v a 2 s . c o m } List<String> patternNames = new ArrayList<>(); List<MoaPattern> patterns = new ArrayList<>(); List<String> stringsToCheck = new ArrayList<>(); if (cmd.hasOption("r")) { String regexStr = cmd.getOptionValue("r"); try { patterns.add(MoaPattern.compile(regexStr)); patternNames.add(regexStr); } catch (Exception e) { System.out.println(e.getMessage()); } } if (cmd.hasOption("rf")) { String fileName = cmd.getOptionValue("rf"); List<String> regexFileContents = readFileContents(new File(fileName)); int emptyLineCountAfterRegex = 0; StringBuilder regexStr = new StringBuilder(); for (String line : regexFileContents) { if (emptyLineCountAfterRegex >= 1) { if (regexStr.length() > 0) { patterns.add(MoaPattern.compile(regexStr.toString())); patternNames.add(regexStr.toString()); } regexStr.setLength(0); emptyLineCountAfterRegex = 0; } if (line.trim().equals("")) { if (regexStr.length() > 0) { ++emptyLineCountAfterRegex; } } else { regexStr.append(line); } } if (regexStr.length() > 0) { try { patterns.add(MoaPattern.compile(regexStr.toString())); patternNames.add(regexStr.toString()); } catch (Exception e) { System.out.println(e.getMessage()); return; } regexStr.setLength(0); } } if (cmd.hasOption("mf")) { String fileName = cmd.getOptionValue("mf"); File file = new File(fileName); if (file.isDirectory()) { System.out.println(fileName + " is a directory, using all *.moar files as patterns"); File[] moarFiles = file.listFiles(pathname -> pathname.getName().endsWith(".moar")); for (File moar : moarFiles) { String jsonString = readWholeFile(moar); patterns.add(MoarJSONSerializer.fromJSON(jsonString)); patternNames.add(moar.getAbsolutePath()); } } else { System.out.println(fileName + " is a single file. using it directly (no check for *.moar suffix)"); String jsonString = readWholeFile(file); patterns.add(MoarJSONSerializer.fromJSON(jsonString)); patternNames.add(fileName); } } if (cmd.hasOption("s")) { String str = cmd.getOptionValue("s"); stringsToCheck.add(str); } if (cmd.hasOption("sf")) { boolean treatLineAsString = cmd.hasOption("ls"); boolean trim = cmd.hasOption("t"); String fileName = cmd.getOptionValue("sf"); StringBuilder stringBuilder = new StringBuilder(); boolean firstLine = true; for (String str : readFileContents(new File(fileName))) { if (treatLineAsString) { if (trim) { str = str.trim(); if (str.length() == 0) { continue; } } stringsToCheck.add(str); } else { if (!firstLine) { stringBuilder.append("\n"); } if (firstLine) { firstLine = false; } stringBuilder.append(str); } } if (!treatLineAsString) { stringsToCheck.add(stringBuilder.toString()); } } if (cmd.hasOption("d")) { //at this point we have already built the Patterns //so just give the user a short note. System.out.println("All Regexes seem to be deterministic."); return; } if (patterns.size() == 0) { System.out.println("no patterns to check"); return; } if (cmd.hasOption("mo")) { String folder = cmd.getOptionValue("mo"); File folderFile = new File(folder); if (!folderFile.exists()) { System.out.println(folder + " does not exist. creating..."); if (!folderFile.mkdirs()) { System.out.println("folder " + folder + " could not be created"); } } int cnt = 0; for (MoaPattern pattern : patterns) { String patternAsJSON = MoarJSONSerializer.toJSON(pattern); try (BufferedWriter writer = new BufferedWriter( new FileWriter(new File(folderFile, "pattern" + ++cnt + ".moar")))) { writer.write(patternAsJSON); } } System.out.println("stored " + cnt + " patterns in " + folder); } if (stringsToCheck.size() == 0) { System.out.println("no strings to check"); return; } boolean multiline = cmd.hasOption("m"); for (String string : stringsToCheck) { int curPattern = 0; for (MoaPattern pattern : patterns) { MoaMatcher matcher = pattern.matcher(string); if (!multiline) { if (matcher.matches()) { System.out.println("\"" + patternNames.get(curPattern) + "\" matches \"" + string + "\""); } else { System.out.println( "\"" + patternNames.get(curPattern) + "\" does not match \"" + string + "\""); } } else { StringBuilder buffer = new StringBuilder(string); int additionalCharsPerMatch = ("<match>" + "</match>").length(); int matchCount = 0; while (matcher.nextMatch()) { buffer.replace(matcher.getStart() + matchCount * additionalCharsPerMatch, matcher.getEnd() + matchCount * additionalCharsPerMatch, "<match>" + string.substring(matcher.getStart(), matcher.getEnd()) + "</match>"); ++matchCount; } System.out.println(buffer.toString()); } } ++curPattern; } }
From source file:edu.upenn.egricelab.AlignerBoost.FilterSAMAlignPE.java
public static void main(String[] args) { if (args.length == 0) { printUsage();//from www .ja va 2s . c o m return; } try { parseOptions(args); } catch (IllegalArgumentException e) { System.err.println("Error: " + e.getMessage()); printUsage(); return; } // Read in chrList, if specified if (chrFile != null) { chrFilter = new HashSet<String>(); try { BufferedReader chrFilterIn = new BufferedReader(new FileReader(chrFile)); String chr = null; while ((chr = chrFilterIn.readLine()) != null) chrFilter.add(chr); chrFilterIn.close(); if (verbose > 0) System.err.println( "Only looking at alignments on " + chrFilter.size() + " specified chromosomes"); } catch (IOException e) { System.err.println("Error: " + e.getMessage()); return; } } if (verbose > 0) { // Start the processMonitor processMonitor = new Timer(); // Start the ProcessStatusTask statusTask = new ProcessStatusTask(); // Schedule to show the status every 1 second processMonitor.scheduleAtFixedRate(statusTask, 0, statusFreq); } // Read in known SNP file, if specified if (knownSnpFile != null) { if (verbose > 0) System.err.println("Checking known SNPs from user specified VCF file"); knownVCF = new VCFFileReader(new File(knownSnpFile)); } SamReaderFactory readerFac = SamReaderFactory.makeDefault(); SAMFileWriterFactory writerFac = new SAMFileWriterFactory(); if (!isSilent) readerFac.validationStringency(ValidationStringency.LENIENT); // use LENIENT stringency else readerFac.validationStringency(ValidationStringency.SILENT); // use SILENT stringency SamReader in = readerFac.open(new File(inFile)); SAMFileHeader inHeader = in.getFileHeader(); if (inHeader.getGroupOrder() == GroupOrder.reference && inHeader.getSortOrder() == SortOrder.coordinate) System.err.println("Warning: Input file '" + inFile + "' might be sorted by coordinate and cannot be correctly processed!"); SAMFileHeader header = inHeader.clone(); // copy the inFile header as outFile header // Add new programHeader SAMProgramRecord progRec = new SAMProgramRecord(progName); progRec.setProgramName(progName); progRec.setProgramVersion(progVer); progRec.setCommandLine(StringUtils.join(" ", args)); header.addProgramRecord(progRec); //System.err.println(inFile + " groupOrder: " + in.getFileHeader().getGroupOrder() + " sortOrder: " + in.getFileHeader().getSortOrder()); // reset the orders header.setGroupOrder(groupOrder); header.setSortOrder(sortOrder); // write SAMHeader String prevID = null; SAMRecord prevRecord = null; List<SAMRecord> alnList = new ArrayList<SAMRecord>(); List<SAMRecordPair> alnPEList = null; // Estimate fragment length distribution by scan one-pass through the alignments SAMRecordIterator results = in.iterator(); if (!NO_ESTIMATE) { if (verbose > 0) { System.err.println("Estimating insert fragment size distribution ..."); statusTask.reset(); statusTask.setInfo("alignments scanned"); } long N = 0; double fragL_S = 0; // fragLen sum double fragL_SS = 0; // fragLen^2 sum while (results.hasNext()) { SAMRecord record = results.next(); if (verbose > 0) statusTask.updateStatus(); if (record.getFirstOfPairFlag() && !record.isSecondaryOrSupplementary()) { double fragLen = Math.abs(record.getInferredInsertSize()); if (fragLen != 0 && fragLen >= MIN_FRAG_LEN && fragLen <= MAX_FRAG_LEN) { // only consider certain alignments N++; fragL_S += fragLen; fragL_SS += fragLen * fragLen; } // stop estimate if already enough if (MAX_ESTIMATE_SCAN > 0 && N >= MAX_ESTIMATE_SCAN) break; } } if (verbose > 0) statusTask.finish(); // estimate fragment size if (N >= MIN_ESTIMATE_BASE) { // override command line values MEAN_FRAG_LEN = fragL_S / N; SD_FRAG_LEN = Math.sqrt((N * fragL_SS - fragL_S * fragL_S) / (N * (N - 1))); String estStr = String.format("Estimated fragment size distribution: N(%.1f, %.1f)", MEAN_FRAG_LEN, SD_FRAG_LEN); if (verbose > 0) System.err.println(estStr); // also add the estimation to comment header.addComment(estStr); } else { System.err.println( "Unable to estimate the fragment size distribution due to too few observed alignments"); System.err.println( "You have to specify the '--mean-frag-len' and '--sd-frag-len' on the command line and re-run this step"); statusTask.cancel(); processMonitor.cancel(); return; } // Initiate the normal model normModel = new NormalDistribution(MEAN_FRAG_LEN, SD_FRAG_LEN); // reset the iterator, if necessary if (in.type() == SamReader.Type.SAM_TYPE) { try { in.close(); } catch (IOException e) { System.err.println(e.getMessage()); } in = readerFac.open(new File(inFile)); } results.close(); results = in.iterator(); } // end of NO_ESTIMATE SAMFileWriter out = OUT_IS_SAM ? writerFac.makeSAMWriter(header, false, new File(outFile)) : writerFac.makeBAMWriter(header, false, new File(outFile)); // check each alignment again if (verbose > 0) { System.err.println("Filtering alignments ..."); statusTask.reset(); statusTask.setInfo("alignments processed"); } while (results.hasNext()) { SAMRecord record = results.next(); if (verbose > 0) statusTask.updateStatus(); String ID = record.getReadName(); // fix read and quality string for this read, if is a secondary hit from multiple hits, used for BWA alignment if (ID.equals(prevID) && record.getReadLength() == 0) SAMAlignFixer.fixSAMRecordRead(record, prevRecord); if (chrFilter != null && !chrFilter.contains(record.getReferenceName())) { prevID = ID; prevRecord = record; continue; } // fix MD:Z string for certain aligners with invalid format (i.e. seqAlto) if (fixMD) SAMAlignFixer.fixMisStr(record); // fix alignment, ignore if failed (unmapped or empty) if (!SAMAlignFixer.fixSAMRecord(record, knownVCF, DO_1DP)) { prevID = ID; prevRecord = record; continue; } if (!record.getReadPairedFlag()) { System.err.println("Error: alignment is not from a paired-end read at\n" + record.getSAMString()); out.close(); statusTask.cancel(); processMonitor.cancel(); return; } if (!ID.equals(prevID) && prevID != null || !results.hasNext()) { // a non-first new ID meet, or end of alignments // create alnPEList from filtered alnList alnPEList = createAlnPEListFromAlnList(alnList); //System.err.printf("%d alignments for %s transformed to %d alnPairs%n", alnList.size(), prevID, alnPEList.size()); int totalPair = alnPEList.size(); // filter highly unlikely PEhits filterPEHits(alnPEList, MIN_ALIGN_RATE, MIN_IDENTITY); // calculate posterior mapQ for each pair calcPEHitPostP(alnPEList, totalPair, MAX_HIT); // filter hits by mapQ if (MIN_MAPQ > 0) filterPEHits(alnPEList, MIN_MAPQ); // sort the list first with an anonymous class of comparator, with DESCREASING order Collections.sort(alnPEList, Collections.reverseOrder()); // control max-best if (MAX_BEST != 0 && alnPEList.size() > MAX_BEST) { // potential too much best hits int nBestStratum = 0; int bestMapQ = alnPEList.get(0).getPEMapQ(); // best mapQ from first PE for (SAMRecordPair pr : alnPEList) if (pr.getPEMapQ() == bestMapQ) nBestStratum++; else break; // stop searching for sorted list if (nBestStratum > MAX_BEST) alnPEList.clear(); } // filter alignments with auxiliary filters if (!MAX_SENSITIVITY) filterPEHits(alnPEList, MAX_SEED_MIS, MAX_SEED_INDEL, MAX_ALL_MIS, MAX_ALL_INDEL); // report remaining secondary alignments, up-to MAX_REPORT for (int i = 0; i < alnPEList.size() && (MAX_REPORT == 0 || i < MAX_REPORT); i++) { SAMRecordPair repPair = alnPEList.get(i); if (doUpdateBit) repPair.setNotPrimaryAlignmentFlags(i != 0); int nReport = MAX_REPORT == 0 ? Math.min(alnPEList.size(), MAX_REPORT) : alnPEList.size(); int nFiltered = alnPEList.size(); if (repPair.fwdRecord != null) { repPair.fwdRecord.setAttribute("NH", nReport); repPair.fwdRecord.setAttribute("XN", nFiltered); out.addAlignment(repPair.fwdRecord); } if (repPair.revRecord != null) { repPair.revRecord.setAttribute("NH", nReport); repPair.revRecord.setAttribute("XN", nFiltered); out.addAlignment(repPair.revRecord); } } // reset list alnList.clear(); alnPEList.clear(); } // update if (!ID.equals(prevID)) { prevID = ID; prevRecord = record; } alnList.add(record); } // end while try { in.close(); out.close(); } catch (IOException e) { System.err.println(e.getMessage()); } // Terminate the monitor task and monitor if (verbose > 0) { statusTask.cancel(); statusTask.finish(); processMonitor.cancel(); } }
From source file:com.oltpbenchmark.DBWorkload.java
/** * @param args//w w w. j a va 2s .c o m * @throws Exception */ public static void main(String[] args) throws Exception { // Initialize log4j String log4jPath = System.getProperty("log4j.configuration"); if (log4jPath != null) { org.apache.log4j.PropertyConfigurator.configure(log4jPath); } else { throw new RuntimeException("Missing log4j.properties file"); } // create the command line parser CommandLineParser parser = new PosixParser(); XMLConfiguration pluginConfig = null; try { pluginConfig = new XMLConfiguration("config/plugin.xml"); } catch (ConfigurationException e1) { LOG.info("Plugin configuration file config/plugin.xml is missing"); e1.printStackTrace(); } pluginConfig.setExpressionEngine(new XPathExpressionEngine()); Options options = new Options(); options.addOption("b", "bench", true, "[required] Benchmark class. Currently supported: " + pluginConfig.getList("/plugin//@name")); options.addOption("c", "config", true, "[required] Workload configuration file"); options.addOption(null, "create", true, "Initialize the database for this benchmark"); options.addOption(null, "clear", true, "Clear all records in the database for this benchmark"); options.addOption(null, "load", true, "Load data using the benchmark's data loader"); options.addOption(null, "execute", true, "Execute the benchmark workload"); options.addOption(null, "runscript", true, "Run an SQL script"); options.addOption(null, "upload", true, "Upload the result"); options.addOption("v", "verbose", false, "Display Messages"); options.addOption("h", "help", false, "Print this help"); options.addOption("s", "sample", true, "Sampling window"); options.addOption("ss", false, "Verbose Sampling per Transaction"); options.addOption("o", "output", true, "Output file (default System.out)"); options.addOption("d", "directory", true, "Base directory for the result files, default is current directory"); options.addOption("t", "timestamp", false, "Each result file is prepended with a timestamp for the beginning of the experiment"); options.addOption(null, "histograms", false, "Print txn histograms"); options.addOption(null, "dialects-export", true, "Export benchmark SQL to a dialects file"); // parse the command line arguments CommandLine argsLine = parser.parse(options, args); if (argsLine.hasOption("h")) { printUsage(options); return; } else if (argsLine.hasOption("c") == false) { LOG.error("Missing Configuration file"); printUsage(options); return; } else if (argsLine.hasOption("b") == false) { LOG.fatal("Missing Benchmark Class to load"); printUsage(options); return; } // If an output directory is used, store the information String outputDirectory = ""; if (argsLine.hasOption("d")) { outputDirectory = argsLine.getOptionValue("d"); } String timestampValue = ""; if (argsLine.hasOption("t")) { timestampValue = String.valueOf(TimeUtil.getCurrentTime().getTime()) + "_"; } // ------------------------------------------------------------------- // GET PLUGIN LIST // ------------------------------------------------------------------- String plugins = argsLine.getOptionValue("b"); String[] pluginList = plugins.split(","); List<BenchmarkModule> benchList = new ArrayList<BenchmarkModule>(); // Use this list for filtering of the output List<TransactionType> activeTXTypes = new ArrayList<TransactionType>(); String configFile = argsLine.getOptionValue("c"); XMLConfiguration xmlConfig = new XMLConfiguration(configFile); xmlConfig.setExpressionEngine(new XPathExpressionEngine()); int lastTxnId = 0; for (String plugin : pluginList) { // ---------------------------------------------------------------- // WORKLOAD CONFIGURATION // ---------------------------------------------------------------- String pluginTest = ""; pluginTest = "[@bench='" + plugin + "']"; WorkloadConfiguration wrkld = new WorkloadConfiguration(); wrkld.setBenchmarkName(plugin); wrkld.setXmlConfig(xmlConfig); wrkld.setDBType(DatabaseType.get(xmlConfig.getString("dbtype"))); wrkld.setDBDriver(xmlConfig.getString("driver")); wrkld.setDBConnection(xmlConfig.getString("DBUrl")); wrkld.setDBName(xmlConfig.getString("DBName")); wrkld.setDBUsername(xmlConfig.getString("username")); wrkld.setDBPassword(xmlConfig.getString("password")); int terminals = xmlConfig.getInt("terminals[not(@bench)]", 0); terminals = xmlConfig.getInt("terminals" + pluginTest, terminals); wrkld.setTerminals(terminals); wrkld.setIsolationMode(xmlConfig.getString("isolation", "TRANSACTION_SERIALIZABLE")); wrkld.setScaleFactor(xmlConfig.getDouble("scalefactor", 1.0)); wrkld.setRecordAbortMessages(xmlConfig.getBoolean("recordabortmessages", false)); int size = xmlConfig.configurationsAt("/works/work").size(); for (int i = 1; i < size + 1; i++) { SubnodeConfiguration work = xmlConfig.configurationAt("works/work[" + i + "]"); List<String> weight_strings; // use a workaround if there multiple workloads or single // attributed workload if (pluginList.length > 1 || work.containsKey("weights[@bench]")) { weight_strings = get_weights(plugin, work); } else { weight_strings = work.getList("weights[not(@bench)]"); } int rate = 1; boolean rateLimited = true; boolean disabled = false; // can be "disabled", "unlimited" or a number String rate_string; rate_string = work.getString("rate[not(@bench)]", ""); rate_string = work.getString("rate" + pluginTest, rate_string); if (rate_string.equals(RATE_DISABLED)) { disabled = true; } else if (rate_string.equals(RATE_UNLIMITED)) { rateLimited = false; } else if (rate_string.isEmpty()) { LOG.fatal(String.format("Please specify the rate for phase %d and workload %s", i, plugin)); System.exit(-1); } else { try { rate = Integer.parseInt(rate_string); if (rate < 1) { LOG.fatal("Rate limit must be at least 1. Use unlimited or disabled values instead."); System.exit(-1); } } catch (NumberFormatException e) { LOG.fatal(String.format("Rate string must be '%s', '%s' or a number", RATE_DISABLED, RATE_UNLIMITED)); System.exit(-1); } } Phase.Arrival arrival = Phase.Arrival.REGULAR; String arrive = work.getString("@arrival", "regular"); if (arrive.toUpperCase().equals("POISSON")) arrival = Phase.Arrival.POISSON; int activeTerminals; activeTerminals = work.getInt("active_terminals[not(@bench)]", terminals); activeTerminals = work.getInt("active_terminals" + pluginTest, activeTerminals); if (activeTerminals > terminals) { System.out.println("Configuration error in work " + i + ": number of active terminals" + "" + "is bigger than the total number of terminals"); System.exit(-1); } wrkld.addWork(work.getInt("/time"), rate, weight_strings, rateLimited, disabled, activeTerminals, arrival); } // FOR int numTxnTypes = xmlConfig.configurationsAt("transactiontypes" + pluginTest + "/transactiontype") .size(); if (numTxnTypes == 0 && pluginList.length == 1) { //if it is a single workload run, <transactiontypes /> w/o attribute is used pluginTest = "[not(@bench)]"; numTxnTypes = xmlConfig.configurationsAt("transactiontypes" + pluginTest + "/transactiontype") .size(); } wrkld.setNumTxnTypes(numTxnTypes); // CHECKING INPUT PHASES int j = 0; for (Phase p : wrkld.getAllPhases()) { j++; if (p.getWeightCount() != wrkld.getNumTxnTypes()) { LOG.fatal(String.format( "Configuration files is inconsistent, phase %d contains %d weights but you defined %d transaction types", j, p.getWeightCount(), wrkld.getNumTxnTypes())); System.exit(-1); } } // FOR // Generate the dialect map wrkld.init(); assert (wrkld.getNumTxnTypes() >= 0); assert (xmlConfig != null); // ---------------------------------------------------------------- // BENCHMARK MODULE // ---------------------------------------------------------------- String classname = pluginConfig.getString("/plugin[@name='" + plugin + "']"); if (classname == null) { throw new ParseException("Plugin " + plugin + " is undefined in config/plugin.xml"); } BenchmarkModule bench = ClassUtil.newInstance(classname, new Object[] { wrkld }, new Class<?>[] { WorkloadConfiguration.class }); assert (benchList.get(0) != null); Map<String, Object> initDebug = new ListOrderedMap<String, Object>(); initDebug.put("Benchmark", String.format("%s {%s}", plugin.toUpperCase(), classname)); initDebug.put("Configuration", configFile); initDebug.put("Type", wrkld.getDBType()); initDebug.put("Driver", wrkld.getDBDriver()); initDebug.put("URL", wrkld.getDBConnection()); initDebug.put("Isolation", xmlConfig.getString("isolation", "TRANSACTION_SERIALIZABLE [DEFAULT]")); initDebug.put("Scale Factor", wrkld.getScaleFactor()); INIT_LOG.info(SINGLE_LINE + "\n\n" + StringUtil.formatMaps(initDebug)); INIT_LOG.info(SINGLE_LINE); // Load TransactionTypes List<TransactionType> ttypes = new ArrayList<TransactionType>(); // Always add an INVALID type for Carlo ttypes.add(TransactionType.INVALID); int txnIdOffset = lastTxnId; for (int i = 1; i < wrkld.getNumTxnTypes() + 1; i++) { String key = "transactiontypes" + pluginTest + "/transactiontype[" + i + "]"; String txnName = xmlConfig.getString(key + "/name"); int txnId = i + 1; if (xmlConfig.containsKey(key + "/id")) { txnId = xmlConfig.getInt(key + "/id"); } TransactionType tmpType = bench.initTransactionType(txnName, txnId + txnIdOffset); // Keep a reference for filtering activeTXTypes.add(tmpType); // Add a reference for the active TTypes in this benchmark ttypes.add(tmpType); lastTxnId = i; } // FOR TransactionTypes tt = new TransactionTypes(ttypes); wrkld.setTransTypes(tt); LOG.debug("Using the following transaction types: " + tt); benchList.add(bench); } // Export StatementDialects if (isBooleanOptionSet(argsLine, "dialects-export")) { BenchmarkModule bench = benchList.get(0); if (bench.getStatementDialects() != null) { LOG.info("Exporting StatementDialects for " + bench); String xml = bench.getStatementDialects().export(bench.getWorkloadConfiguration().getDBType(), bench.getProcedures().values()); System.out.println(xml); System.exit(0); } throw new RuntimeException("No StatementDialects is available for " + bench); } @Deprecated boolean verbose = argsLine.hasOption("v"); // Create the Benchmark's Database if (isBooleanOptionSet(argsLine, "create")) { for (BenchmarkModule benchmark : benchList) { CREATE_LOG.info("Creating new " + benchmark.getBenchmarkName().toUpperCase() + " database..."); runCreator(benchmark, verbose); CREATE_LOG.info("Finished!"); CREATE_LOG.info(SINGLE_LINE); } } else if (CREATE_LOG.isDebugEnabled()) { CREATE_LOG.debug("Skipping creating benchmark database tables"); CREATE_LOG.info(SINGLE_LINE); } // Clear the Benchmark's Database if (isBooleanOptionSet(argsLine, "clear")) { for (BenchmarkModule benchmark : benchList) { CREATE_LOG.info("Resetting " + benchmark.getBenchmarkName().toUpperCase() + " database..."); benchmark.clearDatabase(); CREATE_LOG.info("Finished!"); CREATE_LOG.info(SINGLE_LINE); } } else if (CREATE_LOG.isDebugEnabled()) { CREATE_LOG.debug("Skipping creating benchmark database tables"); CREATE_LOG.info(SINGLE_LINE); } // Execute Loader if (isBooleanOptionSet(argsLine, "load")) { for (BenchmarkModule benchmark : benchList) { LOAD_LOG.info("Loading data into " + benchmark.getBenchmarkName().toUpperCase() + " database..."); runLoader(benchmark, verbose); LOAD_LOG.info("Finished!"); LOAD_LOG.info(SINGLE_LINE); } } else if (LOAD_LOG.isDebugEnabled()) { LOAD_LOG.debug("Skipping loading benchmark database records"); LOAD_LOG.info(SINGLE_LINE); } // Execute a Script if (argsLine.hasOption("runscript")) { for (BenchmarkModule benchmark : benchList) { String script = argsLine.getOptionValue("runscript"); SCRIPT_LOG.info("Running a SQL script: " + script); runScript(benchmark, script); SCRIPT_LOG.info("Finished!"); SCRIPT_LOG.info(SINGLE_LINE); } } // Execute Workload if (isBooleanOptionSet(argsLine, "execute")) { // Bombs away! Results r = null; try { r = runWorkload(benchList, verbose); } catch (Throwable ex) { LOG.error("Unexpected error when running benchmarks.", ex); System.exit(1); } assert (r != null); PrintStream ps = System.out; PrintStream rs = System.out; ResultUploader ru = new ResultUploader(r, xmlConfig, argsLine); if (argsLine.hasOption("o")) { // Check if directory needs to be created if (outputDirectory.length() > 0) { FileUtil.makeDirIfNotExists(outputDirectory.split("/")); } // Build the complex path String baseFile = timestampValue + argsLine.getOptionValue("o"); // Increment the filename for new results String nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".res")); ps = new PrintStream(new File(nextName)); EXEC_LOG.info("Output into file: " + nextName); nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".raw")); rs = new PrintStream(new File(nextName)); EXEC_LOG.info("Output Raw data into file: " + nextName); nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".summary")); PrintStream ss = new PrintStream(new File(nextName)); EXEC_LOG.info("Output summary data into file: " + nextName); ru.writeSummary(ss); ss.close(); nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".db.cnf")); ss = new PrintStream(new File(nextName)); EXEC_LOG.info("Output db config into file: " + nextName); ru.writeDBParameters(ss); ss.close(); nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".ben.cnf")); ss = new PrintStream(new File(nextName)); EXEC_LOG.info("Output benchmark config into file: " + nextName); ru.writeBenchmarkConf(ss); ss.close(); } else if (EXEC_LOG.isDebugEnabled()) { EXEC_LOG.debug("No output file specified"); } if (argsLine.hasOption("s")) { int windowSize = Integer.parseInt(argsLine.getOptionValue("s")); EXEC_LOG.info("Grouped into Buckets of " + windowSize + " seconds"); r.writeCSV(windowSize, ps); if (isBooleanOptionSet(argsLine, "upload")) { ru.uploadResult(); } // Allow more detailed reporting by transaction to make it easier to check if (argsLine.hasOption("ss")) { for (TransactionType t : activeTXTypes) { PrintStream ts = ps; if (ts != System.out) { // Get the actual filename for the output String baseFile = timestampValue + argsLine.getOptionValue("o") + "_" + t.getName(); String prepended = outputDirectory + timestampValue; String nextName = FileUtil .getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".res")); ts = new PrintStream(new File(nextName)); r.writeCSV(windowSize, ts, t); ts.close(); } } } } else if (EXEC_LOG.isDebugEnabled()) { EXEC_LOG.warn("No bucket size specified"); } if (argsLine.hasOption("histograms")) { EXEC_LOG.info(SINGLE_LINE); EXEC_LOG.info("Completed Transactions:\n" + r.getTransactionSuccessHistogram() + "\n"); EXEC_LOG.info("Aborted Transactions:\n" + r.getTransactionAbortHistogram() + "\n"); EXEC_LOG.info("Rejected Transactions:\n" + r.getTransactionRetryHistogram()); EXEC_LOG.info("Unexpected Errors:\n" + r.getTransactionErrorHistogram()); if (r.getTransactionAbortMessageHistogram().isEmpty() == false) EXEC_LOG.info( "User Aborts:\n" + StringUtil.formatMaps(r.getTransactionAbortMessageHistogram())); } else if (EXEC_LOG.isDebugEnabled()) { EXEC_LOG.warn("No bucket size specified"); } r.writeAllCSVAbsoluteTiming(rs); ps.close(); rs.close(); } else { EXEC_LOG.info("Skipping benchmark workload execution"); } }
From source file:be.ibridge.kettle.spoon.Spoon.java
/** * This is the main procedure for Spoon. * /* w w w. jav a2 s . c o m*/ * @param a Arguments are available in the "Get System Info" step. */ public static void main(String[] a) throws KettleException { EnvUtil.environmentInit(); ArrayList args = new ArrayList(); for (int i = 0; i < a.length; i++) args.add(a[i]); Display display = new Display(); Splash splash = new Splash(display); StringBuffer optionRepname, optionUsername, optionPassword, optionJobname, optionTransname, optionFilename, optionDirname, optionLogfile, optionLoglevel; CommandLineOption options[] = new CommandLineOption[] { new CommandLineOption("rep", "Repository name", optionRepname = new StringBuffer()), new CommandLineOption("user", "Repository username", optionUsername = new StringBuffer()), new CommandLineOption("pass", "Repository password", optionPassword = new StringBuffer()), new CommandLineOption("job", "The name of the job to launch", optionJobname = new StringBuffer()), new CommandLineOption("trans", "The name of the transformation to launch", optionTransname = new StringBuffer()), new CommandLineOption("dir", "The directory (don't forget the leading /)", optionDirname = new StringBuffer()), new CommandLineOption("file", "The filename (Transformation in XML) to launch", optionFilename = new StringBuffer()), new CommandLineOption("level", "The logging level (Basic, Detailed, Debug, Rowlevel, Error, Nothing)", optionLoglevel = new StringBuffer()), new CommandLineOption("logfile", "The logging file to write to", optionLogfile = new StringBuffer()), new CommandLineOption("log", "The logging file to write to (deprecated)", optionLogfile = new StringBuffer(), false, true), }; // Parse the options... CommandLineOption.parseArguments(args, options); String kettleRepname = Const.getEnvironmentVariable("KETTLE_REPOSITORY", null); String kettleUsername = Const.getEnvironmentVariable("KETTLE_USER", null); String kettlePassword = Const.getEnvironmentVariable("KETTLE_PASSWORD", null); if (!Const.isEmpty(kettleRepname)) optionRepname = new StringBuffer(kettleRepname); if (!Const.isEmpty(kettleUsername)) optionUsername = new StringBuffer(kettleUsername); if (!Const.isEmpty(kettlePassword)) optionPassword = new StringBuffer(kettlePassword); // Before anything else, check the runtime version!!! String version = Const.JAVA_VERSION; if ("1.4".compareToIgnoreCase(version) > 0) { System.out.println("The System is running on Java version " + version); System.out.println("Unfortunately, it needs version 1.4 or higher to run."); return; } // Set default Locale: Locale.setDefault(Const.DEFAULT_LOCALE); LogWriter log; LogWriter.setConsoleAppenderDebug(); if (Const.isEmpty(optionLogfile)) { log = LogWriter.getInstance(Const.SPOON_LOG_FILE, false, LogWriter.LOG_LEVEL_BASIC); } else { log = LogWriter.getInstance(optionLogfile.toString(), true, LogWriter.LOG_LEVEL_BASIC); } if (log.getRealFilename() != null) log.logBasic(APP_NAME, Messages.getString("Spoon.Log.LoggingToFile") + log.getRealFilename());//"Logging goes to " if (!Const.isEmpty(optionLoglevel)) { log.setLogLevel(optionLoglevel.toString()); log.logBasic(APP_NAME, Messages.getString("Spoon.Log.LoggingAtLevel") + log.getLogLevelDesc());//"Logging is at level : " } /* Load the plugins etc.*/ StepLoader stloader = StepLoader.getInstance(); if (!stloader.read()) { log.logError(APP_NAME, Messages.getString("Spoon.Log.ErrorLoadingAndHaltSystem"));//Error loading steps & plugins... halting Spoon! return; } /* Load the plugins etc. we need to load jobentry*/ JobEntryLoader jeloader = JobEntryLoader.getInstance(); if (!jeloader.read()) { log.logError("Spoon", "Error loading job entries & plugins... halting Kitchen!"); return; } final Spoon spoon = new Spoon(log, display, null); staticSpoon = spoon; spoon.setDestroy(true); spoon.setArguments((String[]) args.toArray(new String[args.size()])); log.logBasic(APP_NAME, Messages.getString("Spoon.Log.MainWindowCreated"));//Main window is created. RepositoryMeta repositoryMeta = null; UserInfo userinfo = null; if (Const.isEmpty(optionRepname) && Const.isEmpty(optionFilename) && spoon.props.showRepositoriesDialogAtStartup()) { log.logBasic(APP_NAME, Messages.getString("Spoon.Log.AskingForRepository"));//"Asking for repository" int perms[] = new int[] { PermissionMeta.TYPE_PERMISSION_TRANSFORMATION, PermissionMeta.TYPE_PERMISSION_JOB }; splash.hide(); RepositoriesDialog rd = new RepositoriesDialog(spoon.disp, perms, Messages.getString("Spoon.Application.Name"));//"Spoon" if (rd.open()) { repositoryMeta = rd.getRepository(); userinfo = rd.getUser(); if (!userinfo.useTransformations()) { MessageBox mb = new MessageBox(spoon.shell, SWT.OK | SWT.ICON_ERROR); mb.setMessage(Messages.getString("Spoon.Dialog.RepositoryUserCannotWork.Message"));//"Sorry, this repository user can't work with transformations from the repository." mb.setText(Messages.getString("Spoon.Dialog.RepositoryUserCannotWork.Title"));//"Error!" mb.open(); userinfo = null; repositoryMeta = null; } } else { // Exit point: user pressed CANCEL! if (rd.isCancelled()) { splash.dispose(); spoon.quitFile(); return; } } } try { // Read kettle transformation specified on command-line? if (!Const.isEmpty(optionRepname) || !Const.isEmpty(optionFilename)) { if (!Const.isEmpty(optionRepname)) { RepositoriesMeta repsinfo = new RepositoriesMeta(log); if (repsinfo.readData()) { repositoryMeta = repsinfo.findRepository(optionRepname.toString()); if (repositoryMeta != null) { // Define and connect to the repository... spoon.rep = new Repository(log, repositoryMeta, userinfo); if (spoon.rep.connect(Messages.getString("Spoon.Application.Name")))//"Spoon" { if (Const.isEmpty(optionDirname)) optionDirname = new StringBuffer(RepositoryDirectory.DIRECTORY_SEPARATOR); // Check username, password spoon.rep.userinfo = new UserInfo(spoon.rep, optionUsername.toString(), optionPassword.toString()); if (spoon.rep.userinfo.getID() > 0) { // Options /file, /job and /trans are mutually exclusive int t = (Const.isEmpty(optionFilename) ? 0 : 1) + (Const.isEmpty(optionJobname) ? 0 : 1) + (Const.isEmpty(optionTransname) ? 0 : 1); if (t > 1) { log.logError(APP_NAME, Messages.getString("Spoon.Log.MutuallyExcusive")); // "More then one mutually exclusive options /file, /job and /trans are specified." } else if (t == 1) { if (!Const.isEmpty(optionFilename)) { spoon.openFile(optionFilename.toString(), false); } else { // OK, if we have a specified job or transformation, try to load it... // If not, keep the repository logged in. RepositoryDirectory repdir = spoon.rep.getDirectoryTree() .findDirectory(optionDirname.toString()); if (repdir == null) { log.logError(APP_NAME, Messages.getString( "Spoon.Log.UnableFindDirectory", optionDirname.toString())); //"Can't find directory ["+dirname+"] in the repository." } else { if (!Const.isEmpty(optionTransname)) { TransMeta transMeta = new TransMeta(spoon.rep, optionTransname.toString(), repdir); transMeta.setFilename(optionRepname.toString()); transMeta.clearChanged(); spoon.addSpoonGraph(transMeta); } else { // Try to load a specified job if any JobMeta jobMeta = new JobMeta(log, spoon.rep, optionJobname.toString(), repdir); jobMeta.setFilename(optionRepname.toString()); jobMeta.clearChanged(); spoon.addChefGraph(jobMeta); } } } } } else { log.logError(APP_NAME, Messages.getString("Spoon.Log.UnableVerifyUser"));//"Can't verify username and password." spoon.rep.disconnect(); spoon.rep = null; } } else { log.logError(APP_NAME, Messages.getString("Spoon.Log.UnableConnectToRepository"));//"Can't connect to the repository." } } else { log.logError(APP_NAME, Messages.getString("Spoon.Log.NoRepositoryRrovided"));//"No repository provided, can't load transformation." } } else { log.logError(APP_NAME, Messages.getString("Spoon.Log.NoRepositoriesDefined"));//"No repositories defined on this system." } } else if (!Const.isEmpty(optionFilename)) { spoon.openFile(optionFilename.toString(), false); } } else // Normal operations, nothing on the commandline... { // Can we connect to the repository? if (repositoryMeta != null && userinfo != null) { spoon.rep = new Repository(log, repositoryMeta, userinfo); if (!spoon.rep.connect(Messages.getString("Spoon.Application.Name"))) //"Spoon" { spoon.rep = null; } } if (spoon.props.openLastFile()) { log.logDetailed(APP_NAME, Messages.getString("Spoon.Log.TryingOpenLastUsedFile"));//"Trying to open the last file used." List lastUsedFiles = spoon.props.getLastUsedFiles(); if (lastUsedFiles.size() > 0) { LastUsedFile lastUsedFile = (LastUsedFile) lastUsedFiles.get(0); spoon.loadLastUsedFile(lastUsedFile, repositoryMeta); } } } } catch (KettleException ke) { log.logError(APP_NAME, Messages.getString("Spoon.Log.ErrorOccurred") + Const.CR + ke.getMessage());//"An error occurred: " spoon.rep = null; // ke.printStackTrace(); } spoon.open(); splash.dispose(); try { while (!spoon.isDisposed()) { if (!spoon.readAndDispatch()) spoon.sleep(); } } catch (Throwable e) { log.logError(APP_NAME, Messages.getString("Spoon.Log.UnexpectedErrorOccurred") + Const.CR + e.getMessage());//"An unexpected error occurred in Spoon: probable cause: please close all windows before stopping Spoon! " e.printStackTrace(); } spoon.dispose(); log.logBasic(APP_NAME, APP_NAME + " " + Messages.getString("Spoon.Log.AppHasEnded"));//" has ended." // Close the logfile log.close(); // Kill all remaining things in this VM! System.exit(0); }
From source file:Main.java
public static <E> E last(List<E> list) { return list.get(list.size() - 1); }
From source file:Main.java
public static Object last(List<Object> elements) { return elements.get(elements.size()); }