List of usage examples for java.util Calendar getTime
public final Date getTime()
Date object representing this Calendar's time value (millisecond offset from the Epoch"). From source file:com.microsoft.windowsazure.services.media.samples.contentprotection.playreadywidevine.Program.java
public static void main(String[] args) { try {/*from w w w. ja va2s . c o m*/ // Set up the MediaContract object to call into the Media Services account Configuration configuration = MediaConfiguration.configureWithOAuthAuthentication(mediaServiceUri, oAuthUri, clientId, clientSecret, scope); mediaService = MediaService.create(configuration); System.out.println("Azure SDK for Java - PlayReady & Widevine Dynamic Encryption Sample"); // Upload a local file to a media asset. AssetInfo uploadAsset = uploadFileAndCreateAsset("Azure-Video.wmv"); System.out.println("Uploaded Asset Id: " + uploadAsset.getId()); // Transform the asset. AssetInfo encodedAsset = encode(uploadAsset); System.out.println("Encoded Asset Id: " + encodedAsset.getId()); // Create the ContentKey ContentKeyInfo contentKeyInfo = createCommonTypeContentKey(encodedAsset); System.out.println("Common Encryption Content Key: " + contentKeyInfo.getId()); // Create the ContentKeyAuthorizationPolicy String tokenTemplateString = null; if (tokenRestriction) { tokenTemplateString = addTokenRestrictedAuthorizationPolicy(contentKeyInfo, tokenType); } else { addOpenAuthorizationPolicy(contentKeyInfo); } // Create the AssetDeliveryPolicy createAssetDeliveryPolicy(encodedAsset, contentKeyInfo); if (tokenTemplateString != null) { // Deserializes a string containing the XML representation of the TokenRestrictionTemplate TokenRestrictionTemplate tokenTemplate = TokenRestrictionTemplateSerializer .deserialize(tokenTemplateString); // Generate a test token based on the the data in the given // TokenRestrictionTemplate. // Note: You need to pass the key id Guid because we specified // TokenClaim.ContentKeyIdentifierClaim in during the creation // of TokenRestrictionTemplate. UUID rawKey = UUID.fromString(contentKeyInfo.getId().substring("nb:kid:UUID:".length())); // Token expiration: 1-year Calendar date = Calendar.getInstance(); date.setTime(new Date()); date.add(Calendar.YEAR, 1); // Generate token String testToken = TokenRestrictionTemplateSerializer.generateTestToken(tokenTemplate, null, rawKey, date.getTime(), null); System.out.println(tokenTemplate.getTokenType().toString() + " Test Token: Bearer " + testToken); } // Create the Streaming Origin Locator String url = getStreamingOriginLocator(encodedAsset); System.out.println("Origin Locator Url: " + url); System.out.println("Sample completed!"); } catch (ServiceException se) { System.out.println("ServiceException encountered."); System.out.println(se.toString()); } catch (Exception e) { System.out.println("Exception encountered."); System.out.println(e.toString()); } }
From source file:search.java
public static void main(String argv[]) { int optind;/*from w w w. j ava 2 s . com*/ String subject = null; String from = null; boolean or = false; boolean today = false; int size = -1; for (optind = 0; optind < argv.length; optind++) { if (argv[optind].equals("-T")) { protocol = argv[++optind]; } else if (argv[optind].equals("-H")) { host = argv[++optind]; } else if (argv[optind].equals("-U")) { user = argv[++optind]; } else if (argv[optind].equals("-P")) { password = argv[++optind]; } else if (argv[optind].equals("-or")) { or = true; } else if (argv[optind].equals("-D")) { debug = true; } else if (argv[optind].equals("-f")) { mbox = argv[++optind]; } else if (argv[optind].equals("-L")) { url = argv[++optind]; } else if (argv[optind].equals("-subject")) { subject = argv[++optind]; } else if (argv[optind].equals("-from")) { from = argv[++optind]; } else if (argv[optind].equals("-today")) { today = true; } else if (argv[optind].equals("-size")) { size = Integer.parseInt(argv[++optind]); } else if (argv[optind].equals("--")) { optind++; break; } else if (argv[optind].startsWith("-")) { System.out.println("Usage: search [-D] [-L url] [-T protocol] [-H host] " + "[-U user] [-P password] [-f mailbox] " + "[-subject subject] [-from from] [-or] [-today]"); System.exit(1); } else { break; } } try { if ((subject == null) && (from == null) && !today && size < 0) { System.out.println("Specify either -subject, -from, -today, or -size"); System.exit(1); } // Get a Properties object Properties props = System.getProperties(); // Get a Session object Session session = Session.getInstance(props, null); session.setDebug(debug); // Get a Store object Store store = null; if (url != null) { URLName urln = new URLName(url); store = session.getStore(urln); store.connect(); } else { if (protocol != null) store = session.getStore(protocol); else store = session.getStore(); // Connect if (host != null || user != null || password != null) store.connect(host, user, password); else store.connect(); } // Open the Folder Folder folder = store.getDefaultFolder(); if (folder == null) { System.out.println("Cant find default namespace"); System.exit(1); } folder = folder.getFolder(mbox); if (folder == null) { System.out.println("Invalid folder"); System.exit(1); } folder.open(Folder.READ_ONLY); SearchTerm term = null; if (subject != null) term = new SubjectTerm(subject); if (from != null) { FromStringTerm fromTerm = new FromStringTerm(from); if (term != null) { if (or) term = new OrTerm(term, fromTerm); else term = new AndTerm(term, fromTerm); } else term = fromTerm; } if (today) { Calendar c = Calendar.getInstance(); c.set(Calendar.HOUR, 0); c.set(Calendar.MINUTE, 0); c.set(Calendar.SECOND, 0); c.set(Calendar.MILLISECOND, 0); c.set(Calendar.AM_PM, Calendar.AM); ReceivedDateTerm startDateTerm = new ReceivedDateTerm(ComparisonTerm.GE, c.getTime()); c.add(Calendar.DATE, 1); // next day ReceivedDateTerm endDateTerm = new ReceivedDateTerm(ComparisonTerm.LT, c.getTime()); SearchTerm dateTerm = new AndTerm(startDateTerm, endDateTerm); if (term != null) { if (or) term = new OrTerm(term, dateTerm); else term = new AndTerm(term, dateTerm); } else term = dateTerm; } if (size >= 0) { SizeTerm sizeTerm = new SizeTerm(ComparisonTerm.GT, size); if (term != null) { if (or) term = new OrTerm(term, sizeTerm); else term = new AndTerm(term, sizeTerm); } else term = sizeTerm; } Message[] msgs = folder.search(term); System.out.println("FOUND " + msgs.length + " MESSAGES"); if (msgs.length == 0) // no match System.exit(1); // Use a suitable FetchProfile FetchProfile fp = new FetchProfile(); fp.add(FetchProfile.Item.ENVELOPE); folder.fetch(msgs, fp); for (int i = 0; i < msgs.length; i++) { System.out.println("--------------------------"); System.out.println("MESSAGE #" + (i + 1) + ":"); dumpPart(msgs[i]); } folder.close(false); store.close(); } catch (Exception ex) { System.out.println("Oops, got exception! " + ex.getMessage()); ex.printStackTrace(); } System.exit(1); }
From source file:de.codesourcery.eve.skills.ui.utils.CalendarWidget.java
public static void main(String[] args) throws Exception { final SimpleDateFormat DF = new SimpleDateFormat("dd.MM"); final Calendar specialDate = Calendar.getInstance(); specialDate.add(Calendar.DAY_OF_MONTH, 5); final AtomicBoolean doStuff = new AtomicBoolean(false); final ICalendarRenderer renderer = new ICalendarRenderer() { @Override/*from w w w .j ava 2 s .co m*/ public String getDateLabel(Date date) { return DF.format(date); } @Override public String getText(Date date) { if (DateUtils.isSameDay(date, specialDate.getTime()) && doStuff.get()) { return "SPECIAL !!!"; } return "some\nmultiline\ntext"; } @Override public String getToolTip(Date date) { return getText(date); } @Override public Color getTextColor(Date date) { return Color.RED; } }; final JFrame frame = new JFrame(); frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); frame.getContentPane().setLayout(new GridBagLayout()); final CalendarWidget widget = new CalendarWidget(new Date(), renderer); widget.addSelectionListener(new ISelectionListener<Date>() { @Override public void selectionChanged(Date selected) { System.out.println("Selected date > " + selected); } }); frame.getContentPane().add(widget, new ConstraintsBuilder().end()); frame.pack(); frame.setVisible(true); java.lang.Thread.sleep(2 * 1000); doStuff.set(true); widget.refreshDateLabel(specialDate.getTime()); }
From source file:com.glaf.core.util.DateUtils.java
public static void main(String[] args) { System.out.println(DateUtils.getDate(new Date())); System.out.println(DateUtils.getDateTime(new Date())); System.out.println(DateUtils.getYearMonthDay(new Date())); Calendar calendar = Calendar.getInstance(); calendar.setTime(new Date()); int year = calendar.get(Calendar.YEAR); int month = calendar.get(Calendar.MONTH) + 1; int day = calendar.get(Calendar.DAY_OF_MONTH); calendar.set(year, month, day - 7);// ww w. j a v a2s .com Date dateBefore = calendar.getTime(); System.out.println("day:" + year + "-" + month + "-" + day); System.out.println(DateUtils.getDate(dateBefore)); calendar.set(year - 1, month, day); dateBefore = calendar.getTime(); System.out.println(DateUtils.getDate(dateBefore)); calendar.set(year, month - 15, day); dateBefore = calendar.getTime(); System.out.println(">>" + DateUtils.getDate(dateBefore)); calendar.set(year, month, day - 365); dateBefore = calendar.getTime(); System.out.println(">>>>" + DateUtils.getDate(dateBefore)); String[] parsePatterns = { DateUtils.DATE_TIME_PATTERN, DateUtils.DATE_PATTERN }; System.out.println(DateUtils.parseDate("2009-12-25", parsePatterns)); System.out.println(DateUtils.parseDate("2009-12-25 12:00:05", parsePatterns)); System.out.println(DateUtils.parseDate("2009-12-25 23:59:59", parsePatterns)); System.out.println(DateUtils.parseDate("2009-12-25 00:00:00", parsePatterns)); System.out.println(Locale.getDefault()); System.out.println(DateUtils.toDate("2009")); System.out.println(DateUtils.toDate("2009-12")); System.out.println(DateUtils.toDate("2009-12-25")); System.out.println(DateUtils.toDate("2009-12-25 13")); System.out.println(DateUtils.toDate("2009-12-25 10:45")); System.out.println(DateUtils.toDate("2009-12-25 22:45:50")); System.out.println(dateDiff(DateUtils.toDate("2013-10-25"), DateUtils.toDate("2013-10-29"))); Date toDate = DateUtils.getDateAfter(DateUtils.toDate("2013-03-21"), 60); System.out.println(getDateTime(toDate)); long daysDiff = DateUtils.dateDiff(new Date(), toDate); System.out.println(daysDiff); System.out.println(DateUtils.getYearDays(2015)); }
From source file:io.druid.examples.rabbitmq.RabbitMQProducerMain.java
public static void main(String[] args) throws Exception { // We use a List to keep track of option insertion order. See below. final List<Option> optionList = new ArrayList<Option>(); optionList.add(OptionBuilder.withLongOpt("help").withDescription("display this help message").create("h")); optionList.add(OptionBuilder.withLongOpt("hostname").hasArg() .withDescription("the hostname of the AMQP broker [defaults to AMQP library default]").create("b")); optionList.add(OptionBuilder.withLongOpt("port").hasArg() .withDescription("the port of the AMQP broker [defaults to AMQP library default]").create("n")); optionList.add(OptionBuilder.withLongOpt("username").hasArg() .withDescription("username to connect to the AMQP broker [defaults to AMQP library default]") .create("u")); optionList.add(OptionBuilder.withLongOpt("password").hasArg() .withDescription("password to connect to the AMQP broker [defaults to AMQP library default]") .create("p")); optionList.add(OptionBuilder.withLongOpt("vhost").hasArg() .withDescription("name of virtual host on the AMQP broker [defaults to AMQP library default]") .create("v")); optionList.add(OptionBuilder.withLongOpt("exchange").isRequired().hasArg() .withDescription("name of the AMQP exchange [required - no default]").create("e")); optionList.add(OptionBuilder.withLongOpt("key").hasArg() .withDescription("the routing key to use when sending messages [default: 'default.routing.key']") .create("k")); optionList.add(OptionBuilder.withLongOpt("type").hasArg() .withDescription("the type of exchange to create [default: 'topic']").create("t")); optionList.add(OptionBuilder.withLongOpt("durable") .withDescription("if set, a durable exchange will be declared [default: not set]").create("d")); optionList.add(OptionBuilder.withLongOpt("autodelete") .withDescription("if set, an auto-delete exchange will be declared [default: not set]") .create("a")); optionList.add(OptionBuilder.withLongOpt("single") .withDescription("if set, only a single message will be sent [default: not set]").create("s")); optionList.add(OptionBuilder.withLongOpt("start").hasArg() .withDescription("time to use to start sending messages from [default: 2010-01-01T00:00:00]") .create());// ww w. j a v a 2s . c o m optionList.add(OptionBuilder.withLongOpt("stop").hasArg().withDescription( "time to use to send messages until (format: '2013-07-18T23:45:59') [default: current time]") .create()); optionList.add(OptionBuilder.withLongOpt("interval").hasArg() .withDescription("the interval to add to the timestamp between messages in seconds [default: 10]") .create()); optionList.add(OptionBuilder.withLongOpt("delay").hasArg() .withDescription("the delay between sending messages in milliseconds [default: 100]").create()); // An extremely silly hack to maintain the above order in the help formatting. HelpFormatter formatter = new HelpFormatter(); // Add a comparator to the HelpFormatter using the ArrayList above to sort by insertion order. formatter.setOptionComparator(new Comparator() { @Override public int compare(Object o1, Object o2) { // I know this isn't fast, but who cares! The list is short. return optionList.indexOf(o1) - optionList.indexOf(o2); } }); // Now we can add all the options to an Options instance. This is dumb! Options options = new Options(); for (Option option : optionList) { options.addOption(option); } CommandLine cmd = null; try { cmd = new BasicParser().parse(options, args); } catch (ParseException e) { formatter.printHelp("RabbitMQProducerMain", e.getMessage(), options, null); System.exit(1); } if (cmd.hasOption("h")) { formatter.printHelp("RabbitMQProducerMain", options); System.exit(2); } ConnectionFactory factory = new ConnectionFactory(); if (cmd.hasOption("b")) { factory.setHost(cmd.getOptionValue("b")); } if (cmd.hasOption("u")) { factory.setUsername(cmd.getOptionValue("u")); } if (cmd.hasOption("p")) { factory.setPassword(cmd.getOptionValue("p")); } if (cmd.hasOption("v")) { factory.setVirtualHost(cmd.getOptionValue("v")); } if (cmd.hasOption("n")) { factory.setPort(Integer.parseInt(cmd.getOptionValue("n"))); } String exchange = cmd.getOptionValue("e"); String routingKey = "default.routing.key"; if (cmd.hasOption("k")) { routingKey = cmd.getOptionValue("k"); } boolean durable = cmd.hasOption("d"); boolean autoDelete = cmd.hasOption("a"); String type = cmd.getOptionValue("t", "topic"); boolean single = cmd.hasOption("single"); int interval = Integer.parseInt(cmd.getOptionValue("interval", "10")); int delay = Integer.parseInt(cmd.getOptionValue("delay", "100")); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); Date stop = sdf.parse(cmd.getOptionValue("stop", sdf.format(new Date()))); Random r = new Random(); Calendar timer = Calendar.getInstance(); timer.setTime(sdf.parse(cmd.getOptionValue("start", "2010-01-01T00:00:00"))); String msg_template = "{\"utcdt\": \"%s\", \"wp\": %d, \"gender\": \"%s\", \"age\": %d}"; Connection connection = factory.newConnection(); Channel channel = connection.createChannel(); channel.exchangeDeclare(exchange, type, durable, autoDelete, null); do { int wp = (10 + r.nextInt(90)) * 100; String gender = r.nextBoolean() ? "male" : "female"; int age = 20 + r.nextInt(70); String line = String.format(msg_template, sdf.format(timer.getTime()), wp, gender, age); channel.basicPublish(exchange, routingKey, null, line.getBytes()); System.out.println("Sent message: " + line); timer.add(Calendar.SECOND, interval); Thread.sleep(delay); } while ((!single && stop.after(timer.getTime()))); connection.close(); }
From source file:com.krawler.common.util.StringUtil.java
public static void main(String args[]) { // dump("this is a test"); // dump("this is 'a nother' test"); // dump("this is\\ test"); // dump("first Roland last 'Schemers' full 'Roland Schemers'"); // dump("multi 'Roland\\nSchemers'"); // dump("a"); // dump(""); // dump("\\ \\ "); // dump("backslash \\\\"); // dump("backslash \\f"); // dump("a b"); // TimeZone.setDefault(TimeZone.getTimeZone("GMT")); //// getUserTimeZoneDate("CST"); ///*ww w . j a v a2s . co m*/ // String olddate = "Tue Sep 29 06:33:19 GMT 2010"; // // System.out.println(issameDatesExTime(new Date(),olddate)); Calendar cal = Calendar.getInstance(); Calendar cal1 = new GregorianCalendar(TimeZone.getTimeZone("GMT" + "+5:30")); try { Date dt = new Date(); cal1.setTimeInMillis(dt.getTime()); cal.set(Calendar.YEAR, cal1.get(Calendar.YEAR)); cal.set(Calendar.MONTH, cal1.get(Calendar.MONTH)); cal.set(Calendar.DAY_OF_MONTH, cal1.get(Calendar.DAY_OF_MONTH)); cal.set(Calendar.HOUR_OF_DAY, cal1.get(Calendar.HOUR_OF_DAY)); cal.set(Calendar.MINUTE, cal1.get(Calendar.MINUTE)); cal.set(Calendar.SECOND, cal1.get(Calendar.SECOND)); cal.set(Calendar.MILLISECOND, cal1.get(Calendar.MILLISECOND)); System.out.println("IST timezone dt: " + cal.getTime() + " Date is: " + dt); } catch (Exception ex) { System.out.println(ex); } finally { //return cal.getTime(); } /*Date date = new Date(); System.out.println(date); SimpleDateFormat sdf = new SimpleDateFormat("EEE MMM dd HH:mm:ss z yyyy"); //SimpleDateFormat sdf = new SimpleDateFormat("MM-dd-yyyy HH:mm:ss"); sdf.setTimeZone(TimeZone.getTimeZone("CST")); // replace timezine with user timezone String datestr = sdf.format(date); System.out.println(datestr); String[] datearr = datestr.split(" "); String newstr = datestr.replace(datearr[4], " "); Date newdate = new Date(newstr); System.out.println(newdate);*/ // datearr[datearr.length-1]="GMT"; // String newstr = ""; // for (int i = 0; i < datearr.length; i++) { // if (i != 4) { // newstr += datearr[i] + " "; // }else{ // newstr += "GMT "; // } // } // newstr = newstr.substring(0, newstr.length() - 1); }
From source file:com.adobe.aem.demo.communities.Loader.java
public static void main(String[] args) { String hostname = null;// ww w . j a v a 2 s .co m String port = null; String altport = null; String csvfile = null; String location = null; String language = "en"; String analytics = null; String adminPassword = "admin"; String[] url = new String[10]; // Handling 10 levels maximum for nested comments boolean reset = false; boolean configure = false; int urlLevel = 0; int row = 0; HashMap<String, ArrayList<String>> learningpaths = new HashMap<String, ArrayList<String>>(); // Command line options for this tool Options options = new Options(); options.addOption("h", true, "Hostname"); options.addOption("p", true, "Port"); options.addOption("a", true, "Alternate Port"); options.addOption("f", true, "CSV file"); options.addOption("r", false, "Reset"); options.addOption("u", true, "Admin Password"); options.addOption("c", false, "Configure"); options.addOption("s", true, "Analytics Endpoint"); options.addOption("t", false, "Analytics Tracking"); CommandLineParser parser = new BasicParser(); try { CommandLine cmd = parser.parse(options, args); if (cmd.hasOption("h")) { hostname = cmd.getOptionValue("h"); } if (cmd.hasOption("p")) { port = cmd.getOptionValue("p"); } if (cmd.hasOption("a")) { altport = cmd.getOptionValue("a"); } if (cmd.hasOption("f")) { csvfile = cmd.getOptionValue("f"); } if (cmd.hasOption("u")) { adminPassword = cmd.getOptionValue("u"); } if (cmd.hasOption("t")) { if (cmd.hasOption("s")) { analytics = cmd.getOptionValue("s"); } } if (cmd.hasOption("r")) { reset = true; } if (cmd.hasOption("c")) { configure = true; } if (csvfile == null || port == null || hostname == null) { System.out.println( "Request parameters: -h hostname -p port -a alternateport -u adminPassword -f path_to_CSV_file -r (true|false, delete content before import) -c (true|false, post additional properties)"); System.exit(-1); } } catch (ParseException ex) { logger.error(ex.getMessage()); } String componentType = null; try { logger.debug("AEM Demo Loader: Processing file " + csvfile); // Reading the CSV file, line by line Reader in = new FileReader(csvfile); Iterable<CSVRecord> records = CSVFormat.EXCEL.parse(in); for (CSVRecord record : records) { row = row + 1; logger.info("Row: " + row + ", new record: " + record.get(0)); // Let's see if we deal with a comment if (record.get(0).startsWith("#")) { // We can ignore the comment line and move on continue; } // Let's see if we need to terminate this process if (record.get(0).equals(KILL)) { System.exit(1); } // Let's see if we need to create a new Community site if (record.get(0).equals(SITE)) { // Building the form entity to be posted MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.addTextBody(":operation", "social:createSite", ContentType.create("text/plain", MIME.UTF8_CHARSET)); builder.addTextBody("_charset_", "UTF-8", ContentType.create("text/plain", MIME.UTF8_CHARSET)); String urlName = null; for (int i = 2; i < record.size() - 1; i = i + 2) { if (record.get(i) != null && record.get(i + 1) != null && record.get(i).length() > 0) { String name = record.get(i).trim(); String value = record.get(i + 1).trim(); if (value.equals("TRUE")) { value = "true"; } if (value.equals("FALSE")) { value = "false"; } if (name.equals("urlName")) { urlName = value; } if (name.equals(LANGUAGE)) { language = value; } if (name.equals(BANNER)) { File attachment = new File( csvfile.substring(0, csvfile.indexOf(".csv")) + File.separator + value); builder.addBinaryBody(BANNER, attachment, ContentType.MULTIPART_FORM_DATA, attachment.getName()); } else if (name.equals(THUMBNAIL)) { File attachment = new File( csvfile.substring(0, csvfile.indexOf(".csv")) + File.separator + value); builder.addBinaryBody(THUMBNAIL, attachment, ContentType.MULTIPART_FORM_DATA, attachment.getName()); } else { builder.addTextBody(name, value, ContentType.create("text/plain", MIME.UTF8_CHARSET)); } } } // Site creation String siteId = doPost(hostname, port, "/content.social.json", "admin", adminPassword, builder.build(), "response/siteId"); // Site publishing, if there's a publish instance to publish to if (!port.equals(altport)) { List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(); nameValuePairs.add(new BasicNameValuePair("id", "nobot")); nameValuePairs.add(new BasicNameValuePair(":operation", "social:publishSite")); nameValuePairs .add(new BasicNameValuePair("path", "/content/sites/" + urlName + "/" + language)); doPost(hostname, port, "/communities/sites.html", "admin", adminPassword, new UrlEncodedFormEntity(nameValuePairs), null); // Wait for site to be available on Publish doWait(hostname, altport, "admin", adminPassword, (siteId != null ? siteId : urlName) + "-groupadministrators"); } continue; } // Let's see if we need to create a new Tag if (record.get(0).equals(TAG)) { // Building the form entity to be posted MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.addTextBody("_charset_", "UTF-8", ContentType.create("text/plain", MIME.UTF8_CHARSET)); for (int i = 1; i < record.size() - 1; i = i + 2) { if (record.get(i) != null && record.get(i + 1) != null && record.get(i).length() > 0 && record.get(i + 1).length() > 0) { String name = record.get(i).trim(); String value = record.get(i + 1).trim(); builder.addTextBody(name, value, ContentType.create("text/plain", MIME.UTF8_CHARSET)); } } // Tag creation doPost(hostname, port, "/bin/tagcommand", "admin", adminPassword, builder.build(), null); continue; } // Let's see if we need to create a new Community site template, and if we can do it (script run against author instance) if (record.get(0).equals(SITETEMPLATE)) { // Building the form entity to be posted MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.addTextBody(":operation", "social:createSiteTemplate", ContentType.create("text/plain", MIME.UTF8_CHARSET)); builder.addTextBody("_charset_", "UTF-8", ContentType.create("text/plain", MIME.UTF8_CHARSET)); for (int i = 2; i < record.size() - 1; i = i + 2) { if (record.get(i) != null && record.get(i + 1) != null && record.get(i).length() > 0) { String name = record.get(i).trim(); String value = record.get(i + 1).trim(); builder.addTextBody(name, value, ContentType.create("text/plain", MIME.UTF8_CHARSET)); } } // Site template creation doPost(hostname, port, "/content.social.json", "admin", adminPassword, builder.build(), null); continue; } // Let's see if we need to create a new Community group if (record.get(0).equals(GROUP)) { // Building the form entity to be posted MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.addTextBody(":operation", "social:createCommunityGroup", ContentType.create("text/plain", MIME.UTF8_CHARSET)); builder.addTextBody("_charset_", "UTF-8", ContentType.create("text/plain", MIME.UTF8_CHARSET)); for (int i = 3; i < record.size() - 1; i = i + 2) { if (record.get(i) != null && record.get(i + 1) != null && record.get(i).length() > 0) { String name = record.get(i).trim(); String value = record.get(i + 1).trim(); if (value.equals("TRUE")) { value = "true"; } if (value.equals("FALSE")) { value = "false"; } if (name.equals(IMAGE)) { File attachment = new File( csvfile.substring(0, csvfile.indexOf(".csv")) + File.separator + value); builder.addBinaryBody(IMAGE, attachment, ContentType.MULTIPART_FORM_DATA, attachment.getName()); } else { builder.addTextBody(name, value, ContentType.create("text/plain", MIME.UTF8_CHARSET)); } } } // Group creation String memberGroupId = doPost(hostname, port, record.get(1), getUserName(record.get(2)), getPassword(record.get(2), adminPassword), builder.build(), "response/memberGroupId"); // Wait for group to be available on Publish, if available logger.debug("Waiting for completion of Community Group creation"); doWait(hostname, port, "admin", adminPassword, memberGroupId); continue; } // Let's see if it's simple Sling Delete request if (record.get(0).equals(SLINGDELETE)) { doDelete(hostname, port, record.get(1), "admin", adminPassword); continue; } // Let's see if we need to add users to an AEM Group if ((record.get(0).equals(GROUPMEMBERS) || record.get(0).equals(SITEMEMBERS)) && record.get(GROUP_INDEX_NAME) != null) { // Checking if we have a member group for this site String groupName = record.get(GROUP_INDEX_NAME); if (record.get(0).equals(SITEMEMBERS)) { // Let's fetch the siteId for this Community Site Url String siteConfig = doGet(hostname, port, groupName, "admin", adminPassword, null); try { String siteId = new JSONObject(siteConfig).getString("siteId"); if (siteId != null) groupName = "community-" + siteId + "-members"; logger.debug("Member group name is " + groupName); } catch (Exception e) { logger.error(e.getMessage()); } } // Pause until the group can found doWait(hostname, port, "admin", adminPassword, groupName); List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(); nameValuePairs.add(new BasicNameValuePair("filter", "[{\"operation\":\"like\",\"rep:principalName\":\"" + groupName + "\"}]")); nameValuePairs.add(new BasicNameValuePair("type", "groups")); String groupList = doGet(hostname, port, "/libs/social/console/content/content/userlist.social.0.10.json", "admin", adminPassword, nameValuePairs); logger.debug("List of groups" + groupList); if (groupList.indexOf(groupName) > 0) { logger.debug("Group was found on " + port); try { JSONArray jsonArray = new JSONObject(groupList).getJSONArray("items"); if (jsonArray.length() == 1) { JSONObject jsonObject = jsonArray.getJSONObject(0); String groupPath = jsonObject.getString("path"); logger.debug("Group path is " + groupPath); // Constructing a multi-part POST for group membership MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); List<NameValuePair> groupNameValuePairs = buildNVP(record, 2); for (NameValuePair nameValuePair : groupNameValuePairs) { builder.addTextBody(nameValuePair.getName(), nameValuePair.getValue(), ContentType.create("text/plain", MIME.UTF8_CHARSET)); } // Adding the list of group members doPost(hostname, port, groupPath + ".rw.userprops.html", "admin", adminPassword, builder.build(), null); } else { logger.info("We have more than one match for a group with this name!"); } } catch (Exception e) { logger.error(e.getMessage()); } } continue; } // Let's see if it's user related if (record.get(0).equals(USERS)) { //First we need to get the path to the user node String json = doGet(hostname, port, "/libs/granite/security/currentuser.json", getUserName(record.get(1)), getPassword(record.get(1), adminPassword), null); if (json != null) { try { // Fetching the home property String home = new JSONObject(json).getString("home"); if (record.get(2).equals(PREFERENCES)) { home = home + "/preferences"; } else { home = home + "/profile"; } logger.debug(home); // Now we can post all the preferences or the profile List<NameValuePair> nameValuePairs = buildNVP(record, 3); doPost(hostname, port, home, "admin", adminPassword, new UrlEncodedFormEntity(nameValuePairs), null); } catch (Exception e) { logger.error(e.getMessage()); } } continue; } // Let's see if we deal with a new block of content or just a new entry if (record.get(0).equals(CALENDAR) || record.get(0).equals(SLINGPOST) || record.get(0).equals(RATINGS) || record.get(0).equals(BLOG) || record.get(0).equals(JOURNAL) || record.get(0).equals(COMMENTS) || record.get(0).equals(REVIEWS) || record.get(0).equals(FILES) || record.get(0).equals(SUMMARY) || record.get(0).equals(ACTIVITIES) || record.get(0).equals(JOIN) || record.get(0).equals(FOLLOW) || record.get(0).equals(MESSAGE) || record.get(0).equals(ASSET) || record.get(0).equals(AVATAR) || record.get(0).equals(RESOURCE) || record.get(0).equals(LEARNING) || record.get(0).equals(QNA) || record.get(0).equals(FORUM)) { // New block of content, we need to reset the processing to first Level componentType = record.get(0); url[0] = record.get(1); urlLevel = 0; if (!componentType.equals(SLINGPOST) && reset) { int pos = record.get(1).indexOf("/jcr:content"); if (pos > 0) doDelete(hostname, port, "/content/usergenerated" + record.get(1).substring(0, pos), "admin", adminPassword); } // If the Configure command line flag is set, we try to configure the component with all options enabled if (componentType.equals(SLINGPOST) || configure) { String configurePath = getConfigurePath(record.get(1)); List<NameValuePair> nameValuePairs = buildNVP(record, 2); if (nameValuePairs.size() > 2) // Only do this when really have configuration settings doPost(hostname, port, configurePath, "admin", adminPassword, new UrlEncodedFormEntity(nameValuePairs), null); } // We're done with this line, moving on to the next line in the CSV file continue; } // Let's see if we need to indent the list, if it's a reply or a reply to a reply if (record.get(1).length() != 1) continue; // We need a valid level indicator if (Integer.parseInt(record.get(1)) > urlLevel) { url[++urlLevel] = location; logger.debug("Incremented urlLevel to: " + urlLevel + ", with a new location:" + location); } else if (Integer.parseInt(record.get(1)) < urlLevel) { urlLevel = Integer.parseInt(record.get(1)); logger.debug("Decremented urlLevel to: " + urlLevel); } // Get the credentials or fall back to password String password = getPassword(record.get(0), adminPassword); String userName = getUserName(record.get(0)); // Adding the generic properties for all POST requests List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(); if (!componentType.equals(RESOURCE)) nameValuePairs.add(new BasicNameValuePair("id", "nobot")); nameValuePairs.add(new BasicNameValuePair("_charset_", "UTF-8")); // Setting some specific fields depending on the content type if (componentType.equals(COMMENTS)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createComment")); nameValuePairs.add(new BasicNameValuePair("message", record.get(2))); } // Creates a forum post (or reply) if (componentType.equals(FORUM)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createForumPost")); nameValuePairs.add(new BasicNameValuePair("subject", record.get(2))); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); } // Follows a user (followedId) for the user posting the request if (componentType.equals(FOLLOW)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:follow")); nameValuePairs.add(new BasicNameValuePair("userId", "/social/authors/" + userName)); nameValuePairs.add(new BasicNameValuePair("followedId", "/social/authors/" + record.get(2))); } // Uploading Avatar picture if (componentType.equals(AVATAR)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:changeAvatar")); } // Joins a user (posting the request) to a Community Group (path) if (componentType.equals(JOIN)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:joinCommunityGroup")); int pos = url[0].indexOf("/configuration.social.json"); if (pos > 0) nameValuePairs.add(new BasicNameValuePair("path", url[0].substring(0, pos) + ".html")); else continue; // Invalid record } // Creates a new private message if (componentType.equals(MESSAGE)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createMessage")); nameValuePairs.add(new BasicNameValuePair("sendMail", "Sending...")); nameValuePairs.add(new BasicNameValuePair("content", record.get(4))); nameValuePairs.add(new BasicNameValuePair("subject", record.get(3))); nameValuePairs.add(new BasicNameValuePair("serviceSelector", "/bin/community")); nameValuePairs.add(new BasicNameValuePair("to", "/social/authors/" + record.get(2))); nameValuePairs.add(new BasicNameValuePair("userId", "/social/authors/" + record.get(2))); nameValuePairs.add(new BasicNameValuePair(":redirect", "//messaging.html")); nameValuePairs.add(new BasicNameValuePair(":formid", "generic_form")); nameValuePairs.add(new BasicNameValuePair(":formstart", "/content/sites/communities/messaging/compose/jcr:content/content/primary/start")); } // Creates a file or a folder if (componentType.equals(FILES)) { // Top level is always assumed to be a folder, second level files, and third and subsequent levels comments on files if (urlLevel == 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createFileLibraryFolder")); nameValuePairs.add(new BasicNameValuePair("name", record.get(2))); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); } else if (urlLevel == 1) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createComment")); } } // Creates a question, a reply or mark a reply as the best answer if (componentType.equals(QNA)) { if (urlLevel == 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createQnaPost")); nameValuePairs.add(new BasicNameValuePair("subject", record.get(2))); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); } else if (urlLevel == 1) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createQnaPost")); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); } else if (urlLevel == 2) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:selectAnswer")); } } // Creates an article or a comment if (componentType.equals(JOURNAL) || componentType.equals(BLOG)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createJournalComment")); nameValuePairs.add(new BasicNameValuePair("subject", record.get(2))); StringBuffer message = new StringBuffer("<p>" + record.get(3) + "</p>"); //We might have more paragraphs to add to the blog or journal article for (int i = 6; i < record.size(); i++) { if (record.get(i).length() > 0) { message.append("<p>" + record.get(i) + "</p>"); } } //We might have some tags to add to the blog or journal article if (record.get(5).length() > 0) { nameValuePairs.add(new BasicNameValuePair("tags", record.get(5))); } nameValuePairs.add(new BasicNameValuePair("message", message.toString())); } // Creates a review or a comment if (componentType.equals(REVIEWS)) { nameValuePairs.add(new BasicNameValuePair("message", record.get(2))); // This might be a top level review, or a comment on a review or another comment if (urlLevel == 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createReview")); nameValuePairs.add(new BasicNameValuePair("ratings", record.get(3))); if (record.size() > 4 && record.get(4).length() > 0) { // If we are dealing with a non-existent resource, then the design drives the behavior nameValuePairs.add(new BasicNameValuePair("scf:resourceType", "social/reviews/components/hbs/reviews")); nameValuePairs.add(new BasicNameValuePair("scf:included", record.get(4))); } } else { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createComment")); } } // Creates a rating if (componentType.equals(RATINGS)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:postTallyResponse")); nameValuePairs.add(new BasicNameValuePair("tallyType", "Rating")); nameValuePairs.add(new BasicNameValuePair("response", record.get(2))); } // Creates a DAM asset if (componentType.equals(ASSET) && record.get(ASSET_INDEX_NAME).length() > 0) { nameValuePairs.add(new BasicNameValuePair("fileName", record.get(ASSET_INDEX_NAME))); } // Creates an enablement resource if (componentType.equals(RESOURCE)) { nameValuePairs.add(new BasicNameValuePair(":operation", "se:createResource")); List<NameValuePair> otherNameValuePairs = buildNVP(record, RESOURCE_INDEX_PROPERTIES); nameValuePairs.addAll(otherNameValuePairs); // Adding the site nameValuePairs.add(new BasicNameValuePair("site", "/content/sites/" + record.get(RESOURCE_INDEX_SITE) + "/resources/en")); // Building the cover image fragment if (record.get(RESOURCE_INDEX_THUMBNAIL).length() > 0) { nameValuePairs.add(new BasicNameValuePair("cover-image", doThumbnail(hostname, port, adminPassword, csvfile, record.get(RESOURCE_INDEX_THUMBNAIL)))); } else { nameValuePairs.add(new BasicNameValuePair("cover-image", "")); } // Building the asset fragment String coverPath = "/content/dam/" + record.get(RESOURCE_INDEX_SITE) + "/resource-assets/" + record.get(2) + "/jcr:content/renditions/cq5dam.thumbnail.319.319.png"; String coverSource = "dam"; String assets = "[{\"cover-img-path\":\"" + coverPath + "\",\"thumbnail-source\":\"" + coverSource + "\",\"asset-category\":\"enablementAsset:dam\",\"resource-asset-name\":null,\"state\":\"A\",\"asset-path\":\"/content/dam/" + record.get(RESOURCE_INDEX_SITE) + "/resource-assets/" + record.get(2) + "\"}]"; nameValuePairs.add(new BasicNameValuePair("assets", assets)); logger.debug("assets:" + assets); } // Creates a learning path if (componentType.equals(LEARNING)) { nameValuePairs.add(new BasicNameValuePair(":operation", "se:editLearningPath")); List<NameValuePair> otherNameValuePairs = buildNVP(record, RESOURCE_INDEX_PROPERTIES); nameValuePairs.addAll(otherNameValuePairs); // Adding the site nameValuePairs.add(new BasicNameValuePair("site", "/content/sites/" + record.get(RESOURCE_INDEX_SITE) + "/resources/en")); // Building the cover image fragment if (record.get(RESOURCE_INDEX_THUMBNAIL).length() > 0) { nameValuePairs.add(new BasicNameValuePair("card-image", doThumbnail(hostname, port, adminPassword, csvfile, record.get(RESOURCE_INDEX_THUMBNAIL)))); } // Building the learning path fragment StringBuffer assets = new StringBuffer("[\""); if (learningpaths.get(record.get(2)) != null) { ArrayList<String> paths = learningpaths.get(record.get(2)); int i = 0; for (String path : paths) { assets.append("{\\\"type\\\":\\\"linked-resource\\\",\\\"path\\\":\\\""); assets.append(path); assets.append("\\\"}"); if (i++ < paths.size() - 1) { assets.append("\",\""); } } } else { logger.debug("No asset for this learning path"); } assets.append("\"]"); nameValuePairs.add(new BasicNameValuePair("learningpath-items", assets.toString())); logger.debug("Learning path:" + assets.toString()); } // Creates a calendar event if (componentType.equals(CALENDAR)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createEvent")); try { JSONObject event = new JSONObject(); // Building the JSON fragment for a new calendar event event.accumulate("subject", record.get(2)); event.accumulate("message", record.get(3)); event.accumulate("location", record.get(4)); event.accumulate("tags", ""); event.accumulate("undefined", "update"); String startDate = record.get(5); startDate = startDate.replaceAll("YYYY", Integer.toString(Calendar.getInstance().get(Calendar.YEAR))); startDate = startDate.replaceAll("MM", Integer.toString(1 + Calendar.getInstance().get(Calendar.MONTH))); event.accumulate("start", startDate); String endDate = record.get(6); endDate = endDate.replaceAll("YYYY", Integer.toString(Calendar.getInstance().get(Calendar.YEAR))); endDate = endDate.replaceAll("MM", Integer.toString(1 + Calendar.getInstance().get(Calendar.MONTH))); event.accumulate("end", endDate); nameValuePairs.add(new BasicNameValuePair("event", event.toString())); } catch (Exception ex) { logger.error(ex.getMessage()); } } // Constructing a multi-part POST request MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); for (NameValuePair nameValuePair : nameValuePairs) { builder.addTextBody(nameValuePair.getName(), nameValuePair.getValue(), ContentType.create("text/plain", MIME.UTF8_CHARSET)); } // See if we have attachments for this new post - or some other actions require a form nonetheless if ((componentType.equals(ASSET) || componentType.equals(AVATAR) || componentType.equals(FORUM) || (componentType.equals(JOURNAL)) || componentType.equals(BLOG)) && record.size() > 4 && record.get(ASSET_INDEX_NAME).length() > 0) { File attachment = new File(csvfile.substring(0, csvfile.indexOf(".csv")) + File.separator + record.get(ASSET_INDEX_NAME)); ContentType ct = ContentType.MULTIPART_FORM_DATA; if (record.get(ASSET_INDEX_NAME).indexOf(".mp4") > 0) { ct = ContentType.create("video/mp4", MIME.UTF8_CHARSET); } else if (record.get(ASSET_INDEX_NAME).indexOf(".jpg") > 0 || record.get(ASSET_INDEX_NAME).indexOf(".jpeg") > 0) { ct = ContentType.create("image/jpeg", MIME.UTF8_CHARSET); } else if (record.get(ASSET_INDEX_NAME).indexOf(".png") > 0) { ct = ContentType.create("image/png", MIME.UTF8_CHARSET); } else if (record.get(ASSET_INDEX_NAME).indexOf(".pdf") > 0) { ct = ContentType.create("application/pdf", MIME.UTF8_CHARSET); } else if (record.get(ASSET_INDEX_NAME).indexOf(".zip") > 0) { ct = ContentType.create("application/zip", MIME.UTF8_CHARSET); } builder.addBinaryBody("file", attachment, ct, attachment.getName()); logger.debug("Adding file to payload with name: " + attachment.getName() + " and type: " + ct.getMimeType()); } // If it's a resource or a learning path, we need the path to the resource for subsequent publishing String jsonElement = "location"; if (componentType.equals(RESOURCE)) { jsonElement = "changes/argument"; } if (componentType.equals(LEARNING)) { jsonElement = "path"; } if (componentType.equals(ASSET)) { jsonElement = null; } // This call generally returns the path to the content fragment that was just created location = Loader.doPost(hostname, port, url[urlLevel], userName, password, builder.build(), jsonElement); // If we are loading a DAM asset, we are waiting for all renditions to be generated before proceeding if (componentType.equals(ASSET)) { int pathIndex = url[urlLevel].lastIndexOf(".createasset.html"); if (pathIndex > 0) doWaitPath(hostname, port, adminPassword, url[urlLevel].substring(0, pathIndex) + "/" + record.get(ASSET_INDEX_NAME) + "/jcr:content/renditions", "nt:file"); } // Let's see if it needs to be added to a learning path if (componentType.equals(RESOURCE) && record.get(RESOURCE_INDEX_PATH).length() > 0 && location != null) { // Adding the location to a list of a resources for this particular Learning Path if (learningpaths.get(record.get(RESOURCE_INDEX_PATH)) == null) learningpaths.put(record.get(RESOURCE_INDEX_PATH), new ArrayList<String>()); logger.debug("Adding resource to Learning path: " + record.get(RESOURCE_INDEX_PATH)); ArrayList<String> locations = learningpaths.get(record.get(RESOURCE_INDEX_PATH)); locations.add(location); learningpaths.put(record.get(RESOURCE_INDEX_PATH), locations); } // If it's a Learning Path, we publish it when possible if (componentType.equals(LEARNING) && !port.equals(altport) && location != null) { // Publishing the learning path List<NameValuePair> publishNameValuePairs = new ArrayList<NameValuePair>(); publishNameValuePairs.add(new BasicNameValuePair(":operation", "se:publishEnablementContent")); publishNameValuePairs.add(new BasicNameValuePair("replication-action", "activate")); logger.debug("Publishing a learning path from: " + location); Loader.doPost(hostname, port, location, userName, password, new UrlEncodedFormEntity(publishNameValuePairs), null); // Waiting for the learning path to be published Loader.doWait(hostname, altport, "admin", adminPassword, location.substring(1 + location.lastIndexOf("/")) // Only search for groups with the learning path in it ); // Decorate the resources within the learning path with comments and ratings, randomly generated ArrayList<String> paths = learningpaths.get(record.get(2)); for (String path : paths) { doDecorate(hostname, altport, path, record, analytics); } } // If it's an Enablement Resource, a lot of things need to happen... // Step 1. If it's a SCORM resource, we wait for the SCORM metadata workflow to be complete before proceeding // Step 2. We publish the resource // Step 3. We set a new first published date on the resource (3 weeks earlier) so that reporting data is more meaningful // Step 4. We wait for the resource to be available on publish (checking that associated groups are available) // Step 5. We retrieve the json for the resource on publish to retrieve the Social endpoints // Step 6. We post ratings and comments for each of the enrollees on publish if (componentType.equals(RESOURCE) && !port.equals(altport) && location != null) { // Wait for the data to be fully copied doWaitPath(hostname, port, adminPassword, location + "/assets/asset", "nt:file"); // If we are dealing with a SCORM asset, we wait a little bit before publishing the resource to that the SCORM workflow is completed if (record.get(2).indexOf(".zip") > 0) { doSleep(10000, "SCORM Resource, waiting for workflow to complete"); } // Publishing the resource List<NameValuePair> publishNameValuePairs = new ArrayList<NameValuePair>(); publishNameValuePairs.add(new BasicNameValuePair(":operation", "se:publishEnablementContent")); publishNameValuePairs.add(new BasicNameValuePair("replication-action", "activate")); logger.debug("Publishing a resource from: " + location); Loader.doPost(hostname, port, location, userName, password, new UrlEncodedFormEntity(publishNameValuePairs), null); // Waiting for the resource to be published Loader.doWait(hostname, altport, "admin", adminPassword, location.substring(1 + location.lastIndexOf("/")) // Only search for groups with the resource path in it ); // Setting the first published timestamp so that reporting always comes with 3 weeks of data after building a new demo instance DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); Calendar cal = Calendar.getInstance(); cal.add(Calendar.DATE, REPORTINGDAYS); List<NameValuePair> publishDateNameValuePairs = new ArrayList<NameValuePair>(); publishDateNameValuePairs .add(new BasicNameValuePair("date-first-published", dateFormat.format(cal.getTime()))); logger.debug("Setting the publish date for a resource from: " + location); doPost(hostname, port, location, userName, password, new UrlEncodedFormEntity(publishDateNameValuePairs), null); // Adding comments and ratings for this resource doDecorate(hostname, altport, location, record, analytics); } } } catch (IOException e) { logger.error(e.getMessage()); } }
From source file:com.oneapm.base.SparkAggregation.java
public static void main(String[] args) throws IOException { String configfile = "alert.cnf"; final FlowConstant flowConstant = new FlowConstant(); Properties config = null;// ww w . j a v a 2s. c o m try { config = getConfig(configfile); } catch (IOException e) { LOG.error(configfile + " doesnt exist in /etc/analysis/... exit" + e); System.exit(-1); } final int window = Integer.parseInt(config.getProperty("time.window", "60")); final List<String> percent = Arrays.asList("tryConnectPer", "cBitpacketAccount", "abortConntionCount", "unidirectionalTrafficPer"); final List<String> jsonList = new ArrayList<>(); final Map<String, Tuple2<String, Integer>> stats = new HashMap<String, Tuple2<String, Integer>>(); final Map<ClientKey, BaseData> broadClient = new HashMap<ClientKey, BaseData>(); final Map<HostKey, BaseData> broadHost = new HashMap<HostKey, BaseData>(); final Map<ServiceKey, BaseData> broadService = new HashMap<ServiceKey, BaseData>(); final Map<SubnetKey, BaseData> broadSubnet = new HashMap<SubnetKey, BaseData>(); final Map<VlanKey, BaseData> broadVlan = new HashMap<VlanKey, BaseData>(); final String URL = config.getProperty("alert.url"); final String HEART_BEAT = config.getProperty("alert.heartbeat"); final String RECOVER = config.getProperty("alert.recover"); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/site", new SiteConvertImpl(flowConstant)); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/net", new NetConvertImpl(flowConstant)); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/vlan_sflow", new LinkConvertImpl(flowConstant)); ZookeeperConfigWatcher.startZookeeperWatcherService(getConfig("alert.cnf").getProperty("kafaka.zoo"), "/ni/subnet", new SubnetConvertImpl(flowConstant)); zookeeperClient.connectZookeeper(config.getProperty("kafaka.zoo")); esGet.setNodeClient(); startZookeeperService(flowConstant, zookeeperClient); zookeeperClient.setRules("/ni/caution"); JavaPairReceiverInputDStream<String, byte[]> rawStream = setupRawStreamFromKafka(config, config.getProperty("group.id", "oneapm-alert")); LOG.info("alert config:" + config.toString()); final Broadcast<IPDataInfo> ipDataBroadcastInfo = getJsc().sparkContext().broadcast(new IPDataInfo()); final Broadcast<ProtocalTypeInfo> protocalTypeBroadcastInfo = getJsc().sparkContext() .broadcast(new ProtocalTypeInfo()); JavaPairDStream<TimeAgg, BeforeAgg> aggJavaPairDStream = rawStream .mapToPair(new PairFunction<Tuple2<String, byte[]>, TimeAgg, BeforeAgg>() { private static final long serialVersionUID = -2751318332921803477L; @Override public Tuple2<TimeAgg, BeforeAgg> call(Tuple2<String, byte[]> stringTuple2) throws Exception { String[] s = TAB.split(new String(stringTuple2._2)); String clientIP = s[1]; String serverIP = s[2]; TimeAgg timeAgg = new TimeAgg(); BeforeAgg beforeAgg = new BeforeAgg(); timeAgg.setServer_ip(serverIP); if (s.length >= 60) { //setProtocal_type if (!"-".equals(s[11]) && !"".equals(s[11])) { timeAgg.setProtocal_type(s[11]); } else { if ("TCP".equals(s[12])) { ProtocalTypeInfo protocalTypeInfo = protocalTypeBroadcastInfo.value(); String protocalType = protocalTypeInfo.config.getProperty(s[4]); if (protocalType != null) { timeAgg.setProtocal_type(protocalType); } else { timeAgg.setProtocal_type("TCP_" + s[4]); } } else { timeAgg.setProtocal_type(s[12]); } } timeAgg.setType("tcp"); //setVlan_id String vlanLinkAlias = flowConstant.VLAN_LINK.get(s[13]); String portLinkAlias = flowConstant.PROBE_POTR_MAP.get(s[0] + "_" + s[15]); if (flowConstant.PROBE_HOST_ENABLE) { if (portLinkAlias != null) { timeAgg.setVlan_id(portLinkAlias); } else { timeAgg.setVlan_id(s[0] + "-nic" + s[15]); } } else if (flowConstant.VLAN_LINK_ENABLE) { if (vlanLinkAlias != null) { timeAgg.setVlan_id(vlanLinkAlias); } else { timeAgg.setVlan_id(s[13]); } } else { timeAgg.setVlan_id("Vlan" + s[13]); } if (!"-".equals(s[6])) { timeAgg.setTimestamp(format.format((long) Double.parseDouble(s[6]) * 1000)); } else { timeAgg.setTimestamp(s[6]); } if (!"-".equals(s[7])) { timeAgg.setTimeEnd(format.format((long) Double.parseDouble(s[7]) * 1000)); } else { timeAgg.setTimeEnd(s[7]); } beforeAgg.setPacket_size(Double.parseDouble(s[55]) + Double.parseDouble(s[56])); beforeAgg.setC_packet_size(Double.parseDouble(s[55])); beforeAgg.setS_packet_size(Double.parseDouble(s[56])); beforeAgg.setPacket_count(Double.parseDouble(s[59]) + Double.parseDouble(s[60])); beforeAgg.setLosspacket_count(Double.parseDouble(s[33]) + Double.parseDouble(s[39]) + Double.parseDouble(s[35]) + Double.parseDouble(s[41])); double cRto = (Double.valueOf(s[19]) + Double.valueOf(s[21]) + Double.valueOf(s[23])) * 1000; double sRto = (Double.valueOf(s[20]) + Double.valueOf(s[22]) + Double.valueOf(s[24])) * 1000; beforeAgg.setTotal_rto(cRto + sRto); // ?/TCP????/TCP????/TCP?MTU?MTU/TCP? beforeAgg.setInt_ZWIN_COUNT(Double.parseDouble(s[43])); beforeAgg.setInt_OOR_COUNT(Double.parseDouble(s[46])); beforeAgg.setInt_CONGEST_COUNT(Double.parseDouble(s[47])); beforeAgg.setMTU(Double.parseDouble(s[61]) + Double.parseDouble(s[62])); Boolean hasSynFlag = Boolean.valueOf(s[5]); timeAgg.setHasSyn(hasSynFlag); timeAgg.setBool_FIN(Boolean.valueOf(s[25])); timeAgg.setBool_RST(Boolean.valueOf(s[26])); if (hasSynFlag && "-".equals(s[9])) { beforeAgg.setAbort(1); } else { beforeAgg.setAbort(0); } beforeAgg.setTcpTurns(Integer.parseInt(s[30])); //int_TURN_COUNT beforeAgg.setConnrequest_count(Double.parseDouble(s[48])); beforeAgg.setsAbortConnCount(Integer.valueOf(s[50])); Long cPayLoad = Long.valueOf(s[53]); Long sPayLoad = Long.valueOf(s[54]); long payLoad = cPayLoad + sPayLoad; double sessionJlRtt = 0; if (!"-".equals(s[9])) { sessionJlRtt = Double.valueOf(s[9]) * 1000; beforeAgg.setRtt(sessionJlRtt); beforeAgg.setServerJlRttCount(1); beforeAgg.setClientJlRttCount(1); } if (hasSynFlag && !"-".equals(s[9])) { beforeAgg.setCount(1); if ("true".equals(s[26]) && payLoad == 0) { beforeAgg.setCount(0); } } if (!"-".equals(s[10])) { double clientJlRtt = Double.valueOf(s[10]) * 1000; double serverJlRtt = sessionJlRtt - clientJlRtt; if (clientJlRtt < sessionJlRtt) { beforeAgg.setServer_rtt(clientJlRtt); beforeAgg.setClient_rtt(serverJlRtt); } else { beforeAgg.setServer_rtt(sessionJlRtt / 2); beforeAgg.setClient_rtt(sessionJlRtt / 2); } } if (beforeAgg.tcpTurns > 0) { beforeAgg.setSessionCount(1); if (Double.parseDouble(s[18]) > 0) { beforeAgg.setServer_reponsetime( Double.parseDouble(s[18]) / beforeAgg.tcpTurns * 1000); } if (Double.parseDouble(s[16]) > 0) { beforeAgg.setResponseTransmissionTime( Double.parseDouble(s[16]) / beforeAgg.tcpTurns * 1000); } if (Double.parseDouble(s[17]) > 0) { beforeAgg.setRequestTransmissionTime( Double.parseDouble(s[17]) / beforeAgg.tcpTurns * 1000); } if (beforeAgg.total_rto > 0) { beforeAgg.setTotal_rto(beforeAgg.getTotal_rto() / beforeAgg.tcpTurns); } beforeAgg.setUserResponseTime( beforeAgg.getRtt() + beforeAgg.getRequestTransmissionTime() + beforeAgg.getResponseTransmissionTime() + beforeAgg.getServer_reponsetime() + beforeAgg.total_rto); } else { beforeAgg.setServer_reponsetime(0); beforeAgg.setResponseTransmissionTime(0); beforeAgg.setRequestTransmissionTime(0); beforeAgg.setTotal_rto(0); beforeAgg.setUserResponseTime(0); } beforeAgg.setC_bitpacket_account(Double.parseDouble(s[57]) + Double.parseDouble(s[58])); } else if (s.length <= 28) { if (!"-".equals(s[8]) && !"".equals(s[8])) { timeAgg.setProtocal_type(s[8]); } else { if ("UDP".equals(s[9])) { ProtocalTypeInfo protocalTypeInfo = protocalTypeBroadcastInfo.value(); String protocalType = protocalTypeInfo.config.getProperty(s[4]); if (protocalType != null) { timeAgg.setProtocal_type(protocalType); } else { timeAgg.setProtocal_type("UDP"); } } else { timeAgg.setProtocal_type(s[9]); } } beforeAgg.setCount(0); timeAgg.setType("udp"); timeAgg.setHasSyn(false); timeAgg.setBool_FIN(false); timeAgg.setBool_RST(false); //setVlan_id String vlanLinkAlias = flowConstant.VLAN_LINK.get(s[12]); String portLinkAlias = flowConstant.PROBE_POTR_MAP.get(s[0] + "_" + s[10]); if (flowConstant.PROBE_HOST_ENABLE) { if (portLinkAlias != null) { timeAgg.setVlan_id(portLinkAlias); } else { timeAgg.setVlan_id(s[0] + "-nic" + s[10]); } } else if (flowConstant.VLAN_LINK_ENABLE) { if (vlanLinkAlias != null) { timeAgg.setVlan_id(vlanLinkAlias); } else { timeAgg.setVlan_id(s[12]); } } else { timeAgg.setVlan_id("Vlan" + s[12]); } if (!"-".equals(s[5])) { timeAgg.setTimestamp(format.format((long) Double.parseDouble(s[5]) * 1000)); } else { timeAgg.setTimestamp(s[5]); } if (!"-".equals(s[6])) { timeAgg.setTimeEnd(format.format((long) Double.parseDouble(s[6]) * 1000)); } else { timeAgg.setTimeEnd(s[6]); } beforeAgg.setPacket_size(Double.parseDouble(s[20]) + Double.parseDouble(s[21])); beforeAgg.setPacket_count(Double.parseDouble(s[24]) + Double.parseDouble(s[25])); beforeAgg.setC_packet_size(Double.parseDouble(s[20])); beforeAgg.setS_packet_size(Double.parseDouble(s[21])); beforeAgg.setInt_ZWIN_COUNT(0); beforeAgg.setInt_OOR_COUNT(0); beforeAgg.setInt_CONGEST_COUNT(0); beforeAgg.setMTU(0); beforeAgg.setC_bitpacket_account(Double.parseDouble(s[22]) + Double.parseDouble(s[23])); beforeAgg.setAbort(0); beforeAgg.setClient_rtt(0); beforeAgg.setServer_rtt(0); beforeAgg.setRtt(0); beforeAgg.setServerJlRttCount(0); beforeAgg.setClientJlRttCount(0); beforeAgg.setLosspacket_count(0); beforeAgg.setServer_reponsetime(0); beforeAgg.setTcpTurns(0); beforeAgg.setConnrequest_count(0); beforeAgg.setTotal_rto(0); beforeAgg.setUserResponseTime(0); beforeAgg.setResponseTransmissionTime(0); beforeAgg.setRequestTransmissionTime(0); beforeAgg.setServer_reponsetime(0); beforeAgg.setsAbortConnCount(0); beforeAgg.setSessionCount(0); } String sInOutFlag = IPUtils.isInHomeNet(serverIP, flowConstant.HOMENET); String cInOutFlag = IPUtils.isInHomeNet(clientIP, flowConstant.HOMENET); //setSubnet if ("IN".equals(sInOutFlag)) { String sSubNet = IPUtils.getSubNet(serverIP, flowConstant.NETMASK); timeAgg.setSubnet(sSubNet + "/" + flowConstant.MASKBITS); } else { timeAgg.setSubnet("-"); } if ("255.255.255.255".equals(clientIP)) { timeAgg.setClient_site("-"); } else { String clientSiteInfo = IPUtils.getSiteInfo(clientIP, flowConstant.SITEINFO_MAP); IPDataInfo ipDataInfo = ipDataBroadcastInfo.getValue(); if (clientSiteInfo != null) { String[] clientSiteInfos = clientSiteInfo.split("_", 3); timeAgg.setClient_site(clientSiteInfos[2]); } else { if ("IN".equals(cInOutFlag)) { timeAgg.setClient_site(""); } else { if (ipDataInfo != null) { String[] ipinfo = ipDataInfo.find(clientIP); // if (ipinfo.length < 3) { timeAgg.setClient_site(""); } else { if ("".equals(ipinfo[0])) { timeAgg.setClient_site(""); } else { //,areasite if ("".equals(ipinfo[1])) { ipinfo[1] = ipinfo[0]; } if ("".equals(ipinfo[2])) { ipinfo[2] = ipinfo[1]; } timeAgg.setClient_site(ipinfo[2]); } } } else { timeAgg.setClient_site("?"); } } } } return new Tuple2<>(timeAgg, beforeAgg); } }).filter(new Function<Tuple2<TimeAgg, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { return !v1._1.getTimestamp().equals("-") && !v1._1.getTimestamp().equals(""); } }); // aggJavaPairDStream.foreachRDD(new Function<JavaPairRDD<TimeAgg, BeforeAgg>, Void>() { // @Override // public Void call(JavaPairRDD<TimeAgg, BeforeAgg> v1) throws Exception { // if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es")) && v1 != null) { // JavaRDD<Map<String, ?>> es = v1.map(new Function<Tuple2<TimeAgg, BeforeAgg>, Map<String, // ?>>() { // // @Override // public Map<String, ?> call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { // ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder(); // // TimeAgg a = v1._1; // BeforeAgg b = v1._2; // String todayStr = sdf.format(format.parse(a.getTimestamp())); // builder.put("server_ip", a.server_ip); // builder.put("protocal_type", a.protocal_type); // builder.put("client_site", a.client_site); // builder.put("vlan_id", a.vlan_id); // builder.put("subnet", a.subnet); // builder.put("timestamp", format.parse(a.timestamp)); // if (b.packet_size > 0) { // builder.put("packet_size", b.packet_size); // } // if (b.c_packet_size > 0) { // builder.put("c_packet_size", b.c_packet_size); // } // // if (b.packet_count > 0) { // builder.put("packet_count", b.packet_count); // } // // if (b.losspacket_count > 0) { // builder.put("losspacket_count", b.losspacket_count); // } // if (b.total_rto > 0) { // builder.put("total_rto", b.total_rto); // builder.put("rtoCount", b.rtoCount); // } // // // if (b.tcpTurns > 0) { // builder.put("tcpTurns", b.tcpTurns); // } // if (b.connrequest_count > 0) { // builder.put("connrequest_count", b.connrequest_count); // } // if (b.abort > 0) { // builder.put("abort", b.abort); // } // if (b.client_rtt > 0) { // builder.put("client_rtt", b.client_rtt); // builder.put("clientJlRttCount", b.clientJlRttCount); // } // if (b.server_rtt > 0) { // builder.put("server_rtt", b.server_rtt); // builder.put("serverJlRttCount", b.serverJlRttCount); // } // // if (b.server_reponsetime > 0) { // builder.put("server_reponsetime", b.server_reponsetime); // builder.put("server_reponsetime_count", b.server_reponsetime_count); // } // // if (b.responseTransmissionTime > 0) { // builder.put("responseTransmissionTime", b.responseTransmissionTime); // builder.put("responseTransmissionTimeCount", b.responseTransmissionTimeCount); // } // if (b.requestTransmissionTime > 0) { // builder.put("requestTransmissionTime", b.requestTransmissionTime); // builder.put("requestTransmissionTimeCount", b.requestTransmissionTimeCount); // // } // // if (b.sAbortConnCount > 0) { // builder.put("sAbortConnCount", b.sAbortConnCount); // } // // if (b.userResponseTime > 0) { // builder.put("userResponseTime", b.userResponseTime); // builder.put("userResponseTimeCount", b.userResponseTimeCount); // } // if (b.c_bitpacket_account > 0) { // builder.put("c_bitpacket_account", b.c_bitpacket_account); // } // builder.put("index_name", todayStr); // // return builder.build(); // } // }).cache(); // if (es != null) { // JavaEsSpark.saveToEs(es, "ni-alert-session-{index_name}/alert", ImmutableMap.of // (ConfigurationOptions.ES_MAPPING_EXCLUDE, "index_name")); // } // } // return null; // } // }); JavaPairDStream<TimeAgg, BeforeAgg> reduceByWindow = aggJavaPairDStream .reduceByKeyAndWindow(new Function2<BeforeAgg, BeforeAgg, BeforeAgg>() { @Override public BeforeAgg call(BeforeAgg v1, BeforeAgg v2) throws Exception { BeforeAgg sum = new BeforeAgg(); sum.setPacket_size(v1.getPacket_size() + v2.getPacket_size()); sum.setC_packet_size(v1.getC_packet_size() + v2.getC_packet_size()); sum.setS_packet_size(v1.getS_packet_size() + v2.getS_packet_size()); sum.setPacket_count(v1.getPacket_count() + v2.getPacket_count()); sum.setC_packet_count(v1.getC_packet_count() + v2.getC_packet_count()); sum.setS_packet_count(v1.getS_packet_count() + v2.getS_packet_count()); sum.setLosspacket_count(v1.getLosspacket_count() + v2.getLosspacket_count()); sum.setTotal_rto(v1.getTotal_rto() + v2.getTotal_rto()); sum.setAbort(v1.getAbort() + v2.getAbort()); sum.setRequestTransmissionTime( v1.getRequestTransmissionTime() + v2.getRequestTransmissionTime()); sum.setResponseTransmissionTime( v1.getResponseTransmissionTime() + v2.getResponseTransmissionTime()); sum.setTcpTurns(v1.getTcpTurns() + v2.getTcpTurns()); sum.setConnrequest_count(v1.getConnrequest_count() + v2.getConnrequest_count()); sum.setRtt(v1.getRtt() + v2.getRtt()); sum.setClient_rtt(v1.getClient_rtt() + v2.getClient_rtt()); sum.setServer_rtt(v1.getServer_rtt() + v2.getServer_rtt()); sum.setServer_reponsetime(v1.getServer_reponsetime() + v2.getServer_reponsetime()); sum.setC_bitpacket_account(v1.getC_bitpacket_account() + v2.getC_bitpacket_account()); sum.setClientJlRttCount(v1.getClientJlRttCount() + v2.getClientJlRttCount()); sum.setServerJlRttCount(v1.getServerJlRttCount() + v2.getServerJlRttCount()); sum.setUserResponseTime(v1.getUserResponseTime() + v2.getUserResponseTime()); sum.setSessionCount(v1.sessionCount + v2.sessionCount); sum.setsAbortConnCount(v1.sAbortConnCount + v2.sAbortConnCount); sum.setCount(v1.getCount() + v2.getCount()); sum.setInt_CONGEST_COUNT(v1.int_CONGEST_COUNT + v2.int_CONGEST_COUNT); sum.setInt_OOR_COUNT(v1.int_OOR_COUNT + v2.int_OOR_COUNT); sum.setInt_ZWIN_COUNT(v1.int_ZWIN_COUNT + v2.int_ZWIN_COUNT); sum.setMTU(v1.MTU + v2.MTU); return sum; } }, Durations.seconds(300), Durations.seconds(60)).cache(); reduceByWindow.foreachRDD(new Function<JavaPairRDD<TimeAgg, BeforeAgg>, Void>() { private static final long serialVersionUID = -4144342491397135515L; @Override public Void call(JavaPairRDD<TimeAgg, BeforeAgg> v1) throws Exception { if (v1.count() > 0) { /** * getStartTime */ List<Long> timeList = v1.map(new Function<Tuple2<TimeAgg, BeforeAgg>, Long>() { @Override public Long call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { return format.parse(v1._1.getTimestamp()).getTime(); } }).distinct().collect(); Collections.sort(timeList, new MyComparator()); long a = timeList.get(3); final String time = format.format(new Date(a)); long b = timeList.get(1); final String endTime = format.format(new Date(b)); if (b > 0) { JavaRDD<Map<String, ?>> active = v1 .filter(new Function<Tuple2<TimeAgg, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { String sInOutFlag = IPUtils.isInHomeNet(v1._1.getServer_ip(), flowConstant.HOMENET); return v1._1.getType().equals("tcp") && "IN".equals(sInOutFlag); } }).mapToPair( new PairFunction<Tuple2<TimeAgg, BeforeAgg>, Tuple2<String, String>, ConnectStatus>() { @Override public Tuple2<Tuple2<String, String>, ConnectStatus> call( Tuple2<TimeAgg, BeforeAgg> timeAggBeforeAggTuple2) throws Exception { ConnectStatus connectStatus = new ConnectStatus(); String serverIp = timeAggBeforeAggTuple2._1.getServer_ip(); String protoType = timeAggBeforeAggTuple2._1.getProtocal_type(); TimeAgg a = timeAggBeforeAggTuple2._1; BeforeAgg b = timeAggBeforeAggTuple2._2; // if (format.parse(a.timestamp).getTime() == format.parse(endTime) .getTime() && a.hasSyn) { connectStatus.setNewCreate(b.getCount()); } else { connectStatus.setNewCreate(0); } //?breakreset?break if (format.parse(a.timeEnd).getTime() == format.parse(endTime) .getTime() && (a.bool_FIN || a.bool_RST)) { connectStatus.setCloseConn(b.getCount()); } else { connectStatus.setCloseConn(0); } if (format.parse(a.timestamp).getTime() <= format.parse(endTime) .getTime() && format.parse(a.timeEnd).getTime() > format.parse(endTime) .getTime() && a.hasSyn) { connectStatus.setActiveConn(b.getCount()); } else if (format.parse(a.timestamp).getTime() == format .parse(endTime).getTime() && format.parse(a.timeEnd).getTime() == format .parse(endTime).getTime() && a.hasSyn) { connectStatus.setActiveConn(b.getCount()); } return new Tuple2<>(new Tuple2<>(serverIp, protoType), connectStatus); } }) .reduceByKey(new Function2<ConnectStatus, ConnectStatus, ConnectStatus>() { @Override public ConnectStatus call(ConnectStatus v1, ConnectStatus v2) throws Exception { ConnectStatus connectStatus = new ConnectStatus(); connectStatus.setNewCreate(v1.newCreate + v2.newCreate); connectStatus.setActiveConn(v1.activeConn + v2.activeConn); connectStatus.setCloseConn(v1.closeConn + v2.closeConn); return connectStatus; } }) .map(new Function<Tuple2<Tuple2<String, String>, ConnectStatus>, Map<String, ?>>() { @Override public Map<String, ?> call(Tuple2<Tuple2<String, String>, ConnectStatus> v1) throws Exception { ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder(); String todayStr = sdf.format(format.parse(endTime)); builder.put("server_ip", v1._1._1); builder.put("protocal_type", v1._1._2); builder.put("newCreateConn", v1._2.getNewCreate()); builder.put("closeConn", v1._2.getCloseConn()); builder.put("activeConn", v1._2.getActiveConn()); builder.put("index_name", todayStr); builder.put("timestamp", format.parse(endTime)); return builder.build(); } }).cache(); if (active != null) { JavaEsSpark.saveToEs(active, "ni-active-conn-{index_name}/active", ImmutableMap.of(ConfigurationOptions.ES_MAPPING_EXCLUDE, "index_name")); } } JavaPairRDD<TimeAgg, BeforeAgg> before = v1 .filter(new Function<Tuple2<TimeAgg, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { return v1._1.getTimestamp().equals(time); } }); if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es")) && before != null) { JavaRDD<Map<String, ?>> es = before .map(new Function<Tuple2<TimeAgg, BeforeAgg>, Map<String, ?>>() { @Override public Map<String, ?> call(Tuple2<TimeAgg, BeforeAgg> v1) throws Exception { ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder(); TimeAgg a = v1._1; BeforeAgg b = v1._2; String todayStr = sdf.format(format.parse(a.getTimestamp())); builder.put("server_ip", a.server_ip); builder.put("protocal_type", a.protocal_type); builder.put("client_site", a.client_site); builder.put("vlan_id", a.vlan_id); builder.put("subnet", a.subnet); builder.put("timestamp", format.parse(a.timestamp)); if (b.packet_size > 0) { builder.put("packet_size", b.packet_size); } if (b.c_packet_size > 0) { builder.put("c_packet_size", b.c_packet_size); } builder.put("count", b.count); if (b.packet_count > 0) { builder.put("packet_count", b.packet_count); } if (b.losspacket_count > 0) { builder.put("losspacket_count", b.losspacket_count); } if (b.total_rto > 0) { builder.put("total_rto", b.total_rto); } if (b.tcpTurns > 0) { builder.put("tcpTurns", b.tcpTurns); builder.put("sessionCount", b.sessionCount); } if (b.connrequest_count > 0) { builder.put("connrequest_count", b.connrequest_count); } if (b.abort > 0) { builder.put("abort", b.abort); } if (b.client_rtt > 0) { builder.put("client_rtt", b.client_rtt); builder.put("clientJlRttCount", b.clientJlRttCount); } if (b.server_rtt > 0) { builder.put("server_rtt", b.server_rtt); builder.put("serverJlRttCount", b.serverJlRttCount); } if (b.server_reponsetime > 0) { builder.put("server_reponsetime", b.server_reponsetime); } if (b.responseTransmissionTime > 0) { builder.put("responseTransmissionTime", b.responseTransmissionTime); } if (b.requestTransmissionTime > 0) { builder.put("requestTransmissionTime", b.requestTransmissionTime); } if (b.sAbortConnCount > 0) { builder.put("sAbortConnCount", b.sAbortConnCount); } if (b.userResponseTime > 0) { builder.put("userResponseTime", b.userResponseTime); } if (b.c_bitpacket_account > 0) { builder.put("c_bitpacket_account", b.c_bitpacket_account); } builder.put("index_name", todayStr); return builder.build(); } }).cache(); if (es != null) { JavaEsSpark.saveToEs(es, "ni-alert-session-{index_name}/alert", ImmutableMap.of(ConfigurationOptions.ES_MAPPING_EXCLUDE, "index_name")); } } UrlPostMethod.urlPostMethod(HEART_BEAT, "frequency=" + window); rules = zookeeperClient.getRules(); if (rules != null) { ArrayList<String> flagRules = new ArrayList<String>(); //?ruleType for (final RuleRecover ruleRecover : rules) { final Rule rule = ruleRecover.getRule(); if (rule.isEnable() && !flagRules.contains(rule.getType())) { //ruleType? flagRules.add(rule.getType()); //ruleType? JavaPairRDD<String, AggResult> alert = before.mapToPair( new PairFunction<Tuple2<TimeAgg, BeforeAgg>, String, BeforeAgg>() { @Override public Tuple2<String, BeforeAgg> call( Tuple2<TimeAgg, BeforeAgg> timeAggBeforeAggTuple2) throws Exception { Field field1 = timeAggBeforeAggTuple2._1.getClass() .getDeclaredField(rule.getType()); field1.setAccessible(true); String result1 = (String) field1.get(timeAggBeforeAggTuple2._1); if (rule.getType().equals("server_ip")) { String sInOutFlag = IPUtils.isInHomeNet( timeAggBeforeAggTuple2._1.getServer_ip(), flowConstant.HOMENET); if ("IN".equals(sInOutFlag)) { return new Tuple2<>(result1, timeAggBeforeAggTuple2._2); } else { return new Tuple2<>(result1, null); } } else { return new Tuple2<>(result1, timeAggBeforeAggTuple2._2); } } }).filter(new Function<Tuple2<String, BeforeAgg>, Boolean>() { @Override public Boolean call(Tuple2<String, BeforeAgg> v1) throws Exception { return v1._2 != null && !v1._1.equals("") && !v1._1.equals("-"); } }).reduceByKey(new Function2<BeforeAgg, BeforeAgg, BeforeAgg>() { @Override public BeforeAgg call(BeforeAgg v1, BeforeAgg v2) throws Exception { BeforeAgg sum = new BeforeAgg(); sum.setPacket_size(v1.getPacket_size() + v2.getPacket_size()); sum.setC_packet_size(v1.getC_packet_size() + v2.getC_packet_size()); sum.setS_packet_size(v1.getS_packet_size() + v2.getS_packet_size()); sum.setPacket_count(v1.getPacket_count() + v2.getPacket_count()); sum.setC_packet_count( v1.getC_packet_count() + v2.getC_packet_count()); sum.setS_packet_count( v1.getS_packet_count() + v2.getS_packet_count()); sum.setLosspacket_count( v1.getLosspacket_count() + v2.getLosspacket_count()); sum.setTotal_rto(v1.getTotal_rto() + v2.getTotal_rto()); sum.setAbort(v1.getAbort() + v2.getAbort()); sum.setRequestTransmissionTime(v1.getRequestTransmissionTime() + v2.getRequestTransmissionTime()); sum.setResponseTransmissionTime(v1.getResponseTransmissionTime() + v2.getResponseTransmissionTime()); sum.setTcpTurns(v1.getTcpTurns() + v2.getTcpTurns()); sum.setConnrequest_count( v1.getConnrequest_count() + v2.getConnrequest_count()); sum.setRtt(v1.getRtt() + v2.getRtt()); sum.setClient_rtt(v1.getClient_rtt() + v2.getClient_rtt()); sum.setServer_rtt(v1.getServer_rtt() + v2.getServer_rtt()); sum.setServer_reponsetime( v1.getServer_reponsetime() + v2.getServer_reponsetime()); sum.setC_bitpacket_account( v1.getC_bitpacket_account() + v2.getC_bitpacket_account()); sum.setClientJlRttCount( v1.getClientJlRttCount() + v2.getClientJlRttCount()); sum.setServerJlRttCount( v1.getServerJlRttCount() + v2.getServerJlRttCount()); sum.setUserResponseTime( v1.getUserResponseTime() + v2.getUserResponseTime()); sum.setSessionCount(v1.sessionCount + v2.sessionCount); sum.setsAbortConnCount(v1.sAbortConnCount + v2.sAbortConnCount); return sum; } }).mapToPair( new PairFunction<Tuple2<String, BeforeAgg>, String, AggResult>() { @Override public Tuple2<String, AggResult> call( Tuple2<String, BeforeAgg> stringBeforeAggTuple2) throws Exception { BeforeAgg before = stringBeforeAggTuple2._2; AggResult result = new AggResult(); result.setTimestamp(time); result.setThroughput(before.packet_size * 8 / window); result.setS_throughput(before.s_packet_size * 8 / window); result.setC_throughput( before.c_bitpacket_account * 8 / window); result.setPacketThroughput(before.packet_count / window); result.setLossRate(before.losspacket_count / before.packet_count * 100); if (before.sessionCount > 0) { result.setRetransferTime( before.total_rto / before.sessionCount); } else { result.setRetransferTime(0); } if (before.clientJlRttCount > 0) { result.setClientRoundTripTime( before.client_rtt / before.clientJlRttCount); result.setRtt(before.rtt / before.clientJlRttCount); } else { result.setClientRoundTripTime(0); result.setRtt(0); } if (before.serverJlRttCount > 0) { result.setServerRoundTripTime( before.server_rtt / before.serverJlRttCount); } else { result.setServerRoundTripTime(0); } if (before.sessionCount > 0) { result.setUserRespTime(before.getUserResponseTime() / before.sessionCount); result.setTransmissionTime( (before.requestTransmissionTime + before.responseTransmissionTime) / before.sessionCount); } else { result.setUserRespTime(0); result.setTransmissionTime(0); } if (before.sessionCount > 0) { result.setServerRespTime(before.server_reponsetime / before.sessionCount); } else { result.setServerRespTime(0); } result.setConnectFailedRate(before.abort / window); //@Deprecates result.setTryConnectPer(0); result.setCongest_pre(before.getInt_CONGEST_COUNT() / before.getCount() * 100); result.setOutoforder_pre(before.getInt_OOR_COUNT() / before.getCount() * 100); result.setZerowindow_pre(before.getInt_ZWIN_COUNT() / before.getCount() * 100); result.setMTU_pre( before.getMTU() / before.getCount() * 100); if (before.packet_count > 0) { result.setcBitpacketAccount(before.c_bitpacket_account / before.packet_count * 100); } else { result.setcBitpacketAccount(0); } if (before.connrequest_count - before.abort > 0) { result.setAbortConntionCount(before.sAbortConnCount / (before.connrequest_count - before.abort) * 100); } else { result.setAbortConntionCount(0); } return new Tuple2<>(stringBeforeAggTuple2._1, result); } }) .cache(); if (alert.count() > 0) { List<String> alertList = new ArrayList<>(); for (final RuleRecover newRule : rules) { final Rule sameRule = newRule.getRule(); if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es"))) { System.out.println( "rule:" + sameRule.toString() + "--------------"); } final int recover = newRule.getRecover(); if (sameRule.isEnable() && sameRule.getType().equals(flagRules.get(flagRules.size() - 1))) { if (!sameRule.getThresholdErrType().equals("absolute")) { if (esGet.getClient() == null) { esGet.setNodeClient(); } final Calendar now = Calendar.getInstance(); now.setTime(format.parse(time)); int minute = now.get(Calendar.MINUTE); Key key = null; switch (sameRule.getType()) { case "server_ip": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadHost.isEmpty()) { Map<HostKey, BaseData> service = esGet .setHost(now.getTime(), "ni-base-hostname"); broadHost.putAll(service); } } else { Map<HostKey, BaseData> service = esGet .setHost(now.getTime(), "ni-base-hostname"); broadHost.clear(); broadHost.putAll(service); } key = new HostKey(); // baseData = broadHost.get(key); break; case "protocal_type": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadService.isEmpty()) { Map<ServiceKey, BaseData> service = esGet .setService(now.getTime(), "ni-base-service"); broadService.putAll(service); } } else { Map<ServiceKey, BaseData> service = esGet .setService(now.getTime(), "ni-base-service"); broadService.clear(); broadService.putAll(service); } key = new ServiceKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadService.get(key); break; case "client_site": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadClient.isEmpty()) { Map<ClientKey, BaseData> service = esGet .setClient(now.getTime(), "ni-base-clientsite"); broadClient.putAll(service); } } else { Map<ClientKey, BaseData> service = esGet .setClient(now.getTime(), "ni-base-clientsite"); broadClient.clear(); broadClient.putAll(service); } key = new ClientKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadClient.get(key); break; case "vlan_id": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadVlan.isEmpty()) { Map<VlanKey, BaseData> service = esGet .setVlan(now.getTime(), "ni-base-link"); broadVlan.putAll(service); } } else { Map<VlanKey, BaseData> service = esGet .setVlan(now.getTime(), "ni-base-link"); broadVlan.clear(); broadVlan.putAll(service); } key = new VlanKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadVlan.get(key); break; case "subnet": if (minute % 5 != 0) { now.set(Calendar.MINUTE, minute / 5 * 5); if (broadSubnet.isEmpty()) { Map<SubnetKey, BaseData> service = esGet .setSubnet(now.getTime(), "ni-base-subnet"); broadSubnet.putAll(service); } } else { Map<SubnetKey, BaseData> service = esGet .setSubnet(now.getTime(), "ni-base-subnet"); broadSubnet.clear(); broadSubnet.putAll(service); } key = new SubnetKey(); // key.setKeyWord(stringAggResultTuple2._1); // key.setStart_timestamp(now.getTime()); // baseData = broadSubnet.get(key); break; } final Key finalKey = key; alertList = alert .filter(new Function<Tuple2<String, AggResult>, Boolean>() { @Override public Boolean call(Tuple2<String, AggResult> v1) throws Exception { Field field2 = v1._2.getClass() .getDeclaredField(sameRule.getValue()); field2.setAccessible(true); double result2 = (double) field2.get(v1._2); if (result2 == 0) { return false; } String contain = sameRule.getContain(); if (contain.equals("") && !v1._1.equals("")) { return true; } if (v1._1 == null || v1._1.equals("")) { return false; } return v1._1.contains(sameRule.getContain()); } }).mapToPair( new PairFunction<Tuple2<String, AggResult>, String, String>() { @Override public Tuple2<String, String> call( Tuple2<String, AggResult> stringAggResultTuple2) throws Exception { Field field2 = stringAggResultTuple2._2 .getClass().getDeclaredField( sameRule.getValue()); field2.setAccessible(true); double alertCursor = (double) field2 .get(stringAggResultTuple2._2); JSONObject json = new JSONObject(); BaseData baseData = new BaseData(); finalKey.setKeyWord( stringAggResultTuple2._1); finalKey.setStart_timestamp(now.getTime()); baseData = broadService.get(finalKey); if (baseData != null) { Field field = baseData.getClass() .getDeclaredField( sameRule.getValue()); field.setAccessible(true); double result = (double) field .get(baseData); AlertLevel alertLevel = new AlertLevel(); if (sameRule.getThresholdErrOp() .equals("ge")) { if (alertCursor - result >= sameRule .getMax_cardinality()) { alertLevel.setWarningLevel( "levelBad"); } else if (alertCursor - result >= sameRule .getMin_cardinality()) { alertLevel.setWarningLevel( "levelWarn"); } else { alertLevel.setWarningLevel( "levelNormal"); } } if (sameRule.getThresholdErrOp() .equals("le")) { if (result - alertCursor <= sameRule .getMax_cardinality()) { alertLevel.setWarningLevel( "levelBad"); } else if (result - alertCursor <= sameRule .getMin_cardinality()) { alertLevel.setWarningLevel( "levelWarn"); } else { alertLevel.setWarningLevel( "levelNormal"); } } alertLevel.setResourceName( stringAggResultTuple2._1); if (sameRule.getType() .equals("server_ip")) { alertLevel.setIpAddress( stringAggResultTuple2._1); } else { alertLevel.setIpAddress(""); } alertLevel.setOccureTime( esFormat.format(format.parse( stringAggResultTuple2._2 .getTimestamp()))); alertLevel.setResourceType( sameRule.getType()); alertLevel.setMetricId( sameRule.getValue()); alertLevel.setResourceInstanceId( stringAggResultTuple2._1); // top2 d?>10%? Map<String, Double> top2 = new HashMap<String, Double>(); top2.put("MTU?", stringAggResultTuple2._2 .getMTU_pre()); top2.put("?", stringAggResultTuple2._2 .getCongest_pre()); top2.put("??", stringAggResultTuple2._2 .getOutoforder_pre()); top2.put("??", stringAggResultTuple2._2 .getZerowindow_pre()); List<Map.Entry<String, Double>> list = SortHashMap .sortHashMap(top2); if ("lossRate" .equals(sameRule.getValue())) { if (list.get(0).getValue() > 10 && list.get(1) .getValue() > 10) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + list.get(0) .getKey() + "" + list.get(0) .getValue() + "%25," + list.get(1) .getKey() + "" + list.get(1) .getValue() + "%25." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else { if ("userRespTime".equals( sameRule.getValue())) { if (list.get(0).getValue() > 10 && list.get(1) .getValue() > 10) { alertLevel .setWarningContent( alertLevel .getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "ms," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + list.get( 0) .getKey() + "" + list.get( 0) .getValue() + "%25," + list.get( 1) .getKey() + "" + list.get( 1) .getValue() + "%25." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel .setWarningContent( alertLevel .getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "ms," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "ms,?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else if ("rtt".equals( sameRule.getValue())) { alertLevel.setWarningContent( "RTT" + millToSec( (long) alertCursor) + ",RTT" + millToSec( (long) stringAggResultTuple2._2 .getClientRoundTripTime()) + "ms,?RTT" + millToSec( (long) stringAggResultTuple2._2 .getServerRoundTripTime()) + "ms." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("throughput".equals( sameRule.getType())) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + convertUnit( (long) alertCursor) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getS_throughput()) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getC_throughput()) + "." + convertUnit( (long) result) + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("packetThroughput" .equals(sameRule .getValue())) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + ",????" + stringAggResultTuple2._2 .getS_packetThroughput() + ",????" + stringAggResultTuple2._2 .getC_packetThroughput() + "." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if (percent.contains( sameRule.getValue())) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + sameRule .getMin_cardinality() + "." + result + "." + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "." + result + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } alertLevel.setRuleId( sameRule.getRuleName()); alertLevel.setUniqueMark(sameRule .getRuleName() + "-" + stringAggResultTuple2._1 + "-" + sameRule.getValue()); if (stats.containsKey( alertLevel.getUniqueMark())) { String preLevel = stats .get(alertLevel .getUniqueMark())._1; int num = stats.get(alertLevel .getUniqueMark())._2; boolean preWarning = preLevel .equals("levelWarn") || preLevel .equals("levelBad"); boolean newWarning = alertLevel .getWarningLevel() .equals("levelWarn") || alertLevel .getWarningLevel() .equals("levelBad"); if (preWarning && !newWarning) { num = 1 - num; stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else if (!preWarning && newWarning) { stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); } else if (!preWarning && !preWarning) { num = 1 - num; stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else { num = 0 - num; stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } } else { if (alertLevel.getWarningLevel() .equals("levelWarn") || alertLevel .getWarningLevel() .equals("levelBad")) { stats.put(alertLevel .getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); json = (JSONObject) JSON .toJSON(alertLevel); return new Tuple2<>( stringAggResultTuple2._1, json.toString()); } } } return new Tuple2<>( stringAggResultTuple2._1, ""); } }) .filter(new Function<Tuple2<String, String>, Boolean>() { @Override public Boolean call(Tuple2<String, String> v1) throws Exception { return !v1._2.equals("") && v1._2 != null; } }).map(new Function<Tuple2<String, String>, String>() { @Override public String call(Tuple2<String, String> v1) throws Exception { return v1._2; } }).collect(); } else { alertList = alert .filter(new Function<Tuple2<String, AggResult>, Boolean>() { @Override public Boolean call(Tuple2<String, AggResult> v1) throws Exception { Field field2 = v1._2.getClass() .getDeclaredField(sameRule.getValue()); field2.setAccessible(true); double result2 = (double) field2.get(v1._2); if (result2 == 0.0) { return false; } String contain = sameRule.getContain(); if (contain.equals("") && !v1._1.equals("")) { return true; } if (v1._1 == null || v1._1.equals("")) { return false; } return v1._1.contains(sameRule.getContain()); } }).mapToPair( new PairFunction<Tuple2<String, AggResult>, String, String>() { @Override public Tuple2<String, String> call( Tuple2<String, AggResult> stringAggResultTuple2) throws Exception { Field field2 = stringAggResultTuple2._2 .getClass().getDeclaredField( sameRule.getValue()); field2.setAccessible(true); double alertCursor = (double) field2 .get(stringAggResultTuple2._2); JSONObject json = new JSONObject(); AlertLevel alertLevel = new AlertLevel(); if (alertCursor >= sameRule .getMax_cardinality()) { alertLevel.setWarningLevel("levelBad"); } else if (alertCursor >= sameRule .getMin_cardinality()) { alertLevel.setWarningLevel("levelWarn"); } else { alertLevel .setWarningLevel("levelNormal"); } alertLevel.setResourceName( stringAggResultTuple2._1); if (sameRule.getType() .equals("server_ip")) { alertLevel.setIpAddress( stringAggResultTuple2._1); } else { alertLevel.setIpAddress(""); } alertLevel.setResourceType( sameRule.getType()); alertLevel.setOccureTime( esFormat.format(format.parse( stringAggResultTuple2._2 .getTimestamp()))); alertLevel.setMetricId(sameRule.getValue()); alertLevel.setResourceInstanceId( stringAggResultTuple2._1); // top2 d?>10%? Map<String, Double> top2 = new HashMap<String, Double>(); top2.put("MTU?", stringAggResultTuple2._2 .getMTU_pre()); top2.put("?", stringAggResultTuple2._2 .getCongest_pre()); top2.put("??", stringAggResultTuple2._2 .getOutoforder_pre()); top2.put("??", stringAggResultTuple2._2 .getZerowindow_pre()); List<Map.Entry<String, Double>> list = SortHashMap .sortHashMap(top2); if ("lossRate" .equals(sameRule.getValue())) { if (list.get(0).getValue() > 10 && list .get(1).getValue() > 10) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + list.get(0) .getKey() + "" + list.get(0) .getValue() + "%25," + list.get(1) .getKey() + "" + list.get(1) .getValue() + "%25." + "" + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + df.format( alertCursor) + "%25," + "," + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else if ("userRespTime" .equals(sameRule.getValue())) { if (list.get(0).getValue() > 10 && list .get(1).getValue() > 10) { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + ",?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + list.get(0) .getKey() + "" + list.get(0) .getValue() + "%25," + list.get(1) .getKey() + "" + list.get(1) .getValue() + "%25." + "" + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent( alertLevel.getMetricId() + "" + millToSec( (long) alertCursor) + "," + millToSec( (long) stringAggResultTuple2._2 .getRtt()) + ",?" + millToSec( (long) stringAggResultTuple2._2 .getServerRespTime()) + "," + millToSec( (long) stringAggResultTuple2._2 .getTransmissionTime()) + "?" + millToSec( (long) stringAggResultTuple2._2 .getRetransferTime()) + "." + sameRule .getMin_cardinality() + "," + sameRule .getMax_cardinality()); } } else if ("rtt" .equals(sameRule.getValue())) { alertLevel.setWarningContent("RTT" + millToSec((long) alertCursor) + ",RTT" + millToSec( (long) stringAggResultTuple2._2 .getClientRoundTripTime()) + ",?RTT" + millToSec( (long) stringAggResultTuple2._2 .getServerRoundTripTime()) + "." + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("throughput" .equals(sameRule.getType())) { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + convertUnit( (long) alertCursor) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getS_throughput()) + ",????" + convertUnit( (long) stringAggResultTuple2._2 .getC_throughput()) + "." + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if ("packetThroughput" .equals(sameRule.getValue())) { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + df.format(alertCursor) + ",????" + stringAggResultTuple2._2 .getS_packetThroughput() + ",????" + stringAggResultTuple2._2 .getC_packetThroughput() + "." + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else if (percent .contains(sameRule.getValue())) { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + df.format(alertCursor) + "%25," + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } else { alertLevel.setWarningContent(alertLevel .getMetricId() + "" + df.format(alertCursor) + "," + sameRule.getMin_cardinality() + "," + sameRule .getMax_cardinality()); } alertLevel .setRuleId(sameRule.getRuleName()); alertLevel.setUniqueMark( sameRule.getRuleName() + "-" + stringAggResultTuple2._1 + "-" + sameRule.getValue()); if (stats.containsKey( alertLevel.getUniqueMark())) { String preLevel = stats.get( alertLevel.getUniqueMark())._1; int num = stats.get( alertLevel.getUniqueMark())._2; boolean preWarning = preLevel .equals("levelWarn") || preLevel.equals("levelBad"); boolean newWarning = alertLevel .getWarningLevel() .equals("levelWarn") || alertLevel.getWarningLevel() .equals("levelBad"); if (preWarning && !newWarning) { num = 1 - num; stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else if (!preWarning && newWarning) { stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); } else if (!preWarning && !preWarning) { num = 1 - num; stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } else { num = 0 - num; stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), num)); } } else { if (alertLevel.getWarningLevel() .equals("levelWarn") || alertLevel.getWarningLevel() .equals("levelBad")) { stats.put( alertLevel.getUniqueMark(), new Tuple2<String, Integer>( alertLevel .getWarningLevel(), 0 - recover)); json = (JSONObject) JSON .toJSON(alertLevel); return new Tuple2<>( stringAggResultTuple2._1, json.toString()); } } return new Tuple2<>( stringAggResultTuple2._1, ""); } }) .filter(new Function<Tuple2<String, String>, Boolean>() { private static final long serialVersionUID = 662946729452638751L; @Override public Boolean call(Tuple2<String, String> v1) throws Exception { return !v1._2.equals("") && v1._2 != null; } }).map(new Function<Tuple2<String, String>, String>() { @Override public String call(Tuple2<String, String> v1) throws Exception { return v1._2; } }).collect(); } } jsonList.addAll(alertList); alertList.clear(); } } } } flagRules.clear(); } Iterator<Map.Entry<String, Tuple2<String, Integer>>> iterator = stats.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry<String, Tuple2<String, Integer>> entry = iterator.next(); int num = entry.getValue()._2; if (num == 0) { UrlPostMethod.urlPostMethod(RECOVER, entry.getValue()._1); iterator.remove(); } else if (num < 0) { num = 0 - num; entry.setValue(new Tuple2<String, Integer>(entry.getValue()._1, num)); } else { num = 1 - num; if (num == 0) { UrlPostMethod.urlPostMethod(RECOVER, entry.getValue()._1); iterator.remove(); } else { entry.setValue(new Tuple2<String, Integer>(entry.getValue()._1, num)); } } } if (stats.size() > 200000) { stats.clear(); } if (jsonList.size() > 0) { if (Boolean.parseBoolean(getConfig("alert.cnf").getProperty("save.es"))) { System.out.println( "-------------------" + jsonList.toString() + "-----------------------"); } for (int i = 0; i <= jsonList.size() / 2000; i++) { UrlPostMethod.urlPostMethod(URL, "warnings=" + Arrays.asList(Arrays.copyOfRange( jsonList.toArray(), i * 2000, (i + 1) * 2000 - 1 > jsonList.size() ? jsonList.size() : (i + 1) * 2000 - 1)) .toString()); } jsonList.clear(); } } return null; } }); rawStream.context().start(); rawStream.context().awaitTermination(); }
From source file:Main.java
public static boolean beforeCalender(Calendar a, Calendar b) { return b.getTime().getTime() - a.getTime().getTime() > -1; }
From source file:Main.java
public static Date dateFromCalendar(Calendar calendar) { return calendar.getTime(); }