List of usage examples for java.util Calendar add
public abstract void add(int field, int amount);
From source file:com.microsoft.windowsazure.services.media.samples.contentprotection.playreadywidevine.Program.java
public static void main(String[] args) { try {/*from ww w.ja v a2 s. c o m*/ // Set up the MediaContract object to call into the Media Services account Configuration configuration = MediaConfiguration.configureWithOAuthAuthentication(mediaServiceUri, oAuthUri, clientId, clientSecret, scope); mediaService = MediaService.create(configuration); System.out.println("Azure SDK for Java - PlayReady & Widevine Dynamic Encryption Sample"); // Upload a local file to a media asset. AssetInfo uploadAsset = uploadFileAndCreateAsset("Azure-Video.wmv"); System.out.println("Uploaded Asset Id: " + uploadAsset.getId()); // Transform the asset. AssetInfo encodedAsset = encode(uploadAsset); System.out.println("Encoded Asset Id: " + encodedAsset.getId()); // Create the ContentKey ContentKeyInfo contentKeyInfo = createCommonTypeContentKey(encodedAsset); System.out.println("Common Encryption Content Key: " + contentKeyInfo.getId()); // Create the ContentKeyAuthorizationPolicy String tokenTemplateString = null; if (tokenRestriction) { tokenTemplateString = addTokenRestrictedAuthorizationPolicy(contentKeyInfo, tokenType); } else { addOpenAuthorizationPolicy(contentKeyInfo); } // Create the AssetDeliveryPolicy createAssetDeliveryPolicy(encodedAsset, contentKeyInfo); if (tokenTemplateString != null) { // Deserializes a string containing the XML representation of the TokenRestrictionTemplate TokenRestrictionTemplate tokenTemplate = TokenRestrictionTemplateSerializer .deserialize(tokenTemplateString); // Generate a test token based on the the data in the given // TokenRestrictionTemplate. // Note: You need to pass the key id Guid because we specified // TokenClaim.ContentKeyIdentifierClaim in during the creation // of TokenRestrictionTemplate. UUID rawKey = UUID.fromString(contentKeyInfo.getId().substring("nb:kid:UUID:".length())); // Token expiration: 1-year Calendar date = Calendar.getInstance(); date.setTime(new Date()); date.add(Calendar.YEAR, 1); // Generate token String testToken = TokenRestrictionTemplateSerializer.generateTestToken(tokenTemplate, null, rawKey, date.getTime(), null); System.out.println(tokenTemplate.getTokenType().toString() + " Test Token: Bearer " + testToken); } // Create the Streaming Origin Locator String url = getStreamingOriginLocator(encodedAsset); System.out.println("Origin Locator Url: " + url); System.out.println("Sample completed!"); } catch (ServiceException se) { System.out.println("ServiceException encountered."); System.out.println(se.toString()); } catch (Exception e) { System.out.println("Exception encountered."); System.out.println(e.toString()); } }
From source file:Main.java
public static void main(String[] args) { Calendar calendar = Calendar.getInstance(); // 15-04-2012 calendar.set(Calendar.DAY_OF_MONTH, 15); calendar.set(Calendar.YEAR, 2012); calendar.set(Calendar.HOUR, 0); calendar.set(Calendar.MINUTE, 0); calendar.set(Calendar.SECOND, 0); calendar.set(Calendar.MONTH, 3); Date start = calendar.getTime(); // 15-06-2012 calendar.set(Calendar.MONTH, 5); Date end = calendar.getTime(); calendar.setTime(start);/*from w ww. ja v a 2 s . co m*/ Date d = null; while ((d = calendar.getTime()).before(end) || d.equals(end)) { int day = calendar.get(Calendar.DAY_OF_WEEK); if (day != Calendar.SATURDAY && day != Calendar.SUNDAY) { System.out.println(d); } calendar.add(Calendar.DAY_OF_MONTH, 1); } }
From source file:search.java
public static void main(String argv[]) { int optind;// w ww. j a v a 2 s . c o m String subject = null; String from = null; boolean or = false; boolean today = false; int size = -1; for (optind = 0; optind < argv.length; optind++) { if (argv[optind].equals("-T")) { protocol = argv[++optind]; } else if (argv[optind].equals("-H")) { host = argv[++optind]; } else if (argv[optind].equals("-U")) { user = argv[++optind]; } else if (argv[optind].equals("-P")) { password = argv[++optind]; } else if (argv[optind].equals("-or")) { or = true; } else if (argv[optind].equals("-D")) { debug = true; } else if (argv[optind].equals("-f")) { mbox = argv[++optind]; } else if (argv[optind].equals("-L")) { url = argv[++optind]; } else if (argv[optind].equals("-subject")) { subject = argv[++optind]; } else if (argv[optind].equals("-from")) { from = argv[++optind]; } else if (argv[optind].equals("-today")) { today = true; } else if (argv[optind].equals("-size")) { size = Integer.parseInt(argv[++optind]); } else if (argv[optind].equals("--")) { optind++; break; } else if (argv[optind].startsWith("-")) { System.out.println("Usage: search [-D] [-L url] [-T protocol] [-H host] " + "[-U user] [-P password] [-f mailbox] " + "[-subject subject] [-from from] [-or] [-today]"); System.exit(1); } else { break; } } try { if ((subject == null) && (from == null) && !today && size < 0) { System.out.println("Specify either -subject, -from, -today, or -size"); System.exit(1); } // Get a Properties object Properties props = System.getProperties(); // Get a Session object Session session = Session.getInstance(props, null); session.setDebug(debug); // Get a Store object Store store = null; if (url != null) { URLName urln = new URLName(url); store = session.getStore(urln); store.connect(); } else { if (protocol != null) store = session.getStore(protocol); else store = session.getStore(); // Connect if (host != null || user != null || password != null) store.connect(host, user, password); else store.connect(); } // Open the Folder Folder folder = store.getDefaultFolder(); if (folder == null) { System.out.println("Cant find default namespace"); System.exit(1); } folder = folder.getFolder(mbox); if (folder == null) { System.out.println("Invalid folder"); System.exit(1); } folder.open(Folder.READ_ONLY); SearchTerm term = null; if (subject != null) term = new SubjectTerm(subject); if (from != null) { FromStringTerm fromTerm = new FromStringTerm(from); if (term != null) { if (or) term = new OrTerm(term, fromTerm); else term = new AndTerm(term, fromTerm); } else term = fromTerm; } if (today) { Calendar c = Calendar.getInstance(); c.set(Calendar.HOUR, 0); c.set(Calendar.MINUTE, 0); c.set(Calendar.SECOND, 0); c.set(Calendar.MILLISECOND, 0); c.set(Calendar.AM_PM, Calendar.AM); ReceivedDateTerm startDateTerm = new ReceivedDateTerm(ComparisonTerm.GE, c.getTime()); c.add(Calendar.DATE, 1); // next day ReceivedDateTerm endDateTerm = new ReceivedDateTerm(ComparisonTerm.LT, c.getTime()); SearchTerm dateTerm = new AndTerm(startDateTerm, endDateTerm); if (term != null) { if (or) term = new OrTerm(term, dateTerm); else term = new AndTerm(term, dateTerm); } else term = dateTerm; } if (size >= 0) { SizeTerm sizeTerm = new SizeTerm(ComparisonTerm.GT, size); if (term != null) { if (or) term = new OrTerm(term, sizeTerm); else term = new AndTerm(term, sizeTerm); } else term = sizeTerm; } Message[] msgs = folder.search(term); System.out.println("FOUND " + msgs.length + " MESSAGES"); if (msgs.length == 0) // no match System.exit(1); // Use a suitable FetchProfile FetchProfile fp = new FetchProfile(); fp.add(FetchProfile.Item.ENVELOPE); folder.fetch(msgs, fp); for (int i = 0; i < msgs.length; i++) { System.out.println("--------------------------"); System.out.println("MESSAGE #" + (i + 1) + ":"); dumpPart(msgs[i]); } folder.close(false); store.close(); } catch (Exception ex) { System.out.println("Oops, got exception! " + ex.getMessage()); ex.printStackTrace(); } System.exit(1); }
From source file:io.druid.examples.rabbitmq.RabbitMQProducerMain.java
public static void main(String[] args) throws Exception { // We use a List to keep track of option insertion order. See below. final List<Option> optionList = new ArrayList<Option>(); optionList.add(OptionBuilder.withLongOpt("help").withDescription("display this help message").create("h")); optionList.add(OptionBuilder.withLongOpt("hostname").hasArg() .withDescription("the hostname of the AMQP broker [defaults to AMQP library default]").create("b")); optionList.add(OptionBuilder.withLongOpt("port").hasArg() .withDescription("the port of the AMQP broker [defaults to AMQP library default]").create("n")); optionList.add(OptionBuilder.withLongOpt("username").hasArg() .withDescription("username to connect to the AMQP broker [defaults to AMQP library default]") .create("u")); optionList.add(OptionBuilder.withLongOpt("password").hasArg() .withDescription("password to connect to the AMQP broker [defaults to AMQP library default]") .create("p")); optionList.add(OptionBuilder.withLongOpt("vhost").hasArg() .withDescription("name of virtual host on the AMQP broker [defaults to AMQP library default]") .create("v")); optionList.add(OptionBuilder.withLongOpt("exchange").isRequired().hasArg() .withDescription("name of the AMQP exchange [required - no default]").create("e")); optionList.add(OptionBuilder.withLongOpt("key").hasArg() .withDescription("the routing key to use when sending messages [default: 'default.routing.key']") .create("k")); optionList.add(OptionBuilder.withLongOpt("type").hasArg() .withDescription("the type of exchange to create [default: 'topic']").create("t")); optionList.add(OptionBuilder.withLongOpt("durable") .withDescription("if set, a durable exchange will be declared [default: not set]").create("d")); optionList.add(OptionBuilder.withLongOpt("autodelete") .withDescription("if set, an auto-delete exchange will be declared [default: not set]") .create("a")); optionList.add(OptionBuilder.withLongOpt("single") .withDescription("if set, only a single message will be sent [default: not set]").create("s")); optionList.add(OptionBuilder.withLongOpt("start").hasArg() .withDescription("time to use to start sending messages from [default: 2010-01-01T00:00:00]") .create());/*from www. ja v a 2 s .co m*/ optionList.add(OptionBuilder.withLongOpt("stop").hasArg().withDescription( "time to use to send messages until (format: '2013-07-18T23:45:59') [default: current time]") .create()); optionList.add(OptionBuilder.withLongOpt("interval").hasArg() .withDescription("the interval to add to the timestamp between messages in seconds [default: 10]") .create()); optionList.add(OptionBuilder.withLongOpt("delay").hasArg() .withDescription("the delay between sending messages in milliseconds [default: 100]").create()); // An extremely silly hack to maintain the above order in the help formatting. HelpFormatter formatter = new HelpFormatter(); // Add a comparator to the HelpFormatter using the ArrayList above to sort by insertion order. formatter.setOptionComparator(new Comparator() { @Override public int compare(Object o1, Object o2) { // I know this isn't fast, but who cares! The list is short. return optionList.indexOf(o1) - optionList.indexOf(o2); } }); // Now we can add all the options to an Options instance. This is dumb! Options options = new Options(); for (Option option : optionList) { options.addOption(option); } CommandLine cmd = null; try { cmd = new BasicParser().parse(options, args); } catch (ParseException e) { formatter.printHelp("RabbitMQProducerMain", e.getMessage(), options, null); System.exit(1); } if (cmd.hasOption("h")) { formatter.printHelp("RabbitMQProducerMain", options); System.exit(2); } ConnectionFactory factory = new ConnectionFactory(); if (cmd.hasOption("b")) { factory.setHost(cmd.getOptionValue("b")); } if (cmd.hasOption("u")) { factory.setUsername(cmd.getOptionValue("u")); } if (cmd.hasOption("p")) { factory.setPassword(cmd.getOptionValue("p")); } if (cmd.hasOption("v")) { factory.setVirtualHost(cmd.getOptionValue("v")); } if (cmd.hasOption("n")) { factory.setPort(Integer.parseInt(cmd.getOptionValue("n"))); } String exchange = cmd.getOptionValue("e"); String routingKey = "default.routing.key"; if (cmd.hasOption("k")) { routingKey = cmd.getOptionValue("k"); } boolean durable = cmd.hasOption("d"); boolean autoDelete = cmd.hasOption("a"); String type = cmd.getOptionValue("t", "topic"); boolean single = cmd.hasOption("single"); int interval = Integer.parseInt(cmd.getOptionValue("interval", "10")); int delay = Integer.parseInt(cmd.getOptionValue("delay", "100")); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); Date stop = sdf.parse(cmd.getOptionValue("stop", sdf.format(new Date()))); Random r = new Random(); Calendar timer = Calendar.getInstance(); timer.setTime(sdf.parse(cmd.getOptionValue("start", "2010-01-01T00:00:00"))); String msg_template = "{\"utcdt\": \"%s\", \"wp\": %d, \"gender\": \"%s\", \"age\": %d}"; Connection connection = factory.newConnection(); Channel channel = connection.createChannel(); channel.exchangeDeclare(exchange, type, durable, autoDelete, null); do { int wp = (10 + r.nextInt(90)) * 100; String gender = r.nextBoolean() ? "male" : "female"; int age = 20 + r.nextInt(70); String line = String.format(msg_template, sdf.format(timer.getTime()), wp, gender, age); channel.basicPublish(exchange, routingKey, null, line.getBytes()); System.out.println("Sent message: " + line); timer.add(Calendar.SECOND, interval); Thread.sleep(delay); } while ((!single && stop.after(timer.getTime()))); connection.close(); }
From source file:fr.itinerennes.bundler.integration.onebusaway.GenerateStaticObaApiResults.java
public static void main(String[] args) throws IOException, GtfsException { final String url = args[0]; final String key = args[1]; final String gtfsFile = args[2]; final String out = args[3].replaceAll("/$", ""); final Map<String, String> agencyMapping = new HashMap<String, String>(); agencyMapping.put("1", "2"); final GtfsDao gtfs = GtfsUtils.load(new File(gtfsFile), agencyMapping); oba = new JsonOneBusAwayClient(new DefaultHttpClient(), url, key); gson = OneBusAwayGsonFactory.newInstance(true); final Calendar end = Calendar.getInstance(); end.set(2013, 11, 22, 0, 0);//from ww w . j a va2 s .c o m final Calendar start = Calendar.getInstance(); start.set(2013, 10, 4, 0, 0); final Calendar current = Calendar.getInstance(); current.setTime(start.getTime()); while (current.before(end) || current.equals(end)) { System.out.println(current.getTime()); for (final Stop stop : gtfs.getAllStops()) { final String stopId = stop.getId().toString(); final String dateDir = String.format("%04d/%02d/%02d", current.get(Calendar.YEAR), current.get(Calendar.MONTH) + 1, current.get(Calendar.DAY_OF_MONTH)); final String methodDir = String.format("%s/schedule-for-stop", out); final File outDir = new File(String.format("%s/%s", methodDir, dateDir)); outDir.mkdirs(); final File f = new File(outDir, String.format("%s.json", stopId)); final StopSchedule ss = oba.getScheduleForStop(stopId, current.getTime()); final String json = gson.toJson(ss); final Writer w = new PrintWriter(f); w.write(json); w.close(); } current.add(Calendar.DAY_OF_MONTH, 1); } final File outDir = new File(String.format("%s/trip-details", out)); outDir.mkdirs(); for (final Trip trip : gtfs.getAllTrips()) { final String tripId = trip.getId().toString(); final File f = new File(outDir, String.format("%s.json", tripId)); final TripSchedule ts = oba.getTripDetails(tripId); final String json = gson.toJson(ts); final Writer w = new PrintWriter(f); w.write(json); w.close(); } }
From source file:com.adobe.aem.demo.communities.Loader.java
public static void main(String[] args) { String hostname = null;//from w w w . ja v a2 s. c o m String port = null; String altport = null; String csvfile = null; String location = null; String language = "en"; String analytics = null; String adminPassword = "admin"; String[] url = new String[10]; // Handling 10 levels maximum for nested comments boolean reset = false; boolean configure = false; int urlLevel = 0; int row = 0; HashMap<String, ArrayList<String>> learningpaths = new HashMap<String, ArrayList<String>>(); // Command line options for this tool Options options = new Options(); options.addOption("h", true, "Hostname"); options.addOption("p", true, "Port"); options.addOption("a", true, "Alternate Port"); options.addOption("f", true, "CSV file"); options.addOption("r", false, "Reset"); options.addOption("u", true, "Admin Password"); options.addOption("c", false, "Configure"); options.addOption("s", true, "Analytics Endpoint"); options.addOption("t", false, "Analytics Tracking"); CommandLineParser parser = new BasicParser(); try { CommandLine cmd = parser.parse(options, args); if (cmd.hasOption("h")) { hostname = cmd.getOptionValue("h"); } if (cmd.hasOption("p")) { port = cmd.getOptionValue("p"); } if (cmd.hasOption("a")) { altport = cmd.getOptionValue("a"); } if (cmd.hasOption("f")) { csvfile = cmd.getOptionValue("f"); } if (cmd.hasOption("u")) { adminPassword = cmd.getOptionValue("u"); } if (cmd.hasOption("t")) { if (cmd.hasOption("s")) { analytics = cmd.getOptionValue("s"); } } if (cmd.hasOption("r")) { reset = true; } if (cmd.hasOption("c")) { configure = true; } if (csvfile == null || port == null || hostname == null) { System.out.println( "Request parameters: -h hostname -p port -a alternateport -u adminPassword -f path_to_CSV_file -r (true|false, delete content before import) -c (true|false, post additional properties)"); System.exit(-1); } } catch (ParseException ex) { logger.error(ex.getMessage()); } String componentType = null; try { logger.debug("AEM Demo Loader: Processing file " + csvfile); // Reading the CSV file, line by line Reader in = new FileReader(csvfile); Iterable<CSVRecord> records = CSVFormat.EXCEL.parse(in); for (CSVRecord record : records) { row = row + 1; logger.info("Row: " + row + ", new record: " + record.get(0)); // Let's see if we deal with a comment if (record.get(0).startsWith("#")) { // We can ignore the comment line and move on continue; } // Let's see if we need to terminate this process if (record.get(0).equals(KILL)) { System.exit(1); } // Let's see if we need to create a new Community site if (record.get(0).equals(SITE)) { // Building the form entity to be posted MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.addTextBody(":operation", "social:createSite", ContentType.create("text/plain", MIME.UTF8_CHARSET)); builder.addTextBody("_charset_", "UTF-8", ContentType.create("text/plain", MIME.UTF8_CHARSET)); String urlName = null; for (int i = 2; i < record.size() - 1; i = i + 2) { if (record.get(i) != null && record.get(i + 1) != null && record.get(i).length() > 0) { String name = record.get(i).trim(); String value = record.get(i + 1).trim(); if (value.equals("TRUE")) { value = "true"; } if (value.equals("FALSE")) { value = "false"; } if (name.equals("urlName")) { urlName = value; } if (name.equals(LANGUAGE)) { language = value; } if (name.equals(BANNER)) { File attachment = new File( csvfile.substring(0, csvfile.indexOf(".csv")) + File.separator + value); builder.addBinaryBody(BANNER, attachment, ContentType.MULTIPART_FORM_DATA, attachment.getName()); } else if (name.equals(THUMBNAIL)) { File attachment = new File( csvfile.substring(0, csvfile.indexOf(".csv")) + File.separator + value); builder.addBinaryBody(THUMBNAIL, attachment, ContentType.MULTIPART_FORM_DATA, attachment.getName()); } else { builder.addTextBody(name, value, ContentType.create("text/plain", MIME.UTF8_CHARSET)); } } } // Site creation String siteId = doPost(hostname, port, "/content.social.json", "admin", adminPassword, builder.build(), "response/siteId"); // Site publishing, if there's a publish instance to publish to if (!port.equals(altport)) { List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(); nameValuePairs.add(new BasicNameValuePair("id", "nobot")); nameValuePairs.add(new BasicNameValuePair(":operation", "social:publishSite")); nameValuePairs .add(new BasicNameValuePair("path", "/content/sites/" + urlName + "/" + language)); doPost(hostname, port, "/communities/sites.html", "admin", adminPassword, new UrlEncodedFormEntity(nameValuePairs), null); // Wait for site to be available on Publish doWait(hostname, altport, "admin", adminPassword, (siteId != null ? siteId : urlName) + "-groupadministrators"); } continue; } // Let's see if we need to create a new Tag if (record.get(0).equals(TAG)) { // Building the form entity to be posted MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.addTextBody("_charset_", "UTF-8", ContentType.create("text/plain", MIME.UTF8_CHARSET)); for (int i = 1; i < record.size() - 1; i = i + 2) { if (record.get(i) != null && record.get(i + 1) != null && record.get(i).length() > 0 && record.get(i + 1).length() > 0) { String name = record.get(i).trim(); String value = record.get(i + 1).trim(); builder.addTextBody(name, value, ContentType.create("text/plain", MIME.UTF8_CHARSET)); } } // Tag creation doPost(hostname, port, "/bin/tagcommand", "admin", adminPassword, builder.build(), null); continue; } // Let's see if we need to create a new Community site template, and if we can do it (script run against author instance) if (record.get(0).equals(SITETEMPLATE)) { // Building the form entity to be posted MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.addTextBody(":operation", "social:createSiteTemplate", ContentType.create("text/plain", MIME.UTF8_CHARSET)); builder.addTextBody("_charset_", "UTF-8", ContentType.create("text/plain", MIME.UTF8_CHARSET)); for (int i = 2; i < record.size() - 1; i = i + 2) { if (record.get(i) != null && record.get(i + 1) != null && record.get(i).length() > 0) { String name = record.get(i).trim(); String value = record.get(i + 1).trim(); builder.addTextBody(name, value, ContentType.create("text/plain", MIME.UTF8_CHARSET)); } } // Site template creation doPost(hostname, port, "/content.social.json", "admin", adminPassword, builder.build(), null); continue; } // Let's see if we need to create a new Community group if (record.get(0).equals(GROUP)) { // Building the form entity to be posted MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.addTextBody(":operation", "social:createCommunityGroup", ContentType.create("text/plain", MIME.UTF8_CHARSET)); builder.addTextBody("_charset_", "UTF-8", ContentType.create("text/plain", MIME.UTF8_CHARSET)); for (int i = 3; i < record.size() - 1; i = i + 2) { if (record.get(i) != null && record.get(i + 1) != null && record.get(i).length() > 0) { String name = record.get(i).trim(); String value = record.get(i + 1).trim(); if (value.equals("TRUE")) { value = "true"; } if (value.equals("FALSE")) { value = "false"; } if (name.equals(IMAGE)) { File attachment = new File( csvfile.substring(0, csvfile.indexOf(".csv")) + File.separator + value); builder.addBinaryBody(IMAGE, attachment, ContentType.MULTIPART_FORM_DATA, attachment.getName()); } else { builder.addTextBody(name, value, ContentType.create("text/plain", MIME.UTF8_CHARSET)); } } } // Group creation String memberGroupId = doPost(hostname, port, record.get(1), getUserName(record.get(2)), getPassword(record.get(2), adminPassword), builder.build(), "response/memberGroupId"); // Wait for group to be available on Publish, if available logger.debug("Waiting for completion of Community Group creation"); doWait(hostname, port, "admin", adminPassword, memberGroupId); continue; } // Let's see if it's simple Sling Delete request if (record.get(0).equals(SLINGDELETE)) { doDelete(hostname, port, record.get(1), "admin", adminPassword); continue; } // Let's see if we need to add users to an AEM Group if ((record.get(0).equals(GROUPMEMBERS) || record.get(0).equals(SITEMEMBERS)) && record.get(GROUP_INDEX_NAME) != null) { // Checking if we have a member group for this site String groupName = record.get(GROUP_INDEX_NAME); if (record.get(0).equals(SITEMEMBERS)) { // Let's fetch the siteId for this Community Site Url String siteConfig = doGet(hostname, port, groupName, "admin", adminPassword, null); try { String siteId = new JSONObject(siteConfig).getString("siteId"); if (siteId != null) groupName = "community-" + siteId + "-members"; logger.debug("Member group name is " + groupName); } catch (Exception e) { logger.error(e.getMessage()); } } // Pause until the group can found doWait(hostname, port, "admin", adminPassword, groupName); List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(); nameValuePairs.add(new BasicNameValuePair("filter", "[{\"operation\":\"like\",\"rep:principalName\":\"" + groupName + "\"}]")); nameValuePairs.add(new BasicNameValuePair("type", "groups")); String groupList = doGet(hostname, port, "/libs/social/console/content/content/userlist.social.0.10.json", "admin", adminPassword, nameValuePairs); logger.debug("List of groups" + groupList); if (groupList.indexOf(groupName) > 0) { logger.debug("Group was found on " + port); try { JSONArray jsonArray = new JSONObject(groupList).getJSONArray("items"); if (jsonArray.length() == 1) { JSONObject jsonObject = jsonArray.getJSONObject(0); String groupPath = jsonObject.getString("path"); logger.debug("Group path is " + groupPath); // Constructing a multi-part POST for group membership MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); List<NameValuePair> groupNameValuePairs = buildNVP(record, 2); for (NameValuePair nameValuePair : groupNameValuePairs) { builder.addTextBody(nameValuePair.getName(), nameValuePair.getValue(), ContentType.create("text/plain", MIME.UTF8_CHARSET)); } // Adding the list of group members doPost(hostname, port, groupPath + ".rw.userprops.html", "admin", adminPassword, builder.build(), null); } else { logger.info("We have more than one match for a group with this name!"); } } catch (Exception e) { logger.error(e.getMessage()); } } continue; } // Let's see if it's user related if (record.get(0).equals(USERS)) { //First we need to get the path to the user node String json = doGet(hostname, port, "/libs/granite/security/currentuser.json", getUserName(record.get(1)), getPassword(record.get(1), adminPassword), null); if (json != null) { try { // Fetching the home property String home = new JSONObject(json).getString("home"); if (record.get(2).equals(PREFERENCES)) { home = home + "/preferences"; } else { home = home + "/profile"; } logger.debug(home); // Now we can post all the preferences or the profile List<NameValuePair> nameValuePairs = buildNVP(record, 3); doPost(hostname, port, home, "admin", adminPassword, new UrlEncodedFormEntity(nameValuePairs), null); } catch (Exception e) { logger.error(e.getMessage()); } } continue; } // Let's see if we deal with a new block of content or just a new entry if (record.get(0).equals(CALENDAR) || record.get(0).equals(SLINGPOST) || record.get(0).equals(RATINGS) || record.get(0).equals(BLOG) || record.get(0).equals(JOURNAL) || record.get(0).equals(COMMENTS) || record.get(0).equals(REVIEWS) || record.get(0).equals(FILES) || record.get(0).equals(SUMMARY) || record.get(0).equals(ACTIVITIES) || record.get(0).equals(JOIN) || record.get(0).equals(FOLLOW) || record.get(0).equals(MESSAGE) || record.get(0).equals(ASSET) || record.get(0).equals(AVATAR) || record.get(0).equals(RESOURCE) || record.get(0).equals(LEARNING) || record.get(0).equals(QNA) || record.get(0).equals(FORUM)) { // New block of content, we need to reset the processing to first Level componentType = record.get(0); url[0] = record.get(1); urlLevel = 0; if (!componentType.equals(SLINGPOST) && reset) { int pos = record.get(1).indexOf("/jcr:content"); if (pos > 0) doDelete(hostname, port, "/content/usergenerated" + record.get(1).substring(0, pos), "admin", adminPassword); } // If the Configure command line flag is set, we try to configure the component with all options enabled if (componentType.equals(SLINGPOST) || configure) { String configurePath = getConfigurePath(record.get(1)); List<NameValuePair> nameValuePairs = buildNVP(record, 2); if (nameValuePairs.size() > 2) // Only do this when really have configuration settings doPost(hostname, port, configurePath, "admin", adminPassword, new UrlEncodedFormEntity(nameValuePairs), null); } // We're done with this line, moving on to the next line in the CSV file continue; } // Let's see if we need to indent the list, if it's a reply or a reply to a reply if (record.get(1).length() != 1) continue; // We need a valid level indicator if (Integer.parseInt(record.get(1)) > urlLevel) { url[++urlLevel] = location; logger.debug("Incremented urlLevel to: " + urlLevel + ", with a new location:" + location); } else if (Integer.parseInt(record.get(1)) < urlLevel) { urlLevel = Integer.parseInt(record.get(1)); logger.debug("Decremented urlLevel to: " + urlLevel); } // Get the credentials or fall back to password String password = getPassword(record.get(0), adminPassword); String userName = getUserName(record.get(0)); // Adding the generic properties for all POST requests List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(); if (!componentType.equals(RESOURCE)) nameValuePairs.add(new BasicNameValuePair("id", "nobot")); nameValuePairs.add(new BasicNameValuePair("_charset_", "UTF-8")); // Setting some specific fields depending on the content type if (componentType.equals(COMMENTS)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createComment")); nameValuePairs.add(new BasicNameValuePair("message", record.get(2))); } // Creates a forum post (or reply) if (componentType.equals(FORUM)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createForumPost")); nameValuePairs.add(new BasicNameValuePair("subject", record.get(2))); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); } // Follows a user (followedId) for the user posting the request if (componentType.equals(FOLLOW)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:follow")); nameValuePairs.add(new BasicNameValuePair("userId", "/social/authors/" + userName)); nameValuePairs.add(new BasicNameValuePair("followedId", "/social/authors/" + record.get(2))); } // Uploading Avatar picture if (componentType.equals(AVATAR)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:changeAvatar")); } // Joins a user (posting the request) to a Community Group (path) if (componentType.equals(JOIN)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:joinCommunityGroup")); int pos = url[0].indexOf("/configuration.social.json"); if (pos > 0) nameValuePairs.add(new BasicNameValuePair("path", url[0].substring(0, pos) + ".html")); else continue; // Invalid record } // Creates a new private message if (componentType.equals(MESSAGE)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createMessage")); nameValuePairs.add(new BasicNameValuePair("sendMail", "Sending...")); nameValuePairs.add(new BasicNameValuePair("content", record.get(4))); nameValuePairs.add(new BasicNameValuePair("subject", record.get(3))); nameValuePairs.add(new BasicNameValuePair("serviceSelector", "/bin/community")); nameValuePairs.add(new BasicNameValuePair("to", "/social/authors/" + record.get(2))); nameValuePairs.add(new BasicNameValuePair("userId", "/social/authors/" + record.get(2))); nameValuePairs.add(new BasicNameValuePair(":redirect", "//messaging.html")); nameValuePairs.add(new BasicNameValuePair(":formid", "generic_form")); nameValuePairs.add(new BasicNameValuePair(":formstart", "/content/sites/communities/messaging/compose/jcr:content/content/primary/start")); } // Creates a file or a folder if (componentType.equals(FILES)) { // Top level is always assumed to be a folder, second level files, and third and subsequent levels comments on files if (urlLevel == 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createFileLibraryFolder")); nameValuePairs.add(new BasicNameValuePair("name", record.get(2))); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); } else if (urlLevel == 1) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createComment")); } } // Creates a question, a reply or mark a reply as the best answer if (componentType.equals(QNA)) { if (urlLevel == 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createQnaPost")); nameValuePairs.add(new BasicNameValuePair("subject", record.get(2))); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); } else if (urlLevel == 1) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createQnaPost")); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); } else if (urlLevel == 2) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:selectAnswer")); } } // Creates an article or a comment if (componentType.equals(JOURNAL) || componentType.equals(BLOG)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createJournalComment")); nameValuePairs.add(new BasicNameValuePair("subject", record.get(2))); StringBuffer message = new StringBuffer("<p>" + record.get(3) + "</p>"); //We might have more paragraphs to add to the blog or journal article for (int i = 6; i < record.size(); i++) { if (record.get(i).length() > 0) { message.append("<p>" + record.get(i) + "</p>"); } } //We might have some tags to add to the blog or journal article if (record.get(5).length() > 0) { nameValuePairs.add(new BasicNameValuePair("tags", record.get(5))); } nameValuePairs.add(new BasicNameValuePair("message", message.toString())); } // Creates a review or a comment if (componentType.equals(REVIEWS)) { nameValuePairs.add(new BasicNameValuePair("message", record.get(2))); // This might be a top level review, or a comment on a review or another comment if (urlLevel == 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createReview")); nameValuePairs.add(new BasicNameValuePair("ratings", record.get(3))); if (record.size() > 4 && record.get(4).length() > 0) { // If we are dealing with a non-existent resource, then the design drives the behavior nameValuePairs.add(new BasicNameValuePair("scf:resourceType", "social/reviews/components/hbs/reviews")); nameValuePairs.add(new BasicNameValuePair("scf:included", record.get(4))); } } else { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createComment")); } } // Creates a rating if (componentType.equals(RATINGS)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:postTallyResponse")); nameValuePairs.add(new BasicNameValuePair("tallyType", "Rating")); nameValuePairs.add(new BasicNameValuePair("response", record.get(2))); } // Creates a DAM asset if (componentType.equals(ASSET) && record.get(ASSET_INDEX_NAME).length() > 0) { nameValuePairs.add(new BasicNameValuePair("fileName", record.get(ASSET_INDEX_NAME))); } // Creates an enablement resource if (componentType.equals(RESOURCE)) { nameValuePairs.add(new BasicNameValuePair(":operation", "se:createResource")); List<NameValuePair> otherNameValuePairs = buildNVP(record, RESOURCE_INDEX_PROPERTIES); nameValuePairs.addAll(otherNameValuePairs); // Adding the site nameValuePairs.add(new BasicNameValuePair("site", "/content/sites/" + record.get(RESOURCE_INDEX_SITE) + "/resources/en")); // Building the cover image fragment if (record.get(RESOURCE_INDEX_THUMBNAIL).length() > 0) { nameValuePairs.add(new BasicNameValuePair("cover-image", doThumbnail(hostname, port, adminPassword, csvfile, record.get(RESOURCE_INDEX_THUMBNAIL)))); } else { nameValuePairs.add(new BasicNameValuePair("cover-image", "")); } // Building the asset fragment String coverPath = "/content/dam/" + record.get(RESOURCE_INDEX_SITE) + "/resource-assets/" + record.get(2) + "/jcr:content/renditions/cq5dam.thumbnail.319.319.png"; String coverSource = "dam"; String assets = "[{\"cover-img-path\":\"" + coverPath + "\",\"thumbnail-source\":\"" + coverSource + "\",\"asset-category\":\"enablementAsset:dam\",\"resource-asset-name\":null,\"state\":\"A\",\"asset-path\":\"/content/dam/" + record.get(RESOURCE_INDEX_SITE) + "/resource-assets/" + record.get(2) + "\"}]"; nameValuePairs.add(new BasicNameValuePair("assets", assets)); logger.debug("assets:" + assets); } // Creates a learning path if (componentType.equals(LEARNING)) { nameValuePairs.add(new BasicNameValuePair(":operation", "se:editLearningPath")); List<NameValuePair> otherNameValuePairs = buildNVP(record, RESOURCE_INDEX_PROPERTIES); nameValuePairs.addAll(otherNameValuePairs); // Adding the site nameValuePairs.add(new BasicNameValuePair("site", "/content/sites/" + record.get(RESOURCE_INDEX_SITE) + "/resources/en")); // Building the cover image fragment if (record.get(RESOURCE_INDEX_THUMBNAIL).length() > 0) { nameValuePairs.add(new BasicNameValuePair("card-image", doThumbnail(hostname, port, adminPassword, csvfile, record.get(RESOURCE_INDEX_THUMBNAIL)))); } // Building the learning path fragment StringBuffer assets = new StringBuffer("[\""); if (learningpaths.get(record.get(2)) != null) { ArrayList<String> paths = learningpaths.get(record.get(2)); int i = 0; for (String path : paths) { assets.append("{\\\"type\\\":\\\"linked-resource\\\",\\\"path\\\":\\\""); assets.append(path); assets.append("\\\"}"); if (i++ < paths.size() - 1) { assets.append("\",\""); } } } else { logger.debug("No asset for this learning path"); } assets.append("\"]"); nameValuePairs.add(new BasicNameValuePair("learningpath-items", assets.toString())); logger.debug("Learning path:" + assets.toString()); } // Creates a calendar event if (componentType.equals(CALENDAR)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createEvent")); try { JSONObject event = new JSONObject(); // Building the JSON fragment for a new calendar event event.accumulate("subject", record.get(2)); event.accumulate("message", record.get(3)); event.accumulate("location", record.get(4)); event.accumulate("tags", ""); event.accumulate("undefined", "update"); String startDate = record.get(5); startDate = startDate.replaceAll("YYYY", Integer.toString(Calendar.getInstance().get(Calendar.YEAR))); startDate = startDate.replaceAll("MM", Integer.toString(1 + Calendar.getInstance().get(Calendar.MONTH))); event.accumulate("start", startDate); String endDate = record.get(6); endDate = endDate.replaceAll("YYYY", Integer.toString(Calendar.getInstance().get(Calendar.YEAR))); endDate = endDate.replaceAll("MM", Integer.toString(1 + Calendar.getInstance().get(Calendar.MONTH))); event.accumulate("end", endDate); nameValuePairs.add(new BasicNameValuePair("event", event.toString())); } catch (Exception ex) { logger.error(ex.getMessage()); } } // Constructing a multi-part POST request MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); for (NameValuePair nameValuePair : nameValuePairs) { builder.addTextBody(nameValuePair.getName(), nameValuePair.getValue(), ContentType.create("text/plain", MIME.UTF8_CHARSET)); } // See if we have attachments for this new post - or some other actions require a form nonetheless if ((componentType.equals(ASSET) || componentType.equals(AVATAR) || componentType.equals(FORUM) || (componentType.equals(JOURNAL)) || componentType.equals(BLOG)) && record.size() > 4 && record.get(ASSET_INDEX_NAME).length() > 0) { File attachment = new File(csvfile.substring(0, csvfile.indexOf(".csv")) + File.separator + record.get(ASSET_INDEX_NAME)); ContentType ct = ContentType.MULTIPART_FORM_DATA; if (record.get(ASSET_INDEX_NAME).indexOf(".mp4") > 0) { ct = ContentType.create("video/mp4", MIME.UTF8_CHARSET); } else if (record.get(ASSET_INDEX_NAME).indexOf(".jpg") > 0 || record.get(ASSET_INDEX_NAME).indexOf(".jpeg") > 0) { ct = ContentType.create("image/jpeg", MIME.UTF8_CHARSET); } else if (record.get(ASSET_INDEX_NAME).indexOf(".png") > 0) { ct = ContentType.create("image/png", MIME.UTF8_CHARSET); } else if (record.get(ASSET_INDEX_NAME).indexOf(".pdf") > 0) { ct = ContentType.create("application/pdf", MIME.UTF8_CHARSET); } else if (record.get(ASSET_INDEX_NAME).indexOf(".zip") > 0) { ct = ContentType.create("application/zip", MIME.UTF8_CHARSET); } builder.addBinaryBody("file", attachment, ct, attachment.getName()); logger.debug("Adding file to payload with name: " + attachment.getName() + " and type: " + ct.getMimeType()); } // If it's a resource or a learning path, we need the path to the resource for subsequent publishing String jsonElement = "location"; if (componentType.equals(RESOURCE)) { jsonElement = "changes/argument"; } if (componentType.equals(LEARNING)) { jsonElement = "path"; } if (componentType.equals(ASSET)) { jsonElement = null; } // This call generally returns the path to the content fragment that was just created location = Loader.doPost(hostname, port, url[urlLevel], userName, password, builder.build(), jsonElement); // If we are loading a DAM asset, we are waiting for all renditions to be generated before proceeding if (componentType.equals(ASSET)) { int pathIndex = url[urlLevel].lastIndexOf(".createasset.html"); if (pathIndex > 0) doWaitPath(hostname, port, adminPassword, url[urlLevel].substring(0, pathIndex) + "/" + record.get(ASSET_INDEX_NAME) + "/jcr:content/renditions", "nt:file"); } // Let's see if it needs to be added to a learning path if (componentType.equals(RESOURCE) && record.get(RESOURCE_INDEX_PATH).length() > 0 && location != null) { // Adding the location to a list of a resources for this particular Learning Path if (learningpaths.get(record.get(RESOURCE_INDEX_PATH)) == null) learningpaths.put(record.get(RESOURCE_INDEX_PATH), new ArrayList<String>()); logger.debug("Adding resource to Learning path: " + record.get(RESOURCE_INDEX_PATH)); ArrayList<String> locations = learningpaths.get(record.get(RESOURCE_INDEX_PATH)); locations.add(location); learningpaths.put(record.get(RESOURCE_INDEX_PATH), locations); } // If it's a Learning Path, we publish it when possible if (componentType.equals(LEARNING) && !port.equals(altport) && location != null) { // Publishing the learning path List<NameValuePair> publishNameValuePairs = new ArrayList<NameValuePair>(); publishNameValuePairs.add(new BasicNameValuePair(":operation", "se:publishEnablementContent")); publishNameValuePairs.add(new BasicNameValuePair("replication-action", "activate")); logger.debug("Publishing a learning path from: " + location); Loader.doPost(hostname, port, location, userName, password, new UrlEncodedFormEntity(publishNameValuePairs), null); // Waiting for the learning path to be published Loader.doWait(hostname, altport, "admin", adminPassword, location.substring(1 + location.lastIndexOf("/")) // Only search for groups with the learning path in it ); // Decorate the resources within the learning path with comments and ratings, randomly generated ArrayList<String> paths = learningpaths.get(record.get(2)); for (String path : paths) { doDecorate(hostname, altport, path, record, analytics); } } // If it's an Enablement Resource, a lot of things need to happen... // Step 1. If it's a SCORM resource, we wait for the SCORM metadata workflow to be complete before proceeding // Step 2. We publish the resource // Step 3. We set a new first published date on the resource (3 weeks earlier) so that reporting data is more meaningful // Step 4. We wait for the resource to be available on publish (checking that associated groups are available) // Step 5. We retrieve the json for the resource on publish to retrieve the Social endpoints // Step 6. We post ratings and comments for each of the enrollees on publish if (componentType.equals(RESOURCE) && !port.equals(altport) && location != null) { // Wait for the data to be fully copied doWaitPath(hostname, port, adminPassword, location + "/assets/asset", "nt:file"); // If we are dealing with a SCORM asset, we wait a little bit before publishing the resource to that the SCORM workflow is completed if (record.get(2).indexOf(".zip") > 0) { doSleep(10000, "SCORM Resource, waiting for workflow to complete"); } // Publishing the resource List<NameValuePair> publishNameValuePairs = new ArrayList<NameValuePair>(); publishNameValuePairs.add(new BasicNameValuePair(":operation", "se:publishEnablementContent")); publishNameValuePairs.add(new BasicNameValuePair("replication-action", "activate")); logger.debug("Publishing a resource from: " + location); Loader.doPost(hostname, port, location, userName, password, new UrlEncodedFormEntity(publishNameValuePairs), null); // Waiting for the resource to be published Loader.doWait(hostname, altport, "admin", adminPassword, location.substring(1 + location.lastIndexOf("/")) // Only search for groups with the resource path in it ); // Setting the first published timestamp so that reporting always comes with 3 weeks of data after building a new demo instance DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); Calendar cal = Calendar.getInstance(); cal.add(Calendar.DATE, REPORTINGDAYS); List<NameValuePair> publishDateNameValuePairs = new ArrayList<NameValuePair>(); publishDateNameValuePairs .add(new BasicNameValuePair("date-first-published", dateFormat.format(cal.getTime()))); logger.debug("Setting the publish date for a resource from: " + location); doPost(hostname, port, location, userName, password, new UrlEncodedFormEntity(publishDateNameValuePairs), null); // Adding comments and ratings for this resource doDecorate(hostname, altport, location, record, analytics); } } } catch (IOException e) { logger.error(e.getMessage()); } }
From source file:eu.smartfp7.foursquare.AttendanceCrawler.java
/** * The main takes an undefined number of cities as arguments, then initializes * the specific crawling of all the trending venues of these cities. * The trending venues must have been previously identified using the `DownloadPages` * program.// w ww . j ava 2 s.c om * * Current valid cities are: london, amsterdam, goldcoast, sanfrancisco. * */ public static void main(String[] args) throws Exception { Settings settings = Settings.getInstance(); String folder = settings.getFolder(); // We keep info and error logs, so that we know what happened in case // of incoherence in the time series. Map<String, FileWriter> info_logs = new HashMap<String, FileWriter>(); Map<String, FileWriter> error_logs = new HashMap<String, FileWriter>(); // For each city we monitor, we store the venue IDs that we got from // a previous crawl. Map<String, Collection<String>> city_venues = new HashMap<String, Collection<String>>(); // Contains the epoch time when the last API call has been made for each // venue. Ensures that we get data only once each hour. Map<String, Long> venue_last_call = new HashMap<String, Long>(); // Contains the epoch time when we last checked if time series were broken // for each city. // We do these checks once every day before the batch forecasting begins. Map<String, Long> sanity_checks = new HashMap<String, Long>(); // We also keep in memory the number of checkins for the last hour for // each venue. Map<String, Integer> venue_last_checkin = new HashMap<String, Integer>(); Map<Long, Integer> APICallsCount = new HashMap<Long, Integer>(); DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); int total_venues = 0; long total_calls = 0; long time_spent_on_API = 0; for (String c : args) { settings.checkFileHierarchy(c); city_venues.put(c, loadVenues(c)); total_venues += city_venues.get(c).size(); info_logs.put(c, new FileWriter(folder + c + File.separator + "log" + File.separator + "info.log", true)); error_logs.put(c, new FileWriter(folder + c + File.separator + "log" + File.separator + "error.log", true)); Calendar cal = Calendar.getInstance(); info_logs.get(c).write("[" + df.format(cal.getTime()) + "] Crawler initialization for " + c + ". " + city_venues.get(c).size() + " venues loaded.\n"); info_logs.get(c).flush(); // If we interrupted the program for some reason, we can get back // the in-memory data. // Important: the program must not be interrupted for more than one // hour, or we will lose time series data. for (String venue_id : city_venues.get(c)) { String ts_file = folder + c + File.separator + "attendances_crawl" + File.separator + venue_id + ".ts"; if (new File(ts_file).exists()) { BufferedReader buffer = new BufferedReader(new FileReader(ts_file)); String mem = null, line = null; for (; (line = buffer.readLine()) != null; mem = line) ; buffer.close(); if (mem == null) continue; String[] tmp = mem.split(","); venue_last_call.put(venue_id, df.parse(tmp[0]).getTime()); venue_last_checkin.put(venue_id, Integer.parseInt(tmp[3])); VenueUtil.fixBrokenTimeSeriesVenue(new File(ts_file)); } // if } // for sanity_checks.put(c, cal.getTimeInMillis()); } // for if (total_venues > 5000) { System.out.println( "Too much venues for a single API account (max 5000).\nPlease create a new Foursquare API account and use these credentials.\nExiting now."); return; } while (true) { for (String c : args) { // We create a FIFO queue and pop venue IDs one at a time. LinkedList<String> city_venues_buffer = new LinkedList<String>(city_venues.get(c)); String venue_id = null; // Artificial wait to avoid processors looping at 100% of their capacity // when there is no more venues to crawl for the current hour. Thread.sleep(3000); while ((venue_id = city_venues_buffer.pollFirst()) != null) { // We get the current time according to the city's time zone Calendar cal = Calendar.getInstance(); cal.add(Calendar.MILLISECOND, TimeZone.getTimeZone(settings.getCityTimezone(c)).getOffset(cal.getTime().getTime()) - Calendar.getInstance().getTimeZone().getOffset(cal.getTime().getTime())); //TimeZone.getTimeZone("Europe/London").getOffset(cal.getTime().getTime())); long current_time = DateUtils.truncate(cal.getTime(), Calendar.HOUR).getTime(); // We query Foursquare only once per hour per venue. if (venue_last_call.get(venue_id) != null && current_time < venue_last_call.get(venue_id) + 3600000) continue; intelligentWait(total_venues, cal.getTime().getTime(), (total_calls == 0 ? 0 : Math.round(time_spent_on_API / total_calls))); Venue venue = null; try { long beforeCall = System.currentTimeMillis(); venue = new Venue(getFoursquareVenueById(venue_id, c)); // If there is no last call, this is the beginning of the time series // for this venue. We get the number of people "here now" to initialize // the series. if (venue_last_call.get(venue_id) == null) { /** TODO: by doing this, we keep a representation of the venue dating from the beginning * of the specific crawl. we might want to change this and update this file once * in a while. */ FileWriter info = new FileWriter(folder + c + File.separator + "foursquare_venues" + File.separator + venue_id + ".info"); info.write(venue.getFoursquareJson()); info.close(); FileWriter out = new FileWriter(folder + c + File.separator + "attendances_crawl" + File.separator + venue_id + ".ts"); out.write("Date,here_now,hour_checkins,total_checkins\n"); out.write(df.format(current_time) + "," + venue.getHereNow() + "," + venue.getHereNow() + "," + venue.getCheckincount() + "\n"); out.close(); } else { FileWriter out = new FileWriter(folder + c + File.separator + "attendances_crawl" + File.separator + venue_id + ".ts", true); int checks = venue.getCheckincount() - venue_last_checkin.get(venue_id); out.write(df.format(current_time) + "," + venue.getHereNow() + "," + Integer.toString(checks) + "," + venue.getCheckincount() + "\n"); out.close(); } if (APICallsCount.get(current_time) == null) APICallsCount.put(current_time, 1); else APICallsCount.put(current_time, APICallsCount.get(current_time) + 1); total_calls++; venue_last_call.put(venue_id, current_time); venue_last_checkin.put(venue_id, venue.getCheckincount()); time_spent_on_API += System.currentTimeMillis() - beforeCall; } catch (Exception e) { // If something bad happens (crawler not available, IO error, ...), we put the // venue_id in the FIFO queue so that it gets reevaluated later. //e.printStackTrace(); error_logs.get(c) .write("[" + df.format(cal.getTime().getTime()) + "] Error with venue " + venue_id + " (" + e.getMessage() + "). " + APICallsCount.get(current_time) + " API calls so far this hour, " + city_venues_buffer.size() + " venues remaining in the buffer.\n"); error_logs.get(c).flush(); System.out.println("[" + df.format(cal.getTime().getTime()) + "] " + c + " -- " + APICallsCount.get(current_time) + " API calls // " + city_venues_buffer.size() + " venues remaining " + " (" + e.getMessage() + ")"); if (e instanceof FoursquareAPIException) if (((FoursquareAPIException) e).getHttp_code().equals("400") && ((FoursquareAPIException) e).getError_detail() .equals("Venue " + venue_id + " has been deleted")) { city_venues.get(c).remove(venue_id); removeVenue(venue_id, c); } else city_venues_buffer.add(venue_id); continue; } } // while // Every day between 0am and 2am, we repair all the broken time series (if there // is something to repair). Calendar cal = Calendar.getInstance(); if (city_venues_buffer.peekFirst() == null && (cal.getTimeInMillis() - sanity_checks.get(c)) >= 86400000 && cal.get(Calendar.HOUR_OF_DAY) < 2) { VenueUtil.fixBrokenTimeSeriesCity(c, folder); sanity_checks.put(c, cal.getTimeInMillis()); info_logs.get(c).write("[" + df.format(cal.getTime()) + "] Sanity check OK.\n"); info_logs.get(c).flush(); } } // for } // while }
From source file:Main.java
private static boolean isBeforeMonths(int months, Date aDate) { Calendar calendar = Calendar.getInstance(); calendar.add(MONTH, months); return aDate.compareTo(calendar.getTime()) < 0; }
From source file:Main.java
private static String setDate(int day, Locale locale) { Calendar cal = Calendar.getInstance(); cal.add(Calendar.DATE, +day); DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT, locale); if (df instanceof SimpleDateFormat) { SimpleDateFormat sdf = (SimpleDateFormat) df; // To show Locale specific short date expression with full year String pattern = sdf.toPattern().replaceAll("y+", "yyyy"); sdf.applyPattern(pattern);/*from ww w .j a va2 s. c o m*/ return sdf.format(cal.getTime()); } String formattedDate = df.format(cal.getTime()); return formattedDate; }
From source file:Main.java
public static Calendar getNextDay(Calendar day) { day.add(Calendar.DAY_OF_MONTH, 1); return day; }