List of usage examples for org.apache.commons.csv CSVRecord size
public int size()
From source file:com.adobe.aem.demomachine.communities.Loader.java
private static void doDecorate(String hostname, String altport, String adminPassword, String location, CSVRecord record, String analytics, String rootPath, Version vBundleCommunitiesEnablement) { // Getting the JSON view of the resource String resourceJson = Loader.doGet(hostname, altport, location + ".social.json", "admin", adminPassword, null);//from ww w. j a va 2 s . c o m // Generating random ratings and comments for the resource for each of the enrolled users try { JSONObject resourceJsonObject = new JSONObject(resourceJson); String resourceRatingsEndpoint = location + "/se_social/se_ratings.social.json"; String resourceCommentsEndpoint = location + "/se_social/se_comments.social.json"; String assetPath = "assetProperties"; if (vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT62FP1)) <= 0) { resourceRatingsEndpoint = resourceJsonObject.getString("ratingsEndPoint") + ".social.json"; resourceCommentsEndpoint = resourceJsonObject.getString("commentsEndPoint") + ".social.json"; } else { assetPath = "primaryAsset"; } String resourceID = resourceJsonObject.getString("id"); String resourceType = resourceJsonObject.getJSONObject(assetPath).getString("type"); String referer = "http://" + hostname + ":" + altport + rootPath + "/" + record.get(RESOURCE_INDEX_SITE) + "/en" + (record.get(RESOURCE_INDEX_FUNCTION).length() > 0 ? ("/" + record.get(RESOURCE_INDEX_FUNCTION)) : "") + ".resource.html" + resourceID; // Looking for the list of enrolled users for (int i = 0; i < record.size() - 1; i = i + 1) { if (record.get(i) != null && record.get(i + 1) != null && record.get(i).equals("deltaList")) { JSONObject enrolledJsonObject = new JSONObject(record.get(i + 1)); Iterator<?> iter = enrolledJsonObject.keys(); while (iter.hasNext()) { String key = (String) iter.next(); logger.debug("New Resource Enrollee: " + key); // Getting information about this assignment (user or group?) String isGroup = doWait(hostname, altport, "admin", adminPassword, key, 1); if (isGroup == null) { // Always generating a page view event if (Math.random() < 0.90) doAnalytics(analytics, "event11", referer, resourceID, resourceType); // Sometimes generating a video view event if (Math.random() < 0.75 && resourceType.equals("video/mp4")) doAnalytics(analytics, "event2", referer, resourceID, resourceType); // Posting ratings and comments if (Math.random() < 0.50) doRatings(hostname, altport, adminPassword, key, resourceRatingsEndpoint, referer, resourceID, resourceType, analytics); if (Math.random() < 0.35) doComments(hostname, altport, adminPassword, key, resourceCommentsEndpoint, referer, resourceID, resourceType, analytics); } else { List<NameValuePair> groupNameValuePairs = new ArrayList<NameValuePair>(); groupNameValuePairs.add(new BasicNameValuePair("groupId", key)); groupNameValuePairs.add(new BasicNameValuePair("includeSubGroups", "true")); String memberList = Loader.doGet(hostname, altport, "/content/community-components/en/communitygroupmemberlist/jcr:content/content/communitygroupmember.social.0.100.json", "admin", adminPassword, groupNameValuePairs); JSONArray memberJsonArray = new JSONObject(memberList).getJSONArray("items"); for (int j = 0; j < memberJsonArray.length(); j++) { JSONObject memberJsonObject = memberJsonArray.getJSONObject(j); String email = memberJsonObject.getString("authorizableId"); if (email != null) { // Always generating a page view event if (Math.random() < 0.90) doAnalytics(analytics, "event11", referer, resourceID, "video/mp4"); // Sometimes generating a video view event if (Math.random() < 0.75 && resourceType.equals("video/mp4")) doAnalytics(analytics, "event2", referer, resourceID, resourceType); if (Math.random() < 0.50) doRatings(hostname, altport, adminPassword, email, resourceRatingsEndpoint, referer, resourceID, resourceType, analytics); if (Math.random() < 0.35) doComments(hostname, altport, adminPassword, email, resourceCommentsEndpoint, referer, resourceID, resourceType, analytics); } } // For each group member } // If there's a principal name } // For each enrollee break; // only one possible deltaList attribute for resource and learning paths } } } catch (Exception e) { logger.error(e.getMessage()); } }
From source file:com.adobe.aem.demomachine.communities.Loader.java
public static void processLoading(ResourceResolver rr, Reader in, String hostname, String port, String altport, String adminPassword, String analytics, boolean reset, boolean configure, boolean minimize, boolean noenablement, String csvfile, int maxretries) { String location = null;/* w w w. ja v a 2 s . com*/ String userHome = null; String sitePagePath = null; String analyticsPagePath = null; String resourceType = null; String subComponentType = null; String rootPath = "/content/sites"; String[] url = new String[10]; // Handling 10 levels maximum for nested comments int urlLevel = 0; int row = 0; boolean ignoreUntilNextComponent = false; HashMap<String, ArrayList<String>> learningpaths = new HashMap<String, ArrayList<String>>(); try { String componentType = null; logger.debug("AEM Demo Loader: Loading bundles versions"); String bundlesList = doGet(hostname, port, "/system/console/bundles.json", "admin", adminPassword, null); // Some steps are specific to the version number of the Enablement add-on Version vBundleCommunitiesEnablement = getVersion(bundlesList, "com.adobe.cq.social.cq-social-enablement-impl"); Version vBundleCommunitiesCalendar = getVersion(bundlesList, "com.adobe.cq.social.cq-social-calendar"); if (vBundleCommunitiesCalendar == null) { vBundleCommunitiesCalendar = getVersion(bundlesList, "com.adobe.cq.social.cq-social-calendar-impl"); } Version vBundleCommunitiesNotifications = getVersion(bundlesList, "com.adobe.cq.social.cq-social-notifications-impl"); Version vBundleCommunitiesSCORM = getVersion(bundlesList, "com.adobe.cq.social.cq-social-scorm-dam"); Version vBundleCommunitiesSCF = getVersion(bundlesList, "com.adobe.cq.social.cq-social-scf-impl"); Version vBundleCommunitiesAdvancedScoring = getVersion(bundlesList, "com.adobe.cq.social.cq-social-scoring-advanced-impl"); // Versions related methods boolean isCommunities61 = vBundleCommunitiesSCF != null && vBundleCommunitiesSCF.compareTo(new Version(COMMUNITIES61)) == 0; boolean isCommunities61FP5orlater = vBundleCommunitiesSCF != null && vBundleCommunitiesSCF.compareTo(new Version(COMMUNITIES61FP5)) >= 0; boolean isCommunities61FP6orlater = vBundleCommunitiesSCF != null && vBundleCommunitiesSCF.compareTo(new Version(COMMUNITIES61FP6)) >= 0; boolean isCommunities61FP7orlater = vBundleCommunitiesSCF != null && vBundleCommunitiesSCF.compareTo(new Version(COMMUNITIES61FP7)) >= 0; Iterable<CSVRecord> records = CSVFormat.EXCEL.parse(in); ignoreUntilNextComponent = false; for (CSVRecord record : records) { LinkedList<InputStream> lIs = new LinkedList<InputStream>(); row = row + 1; logger.info("Row: " + row + ", new record: " + record.get(0)); if (record.size() > 2) subComponentType = record.get(2); else logger.info("No subcomponent type to load"); // Let's see if we deal with a comment if (record.get(0).startsWith("#")) { // We can ignore the comment line and move on continue; } // Let's see if we need to terminate this process if (record.get(0).equals(KILL)) { if (rr == null) System.exit(1); else return; } // Let's see if we need to pause a little bit if (record.get(0).equals(SLEEP) && record.get(1).length() > 0) { doSleep(Long.valueOf(record.get(1)).longValue(), "Pausing " + record.get(1) + " ms"); continue; } // Let's see if we need to set the current site path if (record.get(0).equals(SITEPATH)) { sitePagePath = record.get(1); } // Let's see if we need to create a new Community site if (record.get(0).equals(SITE)) { // Building the form entity to be posted MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.addTextBody(":operation", "social:createSite", ContentType.create("text/plain", MIME.UTF8_CHARSET)); builder.addTextBody("_charset_", "UTF-8", ContentType.create("text/plain", MIME.UTF8_CHARSET)); String urlName = null; String[] initialLanguages = null; boolean isValid = true; for (int i = 2; i < record.size() - 1; i = i + 2) { if (record.get(i) != null && record.get(i + 1) != null && record.get(i).length() > 0) { String name = record.get(i).trim(); String value = record.get(i + 1).trim(); if (value.equals("TRUE")) { value = "true"; } if (value.equals("FALSE")) { value = "false"; } if (name.equals("urlName")) { urlName = value; } if (name.equals(ROOT)) { rootPath = value; logger.debug("Rootpath for subsequent processing is: " + rootPath); if (!isResourceAvailable(hostname, port, adminPassword, rootPath)) { logger.warn("Rootpath " + rootPath + " is not available, proceeding to next record"); isValid = false; } else { logger.info("Rootpath " + rootPath + " is available"); } } if (name.equals(BANNER)) { addBinaryBody(builder, lIs, rr, BANNER, csvfile, value); } else if (name.equals(THUMBNAIL)) { addBinaryBody(builder, lIs, rr, THUMBNAIL, csvfile, value); } else if (name.equals(CSS)) { addBinaryBody(builder, lIs, rr, CSS, csvfile, value); } else if (name.equals(LANGUAGE) || name.equals(LANGUAGES)) { // Starting with 6.1 FP5 and 6.2 FP1, we can create multiple languages at once if (isCommunities61FP5orlater) { initialLanguages = value.split(","); for (String initialLanguage : initialLanguages) { builder.addTextBody(LANGUAGES, initialLanguage, ContentType.create("text/plain", MIME.UTF8_CHARSET)); } } else { // Only keep the first language for pre 6.1 FP5 and 6.2 FP1 initialLanguages = new String[1]; initialLanguages[0] = value.split(",")[0]; builder.addTextBody(LANGUAGE, initialLanguages[0], ContentType.create("text/plain", MIME.UTF8_CHARSET)); } } else { // For cloud services, we verify that they are actually available if ((name.equals(OPTION_TRANSLATION) || name.equals(OPTION_ANALYTICS) || name.equals(OPTION_FACEBOOK) || name.equals(OPTION_TWITTER)) && value.equals("true")) { String cloudName = record.get(i + 2).trim(); String cloudValue = record.get(i + 3).trim(); if ((cloudName.equals(CLOUDSERVICE_TRANSLATION) || cloudName.equals(CLOUDSERVICE_FACEBOOK) || cloudName.equals(CLOUDSERVICE_TWITTER) || cloudName.equals(CLOUDSERVICE_ANALYTICS)) && !isResourceAvailable(hostname, port, adminPassword, cloudValue)) { builder.addTextBody(name, "false", ContentType.create("text/plain", MIME.UTF8_CHARSET)); logger.warn("Cloud service: " + cloudValue + " is not available on this instance"); } else { // We have a valid cloud service builder.addTextBody(name, value, ContentType.create("text/plain", MIME.UTF8_CHARSET)); builder.addTextBody(cloudName, cloudValue, ContentType.create("text/plain", MIME.UTF8_CHARSET)); i = i + 2; logger.debug( "Cloud service: " + cloudValue + " available on this instance"); } } else { // All other values just get added as is builder.addTextBody(name, value, ContentType.create("text/plain", MIME.UTF8_CHARSET)); } } } } // Printing site creation settings //ByteArrayOutputStream out = new ByteArrayOutputStream(); //builder.build().writeTo(out); //String string = out.toString(); //logger.debug(string); // Site creation if (isValid) doPost(hostname, port, "/content.social.json", "admin", adminPassword, builder.build(), null, null); else continue; // Waiting for site creation to be complete boolean existingSiteWithLocale = rootPath.indexOf("/" + initialLanguages[0]) > 0; doWaitPath(hostname, port, adminPassword, rootPath + "/" + urlName + (existingSiteWithLocale ? "" : "/" + initialLanguages[0]), maxretries); // Site publishing, if there's a publish instance to publish to if (!port.equals(altport)) { for (String initialLanguage : initialLanguages) { List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(); nameValuePairs.add(new BasicNameValuePair("id", "nobot")); nameValuePairs.add(new BasicNameValuePair(":operation", "social:publishSite")); nameValuePairs.add(new BasicNameValuePair("path", rootPath + "/" + urlName + (existingSiteWithLocale ? "" : "/" + initialLanguage))); logger.debug("Publishing site " + urlName + " for language " + initialLanguage); doPost(hostname, port, "/communities/sites.html", "admin", adminPassword, new UrlEncodedFormEntity(nameValuePairs), null); doWaitPath(hostname, altport, adminPassword, rootPath + "/" + urlName + (existingSiteWithLocale ? "" : "/" + initialLanguage), maxretries); } } continue; } // Let's see if we need to update an existing Community site (this doesn't include republishing the site!) if (record.get(0).equals(SITEUPDATE) && record.get(1) != null && record.get(2) != null) { // Let's set if we need to run based on version number Version vRecord = null; if (record.get(2).startsWith(">") || record.get(2).startsWith("<") || record.get(2).startsWith("=")) { try { vRecord = new Version(record.get(2).substring(1)); } catch (Exception e) { logger.error("Invalid version number specified" + record.get(2)); } } if (vRecord != null && record.get(2).startsWith(">") && vBundleCommunitiesSCF.compareTo(vRecord) <= 0) { logger.info("Ignoring the site update command for this version of AEM" + vBundleCommunitiesSCF.get()); continue; } if (vRecord != null && record.get(2).startsWith("<") && vBundleCommunitiesSCF.compareTo(vRecord) > 0) { logger.info("Ignoring the site update command for this version of AEM" + vBundleCommunitiesSCF.get()); continue; } if (isResourceAvailable(hostname, port, adminPassword, record.get(1))) { logger.debug("Updating a Community Site " + record.get(1)); } else { logger.error("Can't update a Community Site " + record.get(1)); continue; } // Let's fetch the theme for this Community Site Url String siteConfig = doGet(hostname, port, record.get(1), "admin", adminPassword, null); if (siteConfig == null) { logger.error("Can't update a Community Site " + record.get(1)); continue; } // Building the form entity to be posted MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.addTextBody(":operation", "social:updateSite", ContentType.create("text/plain", MIME.UTF8_CHARSET)); builder.addTextBody("_charset_", "UTF-8", ContentType.create("text/plain", MIME.UTF8_CHARSET)); // Adding the mandatory values for being able to save a site via the JSON endpoint List<String> props = Arrays.asList("urlName", "theme", "moderators", "createGroupPermission", "groupAdmin", "twitterconnectoauthid", "fbconnectoauthid", "translationProviderConfig", "translationProvider", "commonStoreLanguage"); try { JSONObject siteprops = new JSONObject(siteConfig).getJSONObject("properties"); for (String prop : props) { if (siteprops.has(prop)) { Object propValue = siteprops.get(prop); if (propValue instanceof JSONArray) { JSONArray propArray = (JSONArray) propValue; for (int i = 0; i < propArray.length(); i++) { builder.addTextBody(prop, propArray.get(i).toString(), ContentType.create("text/plain", MIME.UTF8_CHARSET)); } } else { builder.addTextBody(prop, propValue.toString(), ContentType.create("text/plain", MIME.UTF8_CHARSET)); } } } } catch (Exception e) { logger.error(e.getMessage()); } // Adding the override values from the CSV record boolean isValid = true; for (int i = 3; i < record.size() - 1; i = i + 2) { if (record.get(i) != null && record.get(i + 1) != null && record.get(i).length() > 0) { String name = record.get(i).trim(); String value = record.get(i + 1).trim(); builder.addTextBody(name, value, ContentType.create("text/plain", MIME.UTF8_CHARSET)); // If the template includes some of the enablement features, then it won't work for 6.1 GA if (name.equals("functions") && value.indexOf("assignments") > 0 && vBundleCommunitiesEnablement == null) { logger.info("Site update is not compatible with this version of AEM"); isValid = false; } // If the template includes some of the ideation features, then it won't work until 6.2 FP2 if (name.equals("functions") && value.indexOf("ideation") > 0 && !isCommunities61FP6orlater) { logger.info("Site update is not compatible with this version of AEM"); isValid = false; } } } // Convenient for debugging the site update operation // printPOST(builder.build()); if (isValid) doPost(hostname, port, record.get(1), "admin", adminPassword, builder.build(), null); continue; } // Let's see if we need to publish a site if (record.get(0).equals(SITEPUBLISH) && record.get(1) != null) { if (isResourceAvailable(hostname, port, adminPassword, record.get(1))) { logger.debug("Publishing a Community Site " + record.get(1)); } else { logger.warn("Can't publish a Community Site " + record.get(1)); continue; } if (!port.equals(altport)) { List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(); nameValuePairs.add(new BasicNameValuePair("id", "nobot")); nameValuePairs.add(new BasicNameValuePair(":operation", "social:publishSite")); nameValuePairs.add(new BasicNameValuePair("nestedActivation", "true")); nameValuePairs.add(new BasicNameValuePair("path", record.get(1))); doPost(hostname, port, "/communities/sites.html", "admin", adminPassword, new UrlEncodedFormEntity(nameValuePairs), null); doWaitPath(hostname, altport, adminPassword, record.get(1), maxretries); } continue; } // Let's see if we need to publish a group if (record.get(0).equals(GROUPPUBLISH) && record.get(1) != null) { if (!port.equals(altport)) { List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(); nameValuePairs.add(new BasicNameValuePair("id", "nobot")); nameValuePairs.add(new BasicNameValuePair(":operation", "social:publishCommunityGroup")); nameValuePairs.add(new BasicNameValuePair("nestedActivation", "true")); nameValuePairs.add(new BasicNameValuePair("path", record.get(1) + "/" + record.get(2))); doPost(hostname, port, "/communities/communitygroups.html/" + record.get(1), "admin", adminPassword, new UrlEncodedFormEntity(nameValuePairs), null); } continue; } // Let's see if we need to activate a tree if (record.get(0).equals(ACTIVATE) && record.get(1) != null) { if (!port.equals(altport)) { List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(); nameValuePairs.add(new BasicNameValuePair("cmd", "activate")); nameValuePairs.add(new BasicNameValuePair("ignoreactivated", "true")); nameValuePairs.add(new BasicNameValuePair("path", record.get(1))); doPost(hostname, port, "/etc/replication/treeactivation.html", "admin", adminPassword, new UrlEncodedFormEntity(nameValuePairs), null); } continue; } // Let's see if we need to create a new Tag if (record.get(0).equals(TAG)) { // Building the form entity to be posted MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.addTextBody("_charset_", "UTF-8", ContentType.create("text/plain", MIME.UTF8_CHARSET)); for (int i = 1; i < record.size() - 1; i = i + 2) { if (record.get(i) != null && record.get(i + 1) != null && record.get(i).length() > 0 && record.get(i + 1).length() > 0) { String name = record.get(i).trim(); String value = record.get(i + 1).trim(); builder.addTextBody(name, value, ContentType.create("text/plain", MIME.UTF8_CHARSET)); } } // Tag creation doPost(hostname, port, "/bin/tagcommand", "admin", adminPassword, builder.build(), null); continue; } // Let's see if we need to assign some badges if (record.get(0).equals(BADGE)) { if (vBundleCommunitiesEnablement == null || vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT61FP3)) < 0) { logger.info("Badging operations not available with this version of AEM"); continue; } List<NameValuePair> nameValuePairs = buildNVP(hostname, port, adminPassword, null, record, 2); String badgePath = record.get(1); if (badgePath.startsWith("/etc") && (vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT61FP4)) == 0 || vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT62)) > 0)) { badgePath = badgePath.replaceAll("/jcr:content", ""); nameValuePairs.add(new BasicNameValuePair("sling:resourceType", "social/gamification/components/hbs/badging/rulecollection/rule")); nameValuePairs.add(new BasicNameValuePair("badgingType", "basic")); } if (nameValuePairs.size() > 2) { for (int i = 0; i < nameValuePairs.size(); i = i + 1) { String name = nameValuePairs.get(i).getName(); String value = nameValuePairs.get(i).getValue(); // Special case to accommodate re-factoring of badging images if (name.equals("badgeContentPath") && (vBundleCommunitiesEnablement .compareTo(new Version(ENABLEMENT61FP4)) == 0 || vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT62)) > 0)) { value = value.replaceAll("/jcr:content", ""); nameValuePairs.set(i, new BasicNameValuePair(name, value)); } // Special case to accommodate re-factoring of badging images if (name.startsWith("thresholds") && (vBundleCommunitiesEnablement .compareTo(new Version(ENABLEMENT61FP4)) == 0 || vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT62)) > 0)) { value = value.replaceAll("/jcr:content(.*)", ""); nameValuePairs.set(i, new BasicNameValuePair(name, value)); } // Special case to accommodate re-factoring or scoring and badging resource types if (name.equals("jcr:primaryType") && (vBundleCommunitiesEnablement .compareTo(new Version(ENABLEMENT61FP4)) == 0 || vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT62)) > 0)) { if (value.equals("cq:PageContent") || value.equals("cq:Page")) { value = "nt:unstructured"; nameValuePairs.set(i, new BasicNameValuePair(name, value)); } } // Special case for accommodate advanced scoring being installed or not if (name.endsWith("Rules") && value.contains("adv-") && vBundleCommunitiesAdvancedScoring == null) { nameValuePairs.remove(i--); } } } // Badge rules operation doPost(hostname, port, badgePath, "admin", adminPassword, new UrlEncodedFormEntity(nameValuePairs), null); continue; } // Let's see if we need to create a new Community site template, and if we can do it (script run against author instance) if (record.get(0).equals(SITETEMPLATE) || record.get(0).equals(GROUPTEMPLATE)) { // Building the form entity to be posted MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.addTextBody(":operation", "social:create" + record.get(0), ContentType.create("text/plain", MIME.UTF8_CHARSET)); builder.addTextBody("_charset_", "UTF-8", ContentType.create("text/plain", MIME.UTF8_CHARSET)); boolean isValid = true; for (int i = 2; i < record.size() - 1; i = i + 2) { if (record.get(i) != null && record.get(i + 1) != null && record.get(i).length() > 0) { String name = record.get(i).trim(); String value = record.get(i + 1).trim(); builder.addTextBody(name, value, ContentType.create("text/plain", MIME.UTF8_CHARSET)); // If the template is already there, let's not try to create it if (name.equals("templateName") && (isResourceAvailable(hostname, port, adminPassword, "/etc/community/templates/sites/custom/" + title2name(value)) || isResourceAvailable(hostname, port, adminPassword, "/etc/community/templates/groups/custom/" + title2name(value)))) { logger.info("Template " + value + " is already there"); isValid = false; } // If the template includes some of the enablement features, then it won't work for 6.1 GA if (name.equals("functions") && value.indexOf("assignments") > 0 && vBundleCommunitiesEnablement == null) { logger.info("Template " + record.get(3) + " is not compatible with this version of AEM"); isValid = false; } // If the template includes some of the ideation features, then it won't work until 6.2 FP2 if (name.equals("functions") && value.indexOf("ideation") > 0 && !isCommunities61FP6orlater) { logger.info("Template " + record.get(3) + " is not compatible with this version of AEM"); isValid = false; } // If the group template includes the nested group features, then it won't work until 6.2 FP1 if (record.get(0).equals(GROUPTEMPLATE) && name.equals("functions") && value.indexOf("groups") > 0 && (vBundleCommunitiesEnablement != null && vBundleCommunitiesEnablement .compareTo(new Version(ENABLEMENT62)) <= 0)) { logger.info("Group template " + record.get(3) + " is not compatible with this version of AEM"); isValid = false; } // If the group template includes the blogs or calendars, then it won't work with 6.1GA if (name.equals("functions") && (value.indexOf("blog") > 0 || value.indexOf("calendar") > 0) && vBundleCommunitiesEnablement == null) { logger.info("Template " + record.get(3) + " is not compatible with this version of AEM"); isValid = false; } } } // Site or Group template creation if (isValid) doPost(hostname, port, "/content.social.json", "admin", adminPassword, builder.build(), null); continue; } // Let's see if we need to create a new Community group if (record.get(0).equals(GROUP) || record.get(0).equals(SUBGROUP)) { // SubGroups are only supported with 6.1 FP5 and 6.2 FP1 onwards if (record.get(0).equals(SUBGROUP) && !isCommunities61FP5orlater) { logger.warn("Subgroups are not supported with this version of AEM Communities"); continue; } // Building the form entity to be posted MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.addTextBody(":operation", "social:createCommunityGroup", ContentType.create("text/plain", MIME.UTF8_CHARSET)); builder.addTextBody("_charset_", "UTF-8", ContentType.create("text/plain", MIME.UTF8_CHARSET)); String urlName = null; String groupType = null; for (int i = 3; i < record.size() - 1; i = i + 2) { if (record.get(i) != null && record.get(i + 1) != null && record.get(i).length() > 0) { String name = record.get(i).trim(); String value = record.get(i + 1).trim(); if (value.equals("TRUE")) { value = "true"; } if (value.equals("FALSE")) { value = "false"; } if (name.equals("type")) { groupType = value; } if (name.equals(IMAGE)) { addBinaryBody(builder, lIs, rr, IMAGE, csvfile, value); } else { builder.addTextBody(name, value, ContentType.create("text/plain", MIME.UTF8_CHARSET)); } if (name.equals("urlName")) { urlName = value; } if (name.equals("siteRoot")) { // Some content root has been provided for the Group. It might result from previous actions and might not be there yet - let's wait for it doWaitPath(hostname, port, adminPassword, value, maxretries); } } } // Private groups are only support with 6.1 FP1 onwards if (groupType != null && groupType.equals("Secret") && isCommunities61) { continue; } // Group creation doPost(hostname, port, record.get(1), getUserName(record.get(2)), getPassword(record.get(2), adminPassword), builder.build(), null); // Waiting for group to be available either on publish or author int i = (record.get(1).indexOf("/jcr:content") > 0) ? record.get(1).indexOf("/jcr:content") : record.get(1).indexOf(".social.json"); if (urlName != null && i > 0) { doWaitPath(hostname, port, adminPassword, record.get(1).substring(0, i) + "/" + urlName, maxretries); } else { logger.warn("Not waiting for Group to be fully available"); } continue; } // Let's see if it's simple Sling Delete request if (record.get(0).equals(SLINGDELETE)) { doDelete(hostname, port, record.get(1), "admin", adminPassword); continue; } // Let's see if we need to delete a Community site if (record.get(0).equals(SITEDELETE) && record.get(1) != null) { // Let's fetch the siteId for this Community Site Url String siteConfig = doGet(hostname, port, record.get(1), "admin", adminPassword, null); // No site to Delete if (siteConfig == null) continue; try { String siteRoot = new JSONObject(siteConfig).getString("siteRoot"); String urlName = new JSONObject(siteConfig).getString("urlName"); String siteId = new JSONObject(siteConfig).getString("siteId"); String resourcesRoot = new JSONObject(siteConfig).getString("siteAssetsPath"); if (siteRoot != null && urlName != null && siteId != null && resourcesRoot != null) { // First, deleting the main JCR path for this site, on author and publish doDelete(hostname, port, siteRoot + "/" + urlName, "admin", adminPassword); doDelete(hostname, altport, siteRoot + "/" + urlName, "admin", adminPassword); // Then, deleting the dam resources for this site, on author and publish doDelete(hostname, port, resourcesRoot, "admin", adminPassword); doDelete(hostname, altport, resourcesRoot, "admin", adminPassword); // Then, deleting the main UGC path for this site, on author and publish doDelete(hostname, port, "/content/usergenerated/asi/jcr" + siteRoot + "/" + urlName, "admin", adminPassword); doDelete(hostname, altport, "/content/usergenerated/asi/jcr" + siteRoot + "/" + urlName, "admin", adminPassword); // Finally, deleting the system groups for this site, on author and publish doDelete(hostname, port, "/home/groups/community-" + siteId, "admin", adminPassword); doDelete(hostname, altport, "/home/groups/community-" + siteId, "admin", adminPassword); } } catch (Exception e) { logger.error(e.getMessage()); } } // Let's see if we need to add users to an AEM Group if ((record.get(0).equals(GROUPMEMBERS) || record.get(0).equals(SITEMEMBERS)) && record.get(GROUP_INDEX_NAME) != null) { // Checking if we have a member group for this site String groupName = null; if (record.get(0).equals(SITEMEMBERS)) { String configurationPath = record.get(GROUP_INDEX_NAME); // Let's make sure the configuration .json is there doWaitPath(hostname, port, adminPassword, configurationPath, maxretries); // Let's fetch the siteId for this Community Site Url String siteConfig = doGet(hostname, port, configurationPath, "admin", adminPassword, null); if (siteConfig == null) { logger.error("Can't retrieve site configuration"); continue; } ; String siteId = null; try { siteId = new JSONObject(siteConfig).getString("siteId"); } catch (Exception e) { logger.warn("No site Id available"); } String urlName = null; try { urlName = new JSONObject(siteConfig).getString("urlName"); } catch (Exception e) { logger.error("No site url available"); continue; } if (siteId != null) groupName = "community-" + siteId + "-members"; else groupName = "community-" + urlName + "-members"; logger.debug("Site Member group name is " + groupName); } if (record.get(0).equals(GROUPMEMBERS)) { groupName = record.get(GROUP_INDEX_NAME); } // We can't proceed if the group name wasn't retrieved from the configuration if (groupName == null) continue; // Pause until the group can found String groupList = doWait(hostname, port, "admin", adminPassword, groupName, maxretries); if (groupList != null && groupList.indexOf("\"results\":1") > 0) { logger.debug("Group was found on " + port); try { JSONArray jsonArray = new JSONObject(groupList).getJSONArray("hits"); if (jsonArray.length() == 1) { JSONObject jsonObject = jsonArray.getJSONObject(0); String groupPath = jsonObject.getString("path"); logger.debug("Group path is " + groupPath); // Constructing a multi-part POST for group membership MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); List<NameValuePair> groupNameValuePairs = buildNVP(hostname, port, adminPassword, null, record, 2); for (NameValuePair nameValuePair : groupNameValuePairs) { builder.addTextBody(nameValuePair.getName(), nameValuePair.getValue(), ContentType.create("text/plain", MIME.UTF8_CHARSET)); } // Adding the list of group members doPost(hostname, port, groupPath + ".rw.userprops.html", "admin", adminPassword, builder.build(), null); } else { logger.info("We have more than one match for a group with this name!"); } } catch (Exception e) { logger.error(e.getMessage()); } } else { logger.warn("Group " + groupName + " cannot be updated as expected"); } continue; } // Let's see if it's user related if (record.get(0).equals(USERS)) { //First we need to get the path to the user node String json = doGet(hostname, port, "/libs/granite/security/currentuser.json", getUserName(record.get(1)), getPassword(record.get(1), adminPassword), null); if (json != null) { try { // Fetching the home property String home = new JSONObject(json).getString("home"); if (record.get(2).equals(PREFERENCES)) { home = home + "/preferences"; } else { home = home + "/profile"; } // Now we can post all the preferences or the profile List<NameValuePair> nameValuePairs = buildNVP(hostname, port, adminPassword, null, record, 3); doPost(hostname, port, home, "admin", adminPassword, new UrlEncodedFormEntity(nameValuePairs), null); } catch (Exception e) { logger.error(e.getMessage()); } } continue; } // Let's see if we need to generate analytics events for Assets Insights if (record.get(0).equals(ASSETINSIGHTS) && record.size() > 1 && analytics != null) { logger.debug("Generating Assets Analytics for reportsuite " + analytics); // Generating Impressions int impressions = new Random().nextInt(21) + 5; for (int i = 0; i < impressions; i++) doAssetsAnalytics(analytics, "event1", "list1", record.get(1).replace('|', ','), "o", "Asset Impression Event"); // Generating Clicks for each asset List<String> assetIds = Arrays.asList(record.get(1).split("\\|", -1)); for (String assetId : assetIds) { int clicks = new Random().nextInt(5) + 2; for (int i = 0; i < clicks; i++) doAssetsAnalytics(analytics, "event2", "eVar4", assetId, "e", "Asset Click Event"); } continue; } // Let's see if we deal with a new block of content or just a new entry if (record.get(0).equals(CALENDAR) || record.get(0).equals(SLINGPOST) || record.get(0).equals(RATINGS) || record.get(0).equals(IDEATION) || record.get(0).equals(BLOG) || record.get(0).equals(JOURNAL) || record.get(0).equals(COMMENTS) || record.get(0).equals(REVIEWS) || record.get(0).equals(FILES) || record.get(0).equals(SUMMARY) || record.get(0).equals(ACTIVITIES) || record.get(0).equals(JOIN) || record.get(0).equals(FOLLOW) || record.get(0).equals(NOTIFICATION) || record.get(0).equals(NOTIFICATIONPREFERENCE) || record.get(0).equals(MESSAGE) || record.get(0).equals(ASSET) || record.get(0).equals(AVATAR) || record.get(0).equals(FOLDER) || record.get(0).equals(BADGEIMAGE) || record.get(0).equals(BADGEASSIGN) || record.get(0).equals(FRAGMENT) || record.get(0).equals(RESOURCE) || record.get(0).equals(LEARNING) || record.get(0).equals(QNA) || record.get(0).equals(FORUM)) { // New block of content, we need to reset the processing to first Level componentType = record.get(0); url[0] = record.get(1); urlLevel = 0; // If it's not a SLINGPOST that could result in nodes to be created, let's make sure the end point is really there. if (!record.get(0).equals(SLINGPOST) && record.get(1) != null && !isResourceAvailable(hostname, port, adminPassword, getRootPath(record.get(1)))) { ignoreUntilNextComponent = true; continue; } else { ignoreUntilNextComponent = false; } if (!componentType.equals(SLINGPOST) && reset) { int pos = record.get(1).indexOf("/jcr:content"); if (pos > 0) doDelete(hostname, port, "/content/usergenerated" + record.get(1).substring(0, pos), "admin", adminPassword); } // If the Configure command line flag is set, we try to configure the component with all options enabled if (componentType.equals(SLINGPOST) || configure) { String configurePath = getConfigurePath(record.get(1)); logger.info(configurePath); List<NameValuePair> nameValuePairs = buildNVP(hostname, port, adminPassword, configurePath, record, 2); if (nameValuePairs.size() > 2) { // If we're posting against a jcr:content node, let's make sure the parent folder is there int pos1 = configurePath.indexOf("/jcr:content"); if (pos1 > 0) { if (!isResourceAvailable(hostname, port, adminPassword, configurePath.substring(0, pos1))) continue; } // If we're posting against a configuration node, let's make sure the parent folder is there int pos2 = configurePath.indexOf("configuration"); if (pos2 > 0) { if (!isResourceAvailable(hostname, port, adminPassword, configurePath)) continue; } // If we're posting to fetch analytics data, let's make sure the analytics host is available int pos3 = configurePath.indexOf("analyticsCommunities"); if (pos3 > 0) { if (!Hostname.isReachable("www.adobe.com", "80")) { logger.warn("Analytics cannot be imported since you appear to be offline"); // The things you have to do when coding in airplanes... continue; } } // Only do this when really have configuration settings doPost(hostname, port, configurePath, "admin", adminPassword, new UrlEncodedFormEntity(nameValuePairs), null); } // If the Sling POST touches the system console, then we need to make sure the system is open for business again before we proceed if (record.get(1).indexOf("system/console") > 0) { doSleep(10000, "Waiting after a bundle change/restart"); doWait(hostname, port, "admin", adminPassword, "administrators", maxretries); } } // We're done with this line, moving on to the next line in the CSV file continue; } // Are we processing until the next component because the end point if not available? if (ignoreUntilNextComponent) { logger.info("Ignoring this record because of unavailable component configuration"); continue; } // Let's see if we need to indent the list, if it's a reply or a reply to a reply if (record.get(1) == null || record.get(1).length() != 1) continue; // We need a valid level indicator if (Integer.parseInt(record.get(1)) > urlLevel) { url[++urlLevel] = location; logger.debug("Incrementing urlLevel to: " + urlLevel + ", with a new location:" + location); } else if (Integer.parseInt(record.get(1)) < urlLevel) { urlLevel = Integer.parseInt(record.get(1)); logger.debug("Decrementing urlLevel to: " + urlLevel); } // Special case for 6.1 GA only with forums and files if (vBundleCommunitiesEnablement == null && (!(componentType.equals(FORUM) || componentType.equals(FILES) || componentType.equals(JOIN)))) continue; // Get the credentials or fall back to password String password = getPassword(record.get(0), adminPassword); String userName = getUserName(record.get(0)); // Adding the generic properties for all POST requests MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.setCharset(MIME.UTF8_CHARSET); builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(); if (!componentType.equals(RESOURCE) && !componentType.equals(LEARNING)) nameValuePairs.add(new BasicNameValuePair("id", "nobot")); nameValuePairs.add(new BasicNameValuePair("_charset_", "UTF-8")); if (urlLevel == 0 && (componentType.equals(FORUM) || componentType.equals(FILES) || componentType.equals(QNA) || componentType.equals(IDEATION) || componentType.equals(BLOG) || componentType.equals(CALENDAR))) { // Generating a unique hashkey nameValuePairs.add(new BasicNameValuePair("ugcUrl", slugify(record.get(2)))); } // Setting some specific fields depending on the content type if (componentType.equals(COMMENTS)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createComment")); nameValuePairs.add(new BasicNameValuePair("message", record.get(2))); } // Follows a user (followedId) for the user posting the request if (componentType.equals(FOLLOW)) { if (vBundleCommunitiesNotifications != null && vBundleCommunitiesNotifications.compareTo(new Version("1.0.12")) < 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:follow")); nameValuePairs.add(new BasicNameValuePair("userId", "/social/authors/" + userName)); nameValuePairs .add(new BasicNameValuePair("followedId", "/social/authors/" + record.get(2))); } else { logger.info("Ignoring FOLLOW with this version of AEM Communities"); continue; } } // Notifications if (componentType.equals(NOTIFICATION)) { if (vBundleCommunitiesNotifications != null && vBundleCommunitiesNotifications.compareTo(new Version("1.0.11")) > 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:updatesubscriptions")); nameValuePairs.add(new BasicNameValuePair("types", "following")); nameValuePairs.add(new BasicNameValuePair("types", "notification")); if (vBundleCommunitiesNotifications.compareTo(new Version("1.1.0")) > 0) nameValuePairs.add(new BasicNameValuePair("types", "subscription")); nameValuePairs.add(new BasicNameValuePair("states", record.get(2).toLowerCase())); nameValuePairs.add(new BasicNameValuePair("states", record.get(3).toLowerCase())); if (vBundleCommunitiesNotifications.compareTo(new Version("1.1.0")) > 0) nameValuePairs.add(new BasicNameValuePair("states", record.get(4).toLowerCase())); nameValuePairs.add(new BasicNameValuePair("subscribedId", record.get(5))); } else { logger.info("Ignoring NOTIFICATION with this version of AEM Communities"); continue; } } // Notification preferences if (componentType.equals(NOTIFICATIONPREFERENCE)) { if (vBundleCommunitiesNotifications != null && vBundleCommunitiesNotifications.compareTo(new Version("1.0.11")) > 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:updateUserPreference")); List<NameValuePair> otherNameValuePairs = buildNVP(hostname, port, adminPassword, null, record, 2); nameValuePairs.addAll(otherNameValuePairs); } } // Uploading Avatar picture if (componentType.equals(AVATAR)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:changeAvatar")); // Appending the path to the user profile to the target location String userJson = doGet(hostname, port, "/libs/granite/security/currentuser.json", getUserName(record.get(0)), getPassword(record.get(0), adminPassword), null); userHome = ""; if (userJson != null) { try { // Fetching the home property userHome = new JSONObject(userJson).getString("home"); } catch (Exception e) { logger.error("Couldn't figure out home folder for user " + record.get(0)); } } } // Assigning badge to user if (componentType.equals(BADGEASSIGN)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:assignBadge")); // Special case to accommodate re-factoring of badging images String value = record.get(3); if (vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT61FP4)) == 0 || vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT62)) > 0) { value = value.replaceAll("/jcr:content", ""); } nameValuePairs.add(new BasicNameValuePair("badgeContentPath", value)); // Appending the path to the user profile to the target location String userJson = doGet(hostname, altport, "/libs/granite/security/currentuser.json", getUserName(record.get(2)), getPassword(record.get(2), adminPassword), null); userHome = ""; if (userJson != null) { try { // Fetching the home property userHome = new JSONObject(userJson).getString("home"); } catch (Exception e) { logger.error("Couldn't figure out home folder for user " + record.get(2)); } } } // Uploading Badge image if (componentType.equals(BADGEIMAGE)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createBadge")); nameValuePairs.add(new BasicNameValuePair("jcr:title", record.get(2))); nameValuePairs.add(new BasicNameValuePair("badgeDisplayName", record.get(3))); nameValuePairs.add(new BasicNameValuePair("badgeDescription", record.get(5))); addBinaryBody(builder, lIs, rr, "badgeImage", csvfile, record.get(ASSET_INDEX_NAME)); } // Joins a user (posting the request) to a Community Group (path) if (componentType.equals(JOIN)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:joinCommunityGroup")); int pos = url[0].indexOf("/configuration.social.json"); if (pos > 0) nameValuePairs.add(new BasicNameValuePair("path", url[0].substring(0, pos) + ".html")); else continue; // Invalid record } // Creates a new private message if (componentType.equals(MESSAGE)) { nameValuePairs.add(new BasicNameValuePair("to", "/social/authors/" + record.get(2))); nameValuePairs.add(new BasicNameValuePair("userId", "/social/authors/" + record.get(2))); nameValuePairs.add(new BasicNameValuePair("toId", "")); nameValuePairs.add(new BasicNameValuePair("serviceSelector", "/bin/community")); nameValuePairs.add(new BasicNameValuePair("redirectUrl", "../messaging.html")); nameValuePairs.add(new BasicNameValuePair("attachmentPaths", "")); nameValuePairs.add(new BasicNameValuePair(":operation", "social:createMessage")); nameValuePairs.add(new BasicNameValuePair("subject", record.get(3))); nameValuePairs.add(new BasicNameValuePair("content", record.get(4))); nameValuePairs.add(new BasicNameValuePair("sendMail", "Sending...")); } // Creates a forum post (or a reply) if (componentType.equals(FORUM)) { if (urlLevel == 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createForumPost")); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); nameValuePairs.add(new BasicNameValuePair("subject", subComponentType)); } else if (subComponentType.equals(UGCREPLY)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createForumPost")); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); nameValuePairs.add(new BasicNameValuePair("subject", "")); } } // Creates a file or a folder if (componentType.equals(FILES)) { // Top level is always assumed to be a folder, second level files, and third and subsequent levels comments on files if (urlLevel == 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createFileLibraryFolder")); nameValuePairs.add(new BasicNameValuePair("name", subComponentType)); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); } else if (subComponentType.equals(UGCREPLY)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createComment")); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); } } // Creates a question, a reply or mark a reply as the best answer if (componentType.equals(QNA)) { if (vBundleCommunitiesEnablement == null) { logger.info("QnAs are not compatible with this version of AEM"); continue; } if (urlLevel == 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createQnaPost")); nameValuePairs.add(new BasicNameValuePair("subject", subComponentType)); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); } else if (subComponentType.equals(UGCREPLY)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createQnaPost")); nameValuePairs.add(new BasicNameValuePair("subject", "")); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); } } // Creates a Blog article or a comment if (componentType.equals(JOURNAL) || componentType.equals(BLOG)) { if (vBundleCommunitiesEnablement == null) { logger.info("Blogs are not compatible with this version of AEM"); continue; } if (urlLevel == 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createJournalComment")); nameValuePairs.add(new BasicNameValuePair("subject", subComponentType)); StringBuffer message = new StringBuffer("<p>" + record.get(3) + "</p>"); //We might have more paragraphs to add to the blog or journal article for (int i = 6; i < record.size(); i++) { if (record.get(i).length() > 0) { if (record.get(i).startsWith("isDraft")) { nameValuePairs.add(new BasicNameValuePair("isDraft", "true")); } else { message.append("<p>" + record.get(i) + "</p>"); } } } //We might have some tags to add to the blog or journal article if (record.get(5).length() > 0) { nameValuePairs.add(new BasicNameValuePair("tags", record.get(5))); } nameValuePairs.add(new BasicNameValuePair("message", message.toString())); } else if (subComponentType.equals(UGCREPLY)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createJournalComment")); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); nameValuePairs.add(new BasicNameValuePair("subject", "")); } } // Creates an Idea or a comment if (componentType.equals(IDEATION)) { if (!isCommunities61FP6orlater) { logger.info("Ideas are not compatible with this version of AEM"); continue; } if (urlLevel == 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createIdeationComment")); nameValuePairs.add(new BasicNameValuePair("subject", subComponentType)); StringBuffer message = new StringBuffer(""); //We might have more paragraphs to add to the idea for (int i = 6; i < record.size(); i++) { if (record.get(i).length() > 0) { message.append("<p>" + record.get(i) + "</p>"); } } if (record.get(5).equals("TRUE")) { nameValuePairs.add(new BasicNameValuePair("isDraft", "true")); } else { nameValuePairs.add(new BasicNameValuePair("isDraft", "false")); } //We might have some tags to add to the blog or journal article if (record.get(3).length() > 0) { nameValuePairs.add(new BasicNameValuePair("tags", record.get(5))); } nameValuePairs.add(new BasicNameValuePair("message", message.toString())); } else if (subComponentType.equals(UGCREPLY)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createIdeationComment")); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); nameValuePairs.add(new BasicNameValuePair("subject", "")); } } // Taking care of moderation actions for all types if (urlLevel >= 1 && !subComponentType.equals(UGCREPLY)) { if (subComponentType.equals(UGCPIN) && !isCommunities61FP5orlater) { logger.warn("This feature is not supported by this version of AEM"); continue; } if ((subComponentType.equals(UGCFEATURE) || subComponentType.equals(UGCLIKE)) && !isCommunities61FP6orlater) { logger.warn("This feature is not supported by this version of AEM"); continue; } if (subComponentType.equals(UGCANSWER)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:selectAnswer")); } if (subComponentType.equals(UGCDENY)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:deny")); } if (subComponentType.equals(UGCFLAG)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:flag")); nameValuePairs.add(new BasicNameValuePair("social:flagformtext", "Marked as spam")); nameValuePairs.add(new BasicNameValuePair("social:doFlag", "true")); } if (subComponentType.equals(UGCFEATURE)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:featured")); nameValuePairs.add(new BasicNameValuePair("social:markFeatured", "true")); } if (subComponentType.equals(UGCPIN)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:pin")); nameValuePairs.add(new BasicNameValuePair("social:doPin", "true")); } if (subComponentType.equals(UGCUPVOTE)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:postTallyResponse")); nameValuePairs.add(new BasicNameValuePair("response", "1")); nameValuePairs.add(new BasicNameValuePair("tallyType", "Voting")); } if (subComponentType.equals(UGCDOWNVOTE)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:postTallyResponse")); nameValuePairs.add(new BasicNameValuePair("response", "-1")); nameValuePairs.add(new BasicNameValuePair("tallyType", "Voting")); } if (subComponentType.equals(UGCLIKE)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:postTallyResponse")); nameValuePairs.add(new BasicNameValuePair("response", "1")); nameValuePairs.add(new BasicNameValuePair("tallyType", "Liking")); } } // Creates a review or a comment if (componentType.equals(REVIEWS)) { nameValuePairs.add(new BasicNameValuePair("message", record.get(2))); // This might be a top level review, or a comment on a review or another comment if (urlLevel == 0) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createReview")); nameValuePairs.add(new BasicNameValuePair("ratings", record.get(3))); if (record.size() > 4 && record.get(4).length() > 0) { nameValuePairs.add(new BasicNameValuePair("scf:included", record.get(4))); if (record.size() > 5 && record.get(5).length() > 0) { nameValuePairs.add(new BasicNameValuePair("scf:resourceType", record.get(5))); } else { nameValuePairs.add(new BasicNameValuePair("scf:resourceType", "social/reviews/components/hbs/reviews")); } } } else { nameValuePairs.add(new BasicNameValuePair(":operation", "social:createComment")); } } // Creates a rating if (componentType.equals(RATINGS)) { nameValuePairs.add(new BasicNameValuePair(":operation", "social:postTallyResponse")); nameValuePairs.add(new BasicNameValuePair("tallyType", "Rating")); nameValuePairs.add(new BasicNameValuePair("response", subComponentType)); } // Creates a DAM asset if (componentType.equals(ASSET) && record.get(ASSET_INDEX_NAME).length() > 0) { nameValuePairs.add(new BasicNameValuePair("fileName", record.get(ASSET_INDEX_NAME))); } // Creates a simple Folder if (componentType.equals(FOLDER)) { nameValuePairs.add(new BasicNameValuePair("./jcr:content/jcr:title", record.get(2))); nameValuePairs.add(new BasicNameValuePair(":name", record.get(3))); nameValuePairs.add(new BasicNameValuePair("./jcr:primaryType", "sling:Folder")); nameValuePairs.add(new BasicNameValuePair("./jcr:content/jcr:primaryType", "nt:unstructured")); } // Creates a simple Text Fragment if (componentType.equals(FRAGMENT)) { nameValuePairs.add(new BasicNameValuePair("template", "/libs/settings/dam/cfm/templates/simple/jcr:content")); nameValuePairs.add(new BasicNameValuePair("name", record.get(2))); nameValuePairs.add(new BasicNameValuePair("parentPath", record.get(3))); nameValuePairs.add(new BasicNameValuePair("./jcr:title", record.get(4))); nameValuePairs.add(new BasicNameValuePair("description", record.get(5))); nameValuePairs.add(new BasicNameValuePair("author", record.get(0))); //We might have some tags to add to the content fragment if (record.get(5).length() > 0) { nameValuePairs.add(new BasicNameValuePair("tags", record.get(6))); nameValuePairs.add(new BasicNameValuePair("tags@TypeHint", "String[]")); nameValuePairs.add(new BasicNameValuePair("tags@Delete", "")); } } // Creates an Enablement resource if (componentType.equals(RESOURCE)) { // Making sure it's referencing some existing file if (rr == null) { File attachment = new File( csvfile.substring(0, csvfile.indexOf(".csv")) + File.separator + record.get(2)); if (!attachment.exists()) { logger.error( "Resource cannot be created as the referenced file is missing on the file system"); continue; } } else { Resource res = rr.getResource(csvfile + "/attachments/" + record.get(2) + "/jcr:content"); if (res == null) { logger.error("A non existent resource named " + record.get(2) + "was referenced"); continue; } } String createResourceOpName = "se:createResource"; String enablementType = "social/enablement/components/hbs/resource"; if (vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT61FP2)) > 0) createResourceOpName = "social:createResource"; if (vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT62FP1)) > 0) createResourceOpName = "social:createEnablementResourceModel"; nameValuePairs.add(new BasicNameValuePair(":operation", createResourceOpName)); List<NameValuePair> otherNameValuePairs = buildNVP(hostname, port, adminPassword, null, record, RESOURCE_INDEX_PROPERTIES); nameValuePairs.addAll(otherNameValuePairs); // Assignments only make sense when SCORM is configured if (vBundleCommunitiesSCORM == null) { nameValuePairs.remove("add-learners"); nameValuePairs.remove("deltaList"); logger.warn("SCORM not configured on this instance, not assigning a resource"); } // Special processing of lists with multiple users, need to split a String into multiple entries if (vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT61FP2)) > 0) { // Author, contact and experts always make sense nameValuePairs = convertArrays(nameValuePairs, "add-learners"); nameValuePairs = convertArrays(nameValuePairs, "resource-author"); nameValuePairs = convertArrays(nameValuePairs, "resource-contact"); nameValuePairs = convertArrays(nameValuePairs, "resource-expert"); } if (vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT62FP1)) > 0) { nameValuePairs.add(new BasicNameValuePair("sling:resourceType", "social/enablement/components/hbs/resource/model")); nameValuePairs = convertKeyName(nameValuePairs, "add-learners", "resource-assignees"); nameValuePairs = convertKeyName(nameValuePairs, "jcr:title", "resource-name"); nameValuePairs = convertKeyName(nameValuePairs, "resourceTags", "resource-tags"); nameValuePairs = convertKeyName(nameValuePairs, "id", "resource-uid"); enablementType = "resource"; } nameValuePairs.add(new BasicNameValuePair("enablement-type", enablementType)); // Adding the site nameValuePairs.add(new BasicNameValuePair("site", url[0])); // Building the cover image fragment if (record.get(RESOURCE_INDEX_THUMBNAIL).length() > 0) { nameValuePairs.add(new BasicNameValuePair("cover-image", doThumbnail(rr, lIs, hostname, port, adminPassword, csvfile, record.get(RESOURCE_INDEX_THUMBNAIL), record.get(RESOURCE_INDEX_SITE), maxretries))); } else { nameValuePairs.add(new BasicNameValuePair("cover-image", "")); } // Building the asset fragment String assetFileName = record.get(2); // Replacing videos with images in case it's a minimized installation int assetFileNamePos = assetFileName.indexOf(".mp4"); if (assetFileNamePos > 0 && minimize) { assetFileName = assetFileName.substring(0, assetFileNamePos) + ".jpg"; } // Not processing SCORM files if the ignore option is there if (assetFileName.endsWith(".zip") && noenablement) { logger.info("Not processing a SCORM resource for this scenario"); continue; } String coverPath = "/content/dam/resources/" + record.get(RESOURCE_INDEX_SITE) + "/" + record.get(2) + "/jcr:content/renditions/cq5dam.thumbnail.319.319.png"; String coverSource = "dam"; String assets = "[{\"cover-img-path\":\"" + coverPath + "\",\"thumbnail-source\":\"" + coverSource + "\",\"asset-category\":\"enablementAsset:dam\",\"resource-asset-name\":null,\"state\":\"A\",\"asset-path\":\"/content/dam/resources/" + record.get(RESOURCE_INDEX_SITE) + "/" + assetFileName + "\"}]"; nameValuePairs.add(new BasicNameValuePair("assets", assets)); // If it's a SCORM asset, making sure the output is available before processing if (assetFileName.endsWith(".zip")) { doWaitPath( hostname, port, adminPassword, "/content/dam/resources/" + record.get(RESOURCE_INDEX_SITE) + "/" + record.get(2) + "/output", maxretries); } } // Creates a learning path if (componentType.equals(LEARNING)) { if (vBundleCommunitiesSCORM == null || noenablement) { logger.info("Ignoring a learning path"); continue; } String createResourceOpName = "se:editLearningPath"; String enablementType = "social/enablement/components/hbs/learningpath"; String resourceList = "learningpath-items"; if (vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT61FP3)) > 0) createResourceOpName = "social:editLearningPath"; if (vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT62FP1)) > 0) createResourceOpName = "social:createEnablementLearningPathModel"; nameValuePairs.add(new BasicNameValuePair(":operation", createResourceOpName)); List<NameValuePair> otherNameValuePairs = buildNVP(hostname, port, adminPassword, null, record, RESOURCE_INDEX_PROPERTIES); nameValuePairs.addAll(otherNameValuePairs); // Special processing of lists with multiple users, need to split a String into multiple entries if (vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT61FP3)) > 0) { nameValuePairs = convertArrays(nameValuePairs, "add-learners"); nameValuePairs = convertArrays(nameValuePairs, "resource-author"); nameValuePairs = convertArrays(nameValuePairs, "resource-contact"); nameValuePairs = convertArrays(nameValuePairs, "resource-expert"); } if (vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT62FP1)) > 0) { nameValuePairs.add(new BasicNameValuePair("sling:resourceType", "social/enablement/components/hbs/model/learningpath")); nameValuePairs = convertKeyName(nameValuePairs, "add-learners", "resource-assignees"); nameValuePairs = convertKeyName(nameValuePairs, "jcr:title", "resource-name"); nameValuePairs = convertKeyName(nameValuePairs, "resourceTags", "resource-tags"); nameValuePairs = convertKeyName(nameValuePairs, "id", "resource-uid"); enablementType = "learningpath"; resourceList = "resourcelist"; } nameValuePairs.add(new BasicNameValuePair("enablement-type", enablementType)); // Adding the site nameValuePairs.add(new BasicNameValuePair("site", url[0])); // Building the cover image fragment if (record.get(RESOURCE_INDEX_THUMBNAIL).length() > 0) { nameValuePairs.add(new BasicNameValuePair( vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT61FP3)) > 0 ? "cover-image" : "card-image", doThumbnail(rr, lIs, hostname, port, adminPassword, csvfile, record.get(RESOURCE_INDEX_THUMBNAIL), record.get(RESOURCE_INDEX_SITE), maxretries))); } // Building the learning path fragment StringBuffer assets = new StringBuffer("["); if (learningpaths.get(record.get(2)) != null) { assets.append("\""); ArrayList<String> paths = learningpaths.get(record.get(2)); int i = 0; for (String path : paths) { assets.append("{\\\"type\\\":\\\"linked-resource\\\",\\\"path\\\":\\\""); assets.append(path); assets.append("\\\"}"); if (i++ < paths.size() - 1) { assets.append("\",\""); } } assets.append("\""); } else { logger.warn("No asset for this learning path"); } assets.append("]"); nameValuePairs.add(new BasicNameValuePair(resourceList, assets.toString())); } // Creates a calendar event if (componentType.equals(CALENDAR)) { if (vBundleCommunitiesEnablement == null) { logger.info("Calendars are not compatible with this version of AEM"); continue; } String startDate = computeDate(record.get(5), record.get(7)); String endDate = computeDate(record.get(6), record.get(7)); nameValuePairs.add(new BasicNameValuePair(":operation", "social:createEvent")); if (vBundleCommunitiesCalendar != null && vBundleCommunitiesCalendar.compareTo(new Version("1.2.29")) > 0) { // Post AEM Communities 6.1 FP3 nameValuePairs.add(new BasicNameValuePair("subject", record.get(2))); nameValuePairs.add(new BasicNameValuePair("message", record.get(3))); nameValuePairs.add(new BasicNameValuePair("location", record.get(4))); nameValuePairs.add(new BasicNameValuePair("tags", "")); nameValuePairs.add(new BasicNameValuePair("address", "")); nameValuePairs.add(new BasicNameValuePair("isDate", "false")); nameValuePairs.add(new BasicNameValuePair("start", startDate)); nameValuePairs.add(new BasicNameValuePair("end", endDate)); // Let's see if we have tags if (record.size() > CALENDAR_INDEX_TAGS && record.get(CALENDAR_INDEX_TAGS).length() > 0) { nameValuePairs.add(new BasicNameValuePair("tags", record.get(CALENDAR_INDEX_TAGS))); } // Let's see if we have a cover image if (record.size() > CALENDAR_INDEX_THUMBNAIL && record.get(CALENDAR_INDEX_THUMBNAIL).length() > 0) { addBinaryBody(builder, lIs, rr, "coverimage", csvfile, record.get(CALENDAR_INDEX_THUMBNAIL)); } } else { // Pre AEM Communities 6.1 FP3 try { JSONObject event = new JSONObject(); // Building the JSON fragment for a new calendar event event.accumulate("subject", record.get(2)); event.accumulate("message", record.get(3)); event.accumulate("location", record.get(4)); event.accumulate("tags", ""); event.accumulate("undefined", "update"); event.accumulate("start", startDate); event.accumulate("end", endDate); nameValuePairs.add(new BasicNameValuePair("event", event.toString())); } catch (Exception ex) { logger.error(ex.getMessage()); } } } for (NameValuePair nameValuePair : nameValuePairs) { builder.addTextBody(nameValuePair.getName(), nameValuePair.getValue(), ContentType.create("text/plain", MIME.UTF8_CHARSET)); } // See if we have attachments for this new post - or some other actions require a form nonetheless if ((componentType.equals(ASSET) || componentType.equals(AVATAR) || componentType.equals(FORUM) || componentType.equals(IDEATION) || componentType.equals(QNA) || (componentType.equals(JOURNAL)) || componentType.equals(BLOG)) && record.size() > 4 && record.get(ASSET_INDEX_NAME).length() > 0) { addBinaryBody(builder, lIs, rr, "file", csvfile, record.get(ASSET_INDEX_NAME)); } // If it's a resource or a learning path, we need the path to the resource for subsequent publishing Map<String, String> elements = new HashMap<String, String>(); String jsonElement = "location"; String referrer = null; if (componentType.equals(RESOURCE) && vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT61FP2)) <= 0) { jsonElement = "changes/argument"; } if (componentType.equals(LEARNING) && vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT61FP3)) <= 0) { jsonElement = "path"; } if (componentType.equals(RESOURCE) || componentType.equals(LEARNING)) { // Useful for debugging complex POST requests //printPOST(builder.build()); } if (!(componentType.equals(ASSET) || componentType.equals(BADGEASSIGN) || componentType.equals(MESSAGE) || componentType.equals(AVATAR))) { // Creating an asset doesn't return a JSON string elements.put(jsonElement, ""); elements.put("response/resourceType", ""); elements.put("response/id", ""); } // This call generally returns the path to the content fragment that was just created int returnCode = Loader.doPost(hostname, port, getPostURL(componentType, subComponentType, url[urlLevel], userHome), userName, password, builder.build(), elements, null); // Again, Assets being a particular case if (!(componentType.equals(ASSET) || componentType.equals(AVATAR))) { location = elements.get(jsonElement); referrer = elements.get("response/id"); if (Integer.parseInt(record.get(1)) == 0) { analyticsPagePath = location; resourceType = elements.get("response/resourceType"); } } // In case of Assets or Resources, we are waiting for all workflows to be completed if (componentType.equals(ASSET) && returnCode < 400) { doSleep(1000, "Pausing 1s after submitting asset"); doWaitWorkflows(hostname, port, adminPassword, "asset", maxretries); } // If we are loading a content fragment, we need to post the actual content next if (componentType.equals(FRAGMENT)) { // Publishing the learning path List<NameValuePair> fragmentNameValuePairs = new ArrayList<NameValuePair>(); fragmentNameValuePairs.add(new BasicNameValuePair("contentType", "text/html")); StringBuffer message = new StringBuffer("<p>" + record.get(7) + "</p>"); //We might have more paragraphs to add to the fragment if (record.size() > 8) { for (int i = 8; i < record.size(); i++) { if (record.get(i).length() > 0) { message.append("<p>" + record.get(i) + "</p>"); } } } fragmentNameValuePairs.add(new BasicNameValuePair("content", message.toString())); Loader.doPost(hostname, port, record.get(3) + "/" + record.get(2) + ".cfm.content.json", userName, password, new UrlEncodedFormEntity(fragmentNameValuePairs), null); } // Let's see if it needs to be added to a learning path if (componentType.equals(RESOURCE) && record.get(RESOURCE_INDEX_PATH).length() > 0 && location != null) { // Adding the location to a list of a resources for this particular Learning Path if (learningpaths.get(record.get(RESOURCE_INDEX_PATH)) == null) learningpaths.put(record.get(RESOURCE_INDEX_PATH), new ArrayList<String>()); logger.debug("Adding resource to Learning path: " + record.get(RESOURCE_INDEX_PATH)); ArrayList<String> locations = learningpaths.get(record.get(RESOURCE_INDEX_PATH)); locations.add(location); learningpaths.put(record.get(RESOURCE_INDEX_PATH), locations); } // If it's a Learning Path, we publish it when possible if (componentType.equals(LEARNING) && !port.equals(altport) && location != null && vBundleCommunitiesSCORM != null) { // Publishing the learning path List<NameValuePair> publishNameValuePairs = new ArrayList<NameValuePair>(); String publishOpName = "se:publishEnablementContent"; if (vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT62FP1)) > 0) publishOpName = "social:publishEnablementLearningPathModel"; publishNameValuePairs.add(new BasicNameValuePair(":operation", publishOpName)); publishNameValuePairs.add(new BasicNameValuePair("replication-action", "activate")); logger.debug("Publishing a learning path from: " + location); Loader.doPost(hostname, port, location, userName, password, new UrlEncodedFormEntity(publishNameValuePairs), null); // Waiting for the learning path to be published doWaitPath(hostname, altport, adminPassword, location, maxretries); // Decorate the resources within the learning path with comments and ratings, randomly generated ArrayList<String> paths = learningpaths.get(record.get(2)); for (String path : paths) { doDecorate(hostname, altport, adminPassword, path, record, analytics, sitePagePath, vBundleCommunitiesEnablement); } } // If it's an Enablement Resource that is not part of a learning path, a lot of things need to happen... // Step 1. If it's a SCORM resource, we wait for the SCORM metadata workflow to be complete before proceeding // Step 2. We publish the resource // Step 3. We set a new first published date on the resource (3 weeks earlier) so that reporting data is more meaningful // Step 4. We wait for the resource to be available on publish (checking that associated groups are available) // Step 5. We retrieve the json for the resource on publish to retrieve the Social endpoints // Step 6. We post ratings and comments for each of the enrollees on publish if (componentType.equals(RESOURCE) && !port.equals(altport) && location != null && !location.equals("")) { // Wait for the workflows to be completed doWaitWorkflows(hostname, port, adminPassword, "resource", maxretries); String resourcePath = "/assets/asset"; if (vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT62FP1)) > 0) { resourcePath = "/se_assets/se_primary"; } // Wait for the data to be fully copied doWaitPath(hostname, port, adminPassword, location + resourcePath, maxretries); // If we are dealing with a SCORM asset, we wait for the SCORM workflow to be completed before publishing the resource if (record.get(2).indexOf(".zip") > 0) { // Wait for the output to be available doWaitPath(hostname, port, adminPassword, location + resourcePath + "/" + record.get(2) + "/output", maxretries); // Wait for 10 seconds doSleep(10000, "Processing a SCORM resource"); } // Wait for the workflows to be completed before publishing the resource doWaitWorkflows(hostname, port, adminPassword, "resource", maxretries); List<NameValuePair> publishNameValuePairs = new ArrayList<NameValuePair>(); String publishOpName = "se:publishEnablementContent"; if (vBundleCommunitiesEnablement.compareTo(new Version(ENABLEMENT62FP1)) > 0) publishOpName = "social:publishEnablementResourceModel"; publishNameValuePairs.add(new BasicNameValuePair(":operation", publishOpName)); publishNameValuePairs.add(new BasicNameValuePair("replication-action", "activate")); logger.debug("Publishing a Resource from: " + location); Loader.doPost(hostname, port, location, userName, password, new UrlEncodedFormEntity(publishNameValuePairs), null); // Waiting for the resource to be published doWaitPath(hostname, altport, adminPassword, location, maxretries); // Adding comments and ratings for this resource logger.debug("Decorating the resource with comments and ratings"); doDecorate(hostname, altport, adminPassword, location, record, analytics, sitePagePath, vBundleCommunitiesEnablement); // Setting the first published timestamp so that reporting always comes with 3 weeks of data after building a new demo instance DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); Calendar cal = Calendar.getInstance(); cal.add(Calendar.DATE, REPORTINGDAYS); List<NameValuePair> publishDateNameValuePairs = new ArrayList<NameValuePair>(); publishDateNameValuePairs .add(new BasicNameValuePair("date-first-published", dateFormat.format(cal.getTime()))); logger.debug("Setting the publish date for a resource at: " + location); doPost(hostname, port, location, userName, password, new UrlEncodedFormEntity(publishDateNameValuePairs), null); } // Generating Analytics when needed for the new fragment of UGC content if (analytics != null && referrer != null) { logger.debug("Component type: " + componentType + ", Analytics page path: " + analyticsPagePath + ", referrer: " + referrer); logger.debug("Analytics: " + analytics + ", resourceType: " + resourceType + ", sitePagePath: " + sitePagePath + ", userName: " + userName); if (analyticsPagePath != null && (componentType.equals(FORUM) || componentType.equals(FILES) || componentType.equals(QNA) || componentType.equals(BLOG) || componentType.equals(IDEATION) || componentType.equals(CALENDAR))) { logger.debug("level: " + Integer.parseInt(record.get(1))); if (Integer.parseInt(record.get(1)) == 0) { // We just created a UGC page that gets viewed. simulate view events. int views = new Random().nextInt(21) + 10; for (int i = 0; i < views; i++) { doUGCAnalytics(analytics, "event11", analyticsPagePath, resourceType, sitePagePath, userName, referrer); } } else { // We just posted to a UGC page (comment, reply, etc.). simulate post event. doUGCAnalytics(analytics, "event13", analyticsPagePath, resourceType, sitePagePath, userName, referrer); } } } // Closing all the input streams where applicable for (InputStream is : lIs) { try { is.close(); } catch (IOException e) { //Omitted } } } } catch (Exception e) { logger.error(e.getMessage()); } }
From source file:net.tradelib.core.Series.java
static public Series fromCsv(String path, boolean header, DateTimeFormatter dtf, LocalTime lt) throws Exception { if (dtf == null) { if (lt == null) dtf = DateTimeFormatter.ISO_LOCAL_DATE_TIME; else//from w w w . j ava 2 s.co m dtf = DateTimeFormatter.ISO_DATE; } // Parse and import the csv CSVFormat csvFmt = CSVFormat.DEFAULT.withCommentMarker('#').withIgnoreSurroundingSpaces(); if (header) csvFmt = csvFmt.withHeader(); CSVParser csv = csvFmt.parse(new BufferedReader(new FileReader(path))); int ncols = -1; Series result = null; double[] values = null; for (CSVRecord rec : csv.getRecords()) { if (result == null) { ncols = rec.size() - 1; values = new double[ncols]; result = new Series(ncols); } for (int ii = 0; ii < ncols; ++ii) { values[ii] = Double.parseDouble(rec.get(ii + 1)); } LocalDateTime ldt; if (lt != null) { ldt = LocalDate.parse(rec.get(0), dtf).atTime(lt); } else { ldt = LocalDateTime.parse(rec.get(0), dtf); } result.append(ldt, values); } if (header) { Map<String, Integer> headerMap = csv.getHeaderMap(); result.clearNames(); for (Map.Entry<String, Integer> me : headerMap.entrySet()) { if (me.getValue() > 0) result.setName(me.getKey(), me.getValue() - 1); } } return result; }
From source file:nl.paston.bonita.importfile.Main.java
protected static Map<String, Serializable> parseRecord(CSVRecord record, CSVRecord fullHeader) { if (record == null) { log.warn("Record is null."); return null; }/* ww w.jav a 2 s . c o m*/ log.info("Parsing record number: " + (record.getRecordNumber() - 1)); log.debug(" with content: " + record.toString()); final Map<String, Serializable> map = new HashMap<>(); for (int i = 0; i < record.size(); i++) { String headerFieldType = getHeaderFieldType(fullHeader.get(i)); Object recordField = getRecordField(headerFieldType, record.get(i)); if (recordField != null) { String headerField = getHeaderField(fullHeader.get(i)); String[] headerFieldParts = headerField.split("\\."); Map<String, Serializable> targetMap = map; for (int j = 0; j < headerFieldParts.length - 1; j++) { try { targetMap = (Map<String, Serializable>) targetMap.computeIfAbsent(headerFieldParts[j], x -> new HashMap<>()); } catch (ClassCastException ex) { log.debug("Problem parsing: " + headerFieldParts[j]); } } String currentHeaderField = headerFieldParts[headerFieldParts.length - 1]; Matcher listMatcher = Pattern.compile("\\[(.*)\\]").matcher(currentHeaderField); if (listMatcher.find()) { String parametersString = listMatcher.group(1); Matcher chfNameMatcher = Pattern.compile("(.*)\\[").matcher(currentHeaderField); if (chfNameMatcher.find()) { String chfName = chfNameMatcher.group(1); String[] parameters = parametersString.split("&"); List<Serializable> list = (List<Serializable>) targetMap.computeIfAbsent(chfName, x -> new ArrayList<>()); if (parametersString.isEmpty()) { list.add((Serializable) recordField); } else { Map<String, Serializable> subMap = new HashMap<>(); for (String parameter : parameters) { String[] parameterKeys = parameter.split("="); if (parameterKeys.length == 2) { subMap.put(parameterKeys[0], parameterKeys[1]); } else if (parameterKeys.length == 1) { subMap.put(parameterKeys[0], (Serializable) recordField); } else { log.error("Wrong number of parameters for: " + currentHeaderField); System.exit(1); } } list.add((Serializable) subMap); } } } else { targetMap.put(currentHeaderField, (Serializable) recordField); } } else { log.warn("Skipped value for record item: " + record.get(i)); } } return map; }
From source file:no.packdrill.android.sparkledroid.lib.parse.csvtsv.CSVParse.java
@Override public Iterator<Map<String, Cell>> iterator() //-------------------------------- { return new Iterator<Map<String, Cell>>() //======================================== {/*ww w . j a v a 2 s . c om*/ Map<String, Cell> m = new HashMap<String, Cell>(); @Override public boolean hasNext() { return it.hasNext(); } @Override public Map<String, Cell> next() //-------------------------------- { CSVRecord record = null; try { record = it.next(); m.clear(); for (int i = 0; i < record.size(); i++) { String k = columns[i]; final String v = record.get(i); if (v == null) continue; if (v.trim().startsWith("_:")) { int p = v.indexOf("_:"); String name; try { name = v.substring(p + 2); } catch (Exception _e) { name = ""; } Cell ri = new Cell(true, name); ri.setUnparsedValue(v); m.put(k, ri); continue; } if (v.trim().toLowerCase().startsWith("missing ")) { Object o = null; Cell ri = new Cell("null", o); ri.setUnparsedValue(v); m.put(k, ri); continue; } URI uri = null; try { uri = new URI(v); } catch (Exception _e) { uri = null; } if (uri != null) { Cell ri = processURI(v); if (ri != null) { m.put(k, ri); continue; } } Cell ri = new Cell(v); ri.setUnparsedValue(v); m.put(k, ri); } } catch (NoSuchElementException e) { lastError = (record == null) ? "" : record.toString(); lastException = e; Log.e(LOGTAG, lastError, e); return null; } return m; } @Override public void remove() { throw new UnsupportedOperationException( "no.packdrill.android.SPARQLClient.parse.csvtsv.iterator.remove"); } }; }
From source file:no.packdrill.android.sparkledroid.lib.parse.csvtsv.TSVParse.java
@Override public Iterator<Map<String, Cell>> iterator() //-------------------------------- { return new Iterator<Map<String, Cell>>() //======================================== {//from w ww .j a v a2 s.c o m Map<String, Cell> m = new HashMap<String, Cell>(); final Pattern languagePattern = Pattern.compile("\"(.+)\"@(\\w\\w)$"); @Override public boolean hasNext() { return it.hasNext(); } @Override public Map<String, Cell> next() //-------------------------------- { CSVRecord record = null; int p; try { record = it.next(); m.clear(); for (int i = 0; i < record.size(); i++) { String k = columns[i]; String v = record.get(i); if (v == null) continue; if (v.trim().startsWith("_:")) { p = v.indexOf("_:"); String name; try { name = v.substring(p + 2); } catch (Exception _e) { name = ""; } Cell ri = new Cell(true, name); ri.setUnparsedValue(v); m.put(k, ri); continue; } if ((v.trim().startsWith("<")) && (v.trim().endsWith(">"))) { p = v.indexOf('<'); int p1 = v.indexOf('>'); if ((p >= 0) && (p1 > 0)) v = v.substring(p + 1, p1); URI uri = null; try { uri = new URI(v); } catch (Exception _e) { uri = null; } if (uri != null) { Cell ri = processURI(v); if (ri != null) { m.put(k, ri); continue; } } Matcher patmatch = languagePattern.matcher(v); if ((patmatch.matches()) && (patmatch.groupCount() > 0)) { String s = patmatch.group(1); String lang = null; if (patmatch.groupCount() > 1) lang = patmatch.group(2); if ((s != null) && (lang != null)) { Cell ri = new Cell(s, lang); ri.setUnparsedValue(v); m.put(k, ri); continue; } } } Cell ri = new Cell(v); ri.setUnparsedValue(v); m.put(k, ri); } } catch (Exception e) { lastError = (record == null) ? "" : record.toString(); lastException = e; Log.e(LOGTAG, lastError, e); return null; } return m; } @Override public void remove() { throw new UnsupportedOperationException( "no.packdrill.android.SPARQLClient.parse.csvtsv.iterator.remove"); } }; }
From source file:norbert.mynemo.dataimport.fileformat.input.MovieLensIdConverter.java
/** * Loads the mapping file./*www.ja v a 2 s . co m*/ * * <p> * The columns of the mapping file are: * <ol> * <li>MovieLens id of the movie</li> * <li>rating</li> * <li>average</li> * <li>IMDb id of the movie</li> * <li>title and year</li> * </ol> * * @param mappingFilepath the file that contains the mapping */ public MovieLensIdConverter(String mappingFilepath) throws IOException { checkArgument(new File(mappingFilepath).exists(), "The mapping file must exist."); CSVParser parser = new CSVParser( new CleanRatingsReader(new BufferedReader(new FileReader(mappingFilepath))), CSVFormat.MYSQL); mappings = new HashMap<>(); for (CSVRecord record : parser) { if (record.size() != RECORD_SIZE) { parser.close(); throw new IllegalStateException("Error: unable to parse the movie file \"" + mappingFilepath + "\". A list of five tab separated values is expected. Approximate" + " line number: " + record.getRecordNumber()); } mappings.put(record.get(MOVIELENS_MOVIE_ID_INDEX), record.get(IMDB_MOVIE_ID_INDEX)); } parser.close(); }
From source file:norbert.mynemo.dataimport.fileformat.input.MovieLensRatingImporter.java
/** * Returns <code>true</code> if the given file can be parsed, <code>false</code> otherwise. *//*from ww w . j a va2 s . c o m*/ public static boolean canImport(String filepath) throws IOException { checkArgument(new File(filepath).exists(), "The file must exist."); if (!CleanRatingsReader.canClean(filepath)) { return false; } try (CSVParser parser = createParser(filepath)) { for (CSVRecord record : parser) { // try to create just one rating return record.size() == RECORD_SIZE && MynemoRating.isValid(TEST_USER, record.get(MOVIE_INDEX), record.get(VALUE_INDEX)); } } // everything seems right return true; }
From source file:notaql.engines.csv.CSVEngineEvaluator.java
/** * Evaluates the given transformation.//w w w.ja va 2 s. c o m * * This first parses the document (with the first line being the header) and then evaluates on our framework. * * TODO: this assumes a header line. It might happen that it is not provided. * * @param transformation * @return */ @Override public JavaRDD<ObjectValue> evaluate(Transformation transformation) { final SparkTransformationEvaluator evaluator = new SparkTransformationEvaluator(transformation); final JavaSparkContext sc = NotaQL.SparkFactory.getSparkContext(); final CSVFormat format = CSVFormat.DEFAULT; final JavaRDD<String> csv = sc.textFile(path); final String first = csv.first(); final CSVRecord header; try { header = format.parse(new StringReader(first)).iterator().next(); } catch (IOException e) { e.printStackTrace(); throw new AssertionError("Header could not be read for some reason."); } String[] headerCols = new String[header.size()]; for (int i = 0; i < header.size(); i++) { headerCols[i] = header.get(i); } final CSVFormat headerFormat = CSVFormat.DEFAULT.withHeader(headerCols); final JavaRDD<CSVRecord> records = csv.filter(f -> !f.equals(first)) .map(line -> headerFormat.parse(new StringReader(line)).iterator().next()); final JavaRDD<Value> converted = records.map(ValueConverter::convertToNotaQL); final JavaRDD<Value> filtered = converted.filter(o -> transformation.satisfiesInPredicate((ObjectValue) o)); return evaluator.process(filtered); }
From source file:nz.ac.waikato.cms.supernova.gui.Supernova.java
/** * Generates the output of the "batch" tab. *//*www . j a v a2 s.c o m*/ protected void generateBatchOutput() { String cls; AbstractOutputGenerator generator; int colID; int colMeasure; int colScore; int colPercentile; Reader reader; CSVParser csvparser; String oldID; Map<String, List<Double>> test; String id; File outfile; String msg; String measure; double score; double percentile; String error; m_BatchLog.setText(""); m_BatchGenerate.setEnabled(false); try { cls = AbstractOutputGenerator.class.getPackage().getName() + "." + m_SingleGenerator.getSelectedItem(); generator = (AbstractOutputGenerator) Class.forName(cls).newInstance(); } catch (Exception e) { batchLog("Failed to instantiate output generator - falling back on PNG", e); generator = new PNG(); } try { colID = 0; colMeasure = 1; colScore = 2; colPercentile = 3; reader = new FileReader(m_BatchCSV.getCurrent()); csvparser = new CSVParser(reader, CSVFormat.EXCEL.withHeader()); oldID = ""; test = new HashMap<>(); for (CSVRecord rec : csvparser) { if (rec.size() < 4) continue; id = rec.get(colID); if (!id.equals(oldID)) { if (!test.isEmpty()) { outfile = new File(m_BatchOutput.getCurrent() + File.separator + oldID + "." + generator.getExtension()); batchLog("Generating: " + outfile, false); batchLog("Using: " + test, false); msg = generator.generate(test, outfile); if (msg != null) { error = "Failed to generate output for ID: " + oldID; batchLog(error, true); } } test.clear(); oldID = id; } measure = rec.get(colMeasure); score = Double.parseDouble(rec.get(colScore)); percentile = Double.parseDouble(rec.get(colPercentile)); test.put(measure, new ArrayList<>(Arrays.asList(new Double[] { score, percentile }))); } if (!test.isEmpty()) { outfile = new File( m_BatchOutput.getCurrent() + File.separator + oldID + "." + generator.getExtension()); batchLog("Generating: " + outfile, false); batchLog("Using: " + test, false); msg = generator.generate(test, outfile); if (msg != null) { error = "Failed to generate output for ID: " + oldID; batchLog(error, true); } } } catch (Exception e) { batchLog("Failed to generate output!", e); } m_BatchGenerate.setEnabled(true); }