List of usage examples for java.util TreeMap keySet
public Set<K> keySet()
From source file:com.netflix.ice.processor.BillingFileProcessor.java
@Override protected void poll() throws Exception { TreeMap<DateTime, List<BillingFile>> filesToProcess = Maps.newTreeMap(); Map<DateTime, List<BillingFile>> monitorFilesToProcess = Maps.newTreeMap(); // list the tar.gz file in billing file folder for (int i = 0; i < config.billingS3BucketNames.length; i++) { String billingS3BucketName = config.billingS3BucketNames[i]; String billingS3BucketPrefix = config.billingS3BucketPrefixes.length > i ? config.billingS3BucketPrefixes[i] : ""; String accountId = config.billingAccountIds.length > i ? config.billingAccountIds[i] : ""; String billingAccessRoleName = config.billingAccessRoleNames.length > i ? config.billingAccessRoleNames[i] : ""; String billingAccessExternalId = config.billingAccessExternalIds.length > i ? config.billingAccessExternalIds[i] : ""; logger.info("trying to list objects in billing bucket " + billingS3BucketName + " using assume role, and external id " + billingAccessRoleName + " " + billingAccessExternalId); List<S3ObjectSummary> objectSummaries = AwsUtils.listAllObjects(billingS3BucketName, billingS3BucketPrefix, accountId, billingAccessRoleName, billingAccessExternalId); logger.info("found " + objectSummaries.size() + " in billing bucket " + billingS3BucketName); TreeMap<DateTime, S3ObjectSummary> filesToProcessInOneBucket = Maps.newTreeMap(); Map<DateTime, S3ObjectSummary> monitorFilesToProcessInOneBucket = Maps.newTreeMap(); // for each file, download&process if not needed for (S3ObjectSummary objectSummary : objectSummaries) { String fileKey = objectSummary.getKey(); DateTime dataTime = AwsUtils.getDateTimeFromFileNameWithTags(fileKey); boolean withTags = true; if (dataTime == null) { dataTime = AwsUtils.getDateTimeFromFileName(fileKey); withTags = false;/*from w w w .j av a2s.com*/ } if (dataTime != null && !dataTime.isBefore(config.startDate)) { if (!filesToProcessInOneBucket.containsKey(dataTime) || withTags && config.resourceService != null || !withTags && config.resourceService == null) filesToProcessInOneBucket.put(dataTime, objectSummary); else logger.info("ignoring file " + objectSummary.getKey()); } else { logger.info("ignoring file " + objectSummary.getKey()); } } for (S3ObjectSummary objectSummary : objectSummaries) { String fileKey = objectSummary.getKey(); DateTime dataTime = AwsUtils.getDateTimeFromFileNameWithMonitoring(fileKey); if (dataTime != null && !dataTime.isBefore(config.startDate)) { monitorFilesToProcessInOneBucket.put(dataTime, objectSummary); } } for (DateTime key : filesToProcessInOneBucket.keySet()) { List<BillingFile> list = filesToProcess.get(key); if (list == null) { list = Lists.newArrayList(); filesToProcess.put(key, list); } list.add(new BillingFile(filesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix)); } for (DateTime key : monitorFilesToProcessInOneBucket.keySet()) { List<BillingFile> list = monitorFilesToProcess.get(key); if (list == null) { list = Lists.newArrayList(); monitorFilesToProcess.put(key, list); } list.add(new BillingFile(monitorFilesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix)); } } for (DateTime dataTime : filesToProcess.keySet()) { startMilli = endMilli = dataTime.getMillis(); init(); boolean hasNewFiles = false; boolean hasTags = false; long lastProcessed = lastProcessTime(AwsUtils.monthDateFormat.print(dataTime)); for (BillingFile billingFile : filesToProcess.get(dataTime)) { S3ObjectSummary objectSummary = billingFile.s3ObjectSummary; if (objectSummary.getLastModified().getTime() < lastProcessed) { logger.info("data has been processed. ignoring " + objectSummary.getKey() + "..."); continue; } hasNewFiles = true; } if (!hasNewFiles) { logger.info("data has been processed. ignoring all files at " + AwsUtils.monthDateFormat.print(dataTime)); continue; } long processTime = new DateTime(DateTimeZone.UTC).getMillis(); for (BillingFile billingFile : filesToProcess.get(dataTime)) { S3ObjectSummary objectSummary = billingFile.s3ObjectSummary; String fileKey = objectSummary.getKey(); File file = new File(config.localDir, fileKey.substring(billingFile.prefix.length())); logger.info("trying to download " + fileKey + "..."); boolean downloaded = AwsUtils.downloadFileIfChangedSince(objectSummary.getBucketName(), billingFile.prefix, file, lastProcessed, billingFile.accountId, billingFile.accessRoleName, billingFile.externalId); if (downloaded) logger.info("downloaded " + fileKey); else { logger.info("file already downloaded " + fileKey + "..."); } logger.info("processing " + fileKey + "..."); boolean withTags = fileKey.contains("with-resources-and-tags"); hasTags = hasTags || withTags; processingMonitor = false; processBillingZipFile(file, withTags); logger.info("done processing " + fileKey); } if (monitorFilesToProcess.get(dataTime) != null) { for (BillingFile monitorBillingFile : monitorFilesToProcess.get(dataTime)) { S3ObjectSummary monitorObjectSummary = monitorBillingFile.s3ObjectSummary; if (monitorObjectSummary != null) { String monitorFileKey = monitorObjectSummary.getKey(); logger.info("processing " + monitorFileKey + "..."); File monitorFile = new File(config.localDir, monitorFileKey.substring(monitorFileKey.lastIndexOf("/") + 1)); logger.info("trying to download " + monitorFileKey + "..."); boolean downloaded = AwsUtils.downloadFileIfChangedSince( monitorObjectSummary.getBucketName(), monitorBillingFile.prefix, monitorFile, lastProcessed, monitorBillingFile.accountId, monitorBillingFile.accessRoleName, monitorBillingFile.externalId); if (downloaded) logger.info("downloaded " + monitorFile); else logger.warn(monitorFile + "already downloaded..."); FileInputStream in = new FileInputStream(monitorFile); try { processingMonitor = true; processBillingFile(monitorFile.getName(), in, true); } catch (Exception e) { logger.error("Error processing " + monitorFile, e); } finally { in.close(); } } } } if (dataTime.equals(filesToProcess.lastKey())) { int hours = (int) ((endMilli - startMilli) / 3600000L); logger.info("cut hours to " + hours); cutData(hours); } // now get reservation capacity to calculate upfront and un-used cost for (Ec2InstanceReservationPrice.ReservationUtilization utilization : Ec2InstanceReservationPrice.ReservationUtilization .values()) processReservations(utilization); if (hasTags && config.resourceService != null) config.resourceService.commit(); logger.info("archiving results for " + dataTime + "..."); archive(); logger.info("done archiving " + dataTime); updateProcessTime(AwsUtils.monthDateFormat.print(dataTime), processTime); if (dataTime.equals(filesToProcess.lastKey())) { sendOndemandCostAlert(); } } logger.info("AWS usage processed."); }
From source file:org.exoplatform.addon.pulse.service.ws.RestActivitiesStatistic.java
private ChartData buildStatisticByFilter(String maxColumn, String filter, Date fromDate) throws Exception { int totalDataCoulumn = 5; try {//from www .j a va2s. c o m totalDataCoulumn = Integer.parseInt(maxColumn); } catch (Exception e) { //do nothing } if (filter.equalsIgnoreCase(FILTER_BY_DAY)) { Calendar calendar = Calendar.getInstance(); calendar.setTime(fromDate); calendar.add(Calendar.DATE, totalDataCoulumn - 1); Date toDate = calendar.getTime(); List<ActivityStatisticBean> list = service.getListActivityStatisticByDate(fromDate, toDate); TreeMap<Date, ActivityStatisticBean> dateData = new TreeMap<Date, ActivityStatisticBean>(); //init empty-data for (int i = 0; i < totalDataCoulumn; i++) { calendar.clear(); calendar.setTime(fromDate); calendar.add(Calendar.DATE, i); Date nextDate = parseDate(partString(calendar.getTime(), "dd/MM/yyyy"), "dd/MM/yyyy"); dateData.put(nextDate, null); } List<String> listTitle = new ArrayList<String>(); List<Long> newUsersData = new ArrayList<Long>(); List<Long> loginCountData = new ArrayList<Long>(); List<Long> forumActiveUsersData = new ArrayList<Long>(); List<Long> newForumPostsData = new ArrayList<Long>(); List<Long> userConnectionData = new ArrayList<Long>(); List<Long> socialPostData = new ArrayList<Long>(); List<Long> emailNotificationData = new ArrayList<Long>(); ChartData chartData = new ChartData(); for (ActivityStatisticBean bean : list) { dateData.put(parseDate(partString(bean.getCreatedDate(), "dd/MM/yyyy"), "dd/MM/yyyy"), bean); } for (Date key : dateData.keySet()) { ActivityStatisticBean bean = dateData.get(key); if (bean != null) { listTitle.add(partString(bean.getCreatedDate(), "dd-MM-yyyy")); newUsersData.add(bean.getNewUserToday()); loginCountData.add(bean.getLoginCountToday()); forumActiveUsersData.add(bean.getForumActiveUserToday()); newForumPostsData.add(bean.getForumPostToday()); userConnectionData.add(bean.getUserConnectionCountToday()); socialPostData.add(bean.getSocialPostCountToday()); emailNotificationData.add(bean.getEmailNotificationCountToday()); } else { listTitle.add(partString(key, "dd-MM-yyyy")); newUsersData.add(0L); loginCountData.add(0L); forumActiveUsersData.add(0L); newForumPostsData.add(0L); userConnectionData.add(0L); socialPostData.add(0L); emailNotificationData.add(0L); } } chartData.setListTitle(listTitle); chartData.setNewUsersData(newUsersData); chartData.setLoginCountData(loginCountData); chartData.setForumActiveUsersData(forumActiveUsersData); chartData.setNewForumPostsData(newForumPostsData); chartData.setUserConnectionData(userConnectionData); chartData.setSocialPostData(socialPostData); chartData.setEmailNotificationData(emailNotificationData); return chartData; } if (filter.equalsIgnoreCase(FILTER_BY_WEEK)) { Calendar calendar = Calendar.getInstance(); calendar.clear(); calendar.setTime(fromDate); calendar.add(Calendar.WEEK_OF_YEAR, totalDataCoulumn - 1); Date nextFewWeek = calendar.getTime(); List<ActivityStatisticBean> list = service.getListActivityStatisticByDate(fromDate, nextFewWeek); List<String> listTitle = new ArrayList<String>(); List<Long> newUsersData = new ArrayList<Long>(); List<Long> loginCountData = new ArrayList<Long>(); List<Long> forumActiveUsersData = new ArrayList<Long>(); List<Long> newForumPostsData = new ArrayList<Long>(); List<Long> userConnectionData = new ArrayList<Long>(); List<Long> socialPostData = new ArrayList<Long>(); List<Long> emailNotificationData = new ArrayList<Long>(); ChartData chartData = new ChartData(); TreeMap<String, List<ActivityStatisticBean>> weekData = new TreeMap<String, List<ActivityStatisticBean>>(); //init empty-data for (int i = 0; i < totalDataCoulumn; i++) { calendar.clear(); calendar.setTime(fromDate); calendar.add(Calendar.WEEK_OF_YEAR, i); int weekIndex = calendar.get(Calendar.WEEK_OF_YEAR); int monthIndex = calendar.get(Calendar.MONTH); if (monthIndex == Calendar.DECEMBER && weekIndex == 1) weekIndex = 53; int year = calendar.get(Calendar.YEAR); //goto begin of week calendar.clear(); calendar.set(Calendar.WEEK_OF_YEAR, weekIndex); calendar.set(Calendar.YEAR, year); //goto end of week calendar.add(Calendar.DATE, 6); String week = ""; if (calendar.get(Calendar.MONTH) == Calendar.DECEMBER && calendar.get(Calendar.WEEK_OF_YEAR) == 1) { week = 53 + "-" + calendar.get(Calendar.YEAR); } else { week = calendar.get(Calendar.WEEK_OF_YEAR) + "-" + calendar.get(Calendar.YEAR); } week = week.length() < 7 ? calendar.get(Calendar.YEAR) + "-" + "0" + week : calendar.get(Calendar.YEAR) + "-" + week; weekData.put(week, new ArrayList<ActivityStatisticBean>()); } for (ActivityStatisticBean bean : list) { calendar.clear(); calendar.setTime(bean.getCreatedDate()); int weekIndex = calendar.get(Calendar.WEEK_OF_YEAR); int monthIndex = calendar.get(Calendar.MONTH); if (monthIndex == Calendar.DECEMBER && weekIndex == 1) weekIndex = 53; int year = calendar.get(Calendar.YEAR); //goto begin of week calendar.clear(); calendar.set(Calendar.WEEK_OF_YEAR, weekIndex); calendar.set(Calendar.YEAR, year); //goto end of week calendar.add(Calendar.DATE, 6); String week = ""; if (calendar.get(Calendar.MONTH) == Calendar.DECEMBER && calendar.get(Calendar.WEEK_OF_YEAR) == 1) { week = 53 + "-" + calendar.get(Calendar.YEAR); } else { week = calendar.get(Calendar.WEEK_OF_YEAR) + "-" + calendar.get(Calendar.YEAR); } week = week.length() < 7 ? calendar.get(Calendar.YEAR) + "-" + "0" + week : calendar.get(Calendar.YEAR) + "-" + week; if (weekData.containsKey(week)) { List<ActivityStatisticBean> listValueOfNode = weekData.get(week); listValueOfNode.add(bean); } else { List<ActivityStatisticBean> listValueOfNode = new ArrayList<ActivityStatisticBean>(); listValueOfNode.add(bean); weekData.put(week, listValueOfNode); } } for (String key : weekData.keySet()) { List<ActivityStatisticBean> listValueOfNode = weekData.get(key); Long weekNewUsersValue = 0L; Long weekLoginCountValue = 0L; Long weekForumActiveUsersValue = 0L; Long weekNewForumPostsValue = 0L; Long weekUserConnectionValue = 0L; Long weekSocialPostsValue = 0L; Long weekEmailNotificationValue = 0L; for (ActivityStatisticBean obj : listValueOfNode) { weekNewUsersValue = weekNewUsersValue + obj.getNewUserToday(); weekLoginCountValue = weekLoginCountValue + obj.getLoginCountToday(); weekForumActiveUsersValue = weekForumActiveUsersValue + obj.getForumActiveUserToday(); weekNewForumPostsValue = weekNewForumPostsValue + obj.getForumPostToday(); weekUserConnectionValue = weekUserConnectionValue + obj.getUserConnectionCountToday(); weekSocialPostsValue = weekSocialPostsValue + obj.getSocialPostCountToday(); weekEmailNotificationValue = weekEmailNotificationValue + obj.getEmailNotificationCountToday(); } String weekTitle = "W" + key.substring(5, key.length()); listTitle.add(weekTitle); newUsersData.add(weekNewUsersValue); loginCountData.add(weekLoginCountValue); forumActiveUsersData.add( weekForumActiveUsersValue > 0 ? (Long) (weekForumActiveUsersValue / listValueOfNode.size()) : 0L); newForumPostsData.add(weekNewForumPostsValue); userConnectionData.add(weekUserConnectionValue); socialPostData.add(weekSocialPostsValue); emailNotificationData.add(weekEmailNotificationValue); } chartData.setListTitle(listTitle); chartData.setNewUsersData(newUsersData); chartData.setLoginCountData(loginCountData); chartData.setForumActiveUsersData(forumActiveUsersData); chartData.setNewForumPostsData(newForumPostsData); chartData.setUserConnectionData(userConnectionData); chartData.setSocialPostData(socialPostData); chartData.setEmailNotificationData(emailNotificationData); return chartData; } if (filter.equalsIgnoreCase(FILTER_BY_MONTH)) { Calendar calendar = Calendar.getInstance(); calendar.clear(); calendar.setTime(fromDate); calendar.add(Calendar.MONTH, totalDataCoulumn - 1); Date nextFewMonth = calendar.getTime(); List<ActivityStatisticBean> list = service.getListActivityStatisticByDate(fromDate, nextFewMonth); List<String> listTitle = new ArrayList<String>(); List<Long> newUsersData = new ArrayList<Long>(); List<Long> loginCountData = new ArrayList<Long>(); List<Long> forumActiveUsersData = new ArrayList<Long>(); List<Long> newForumPostsData = new ArrayList<Long>(); List<Long> userConnectionData = new ArrayList<Long>(); List<Long> socialPostData = new ArrayList<Long>(); List<Long> emailNotificationData = new ArrayList<Long>(); ChartData chartData = new ChartData(); TreeMap<String, List<ActivityStatisticBean>> monthData = new TreeMap<String, List<ActivityStatisticBean>>(); //init empty-data for (int i = 0; i < totalDataCoulumn; i++) { calendar.clear(); calendar.setTime(fromDate); calendar.add(Calendar.MONTH, i); String month = calendar.get(Calendar.YEAR) + "-" + partString(calendar.getTime(), "MM") + "-" + partString(calendar.getTime(), "MMM") + "-" + calendar.get(Calendar.YEAR); //get name of Month monthData.put(month, new ArrayList<ActivityStatisticBean>()); } for (ActivityStatisticBean bean : list) { calendar.clear(); calendar.setTime(bean.getCreatedDate()); String month = calendar.get(Calendar.YEAR) + "-" + partString(calendar.getTime(), "MM") + "-" + partString(calendar.getTime(), "MMM") + "-" + calendar.get(Calendar.YEAR); //get name of Month if (monthData.containsKey(month)) { List<ActivityStatisticBean> listValueOfNode = monthData.get(month); listValueOfNode.add(bean); } else { List<ActivityStatisticBean> listValueOfNode = new ArrayList<ActivityStatisticBean>(); listValueOfNode.add(bean); monthData.put(month, listValueOfNode); } } for (String key : monthData.keySet()) { List<ActivityStatisticBean> listValueOfNode = monthData.get(key); Long monthNewUsersValue = 0L; Long monthLoginCountValue = 0L; Long monthForumActiveUsersValue = 0L; Long monthNewForumPostsValue = 0L; Long monthUserConnectionValue = 0L; Long monthSocialPostsValue = 0L; Long monthEmailNotificationValue = 0L; for (ActivityStatisticBean obj : listValueOfNode) { monthNewUsersValue = monthNewUsersValue + obj.getNewUserToday(); monthLoginCountValue = monthLoginCountValue + obj.getLoginCountToday(); monthForumActiveUsersValue = monthForumActiveUsersValue + obj.getForumActiveUserToday(); monthNewForumPostsValue = monthNewForumPostsValue + obj.getForumPostToday(); monthUserConnectionValue = monthUserConnectionValue + obj.getUserConnectionCountToday(); monthSocialPostsValue = monthSocialPostsValue + obj.getSocialPostCountToday(); monthEmailNotificationValue = +monthEmailNotificationValue + obj.getEmailNotificationCountToday(); } listTitle.add(key.substring(8, key.length())); newUsersData.add(monthNewUsersValue); loginCountData.add(monthLoginCountValue); forumActiveUsersData.add(monthForumActiveUsersValue > 0 ? (Long) (monthForumActiveUsersValue / listValueOfNode.size()) : 0L); newForumPostsData.add(monthNewForumPostsValue); userConnectionData.add(monthUserConnectionValue); socialPostData.add(monthSocialPostsValue); emailNotificationData.add(monthEmailNotificationValue); } chartData.setListTitle(listTitle); chartData.setNewUsersData(newUsersData); chartData.setLoginCountData(loginCountData); chartData.setForumActiveUsersData(forumActiveUsersData); chartData.setNewForumPostsData(newForumPostsData); chartData.setUserConnectionData(userConnectionData); chartData.setSocialPostData(socialPostData); chartData.setEmailNotificationData(emailNotificationData); return chartData; } return null; }
From source file:ubic.gemma.persistence.service.association.coexpression.CoexpressionDaoImpl.java
private void saveExperimentLevelLinks(Session sess, LinkCreator c, TreeMap<Long, NonPersistentNonOrderedCoexpLink> links, BioAssaySet bioAssaySet) { int progress = 0; int BATCH_SIZE = 1024; List<ExperimentCoexpressionLink> flippedLinks = new ArrayList<>(); for (Long linkid : links.keySet()) { NonPersistentNonOrderedCoexpLink link = links.get(linkid); ExperimentCoexpressionLink ecl = c.createEELink(bioAssaySet, linkid, link.getFirstGene(), link.getSecondGene());//from ww w. j a va 2s . c o m /* * At same time, create flipped versions, but save them later for ordering. Notice that we use the SAME link * ID - not the one for the flipped version in the gene2gene table. * * Ideally we would ensure that the gene2gene link ID used is the same for all links that are between * the same pair of genes. That would let us be able to easily count the support directly from an * experiment-level query, without going to the supportDetails. I do not believe the current code guarantees * this. */ flippedLinks.add(c.createEELink(bioAssaySet, linkid, link.getSecondGene(), link.getFirstGene())); sess.save(ecl); if (++progress % 50000 == 0) { CoexpressionDaoImpl.log .info("Created " + progress + "/" + links.size() + " experiment-level links..."); } if (progress % BATCH_SIZE == 0) { sess.flush(); sess.clear(); } } sess.flush(); sess.clear(); /* * Sort the flipped links by the first gene */ Collections.sort(flippedLinks, new Comparator<ExperimentCoexpressionLink>() { @Override public int compare(ExperimentCoexpressionLink o1, ExperimentCoexpressionLink o2) { return o1.getFirstGene().compareTo(o2.getFirstGene()); } }); /* * Save the flipped ones. */ progress = 0; for (ExperimentCoexpressionLink fl : flippedLinks) { sess.save(fl); if (++progress % 50000 == 0) { CoexpressionDaoImpl.log .info("Created " + progress + "/" + links.size() + " flipped experiment-level links..."); } if (progress % BATCH_SIZE == 0) { sess.flush(); sess.clear(); } } // one for the road. sess.flush(); sess.clear(); }
From source file:com.sfs.whichdoctor.webservice.RotationServiceImpl.java
/** * Find people based on the supplied parameters. * * @param year the year/* w w w . j a va 2 s. c om*/ * @param program the program * @param training_site_country_code the training_site_country_code * @param state the state * @param primary_site the primary_site * @param training_site the training_site * @param division the division * @param keyword the keyword * @param incl_prep the incl_prep * @param incl_non_prep the incl_non_prep * @return the string */ private List<Integer> findPeople(final int year, final String program, final String training_site_country_code, final String state, final String primary_site, final String training_site, final String division, final String keyword, final int incl_prep, final int incl_non_prep) { List<Integer> personIdentifiers = new ArrayList<Integer>(); TreeMap<Integer, Integer> people = new TreeMap<Integer, Integer>(); SearchBean search = initiateSearch(year, program, training_site_country_code, state, primary_site, training_site, division, keyword); // Load the list of relevant rotations to find the person identifiers SearchResultsBean results = null; try { results = searchDAO.search(search, new BuilderBean()); } catch (WhichDoctorSearchDaoException wse) { logger.error("Error performing search for rotations: " + wse.getMessage()); } if (incl_prep == 0 || incl_non_prep == 0) { // For the next part we need to set the search to neutral so that we may // perform a subsearch within the existing parameters. search.setSearchVectors(results.getSearchVectors()); search = searchPREP(search, incl_prep, incl_non_prep); try { results = searchDAO.search(search, new BuilderBean()); } catch (WhichDoctorSearchDaoException wse) { logger.error("Error performing search for rotations: " + wse.getMessage()); } } if (results != null && results.getSearchResults() != null) { for (Object obj : results.getSearchResults()) { RotationBean rotation = (RotationBean) obj; if (rotation.getPerson() != null) { int id = rotation.getPerson().getPersonIdentifier(); if (!people.containsKey(id)) { people.put(id, id); } } } } for (Integer id : people.keySet()) { personIdentifiers.add(id); } return personIdentifiers; }
From source file:ca.uhn.fhir.context.BaseRuntimeElementCompositeDefinition.java
private void scanCompositeElementForChildren() { Set<String> elementNames = new HashSet<String>(); TreeMap<Integer, BaseRuntimeDeclaredChildDefinition> orderToElementDef = new TreeMap<Integer, BaseRuntimeDeclaredChildDefinition>(); TreeMap<Integer, BaseRuntimeDeclaredChildDefinition> orderToExtensionDef = new TreeMap<Integer, BaseRuntimeDeclaredChildDefinition>(); scanCompositeElementForChildren(elementNames, orderToElementDef, orderToExtensionDef); if (forcedOrder != null) { /* /*from w w w . j a v a 2 s . c om*/ * Find out how many elements don't match any entry in the list * for forced order. Those elements come first. */ TreeMap<Integer, BaseRuntimeDeclaredChildDefinition> newOrderToExtensionDef = new TreeMap<Integer, BaseRuntimeDeclaredChildDefinition>(); int unknownCount = 0; for (BaseRuntimeDeclaredChildDefinition nextEntry : orderToElementDef.values()) { if (!forcedOrder.containsKey(nextEntry.getElementName())) { newOrderToExtensionDef.put(unknownCount, nextEntry); unknownCount++; } } for (BaseRuntimeDeclaredChildDefinition nextEntry : orderToElementDef.values()) { if (forcedOrder.containsKey(nextEntry.getElementName())) { Integer newOrder = forcedOrder.get(nextEntry.getElementName()); newOrderToExtensionDef.put(newOrder + unknownCount, nextEntry); } } orderToElementDef = newOrderToExtensionDef; } // while (orderToElementDef.size() > 0 && orderToElementDef.firstKey() < // 0) { // BaseRuntimeDeclaredChildDefinition elementDef = // orderToElementDef.remove(orderToElementDef.firstKey()); // if (elementDef.getElementName().equals("identifier")) { // orderToElementDef.put(theIdentifierOrder, elementDef); // } else { // throw new ConfigurationException("Don't know how to handle element: " // + elementDef.getElementName()); // } // } TreeSet<Integer> orders = new TreeSet<Integer>(); orders.addAll(orderToElementDef.keySet()); orders.addAll(orderToExtensionDef.keySet()); for (Integer i : orders) { BaseRuntimeChildDefinition nextChild = orderToElementDef.get(i); if (nextChild != null) { this.addChild(nextChild); } BaseRuntimeDeclaredChildDefinition nextExt = orderToExtensionDef.get(i); if (nextExt != null) { this.addExtension((RuntimeChildDeclaredExtensionDefinition) nextExt); } } }
From source file:chibi.gemmaanalysis.CoexpressionAnalysisService.java
/** * Calculates all pairwise correlations between the query and target composite sequences and then takes the median * correlation//from w ww . j a v a 2 s . co m * * @param queryCss * @param targetCss * @param dataMatrix * @return */ private CorrelationSampleSize calculateCorrelation(Collection<CompositeSequence> queryCss, Collection<CompositeSequence> targetCss, ExpressionDataDoubleMatrix dataMatrix, CorrelationMethod method) { TreeMap<Double, Double> correlNumUsedMap = new TreeMap<Double, Double>(); // calculate all pairwise correlations between cs groups for (CompositeSequence queryCs : queryCss) { for (CompositeSequence targetCs : targetCss) { Double[] queryVals = dataMatrix.getRow(queryCs); Double[] targetVals = dataMatrix.getRow(targetCs); if (queryVals != null && targetVals != null) { double[] v1 = new double[queryVals.length]; double[] v2 = new double[targetVals.length]; for (int i = 0; i < queryVals.length; i++) { if (queryVals[i] != null) v1[i] = queryVals[i]; else v1[i] = Double.NaN; } for (int i = 0; i < targetVals.length; i++) { if (targetVals[i] != null) v2[i] = targetVals[i]; else v2[i] = Double.NaN; } int numUsed = 0; for (int i = 0; i < v1.length && i < v2.length; i++) if (!Double.isNaN(v1[i]) && !Double.isNaN(v2[i])) numUsed++; if (numUsed > MIN_NUM_USED) { double correlation; switch (method) { case SPEARMAN: correlation = Distance.spearmanRankCorrelation(new DoubleArrayList(v1), new DoubleArrayList(v2)); break; case PEARSON: default: correlation = CorrelationStats.correl(v1, v2); } correlNumUsedMap.put(correlation, (double) numUsed); } } } } if (correlNumUsedMap.size() == 0) { return null; } List<Double> correlations = new ArrayList<Double>(correlNumUsedMap.keySet()); // take the median correlation Double correlation = correlations.get(correlations.size() / 2); Double sampleSize = correlNumUsedMap.get(correlation); CorrelationSampleSize c = new CorrelationSampleSize(); c.correlation = correlation; c.sampleSize = sampleSize; return c; }
From source file:com.redhat.rhn.taskomatic.task.DailySummary.java
/** * DO NOT CALL FROM OUTSIDE THIS CLASS. Renders the actions email message * @param actions list of recent actions * @return the actions email message//w w w . j a v a 2 s .c om */ public String renderActionsMessage(List<ActionMessage> actions) { int longestActionLength = HEADER_SPACER; int longestStatusLength = 0; StringBuilder hdr = new StringBuilder(); StringBuilder body = new StringBuilder(); StringBuilder legend = new StringBuilder(); StringBuilder msg = new StringBuilder(); LinkedHashSet<String> statusSet = new LinkedHashSet(); TreeMap<String, Map<String, Integer>> nonErrataActions = new TreeMap(); TreeMap<String, Map<String, Integer>> errataActions = new TreeMap(); TreeMap<String, String> errataSynopsis = new TreeMap(); legend.append(LocalizationService.getInstance().getMessage("taskomatic.daily.errata")); legend.append("\n\n"); for (ActionMessage am : actions) { if (!statusSet.contains(am.getStatus())) { statusSet.add(am.getStatus()); if (am.getStatus().length() > longestStatusLength) { longestStatusLength = am.getStatus().length(); } } if (am.getType().equals(ERRATA_UPDATE)) { String advisoryKey = ERRATA_INDENTION + am.getAdvisory(); if (!errataActions.containsKey(advisoryKey)) { errataActions.put(advisoryKey, new HashMap()); if (advisoryKey.length() + HEADER_SPACER > longestActionLength) { longestActionLength = advisoryKey.length() + HEADER_SPACER; } } Map<String, Integer> counts = errataActions.get(advisoryKey); counts.put(am.getStatus(), am.getCount()); if (am.getAdvisory() != null && !errataSynopsis.containsKey(am.getAdvisory())) { errataSynopsis.put(am.getAdvisory(), am.getSynopsis()); } } else { if (!nonErrataActions.containsKey(am.getType())) { nonErrataActions.put(am.getType(), new HashMap()); if (am.getType().length() + HEADER_SPACER > longestActionLength) { longestActionLength = am.getType().length() + HEADER_SPACER; } } Map<String, Integer> counts = nonErrataActions.get(am.getType()); counts.put(am.getStatus(), am.getCount()); } } hdr.append(StringUtils.repeat(" ", longestActionLength)); for (String status : statusSet) { hdr.append(status + StringUtils.repeat(" ", (longestStatusLength + ERRATA_SPACER) - status.length())); } if (!errataActions.isEmpty()) { body.append(ERRATA_UPDATE + ":" + "\n"); } StringBuffer formattedErrataActions = renderActionTree(longestActionLength, longestStatusLength, statusSet, errataActions); body.append(formattedErrataActions); for (String advisory : errataSynopsis.keySet()) { legend.append(ERRATA_INDENTION + advisory + ERRATA_INDENTION + errataSynopsis.get(advisory) + "\n"); } StringBuffer formattedNonErrataActions = renderActionTree(longestActionLength, longestStatusLength, statusSet, nonErrataActions); body.append(formattedNonErrataActions); // finally put all this together msg.append(hdr.toString()); msg.append("\n"); msg.append(body.toString()); msg.append("\n\n"); if (!errataSynopsis.isEmpty()) { msg.append(legend.toString()); } return msg.toString(); }
From source file:uk.ac.leeds.ccg.andyt.projects.moses.process.Comparison.java
/** * Aim is to produce an aggregated data set for comparison totalling males * and females by MSOA to compare with CASUV003DataRecord *//*from w ww. ja v a 2 s . c o m*/ private void run3() throws IOException { boolean aggregateToMSOA = true; // boolean aggregateToMSOA = false; ToyModelDataHandler tToyModelDataHandler = new ToyModelDataHandler(); String startOfFilename = "C:/Work/Projects/MoSeS/Workspace/Leeds/ToyModel_SWR_OA_HSARHP_ISARCEP_0_5_5000_3_30_12_20"; // String startOfFilename = new String( // "C:/Work/Projects/MoSeS/Workspace/Leeds/ToyModel_SWR_OA_HSARHP_ISARCEP_0_5_1000_3_30_12_20" // ); // String startOfFilename = new String( // "C:/Work/Projects/MoSeS/Workspace/Leeds/ToyModel_SWR_OA_ISARHP_ISARCEP_0_5_200_3_30_12_20" // ); File tToyModelDataRecord2CSVFile = new File(startOfFilename + ".csv"); File tToyModelDataRecordMaleFemaleComparisonFile; if (aggregateToMSOA) { tToyModelDataRecordMaleFemaleComparisonFile = new File( startOfFilename + "_MSOAMaleFemaleComparison.csv"); } else { tToyModelDataRecordMaleFemaleComparisonFile = new File(startOfFilename + "_OAMaleFemaleComparison.csv"); } if (!tToyModelDataRecordMaleFemaleComparisonFile.exists()) { tToyModelDataRecordMaleFemaleComparisonFile.createNewFile(); } PrintWriter tToyModelDataRecordMaleFemaleComparisonFilePrintWriter = new PrintWriter( tToyModelDataRecordMaleFemaleComparisonFile); // CASUV003DataHandler tCASUV003DataHandler = new CASUV003DataHandler( // new File( // "C:/Work/Projects/MoSeS/Workspace/Leeds/CASUV003DataRecordsMSOA.dat" // ) ); CASUV003DataHandler tCASUV003DataHandler; CAS001DataHandler tCAS001DataHandler; if (aggregateToMSOA) { tCASUV003DataHandler = new CASUV003DataHandler( new File("C:/Work/Projects/MoSeS/Workspace/Leeds/CASUV003DataRecordsMSOA.dat")); tCAS001DataHandler = new CAS001DataHandler( new File("C:/Work/Projects/MoSeS/Workspace/Leeds/CAS001DataRecordsMSOA.dat")); } else { tCASUV003DataHandler = new CASUV003DataHandler( new File("C:/Work/Projects/MoSeS/Workspace/CASUV003DataRecords.dat")); tCAS001DataHandler = new CAS001DataHandler( new File("C:/Work/Projects/MoSeS/Workspace/CAS001DataRecords.dat")); } CASUV003DataRecord aCASUV003DataRecord; CAS001DataRecord aCAS001DataRecord; BufferedReader tBufferedReader = new BufferedReader( new InputStreamReader(new FileInputStream(tToyModelDataRecord2CSVFile))); StreamTokenizer tStreamTokenizer = new StreamTokenizer(tBufferedReader); Generic_StaticIO.setStreamTokenizerSyntax1(tStreamTokenizer); // Initialise int tMaleCount; int tFemaleCount; int tMaleCEPCount; int tMaleHPCount; int tFemaleCEPCount; int tFemaleHPCount; int tokenType = tStreamTokenizer.nextToken(); ToyModelDataRecord_2 aToyModelDataRecord2; String aZoneCode; HashMap tLookUpMSOAfromOAHashMap = null; CASDataHandler tCASDataHandler = new CASDataHandler(); if (aggregateToMSOA) { tLookUpMSOAfromOAHashMap = tCASDataHandler.get_LookUpMSOAfromOAHashMap(); } Counts aCounts; tToyModelDataRecordMaleFemaleComparisonFilePrintWriter.println( "ZoneCode,CAS001HPFemales,CAS001CEPFemales,CAS001Females,CASUV003Females,ToyModelFemales,ToyModelHPFemales,ToyModelCEPFemales,CAS001HPMales,CAS001CEPMales,CAS001Males,CASUV003Males,ToyModelMales,ToyModelHPMales,ToyModelCEPMales"); TreeMap result = new TreeMap(); while (tokenType != StreamTokenizer.TT_EOF) { switch (tokenType) { case StreamTokenizer.TT_WORD: aToyModelDataRecord2 = new ToyModelDataRecord_2(tToyModelDataHandler, tStreamTokenizer.sval); if (aggregateToMSOA) { aZoneCode = (String) tLookUpMSOAfromOAHashMap .get(new String(aToyModelDataRecord2.getZone_Code())); } else { aZoneCode = String.valueOf(aToyModelDataRecord2.getZone_Code()); } if (aToyModelDataRecord2.SEX == 0) { tFemaleCount = 1; if (aToyModelDataRecord2.tHouseholdID != -9) { tFemaleHPCount = 1; tFemaleCEPCount = 0; } else { tFemaleHPCount = 0; tFemaleCEPCount = 1; } tMaleCount = 0; tMaleHPCount = 0; tMaleCEPCount = 0; } else { tMaleCount = 1; if (aToyModelDataRecord2.tHouseholdID != -9) { tMaleHPCount = 1; tMaleCEPCount = 0; } else { tMaleHPCount = 0; tMaleCEPCount = 1; } tFemaleCount = 0; tFemaleHPCount = 0; tFemaleCEPCount = 0; } if (result.containsKey(aZoneCode)) { aCounts = (Counts) result.get(aZoneCode); result.remove(aZoneCode); aCounts.addToCounts(tMaleCount, tMaleCEPCount, tMaleHPCount, tFemaleCount, tFemaleCEPCount, tFemaleHPCount); result.put(aZoneCode, aCounts); } else { aCounts = new Counts(); aCounts.addToCounts(tMaleCount, tMaleCEPCount, tMaleHPCount, tFemaleCount, tFemaleCEPCount, tFemaleHPCount); result.put(aZoneCode, aCounts); } } tokenType = tStreamTokenizer.nextToken(); } Iterator aIterator = result.keySet().iterator(); Object key; while (aIterator.hasNext()) { key = aIterator.next(); aCounts = (Counts) result.get(key); aZoneCode = (String) key; aCASUV003DataRecord = (CASUV003DataRecord) tCASUV003DataHandler.getDataRecord(aZoneCode); aCAS001DataRecord = (CAS001DataRecord) tCAS001DataHandler.getDataRecord(aZoneCode); tToyModelDataRecordMaleFemaleComparisonFilePrintWriter.println("" + aZoneCode + ", " + aCAS001DataRecord.getHouseholdResidentsFemales() + ", " + aCAS001DataRecord.getCommunalEstablishmentResidentsFemales() + ", " + (aCAS001DataRecord.getHouseholdResidentsFemales() + aCAS001DataRecord.getCommunalEstablishmentResidentsFemales()) + ", " + aCASUV003DataRecord.getFemales() + ", " + aCounts.tFemaleCount + ", " + aCounts.tFemaleHPCount + ", " + aCounts.tFemaleCEPCount + ", " + aCAS001DataRecord.getHouseholdResidentsMales() + ", " + aCAS001DataRecord.getCommunalEstablishmentResidentsMales() + ", " + (aCAS001DataRecord.getHouseholdResidentsMales() + aCAS001DataRecord.getCommunalEstablishmentResidentsMales()) + ", " + aCASUV003DataRecord.getMales() + ", " + aCounts.tMaleCount + ", " + aCounts.tMaleHPCount + ", " + aCounts.tMaleCEPCount); } tBufferedReader.close(); tToyModelDataRecordMaleFemaleComparisonFilePrintWriter.close(); }
From source file:trendanalisis.main.tools.weka.CoreWekaTFIDF.java
/** * determines the dictionary./*w w w .j a v a 2 s . co m*/ */ private void determineDictionary() { // Operate on a per-class basis if class attribute is set int classInd = getInputFormat().classIndex(); int values = 1; if (!m_doNotOperateOnPerClassBasis && (classInd != -1)) { values = getInputFormat().attribute(classInd).numValues(); } // TreeMap dictionaryArr [] = new TreeMap[values]; @SuppressWarnings("unchecked") TreeMap<String, Count>[] dictionaryArr = new TreeMap[values]; for (int i = 0; i < values; i++) { dictionaryArr[i] = new TreeMap<String, Count>(); } // Make sure we know which fields to convert determineSelectedRange(); // Tokenize all training text into an orderedMap of "words". long pruneRate = Math.round((m_PeriodicPruningRate / 100.0) * getInputFormat().numInstances()); for (int i = 0; i < getInputFormat().numInstances(); i++) { Instance instance = getInputFormat().instance(i); int vInd = 0; if (!m_doNotOperateOnPerClassBasis && (classInd != -1)) { vInd = (int) instance.classValue(); } // Iterate through all relevant string attributes of the current instance Hashtable<String, Integer> h = new Hashtable<String, Integer>(); for (int j = 0; j < instance.numAttributes(); j++) { if (m_SelectedRange.isInRange(j) && (instance.isMissing(j) == false)) { // Get tokenizer m_Tokenizer.tokenize(instance.stringValue(j)); // Iterate through tokens, perform stemming, and remove stopwords // (if required) while (m_Tokenizer.hasMoreElements()) { String word = m_Tokenizer.nextElement().intern(); if (this.m_lowerCaseTokens == true) { word = word.toLowerCase(); } word = m_Stemmer.stem(word); if (m_StopwordsHandler.isStopword(word)) { continue; } if (!(h.containsKey(word))) { h.put(word, new Integer(0)); } Count count = dictionaryArr[vInd].get(word); if (count == null) { dictionaryArr[vInd].put(word, new Count(1)); } else { count.count++; } } } } // updating the docCount for the words that have occurred in this // instance(document). Enumeration<String> e = h.keys(); while (e.hasMoreElements()) { String word = e.nextElement(); Count c = dictionaryArr[vInd].get(word); if (c != null) { c.docCount++; } else { System.err.println( "Warning: A word should definitely be in the " + "dictionary.Please check the code"); } } if (pruneRate > 0) { if (i % pruneRate == 0 && i > 0) { for (int z = 0; z < values; z++) { ArrayList<String> d = new ArrayList<String>(1000); Iterator<String> it = dictionaryArr[z].keySet().iterator(); while (it.hasNext()) { String word = it.next(); Count count = dictionaryArr[z].get(word); if (count.count <= 1) { d.add(word); } } Iterator<String> iter = d.iterator(); while (iter.hasNext()) { String word = iter.next(); dictionaryArr[z].remove(word); } } } } } // Figure out the minimum required word frequency int totalsize = 0; int prune[] = new int[values]; for (int z = 0; z < values; z++) { totalsize += dictionaryArr[z].size(); int array[] = new int[dictionaryArr[z].size()]; int pos = 0; Iterator<String> it = dictionaryArr[z].keySet().iterator(); while (it.hasNext()) { String word = it.next(); Count count = dictionaryArr[z].get(word); array[pos] = count.count; pos++; } // sort the array sortArray(array); if (array.length < m_WordsToKeep) { // if there aren't enough words, set the threshold to // minFreq prune[z] = m_minTermFreq; } else { // otherwise set it to be at least minFreq prune[z] = Math.max(m_minTermFreq, array[array.length - m_WordsToKeep]); } } // Convert the dictionary into an attribute index // and create one attribute per word ArrayList<Attribute> attributes = new ArrayList<Attribute>(totalsize + getInputFormat().numAttributes()); // Add the non-converted attributes int classIndex = -1; for (int i = 0; i < getInputFormat().numAttributes(); i++) { if (!m_SelectedRange.isInRange(i)) { if (getInputFormat().classIndex() == i) { classIndex = attributes.size(); } attributes.add((Attribute) getInputFormat().attribute(i).copy()); } } // Add the word vector attributes (eliminating duplicates // that occur in multiple classes) TreeMap<String, Integer> newDictionary = new TreeMap<String, Integer>(); int index = attributes.size(); for (int z = 0; z < values; z++) { Iterator<String> it = dictionaryArr[z].keySet().iterator(); while (it.hasNext()) { String word = it.next(); Count count = dictionaryArr[z].get(word); if (count.count >= prune[z]) { if (newDictionary.get(word) == null) { newDictionary.put(word, new Integer(index++)); attributes.add(new Attribute(m_Prefix + word)); } } } } // Compute document frequencies global_tf = new double[attributes.size()]; df_prob = new double[attributes.size()]; m_DocsCounts = new int[attributes.size()]; IG = new double[attributes.size()]; Iterator<String> it = newDictionary.keySet().iterator(); while (it.hasNext()) { String word = it.next(); int idx = newDictionary.get(word).intValue(); int docsCount = 0; for (int j = 0; j < values; j++) { Count c = dictionaryArr[j].get(word); if (c != null) { docsCount += c.docCount; } } m_DocsCounts[idx] = docsCount; } // Trim vector and set instance variables attributes.trimToSize(); m_Dictionary = newDictionary; m_NumInstances = getInputFormat().numInstances(); // Set the filter's output format Instances outputFormat = new Instances(getInputFormat().relationName(), attributes, 0); outputFormat.setClassIndex(classIndex); setOutputFormat(outputFormat); }
From source file:net.triptech.metahive.service.EmailSenderService.java
/** * Send an email message using the configured Spring sender. On success * record the sent message in the datastore for reporting purposes * * @param email the email/* w w w . j a v a 2s.c om*/ * @param attachments the attachments * @throws ServiceException the service exception */ public final void send(final SimpleMailMessage email, TreeMap<String, Object> attachments) throws ServiceException { // Check to see whether the required fields are set (to, from, message) if (email.getTo() == null) { throw new ServiceException("Error sending email: Recipient " + "address required"); } if (StringUtils.isBlank(email.getFrom())) { throw new ServiceException("Error sending email: Email requires " + "a from address"); } if (StringUtils.isBlank(email.getText())) { throw new ServiceException("Error sending email: No email " + "message specified"); } if (mailSender == null) { throw new ServiceException("The JavaMail sender has not " + "been configured"); } // Prepare the email message MimeMessage message = mailSender.createMimeMessage(); MimeMessageHelper helper = null; boolean htmlMessage = false; if (StringUtils.containsIgnoreCase(email.getText(), "<html")) { htmlMessage = true; try { helper = new MimeMessageHelper(message, true, "UTF-8"); } catch (MessagingException me) { throw new ServiceException("Error preparing email for sending: " + me.getMessage()); } } else { helper = new MimeMessageHelper(message); } try { helper.setTo(email.getTo()); helper.setFrom(email.getFrom()); helper.setSubject(email.getSubject()); if (email.getCc() != null) { helper.setCc(email.getCc()); } if (email.getBcc() != null) { helper.setBcc(email.getBcc()); } if (htmlMessage) { String plainText = email.getText(); try { ConvertHtmlToText htmlToText = new ConvertHtmlToText(); plainText = htmlToText.convert(email.getText()); } catch (Exception e) { logger.error("Error converting HTML to plain text: " + e.getMessage()); } helper.setText(plainText, email.getText()); } else { helper.setText(email.getText()); } if (email.getSentDate() != null) { helper.setSentDate(email.getSentDate()); } else { helper.setSentDate(Calendar.getInstance().getTime()); } } catch (MessagingException me) { throw new ServiceException("Error preparing email for sending: " + me.getMessage()); } // Append any attachments (if an HTML email) if (htmlMessage && attachments != null) { for (String id : attachments.keySet()) { Object reference = attachments.get(id); if (reference instanceof File) { try { FileSystemResource res = new FileSystemResource((File) reference); helper.addInline(id, res); } catch (MessagingException me) { logger.error("Error appending File attachment: " + me.getMessage()); } } if (reference instanceof URL) { try { UrlResource res = new UrlResource((URL) reference); helper.addInline(id, res); } catch (MessagingException me) { logger.error("Error appending URL attachment: " + me.getMessage()); } } } } // Send the email message try { mailSender.send(message); } catch (MailException me) { logger.error("Error sending email: " + me.getMessage()); throw new ServiceException("Error sending email: " + me.getMessage()); } }