List of usage examples for java.util Map entrySet
Set<Map.Entry<K, V>> entrySet();
From source file:joshelser.as2015.query.Query.java
public static void main(String[] args) throws Exception { JCommander commander = new JCommander(); final Opts options = new Opts(); commander.addObject(options);/*from ww w.ja v a2 s . c om*/ commander.setProgramName("Query"); try { commander.parse(args); } catch (ParameterException ex) { commander.usage(); System.err.println(ex.getMessage()); System.exit(1); } ClientConfiguration conf = ClientConfiguration.loadDefault(); if (null != options.clientConfFile) { conf = new ClientConfiguration(new PropertiesConfiguration(options.clientConfFile)); } conf.withInstance(options.instanceName).withZkHosts(options.zookeepers); ZooKeeperInstance inst = new ZooKeeperInstance(conf); Connector conn = inst.getConnector(options.user, new PasswordToken(options.password)); BatchScanner bs = conn.createBatchScanner(options.table, Authorizations.EMPTY, 16); try { bs.setRanges(Collections.singleton(new Range())); final Text categoryText = new Text("category"); bs.fetchColumn(categoryText, new Text("name")); bs.fetchColumn(new Text("review"), new Text("score")); bs.fetchColumn(new Text("review"), new Text("userId")); bs.addScanIterator(new IteratorSetting(50, "wri", WholeRowIterator.class)); final Text colf = new Text(); Map<String, List<Integer>> scoresByUser = new HashMap<>(); for (Entry<Key, Value> entry : bs) { SortedMap<Key, Value> row = WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()); Iterator<Entry<Key, Value>> iter = row.entrySet().iterator(); if (!iter.hasNext()) { // row was empty continue; } Entry<Key, Value> categoryEntry = iter.next(); categoryEntry.getKey().getColumnFamily(colf); if (!colf.equals(categoryText)) { throw new IllegalArgumentException("Unknown!"); } if (!categoryEntry.getValue().toString().equals("books")) { // not a book review continue; } if (!iter.hasNext()) { continue; } Entry<Key, Value> reviewScore = iter.next(); if (!iter.hasNext()) { continue; } Entry<Key, Value> reviewUserId = iter.next(); String userId = reviewUserId.getValue().toString(); if (userId.equals("unknown")) { // filter unknow user id continue; } List<Integer> scores = scoresByUser.get(userId); if (null == scores) { scores = new ArrayList<>(); scoresByUser.put(userId, scores); } scores.add(Float.valueOf(reviewScore.getValue().toString()).intValue()); } for (Entry<String, List<Integer>> entry : scoresByUser.entrySet()) { int sum = 0; for (Integer val : entry.getValue()) { sum += val; } System.out.println(entry.getKey() + " => " + new Float(sum) / entry.getValue().size()); } } finally { bs.close(); } }
From source file:de.tudarmstadt.ukp.experiments.argumentation.convincingness.sampling.Step4MTurkOutputCollector.java
@SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { String inputDirWithArgumentPairs = args[0]; File[] resultFiles;//from ww w . ja v a2 s.c om if (args[1].contains("*")) { File path = new File(args[1]); File directory = path.getParentFile(); String regex = path.getName().replaceAll("\\*", ""); List<File> files = new ArrayList<>(FileUtils.listFiles(directory, new String[] { regex }, false)); resultFiles = new File[files.size()]; for (int i = 0; i < files.size(); i++) { resultFiles[i] = files.get(i); } } else { // result file is a comma-separated list of CSV files from MTurk String[] split = args[1].split(","); resultFiles = new File[split.length]; for (int i = 0; i < split.length; i++) { resultFiles[i] = new File(split[i]); } } File outputDir = new File(args[2]); if (!outputDir.exists()) { if (!outputDir.mkdirs()) { throw new IOException("Cannot create directory " + outputDir); } } // error if output folder not empty to prevent any confusion by mixing files if (!FileUtils.listFiles(outputDir, null, false).isEmpty()) { throw new IllegalArgumentException("Output dir " + outputDir + " is not empty"); } // collected assignments with empty reason for rejections Set<String> assignmentsWithEmptyReason = new HashSet<>(); // parse with first line as header MTurkOutputReader mTurkOutputReader = new MTurkOutputReader(resultFiles); Collection<File> files = FileUtils.listFiles(new File(inputDirWithArgumentPairs), new String[] { "xml" }, false); if (files.isEmpty()) { throw new IOException("No xml files found in " + inputDirWithArgumentPairs); } // statistics: how many hits with how many assignments ; hit ID / assignments Map<String, Map<String, Integer>> assignmentsPerHits = new HashMap<>(); // collect accept/reject statistics for (Map<String, String> record : mTurkOutputReader) { boolean wasRejected = "Rejected".equals(record.get("assignmentstatus")); String hitID = record.get("hitid"); String hitTypeId = record.get("hittypeid"); if (!wasRejected) { // update statistics if (!assignmentsPerHits.containsKey(hitTypeId)) { assignmentsPerHits.put(hitTypeId, new HashMap<String, Integer>()); } if (!assignmentsPerHits.get(hitTypeId).containsKey(hitID)) { assignmentsPerHits.get(hitTypeId).put(hitID, 0); } assignmentsPerHits.get(hitTypeId).put(hitID, assignmentsPerHits.get(hitTypeId).get(hitID) + 1); } } // statistics: how many hits with how many assignments ; hit ID / assignments Map<String, Integer> approvedAssignmentsPerHit = new HashMap<>(); Map<String, Integer> rejectedAssignmentsPerHit = new HashMap<>(); // collect accept/reject statistics for (Map<String, String> record : mTurkOutputReader) { boolean approved = "Approved".equals(record.get("assignmentstatus")); boolean rejected = "Rejected".equals(record.get("assignmentstatus")); String hitID = record.get("hitid"); if (approved) { // update statistics if (!approvedAssignmentsPerHit.containsKey(hitID)) { approvedAssignmentsPerHit.put(hitID, 0); } approvedAssignmentsPerHit.put(hitID, approvedAssignmentsPerHit.get(hitID) + 1); } else if (rejected) { // update statistics if (!rejectedAssignmentsPerHit.containsKey(hitID)) { rejectedAssignmentsPerHit.put(hitID, 0); } rejectedAssignmentsPerHit.put(hitID, rejectedAssignmentsPerHit.get(hitID) + 1); } else { throw new IllegalStateException( "Unknown state: " + record.get("assignmentstatus") + " HITID: " + hitID); } } // System.out.println("Approved: " + approvedAssignmentsPerHit); // System.out.println("Rejected: " + rejectedAssignmentsPerHit); System.out.println("Approved (values): " + new HashSet<>(approvedAssignmentsPerHit.values())); System.out.println("Rejected (values): " + new HashSet<>(rejectedAssignmentsPerHit.values())); // rejection statistics int totalRejected = 0; for (Map.Entry<String, Integer> rejectionEntry : rejectedAssignmentsPerHit.entrySet()) { totalRejected += rejectionEntry.getValue(); } System.out.println("Total rejections: " + totalRejected); /* // generate .success files for adding more annotations for (File resultFile : resultFiles) { String hitTypeID = mTurkOutputReader.getHitTypeIdForFile().get(resultFile); // assignments for that hittypeid (= file) Map<String, Integer> assignments = assignmentsPerHits.get(hitTypeID); prepareUpdateHITsFiles(assignments, hitTypeID, resultFile); } */ int totalSavedPairs = 0; // load all previously prepared argument pairs for (File file : files) { List<ArgumentPair> argumentPairs = (List<ArgumentPair>) XStreamTools.getXStream().fromXML(file); List<AnnotatedArgumentPair> annotatedArgumentPairs = new ArrayList<>(); for (ArgumentPair argumentPair : argumentPairs) { AnnotatedArgumentPair annotatedArgumentPair = new AnnotatedArgumentPair(argumentPair); // is there such an answer? String key = "Answer." + argumentPair.getId(); // iterate only if there is such column to save time if (mTurkOutputReader.getColumnNames().contains(key)) { // now find the results for (Map<String, String> record : mTurkOutputReader) { if (record.containsKey(key)) { // extract the values AnnotatedArgumentPair.MTurkAssignment assignment = new AnnotatedArgumentPair.MTurkAssignment(); boolean wasRejected = "Rejected".equals(record.get("assignmentstatus")); // only non-rejected (if required) if (!wasRejected) { String hitID = record.get("hitid"); String workerID = record.get("workerid"); String assignmentId = record.get("assignmentid"); try { assignment.setAssignmentAcceptTime( DATE_FORMAT.parse(record.get("assignmentaccepttime"))); assignment.setAssignmentSubmitTime( DATE_FORMAT.parse(record.get("assignmentsubmittime"))); assignment.setHitComment(record.get("Answer.feedback")); assignment.setHitID(hitID); assignment.setTurkID(workerID); assignment.setAssignmentId(assignmentId); // and answer specific fields String valueRaw = record.get(key); // so far the label has had format aXXX_aYYY_a1, aXXX_aYYY_a2, or aXXX_aYYY_equal // strip now only true label String label = valueRaw.split("_")[2]; assignment.setValue(label); String reason = record.get(key + "_reason"); // missing reason if (reason == null) { assignmentsWithEmptyReason.add(assignmentId); } else { assignment.setReason(reason); // get worker's stance String stanceRaw = record.get(key + "_stance"); if (stanceRaw != null) { // parse stance String stance = stanceRaw.split("_stance_")[1]; assignment.setWorkerStance(stance); } // we take maximal 5 assignments Collections.sort(annotatedArgumentPair.mTurkAssignments, new Comparator<AnnotatedArgumentPair.MTurkAssignment>() { @Override public int compare(AnnotatedArgumentPair.MTurkAssignment o1, AnnotatedArgumentPair.MTurkAssignment o2) { return o1.getAssignmentAcceptTime() .compareTo(o2.getAssignmentAcceptTime()); } }); if (annotatedArgumentPair.mTurkAssignments .size() < MAXIMUM_ASSIGNMENTS_PER_HIT) { annotatedArgumentPair.mTurkAssignments.add(assignment); } } } catch (IllegalArgumentException | NullPointerException ex) { System.err.println("Malformed annotations for HIT " + hitID + ", worker " + workerID + ", assignment " + assignmentId + "; " + ex.getMessage() + ", full record: " + record); } } } } } // and if there are some annotations, add it to the result set if (!annotatedArgumentPair.mTurkAssignments.isEmpty()) { annotatedArgumentPairs.add(annotatedArgumentPair); } } if (!annotatedArgumentPairs.isEmpty()) { File outputFile = new File(outputDir, file.getName()); XStreamTools.toXML(annotatedArgumentPairs, outputFile); System.out.println("Saved " + annotatedArgumentPairs.size() + " annotated pairs to " + outputFile); totalSavedPairs += annotatedArgumentPairs.size(); } } System.out.println("Total saved " + totalSavedPairs + " pairs"); // print assignments with empty reasons if (!assignmentsWithEmptyReason.isEmpty()) { System.out.println( "== Assignments with empty reason:\nassignmentIdToReject\tassignmentIdToRejectComment"); for (String assignmentId : assignmentsWithEmptyReason) { System.out.println( assignmentId + "\t\"Dear worker, you did not fill the required field with a reason.\""); } } }
From source file:com.mozilla.socorro.RawDumpSizeScan.java
public static void main(String[] args) throws ParseException { String startDateStr = args[0]; String endDateStr = args[1];/*from w w w .jav a 2s . co m*/ // Set both start/end time and start/stop row Calendar startCal = Calendar.getInstance(); Calendar endCal = Calendar.getInstance(); SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); if (!StringUtils.isBlank(startDateStr)) { startCal.setTime(sdf.parse(startDateStr)); } if (!StringUtils.isBlank(endDateStr)) { endCal.setTime(sdf.parse(endDateStr)); } DescriptiveStatistics stats = new DescriptiveStatistics(); long numNullRawBytes = 0L; HTable table = null; Map<String, Integer> rowValueSizeMap = new HashMap<String, Integer>(); try { table = new HTable(TABLE_NAME_CRASH_REPORTS); Scan[] scans = generateScans(startCal, endCal); for (Scan s : scans) { ResultScanner rs = table.getScanner(s); Iterator<Result> iter = rs.iterator(); while (iter.hasNext()) { Result r = iter.next(); ImmutableBytesWritable rawBytes = r.getBytes(); //length = r.getValue(RAW_DATA_BYTES, DUMP_BYTES); if (rawBytes != null) { int length = rawBytes.getLength(); if (length > 20971520) { rowValueSizeMap.put(new String(r.getRow()), length); } stats.addValue(length); } else { numNullRawBytes++; } if (stats.getN() % 10000 == 0) { System.out.println("Processed " + stats.getN()); System.out.println(String.format("Min: %.02f Max: %.02f Mean: %.02f", stats.getMin(), stats.getMax(), stats.getMean())); System.out.println( String.format("1st Quartile: %.02f 2nd Quartile: %.02f 3rd Quartile: %.02f", stats.getPercentile(25.0d), stats.getPercentile(50.0d), stats.getPercentile(75.0d))); System.out.println("Number of large entries: " + rowValueSizeMap.size()); } } rs.close(); } System.out.println("Finished Processing!"); System.out.println(String.format("Min: %.02f Max: %.02f Mean: %.02f", stats.getMin(), stats.getMax(), stats.getMean())); System.out.println(String.format("1st Quartile: %.02f 2nd Quartile: %.02f 3rd Quartile: %.02f", stats.getPercentile(25.0d), stats.getPercentile(50.0d), stats.getPercentile(75.0d))); for (Map.Entry<String, Integer> entry : rowValueSizeMap.entrySet()) { System.out.println(String.format("RowId: %s => Length: %d", entry.getKey(), entry.getValue())); } } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } finally { if (table != null) { try { table.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } }
From source file:de.citec.csra.elancsv.parser.SimpleParser.java
public static void main(String[] args) throws IOException, ParseException { Options opts = new Options(); opts.addOption("file", true, "Tab-separated ELAN export file to load."); opts.addOption("tier", true, "Tier to analyze. Optional: Append ::num to interpret annotations numerically."); opts.addOption("format", true, "How to read information from the file name. %V -> participant, %A -> annoatator, %C -> condition, e.g. \"%V - %A\""); opts.addOption("help", false, "Print this help and exit"); CommandLineParser parser = new BasicParser(); CommandLine cmd = parser.parse(opts, args); if (cmd.hasOption("help")) { helpExit(opts, "where OPTION includes:"); }/*from ww w .j av a 2 s.com*/ String infile = cmd.getOptionValue("file"); if (infile == null) { helpExit(opts, "Error: no file given."); } String format = cmd.getOptionValue("format"); if (format == null) { helpExit(opts, "Error: no format given."); } String tier = cmd.getOptionValue("tier"); if (tier == null) { helpExit(opts, "Error: no tier given."); } // TODO count values in annotations (e.g. search all robot occurrences) String[] tn = tier.split("::"); boolean numeric = false; if (tn.length == 2 && tn[1].equals("num")) { numeric = true; tier = tn[0]; } format = "^" + format + "$"; format = format.replaceFirst("%V", "(?<V>.*?)"); format = format.replaceFirst("%A", "(?<A>.*?)"); format = format.replaceFirst("%C", "(?<C>.*?)"); Pattern pa = Pattern.compile(format); Map<String, Participant> participants = new HashMap<>(); BufferedReader br = new BufferedReader(new FileReader(infile)); String line; int lineno = 0; while ((line = br.readLine()) != null) { String[] parts = line.split("\t"); lineno++; if (parts.length < 5) { System.err.println("WARNING: line '" + lineno + "' too short '" + line + "'"); continue; } Annotation a = new Annotation(Long.valueOf(parts[ElanFormat.START.field]), Long.valueOf(parts[ElanFormat.STOP.field]), Long.valueOf(parts[ElanFormat.DURATION.field]), parts[ElanFormat.VALUE.field]); String tname = parts[ElanFormat.TIER.field]; String file = parts[ElanFormat.FILE.field].replaceAll(".eaf", ""); Matcher m = pa.matcher(file); String vp = file; String condition = "?"; String annotator = "?"; String participantID = vp; if (m.find()) { vp = m.group("V"); if (format.indexOf("<A>") > 0) { annotator = m.group("A"); } if (format.indexOf("<C>") > 0) { condition = m.group("C"); } } participantID = vp + ";" + annotator; if (!participants.containsKey(participantID)) { participants.put(participantID, new Participant(vp, condition, annotator)); } Participant p = participants.get(participantID); if (!p.tiers.containsKey(tname)) { p.tiers.put(tname, new Tier(tname)); } p.tiers.get(tname).annotations.add(a); } Map<String, Map<String, Number>> values = new HashMap<>(); Set<String> rownames = new HashSet<>(); String allCountKey = "c: all values"; String allDurationKey = "d: all values"; String allMeanKey = "m: all values"; for (Map.Entry<String, Participant> e : participants.entrySet()) { // System.out.println(e); Tier t = e.getValue().tiers.get(tier); String participantID = e.getKey(); if (!values.containsKey(participantID)) { values.put(participantID, new HashMap<String, Number>()); } Map<String, Number> row = values.get(participantID); //participant id if (t != null) { row.put(allCountKey, 0l); row.put(allDurationKey, 0l); row.put(allMeanKey, 0l); for (Annotation a : t.annotations) { long countAll = (long) row.get(allCountKey) + 1; long durationAll = (long) row.get(allDurationKey) + a.duration; long meanAll = durationAll / countAll; row.put(allCountKey, countAll); row.put(allDurationKey, durationAll); row.put(allMeanKey, meanAll); if (!numeric) { String countKey = "c: " + a.value; String durationKey = "d: " + a.value; String meanKey = "m: " + a.value; if (!row.containsKey(countKey)) { row.put(countKey, 0l); } if (!row.containsKey(durationKey)) { row.put(durationKey, 0l); } if (!row.containsKey(meanKey)) { row.put(meanKey, 0d); } long count = (long) row.get(countKey) + 1; long duration = (long) row.get(durationKey) + a.duration; double mean = duration * 1.0 / count; row.put(countKey, count); row.put(durationKey, duration); row.put(meanKey, mean); rownames.add(countKey); rownames.add(durationKey); rownames.add(meanKey); } else { String countKey = "c: " + t.name; String sumKey = "s: " + t.name; String meanKey = "m: " + t.name; if (!row.containsKey(countKey)) { row.put(countKey, 0l); } if (!row.containsKey(sumKey)) { row.put(sumKey, 0d); } if (!row.containsKey(meanKey)) { row.put(meanKey, 0d); } double d = 0; try { d = Double.valueOf(a.value); } catch (NumberFormatException ex) { } long count = (long) row.get(countKey) + 1; double sum = (double) row.get(sumKey) + d; double mean = sum / count; row.put(countKey, count); row.put(sumKey, sum); row.put(meanKey, mean); rownames.add(countKey); rownames.add(sumKey); rownames.add(meanKey); } } } } ArrayList<String> list = new ArrayList(rownames); Collections.sort(list); StringBuilder header = new StringBuilder("ID;Annotator;"); header.append(allCountKey); header.append(";"); header.append(allDurationKey); header.append(";"); header.append(allMeanKey); header.append(";"); for (String l : list) { header.append(l); header.append(";"); } System.out.println(header); for (Map.Entry<String, Map<String, Number>> e : values.entrySet()) { StringBuilder row = new StringBuilder(e.getKey()); row.append(";"); if (e.getValue().containsKey(allCountKey)) { row.append(e.getValue().get(allCountKey)); } else { row.append("0"); } row.append(";"); if (e.getValue().containsKey(allDurationKey)) { row.append(e.getValue().get(allDurationKey)); } else { row.append("0"); } row.append(";"); if (e.getValue().containsKey(allMeanKey)) { row.append(e.getValue().get(allMeanKey)); } else { row.append("0"); } row.append(";"); for (String l : list) { if (e.getValue().containsKey(l)) { row.append(e.getValue().get(l)); } else { row.append("0"); } row.append(";"); } System.out.println(row); } }
From source file:de.tudarmstadt.ukp.csniper.resbuild.EvaluationItemFixer.java
public static void main(String[] args) { connect(HOST, DATABASE, USER, PASSWORD); Map<Integer, String> items = new HashMap<Integer, String>(); Map<Integer, String> failed = new HashMap<Integer, String>(); // fetch coveredTexts of dubious items and clean it PreparedStatement select = null; try {/* w w w . j a va 2s. c o m*/ StringBuilder selectQuery = new StringBuilder(); selectQuery.append("SELECT * FROM EvaluationItem "); selectQuery.append("WHERE LOCATE(coveredText, ' ') > 0 "); selectQuery.append("OR LOCATE('" + LRB + "', coveredText) > 0 "); selectQuery.append("OR LOCATE('" + RRB + "', coveredText) > 0 "); selectQuery.append("OR LEFT(coveredText, 1) = ' ' "); selectQuery.append("OR RIGHT(coveredText, 1) = ' ' "); select = connection.prepareStatement(selectQuery.toString()); log.info("Running query [" + selectQuery.toString() + "]."); ResultSet rs = select.executeQuery(); while (rs.next()) { int id = rs.getInt("id"); String coveredText = rs.getString("coveredText"); try { // special handling of double whitespace: in this case, re-fetch the text if (coveredText.contains(" ")) { coveredText = retrieveCoveredText(rs.getString("collectionId"), rs.getString("documentId"), rs.getInt("beginOffset"), rs.getInt("endOffset")); } // replace bracket placeholders and trim the text coveredText = StringUtils.replace(coveredText, LRB, "("); coveredText = StringUtils.replace(coveredText, RRB, ")"); coveredText = coveredText.trim(); items.put(id, coveredText); } catch (IllegalArgumentException e) { failed.put(id, e.getMessage()); } } } catch (SQLException e) { log.error("Exception while selecting: " + e.getMessage()); } finally { closeQuietly(select); } // write logs BufferedWriter bwf = null; BufferedWriter bws = null; try { bwf = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(new File(LOG_FAILED)), "UTF-8")); for (Entry<Integer, String> e : failed.entrySet()) { bwf.write(e.getKey() + " - " + e.getValue() + "\n"); } bws = new BufferedWriter( new OutputStreamWriter(new FileOutputStream(new File(LOG_SUCCESSFUL)), "UTF-8")); for (Entry<Integer, String> e : items.entrySet()) { bws.write(e.getKey() + " - " + e.getValue() + "\n"); } } catch (IOException e) { log.error("Got an IOException while writing the log files."); } finally { IOUtils.closeQuietly(bwf); IOUtils.closeQuietly(bws); } log.info("Texts for [" + items.size() + "] items need to be cleaned up."); // update the dubious items with the cleaned coveredText PreparedStatement update = null; try { String updateQuery = "UPDATE EvaluationItem SET coveredText = ? WHERE id = ?"; update = connection.prepareStatement(updateQuery); int i = 0; for (Entry<Integer, String> e : items.entrySet()) { int id = e.getKey(); String coveredText = e.getValue(); // update item in database update.setString(1, coveredText); update.setInt(2, id); update.executeUpdate(); log.debug("Updating " + id + " with [" + coveredText + "]"); // show percentage of updated items i++; int part = (int) Math.ceil((double) items.size() / 100); if (i % part == 0) { log.info(i / part + "% finished (" + i + "/" + items.size() + ")."); } } } catch (SQLException e) { log.error("Exception while updating: " + e.getMessage()); } finally { closeQuietly(update); } closeQuietly(connection); }
From source file:com.act.lcms.db.io.PrintConstructInfo.java
public static void main(String[] args) throws Exception { Options opts = new Options(); for (Option.Builder b : OPTION_BUILDERS) { opts.addOption(b.build());/*from w w w.j ava 2s . c o m*/ } CommandLine cl = null; try { CommandLineParser parser = new DefaultParser(); cl = parser.parse(opts, args); } catch (ParseException e) { System.err.format("Argument parsing failed: %s\n", e.getMessage()); HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); System.exit(1); } if (cl.hasOption("help")) { HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); return; } File lcmsDir = new File(cl.getOptionValue(OPTION_DIRECTORY)); if (!lcmsDir.isDirectory()) { System.err.format("File at %s is not a directory\n", lcmsDir.getAbsolutePath()); HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); System.exit(1); } try (DB db = DB.openDBFromCLI(cl)) { System.out.print("Loading/updating LCMS scan files into DB\n"); ScanFile.insertOrUpdateScanFilesInDirectory(db, lcmsDir); String construct = cl.getOptionValue(OPTION_CONSTRUCT); List<LCMSWell> lcmsWells = LCMSWell.getInstance().getByConstructID(db, construct); Collections.sort(lcmsWells, new Comparator<LCMSWell>() { @Override public int compare(LCMSWell o1, LCMSWell o2) { return o1.getId().compareTo(o2.getId()); } }); Set<String> uniqueMSIDs = new HashSet<>(); Map<Integer, Plate> platesById = new HashMap<>(); System.out.format("\n\n-- Construct %s --\n\n", construct); List<ChemicalAssociatedWithPathway> pathwayChems = ChemicalAssociatedWithPathway.getInstance() .getChemicalsAssociatedWithPathwayByConstructId(db, construct); System.out.print("Chemicals associated with pathway:\n"); System.out.format(" %-8s%-15s%-45s\n", "index", "kind", "chemical"); for (ChemicalAssociatedWithPathway chem : pathwayChems) { System.out.format(" %-8d%-15s%-45s\n", chem.getIndex(), chem.getKind(), chem.getChemical()); } System.out.print("\nLCMS wells:\n"); System.out.format(" %-15s%-6s%-15s%-15s%-15s\n", "barcode", "well", "msid", "fed", "lcms_count"); for (LCMSWell well : lcmsWells) { uniqueMSIDs.add(well.getMsid()); Plate p = platesById.get(well.getPlateId()); if (p == null) { // TODO: migrate Plate to be a subclass of BaseDBModel. p = Plate.getPlateById(db, well.getPlateId()); platesById.put(p.getId(), p); } String chem = well.getChemical(); List<ScanFile> scanFiles = ScanFile.getScanFileByPlateIDRowAndColumn(db, p.getId(), well.getPlateRow(), well.getPlateColumn()); System.out.format(" %-15s%-6s%-15s%-15s%-15d\n", p.getBarcode(), well.getCoordinatesString(), well.getMsid(), chem == null || chem.isEmpty() ? "--" : chem, scanFiles.size()); System.out.flush(); } List<Integer> plateIds = Arrays.asList(platesById.keySet().toArray(new Integer[platesById.size()])); Collections.sort(plateIds); System.out.print("\nAppears in plates:\n"); for (Integer id : plateIds) { Plate p = platesById.get(id); System.out.format(" %s: %s\n", p.getBarcode(), p.getName()); } List<String> msids = Arrays.asList(uniqueMSIDs.toArray(new String[uniqueMSIDs.size()])); Collections.sort(msids); System.out.format("\nMSIDS: %s\n", StringUtils.join(msids, ", ")); Set<String> availableNegativeControls = new HashSet<>(); for (Map.Entry<Integer, Plate> entry : platesById.entrySet()) { List<LCMSWell> wells = LCMSWell.getInstance().getByPlateId(db, entry.getKey()); for (LCMSWell well : wells) { if (!construct.equals(well.getComposition())) { availableNegativeControls.add(well.getComposition()); } } } // Print available standards for each step w/ plate barcodes and coordinates. System.out.format("\nAvailable Standards:\n"); Map<Integer, Plate> plateCache = new HashMap<>(); for (ChemicalAssociatedWithPathway chem : pathwayChems) { List<StandardWell> matchingWells = StandardWell.getInstance().getStandardWellsByChemical(db, chem.getChemical()); for (StandardWell well : matchingWells) { if (!plateCache.containsKey(well.getPlateId())) { Plate p = Plate.getPlateById(db, well.getPlateId()); plateCache.put(p.getId(), p); } } Map<Integer, List<StandardWell>> standardWellsByPlateId = new HashMap<>(); for (StandardWell well : matchingWells) { List<StandardWell> plateWells = standardWellsByPlateId.get(well.getPlateId()); if (plateWells == null) { plateWells = new ArrayList<>(); standardWellsByPlateId.put(well.getPlateId(), plateWells); } plateWells.add(well); } List<Pair<String, Integer>> plateBarcodes = new ArrayList<>(plateCache.size()); for (Plate p : plateCache.values()) { if (p.getBarcode() == null) { plateBarcodes.add(Pair.of("(no barcode)", p.getId())); } else { plateBarcodes.add(Pair.of(p.getBarcode(), p.getId())); } } Collections.sort(plateBarcodes); System.out.format(" %s:\n", chem.getChemical()); for (Pair<String, Integer> barcodePair : plateBarcodes) { // TODO: hoist this whole sorting/translation step into a utility class. List<StandardWell> wells = standardWellsByPlateId.get(barcodePair.getRight()); if (wells == null) { // Don't print plates that don't apply to this chemical, which can happen because we're caching the plates. continue; } Collections.sort(wells, new Comparator<StandardWell>() { @Override public int compare(StandardWell o1, StandardWell o2) { int c = o1.getPlateRow().compareTo(o2.getPlateRow()); if (c != 0) return c; return o1.getPlateColumn().compareTo(o2.getPlateColumn()); } }); List<String> descriptions = new ArrayList<>(wells.size()); for (StandardWell well : wells) { descriptions.add(String.format("%s in %s%s", well.getCoordinatesString(), well.getMedia(), well.getConcentration() == null ? "" : String.format(" c. %f", well.getConcentration()))); } System.out.format(" %s: %s\n", barcodePair.getLeft(), StringUtils.join(descriptions, ", ")); } } List<String> negativeControlStrains = Arrays .asList(availableNegativeControls.toArray(new String[availableNegativeControls.size()])); Collections.sort(negativeControlStrains); System.out.format("\nAvailable negative controls: %s\n", StringUtils.join(negativeControlStrains, ",")); System.out.print("\n----------\n"); System.out.print("\n\n"); } }
From source file:com.act.lcms.db.io.ExportStandardIonResultsFromDB.java
public static void main(String[] args) throws Exception { Options opts = new Options(); for (Option.Builder b : OPTION_BUILDERS) { opts.addOption(b.build());/* www . j ava 2 s .c o m*/ } CommandLine cl = null; try { CommandLineParser parser = new DefaultParser(); cl = parser.parse(opts, args); } catch (ParseException e) { System.err.format("Argument parsing failed: %s\n", e.getMessage()); HELP_FORMATTER.printHelp(ExportStandardIonResultsFromDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); System.exit(1); } if (cl.hasOption("help")) { HELP_FORMATTER.printHelp(ExportStandardIonResultsFromDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); return; } try (DB db = DB.openDBFromCLI(cl)) { List<String> chemicalNames = new ArrayList<>(); if (cl.hasOption(OPTION_CONSTRUCT)) { // Extract the chemicals in the pathway and their product masses, then look up info on those chemicals List<Pair<ChemicalAssociatedWithPathway, Double>> productMasses = Utils .extractMassesForChemicalsAssociatedWithConstruct(db, cl.getOptionValue(OPTION_CONSTRUCT)); for (Pair<ChemicalAssociatedWithPathway, Double> pair : productMasses) { chemicalNames.add(pair.getLeft().getChemical()); } } if (cl.hasOption(OPTION_CHEMICALS)) { chemicalNames.addAll(Arrays.asList(cl.getOptionValues(OPTION_CHEMICALS))); } if (chemicalNames.size() == 0) { System.err.format("No chemicals can be found from the input query.\n"); System.exit(-1); } List<String> standardIonHeaderFields = new ArrayList<String>() { { add(STANDARD_ION_HEADER_FIELDS.CHEMICAL.name()); add(STANDARD_ION_HEADER_FIELDS.BEST_ION_FROM_ALGO.name()); add(STANDARD_ION_HEADER_FIELDS.MANUAL_PICK.name()); add(STANDARD_ION_HEADER_FIELDS.AUTHOR.name()); add(STANDARD_ION_HEADER_FIELDS.DIAGNOSTIC_PLOTS.name()); add(STANDARD_ION_HEADER_FIELDS.NOTE.name()); } }; String outAnalysis; if (cl.hasOption(OPTION_OUTPUT_PREFIX)) { outAnalysis = cl.getOptionValue(OPTION_OUTPUT_PREFIX) + "." + TSV_FORMAT; } else { outAnalysis = String.join("-", chemicalNames) + "." + TSV_FORMAT; } File lcmsDir = new File(cl.getOptionValue(OPTION_DIRECTORY)); if (!lcmsDir.isDirectory()) { System.err.format("File at %s is not a directory\n", lcmsDir.getAbsolutePath()); HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); System.exit(1); } String plottingDirectory = cl.getOptionValue(OPTION_PLOTTING_DIR); TSVWriter<String, String> resultsWriter = new TSVWriter<>(standardIonHeaderFields); resultsWriter.open(new File(outAnalysis)); // For each chemical, create a TSV row and a corresponding diagnostic plot for (String chemicalName : chemicalNames) { List<String> graphLabels = new ArrayList<>(); List<Double> yMaxList = new ArrayList<>(); String outData = plottingDirectory + "/" + chemicalName + ".data"; String outImg = plottingDirectory + "/" + chemicalName + ".pdf"; // For each diagnostic plot, open a new file stream. try (FileOutputStream fos = new FileOutputStream(outData)) { List<StandardIonResult> getResultByChemicalName = StandardIonResult.getByChemicalName(db, chemicalName); if (getResultByChemicalName != null && getResultByChemicalName.size() > 0) { // PART 1: Get the best metlin ion across all standard ion results for a given chemical String bestGlobalMetlinIon = AnalysisHelper .scoreAndReturnBestMetlinIonFromStandardIonResults(getResultByChemicalName, new HashMap<>(), true, true); // PART 2: Plot all the graphs related to the chemical. The plots are structured as follows: // // Page 1: All graphs (water, MeOH, Yeast) for Global ion picked (best ion among ALL standard ion runs for // the given chemical) by the algorithm // Page 2: All graphs for M+H // Page 3: All graphs for Local ions picked (best ion within a SINGLE standard ion run) + negative controls // for Yeast. // // Each page is demarcated by a blank graph. // Arrange results based on media Map<String, List<StandardIonResult>> categories = StandardIonResult .categorizeListOfStandardWellsByMedia(db, getResultByChemicalName); // This set contains all the best metlin ions corresponding to all the standard ion runs. Set<String> bestLocalIons = new HashSet<>(); bestLocalIons.add(bestGlobalMetlinIon); bestLocalIons.add(DEFAULT_ION); for (StandardIonResult result : getResultByChemicalName) { bestLocalIons.add(result.getBestMetlinIon()); } // We sort the best local ions are follows: // 1) Global best ion spectra 2) M+H spectra 3) Local best ion spectra List<String> bestLocalIonsArray = new ArrayList<>(bestLocalIons); Collections.sort(bestLocalIonsArray, new Comparator<String>() { @Override public int compare(String o1, String o2) { if (o1.equals(bestGlobalMetlinIon) && !o2.equals(bestGlobalMetlinIon)) { return -1; } else if (o1.equals(DEFAULT_ION) && !o2.equals(bestGlobalMetlinIon)) { return -1; } else { return 1; } } }); // This variable stores the index of the array at which all the remaining spectra are contained in one // page. This happens right after the M+H ion spectra. Integer combineAllSpectraIntoPageThreeFromIndex = 0; for (int i = 0; i < bestLocalIonsArray.size(); i++) { if (bestLocalIonsArray.get(i).equals(DEFAULT_ION)) { combineAllSpectraIntoPageThreeFromIndex = i + 1; } } for (int i = 0; i < bestLocalIonsArray.size(); i++) { String ion = bestLocalIonsArray.get(i); for (Map.Entry<String, List<StandardIonResult>> mediaToListOfIonResults : categories .entrySet()) { for (StandardIonResult result : mediaToListOfIonResults.getValue()) { // For every standard ion result, we plot the best global metlin ion and M+H. These plots are in the // pages 1 and 2. For all page 3 (aka miscellaneous spectra), we only plot the best local ion // corresponding to it's spectra and not some other graph's spectra. In the below condition, // we reach the page 3 case with not the same best ion as the spectra, in which case we just continue // and not draw anything on the page. if (i >= combineAllSpectraIntoPageThreeFromIndex && !(result.getBestMetlinIon().equals(ion))) { continue; } StandardWell positiveWell = StandardWell.getInstance().getById(db, result.getStandardWellId()); String positiveControlChemical = positiveWell.getChemical(); ScanData<StandardWell> encapsulatedDataForPositiveControl = AnalysisHelper .getScanDataForWell(db, lcmsDir, positiveWell, positiveControlChemical, positiveControlChemical); Set<String> singletonSet = Collections.singleton(ion); String additionalInfo = generateAdditionalLabelInformation(positiveWell, result, ion); List<String> labels = AnalysisHelper .writeScanData(fos, lcmsDir, MAX_INTENSITY, encapsulatedDataForPositiveControl, false, false, singletonSet) .stream().map(label -> label + additionalInfo) .collect(Collectors.toList()); yMaxList.add(encapsulatedDataForPositiveControl.getMs1ScanResults() .getMaxIntensityForIon(ion)); List<String> negativeLabels = null; // Only do the negative control in the miscellaneous page (page 3) and if the well is in yeast media. if (mediaToListOfIonResults.getKey() .equals(StandardWell.MEDIA_TYPE.YEAST.name()) && (i >= combineAllSpectraIntoPageThreeFromIndex && (result.getBestMetlinIon().equals(ion)))) { //TODO: Change the representative negative well to one that displays the highest noise in the future. // For now, we just use the first index among the negative wells. int representativeIndex = 0; StandardWell representativeNegativeControlWell = StandardWell.getInstance() .getById(db, result.getNegativeWellIds().get(representativeIndex)); ScanData encapsulatedDataForNegativeControl = AnalysisHelper .getScanDataForWell(db, lcmsDir, representativeNegativeControlWell, positiveWell.getChemical(), representativeNegativeControlWell.getChemical()); String negativePlateAdditionalInfo = generateAdditionalLabelInformation( representativeNegativeControlWell, null, null); negativeLabels = AnalysisHelper.writeScanData(fos, lcmsDir, MAX_INTENSITY, encapsulatedDataForNegativeControl, false, false, singletonSet) .stream().map(label -> label + negativePlateAdditionalInfo) .collect(Collectors.toList()); yMaxList.add(encapsulatedDataForNegativeControl.getMs1ScanResults() .getMaxIntensityForIon(ion)); } graphLabels.addAll(labels); if (negativeLabels != null) { graphLabels.addAll(negativeLabels); } } } // Add a blank graph to demarcate pages. if (i < combineAllSpectraIntoPageThreeFromIndex) { graphLabels.addAll(AnalysisHelper.writeScanData(fos, lcmsDir, 0.0, BLANK_SCAN, false, false, new HashSet<>())); yMaxList.add(0.0d); } } // We need to pass the yMax values as an array to the Gnuplotter. Double fontScale = null; if (cl.hasOption(FONT_SCALE)) { try { fontScale = Double.parseDouble(cl.getOptionValue(FONT_SCALE)); } catch (IllegalArgumentException e) { System.err.format("Argument for font-scale must be a floating point number.\n"); System.exit(1); } } Double[] yMaxes = yMaxList.toArray(new Double[yMaxList.size()]); Gnuplotter plotter = fontScale == null ? new Gnuplotter() : new Gnuplotter(fontScale); plotter.plot2D(outData, outImg, graphLabels.toArray(new String[graphLabels.size()]), "time", null, "intensity", "pdf", null, null, yMaxes, outImg + ".gnuplot"); Map<String, String> row = new HashMap<>(); row.put(STANDARD_ION_HEADER_FIELDS.CHEMICAL.name(), chemicalName); row.put(STANDARD_ION_HEADER_FIELDS.BEST_ION_FROM_ALGO.name(), bestGlobalMetlinIon); row.put(STANDARD_ION_HEADER_FIELDS.DIAGNOSTIC_PLOTS.name(), outImg); resultsWriter.append(row); resultsWriter.flush(); } } } resultsWriter.flush(); resultsWriter.close(); } }
From source file:com.twentyn.patentScorer.ScoreMerger.java
public static void main(String[] args) throws Exception { System.out.println("Starting up..."); System.out.flush();//ww w . j a v a 2 s. c om Options opts = new Options(); opts.addOption(Option.builder("h").longOpt("help").desc("Print this help message and exit").build()); opts.addOption(Option.builder("r").longOpt("results").required().hasArg() .desc("A directory of search results to read").build()); opts.addOption(Option.builder("s").longOpt("scores").required().hasArg() .desc("A directory of patent classification scores to read").build()); opts.addOption(Option.builder("o").longOpt("output").required().hasArg() .desc("The output file where results will be written.").build()); HelpFormatter helpFormatter = new HelpFormatter(); CommandLineParser cmdLineParser = new DefaultParser(); CommandLine cmdLine = null; try { cmdLine = cmdLineParser.parse(opts, args); } catch (ParseException e) { System.out.println("Caught exception when parsing command line: " + e.getMessage()); helpFormatter.printHelp("DocumentIndexer", opts); System.exit(1); } if (cmdLine.hasOption("help")) { helpFormatter.printHelp("DocumentIndexer", opts); System.exit(0); } File scoresDirectory = new File(cmdLine.getOptionValue("scores")); if (cmdLine.getOptionValue("scores") == null || !scoresDirectory.isDirectory()) { LOGGER.error("Not a directory of score files: " + cmdLine.getOptionValue("scores")); } File resultsDirectory = new File(cmdLine.getOptionValue("results")); if (cmdLine.getOptionValue("results") == null || !resultsDirectory.isDirectory()) { LOGGER.error("Not a directory of results files: " + cmdLine.getOptionValue("results")); } FileWriter outputWriter = new FileWriter(cmdLine.getOptionValue("output")); ObjectMapper objectMapper = new ObjectMapper(); objectMapper.enable(SerializationFeature.INDENT_OUTPUT); objectMapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY); FilenameFilter jsonFilter = new FilenameFilter() { public final Pattern JSON_PATTERN = Pattern.compile("\\.json$"); public boolean accept(File dir, String name) { return JSON_PATTERN.matcher(name).find(); } }; Map<String, PatentScorer.ClassificationResult> scores = new HashMap<>(); LOGGER.info("Reading scores from directory at " + scoresDirectory.getAbsolutePath()); for (File scoreFile : scoresDirectory.listFiles(jsonFilter)) { BufferedReader reader = new BufferedReader(new FileReader(scoreFile)); int count = 0; String line; while ((line = reader.readLine()) != null) { PatentScorer.ClassificationResult res = objectMapper.readValue(line, PatentScorer.ClassificationResult.class); scores.put(res.docId, res); count++; } LOGGER.info("Read " + count + " scores from " + scoreFile.getAbsolutePath()); } Map<String, List<DocumentSearch.SearchResult>> synonymsToResults = new HashMap<>(); Map<String, List<DocumentSearch.SearchResult>> inchisToResults = new HashMap<>(); LOGGER.info("Reading results from directory at " + resultsDirectory); // With help from http://stackoverflow.com/questions/6846244/jackson-and-generic-type-reference. JavaType resultsType = objectMapper.getTypeFactory().constructCollectionType(List.class, DocumentSearch.SearchResult.class); List<File> resultsFiles = Arrays.asList(resultsDirectory.listFiles(jsonFilter)); Collections.sort(resultsFiles, new Comparator<File>() { @Override public int compare(File o1, File o2) { return o1.getName().compareTo(o2.getName()); } }); for (File resultsFile : resultsFiles) { BufferedReader reader = new BufferedReader(new FileReader(resultsFile)); CharBuffer buffer = CharBuffer.allocate(Long.valueOf(resultsFile.length()).intValue()); int bytesRead = reader.read(buffer); LOGGER.info("Read " + bytesRead + " bytes from " + resultsFile.getName() + " (length is " + resultsFile.length() + ")"); List<DocumentSearch.SearchResult> results = objectMapper.readValue(new CharArrayReader(buffer.array()), resultsType); LOGGER.info("Read " + results.size() + " results from " + resultsFile.getAbsolutePath()); int count = 0; for (DocumentSearch.SearchResult sres : results) { for (DocumentSearch.ResultDocument resDoc : sres.getResults()) { String docId = resDoc.getDocId(); PatentScorer.ClassificationResult classificationResult = scores.get(docId); if (classificationResult == null) { LOGGER.warn("No classification result found for " + docId); } else { resDoc.setClassifierScore(classificationResult.getScore()); } } if (!synonymsToResults.containsKey(sres.getSynonym())) { synonymsToResults.put(sres.getSynonym(), new ArrayList<DocumentSearch.SearchResult>()); } synonymsToResults.get(sres.getSynonym()).add(sres); count++; if (count % 1000 == 0) { LOGGER.info("Processed " + count + " search result documents"); } } } Comparator<DocumentSearch.ResultDocument> resultDocumentComparator = new Comparator<DocumentSearch.ResultDocument>() { @Override public int compare(DocumentSearch.ResultDocument o1, DocumentSearch.ResultDocument o2) { int cmp = o2.getClassifierScore().compareTo(o1.getClassifierScore()); if (cmp != 0) { return cmp; } cmp = o2.getScore().compareTo(o1.getScore()); return cmp; } }; for (Map.Entry<String, List<DocumentSearch.SearchResult>> entry : synonymsToResults.entrySet()) { DocumentSearch.SearchResult newSearchRes = null; // Merge all result documents into a single search result. for (DocumentSearch.SearchResult sr : entry.getValue()) { if (newSearchRes == null) { newSearchRes = sr; } else { newSearchRes.getResults().addAll(sr.getResults()); } } if (newSearchRes == null || newSearchRes.getResults() == null) { LOGGER.error("Search results for " + entry.getKey() + " are null."); continue; } Collections.sort(newSearchRes.getResults(), resultDocumentComparator); if (!inchisToResults.containsKey(newSearchRes.getInchi())) { inchisToResults.put(newSearchRes.getInchi(), new ArrayList<DocumentSearch.SearchResult>()); } inchisToResults.get(newSearchRes.getInchi()).add(newSearchRes); } List<String> sortedKeys = new ArrayList<String>(inchisToResults.keySet()); Collections.sort(sortedKeys); List<GroupedInchiResults> orderedResults = new ArrayList<>(sortedKeys.size()); Comparator<DocumentSearch.SearchResult> synonymSorter = new Comparator<DocumentSearch.SearchResult>() { @Override public int compare(DocumentSearch.SearchResult o1, DocumentSearch.SearchResult o2) { return o1.getSynonym().compareTo(o2.getSynonym()); } }; for (String inchi : sortedKeys) { List<DocumentSearch.SearchResult> res = inchisToResults.get(inchi); Collections.sort(res, synonymSorter); orderedResults.add(new GroupedInchiResults(inchi, res)); } objectMapper.writerWithView(Object.class).writeValue(outputWriter, orderedResults); outputWriter.close(); }
From source file:FaceRatios.java
@SuppressWarnings("serial") public static void main(String[] args) { int r = FSDK.ActivateLibrary(FACE_SDK_LICENSE); if (r == FSDK.FSDKE_OK) { FSDK.Initialize();//from ww w.j a v a 2 s . c o m FSDK.SetFaceDetectionParameters(true, true, 384); Map<String, Map<String, ArrayList<Double>>> faceProperties = new HashMap<>(); for (String directory : new File(FACE_DIRECTORY).list()) { if (new File(FACE_DIRECTORY + directory).isDirectory()) { Map<String, ArrayList<Double>> properties = new HashMap<String, ArrayList<Double>>() { { for (String property : propertyNames) put(property, new ArrayList<Double>()); } }; File[] files = new File(FACE_DIRECTORY + directory).listFiles(); System.out.println("Analyzing " + directory + " with " + files.length + " files\n"); for (File file : files) { if (file.isFile()) { HImage imageHandle = new HImage(); FSDK.LoadImageFromFileW(imageHandle, file.getAbsolutePath()); FSDK.TFacePosition.ByReference facePosition = new FSDK.TFacePosition.ByReference(); if (FSDK.DetectFace(imageHandle, facePosition) == FSDK.FSDKE_OK) { FSDK_Features.ByReference facialFeatures = new FSDK_Features.ByReference(); FSDK.DetectFacialFeaturesInRegion(imageHandle, (FSDK.TFacePosition) facePosition, facialFeatures); Point[] featurePoints = new Point[FSDK.FSDK_FACIAL_FEATURE_COUNT]; for (int i = 0; i < FSDK.FSDK_FACIAL_FEATURE_COUNT; i++) { featurePoints[i] = new Point(0, 0); featurePoints[i].x = facialFeatures.features[i].x; featurePoints[i].y = facialFeatures.features[i].y; } double eyeDistance = featureDistance(featurePoints, FeatureID.LEFT_EYE, FeatureID.RIGHT_EYE); double rightEyeSize = featureDistance(featurePoints, FeatureID.RIGHT_EYE_INNER_CORNER, FeatureID.RIGHT_EYE_OUTER_CORNER); double leftEyeSize = featureDistance(featurePoints, FeatureID.LEFT_EYE_INNER_CORNER, FeatureID.LEFT_EYE_OUTER_CORNER); double averageEyeSize = (rightEyeSize + leftEyeSize) / 2; double mouthLength = featureDistance(featurePoints, FeatureID.MOUTH_RIGHT_CORNER, FeatureID.MOUTH_LEFT_CORNER); double mouthHeight = featureDistance(featurePoints, FeatureID.MOUTH_BOTTOM, FeatureID.MOUTH_TOP); double noseHeight = featureDistance(featurePoints, FeatureID.NOSE_BOTTOM, FeatureID.NOSE_BRIDGE); double chinHeight = featureDistance(featurePoints, FeatureID.CHIN_BOTTOM, FeatureID.MOUTH_BOTTOM); double chinToBridgeHeight = featureDistance(featurePoints, FeatureID.CHIN_BOTTOM, FeatureID.NOSE_BRIDGE); double faceContourLeft = (featurePoints[FeatureID.CHIN_BOTTOM.getIndex()].getY() - featurePoints[FeatureID.FACE_CONTOUR2.getIndex()].getY()) / (featurePoints[FeatureID.CHIN_BOTTOM.getIndex()].getX() - featurePoints[FeatureID.FACE_CONTOUR2.getIndex()].getX()); double faceContourRight = (featurePoints[FeatureID.CHIN_BOTTOM.getIndex()].getY() - featurePoints[FeatureID.FACE_CONTOUR12.getIndex()].getY()) / (featurePoints[FeatureID.CHIN_BOTTOM.getIndex()].getX() - featurePoints[FeatureID.FACE_CONTOUR12.getIndex()].getX()); double bridgeLeftEyeDistance = featureDistance(featurePoints, FeatureID.LEFT_EYE_INNER_CORNER, FeatureID.NOSE_BRIDGE); double bridgeRightEyeDistance = featureDistance(featurePoints, FeatureID.RIGHT_EYE_INNER_CORNER, FeatureID.NOSE_BRIDGE); properties.get("eyeSize/eyeDistance").add(averageEyeSize / eyeDistance); properties.get("eyeSizeDisparity") .add(Math.abs(leftEyeSize - rightEyeSize) / averageEyeSize); properties.get("bridgeToEyeDisparity") .add(Math.abs(bridgeLeftEyeDistance - bridgeRightEyeDistance) / ((bridgeLeftEyeDistance + bridgeRightEyeDistance) / 2)); properties.get("eyeDistance/mouthLength").add(eyeDistance / mouthLength); properties.get("eyeDistance/noseHeight").add(eyeDistance / noseHeight); properties.get("eyeSize/mouthLength").add(eyeDistance / mouthLength); properties.get("eyeSize/noseHeight").add(eyeDistance / noseHeight); properties.get("mouthLength/mouthHeight").add(mouthLength / mouthHeight); properties.get("chinHeight/noseHeight").add(chinHeight / noseHeight); properties.get("chinHeight/chinToBridgeHeight") .add(chinHeight / chinToBridgeHeight); properties.get("noseHeight/chinToBridgeHeight") .add(noseHeight / chinToBridgeHeight); properties.get("mouthHeight/chinToBridgeHeight") .add(mouthHeight / chinToBridgeHeight); properties.get("faceCountourAngle") .add(Math.toDegrees(Math.atan((faceContourLeft - faceContourRight) / (1 + faceContourLeft * faceContourRight)))); } FSDK.FreeImage(imageHandle); } } System.out.format("%32s\t%8s\t%8s\t%3s%n", "Property", "", "", "c"); System.out.println(new String(new char[76]).replace("\0", "-")); ArrayList<Entry<String, ArrayList<Double>>> propertyList = new ArrayList<>( properties.entrySet()); Collections.sort(propertyList, new Comparator<Entry<String, ArrayList<Double>>>() { @Override public int compare(Entry<String, ArrayList<Double>> arg0, Entry<String, ArrayList<Double>> arg1) { DescriptiveStatistics dStats0 = new DescriptiveStatistics(listToArray(arg0.getValue())); DescriptiveStatistics dStats1 = new DescriptiveStatistics(listToArray(arg1.getValue())); return new Double(dStats0.getStandardDeviation() / dStats0.getMean()) .compareTo(dStats1.getStandardDeviation() / dStats1.getMean()); } }); for (Entry<String, ArrayList<Double>> property : propertyList) { DescriptiveStatistics dStats = new DescriptiveStatistics(listToArray(property.getValue())); System.out.format("%32s\t%4f\t%4f\t%3s%n", property.getKey(), dStats.getMean(), dStats.getStandardDeviation(), Math.round(dStats.getStandardDeviation() / dStats.getMean() * 100) + "%"); } System.out.println("\n"); faceProperties.put(directory, properties); } } for (String propertyName : propertyNames) { DefaultBoxAndWhiskerCategoryDataset dataset = new DefaultBoxAndWhiskerCategoryDataset(); for (Entry<String, Map<String, ArrayList<Double>>> face : faceProperties.entrySet()) { dataset.add(face.getValue().get(propertyName), "Default Series", face.getKey()); } PropertyBoxWhisker plot = new PropertyBoxWhisker(propertyName, dataset); plot.pack(); plot.setVisible(true); } } }
From source file:com.moviejukebox.MovieJukebox.java
public static void main(String[] args) throws Throwable { JukeboxStatistics.setTimeStart(System.currentTimeMillis()); // Create the log file name here, so we can change it later (because it's locked System.setProperty("file.name", LOG_FILENAME); PropertyConfigurator.configure("properties/log4j.properties"); LOG.info("Yet Another Movie Jukebox {}", GitRepositoryState.getVersion()); LOG.info("~~~ ~~~~~~~ ~~~~~ ~~~~~~~ {}", StringUtils.repeat("~", GitRepositoryState.getVersion().length())); LOG.info("https://github.com/YAMJ/yamj-v2"); LOG.info("Copyright (c) 2004-2016 YAMJ Members"); LOG.info(""); LOG.info("This software is licensed under the GNU General Public License v3+"); LOG.info("See this page: https://github.com/YAMJ/yamj-v2/wiki/License"); LOG.info(""); LOG.info(" Revision SHA: {} {}", GIT.getCommitId(), GIT.getDirty() ? "(Custom Build)" : ""); LOG.info(" Commit Date: {}", GIT.getCommitTime()); LOG.info(" Build Date: {}", GIT.getBuildTime()); LOG.info(""); LOG.info(" Java Version: {}", GitRepositoryState.getJavaVersion()); LOG.info(""); if (!SystemTools.validateInstallation()) { LOG.info("ABORTING."); return;// w w w . ja va 2 s. co m } String movieLibraryRoot = null; String jukeboxRoot = null; Map<String, String> cmdLineProps = new LinkedHashMap<>(); try { for (int i = 0; i < args.length; i++) { String arg = args[i]; if ("-v".equalsIgnoreCase(arg)) { // We've printed the version, so quit now return; } else if ("-t".equalsIgnoreCase(arg)) { String pin = args[++i]; // load the apikeys.properties file if (!setPropertiesStreamName("./properties/apikeys.properties", Boolean.TRUE)) { return; } // authorize to Trakt.TV TraktTV.getInstance().initialize().authorizeWithPin(pin); // We've authorized access to Trakt.TV, so quit now return; } else if ("-o".equalsIgnoreCase(arg)) { jukeboxRoot = args[++i]; PropertiesUtil.setProperty("mjb.jukeboxRoot", jukeboxRoot); } else if ("-c".equalsIgnoreCase(arg)) { jukeboxClean = Boolean.TRUE; PropertiesUtil.setProperty("mjb.jukeboxClean", TRUE); } else if ("-k".equalsIgnoreCase(arg)) { setJukeboxPreserve(Boolean.TRUE); } else if ("-p".equalsIgnoreCase(arg)) { userPropertiesName = args[++i]; } else if ("-i".equalsIgnoreCase(arg)) { skipIndexGeneration = Boolean.TRUE; PropertiesUtil.setProperty("mjb.skipIndexGeneration", TRUE); } else if ("-h".equalsIgnoreCase(arg)) { skipHtmlGeneration = Boolean.TRUE; PropertiesUtil.setProperty("mjb.skipHtmlGeneration", Boolean.TRUE); } else if ("-dump".equalsIgnoreCase(arg)) { dumpLibraryStructure = Boolean.TRUE; } else if ("-memory".equalsIgnoreCase(arg)) { showMemory = Boolean.TRUE; PropertiesUtil.setProperty("mjb.showMemory", Boolean.TRUE); } else if (arg.startsWith("-D")) { String propLine = arg.length() > 2 ? arg.substring(2) : args[++i]; int propDiv = propLine.indexOf('='); if (-1 != propDiv) { cmdLineProps.put(propLine.substring(0, propDiv), propLine.substring(propDiv + 1)); } } else if (arg.startsWith("-")) { help(); return; } else { movieLibraryRoot = args[i]; } } } catch (Exception error) { LOG.error("Wrong arguments specified"); help(); return; } // Save the name of the properties file for use later setProperty("userPropertiesName", userPropertiesName); LOG.info("Processing started at {}", new Date()); LOG.info(""); // Load the moviejukebox-default.properties file if (!setPropertiesStreamName("./properties/moviejukebox-default.properties", Boolean.TRUE)) { return; } // Load the user properties file "moviejukebox.properties" // No need to abort if we don't find this file // Must be read before the skin, because this may contain an override skin setPropertiesStreamName(userPropertiesName, Boolean.FALSE); // Grab the skin from the command-line properties if (cmdLineProps.containsKey(SKIN_DIR)) { setProperty(SKIN_DIR, cmdLineProps.get(SKIN_DIR)); } // Load the skin.properties file if (!setPropertiesStreamName(getProperty(SKIN_DIR, SKIN_DEFAULT) + "/skin.properties", Boolean.TRUE)) { return; } // Load the skin-user.properties file (ignore the error) setPropertiesStreamName(getProperty(SKIN_DIR, SKIN_DEFAULT) + "/skin-user.properties", Boolean.FALSE); // Load the overlay.properties file (ignore the error) String overlayRoot = getProperty("mjb.overlay.dir", Movie.UNKNOWN); overlayRoot = (PropertiesUtil.getBooleanProperty("mjb.overlay.skinroot", Boolean.TRUE) ? (getProperty(SKIN_DIR, SKIN_DEFAULT) + File.separator) : "") + (StringTools.isValidString(overlayRoot) ? (overlayRoot + File.separator) : ""); setPropertiesStreamName(overlayRoot + "overlay.properties", Boolean.FALSE); // Load the apikeys.properties file if (!setPropertiesStreamName("./properties/apikeys.properties", Boolean.TRUE)) { return; } // This is needed to update the static reference for the API Keys in the pattern formatter // because the formatter is initialised before the properties files are read FilteringLayout.addApiKeys(); // Load the rest of the command-line properties for (Map.Entry<String, String> propEntry : cmdLineProps.entrySet()) { setProperty(propEntry.getKey(), propEntry.getValue()); } // Read the information about the skin SkinProperties.readSkinVersion(); // Display the information about the skin SkinProperties.printSkinVersion(); StringBuilder properties = new StringBuilder("{"); for (Map.Entry<Object, Object> propEntry : PropertiesUtil.getEntrySet()) { properties.append(propEntry.getKey()); properties.append("="); properties.append(propEntry.getValue()); properties.append(","); } properties.replace(properties.length() - 1, properties.length(), "}"); // Print out the properties to the log file. LOG.debug("Properties: {}", properties.toString()); // Check for mjb.skipIndexGeneration and set as necessary // This duplicates the "-i" functionality, but allows you to have it in the property file skipIndexGeneration = PropertiesUtil.getBooleanProperty("mjb.skipIndexGeneration", Boolean.FALSE); if (PropertiesUtil.getBooleanProperty("mjb.people", Boolean.FALSE)) { peopleScan = Boolean.TRUE; peopleScrape = PropertiesUtil.getBooleanProperty("mjb.people.scrape", Boolean.TRUE); peopleMax = PropertiesUtil.getIntProperty("mjb.people.maxCount", 10); popularity = PropertiesUtil.getIntProperty("mjb.people.popularity", 5); // Issue 1947: Cast enhancement - option to save all related files to a specific folder peopleFolder = PropertiesUtil.getProperty("mjb.people.folder", ""); if (isNotValidString(peopleFolder)) { peopleFolder = ""; } else if (!peopleFolder.endsWith(File.separator)) { peopleFolder += File.separator; } StringTokenizer st = new StringTokenizer( PropertiesUtil.getProperty("photo.scanner.photoExtensions", "jpg,jpeg,gif,bmp,png"), ",;| "); while (st.hasMoreTokens()) { PHOTO_EXTENSIONS.add(st.nextToken()); } } // Check for mjb.skipHtmlGeneration and set as necessary // This duplicates the "-h" functionality, but allows you to have it in the property file skipHtmlGeneration = PropertiesUtil.getBooleanProperty("mjb.skipHtmlGeneration", Boolean.FALSE); // Look for the parameter in the properties file if it's not been set on the command line // This way we don't overwrite the setting if it's not found and defaults to FALSE showMemory = PropertiesUtil.getBooleanProperty("mjb.showMemory", Boolean.FALSE); // This duplicates the "-c" functionality, but allows you to have it in the property file jukeboxClean = PropertiesUtil.getBooleanProperty("mjb.jukeboxClean", Boolean.FALSE); MovieFilenameScanner.setSkipKeywords( tokenizeToArray(getProperty("filename.scanner.skip.keywords", ""), ",;| "), PropertiesUtil.getBooleanProperty("filename.scanner.skip.caseSensitive", Boolean.TRUE)); MovieFilenameScanner.setSkipRegexKeywords( tokenizeToArray(getProperty("filename.scanner.skip.keywords.regex", ""), ","), PropertiesUtil.getBooleanProperty("filename.scanner.skip.caseSensitive.regex", Boolean.TRUE)); MovieFilenameScanner.setExtrasKeywords( tokenizeToArray(getProperty("filename.extras.keywords", "trailer,extra,bonus"), ",;| ")); MovieFilenameScanner.setMovieVersionKeywords(tokenizeToArray( getProperty("filename.movie.versions.keywords", "remastered,directors cut,extended cut,final cut"), ",;|")); MovieFilenameScanner.setLanguageDetection( PropertiesUtil.getBooleanProperty("filename.scanner.language.detection", Boolean.TRUE)); final KeywordMap languages = PropertiesUtil.getKeywordMap("filename.scanner.language.keywords", null); if (!languages.isEmpty()) { MovieFilenameScanner.clearLanguages(); for (String lang : languages.getKeywords()) { String values = languages.get(lang); if (values != null) { MovieFilenameScanner.addLanguage(lang, values, values); } else { LOG.info("No values found for language code '{}'", lang); } } } final KeywordMap sourceKeywords = PropertiesUtil.getKeywordMap("filename.scanner.source.keywords", "HDTV,PDTV,DVDRip,DVDSCR,DSRip,CAM,R5,LINE,HD2DVD,DVD,DVD5,DVD9,HRHDTV,MVCD,VCD,TS,VHSRip,BluRay,HDDVD,D-THEATER,SDTV"); MovieFilenameScanner.setSourceKeywords(sourceKeywords.getKeywords(), sourceKeywords); String temp = getProperty("sorting.strip.prefixes"); if (temp != null) { StringTokenizer st = new StringTokenizer(temp, ","); while (st.hasMoreTokens()) { String token = st.nextToken().trim(); if (token.startsWith("\"") && token.endsWith("\"")) { token = token.substring(1, token.length() - 1); } Movie.addSortIgnorePrefixes(token.toLowerCase()); } } enableWatchScanner = PropertiesUtil.getBooleanProperty("watched.scanner.enable", Boolean.TRUE); enableWatchTraktTv = PropertiesUtil.getBooleanProperty("watched.trakttv.enable", Boolean.FALSE); enableCompleteMovies = PropertiesUtil.getBooleanProperty("complete.movies.enable", Boolean.TRUE); // Check to see if don't have a root, check the property file if (StringTools.isNotValidString(movieLibraryRoot)) { movieLibraryRoot = getProperty("mjb.libraryRoot"); if (StringTools.isValidString(movieLibraryRoot)) { LOG.info("Got libraryRoot from properties file: {}", movieLibraryRoot); } else { LOG.error("No library root found!"); help(); return; } } if (jukeboxRoot == null) { jukeboxRoot = getProperty("mjb.jukeboxRoot"); if (jukeboxRoot == null) { LOG.info("jukeboxRoot is null in properties file. Please fix this as it may cause errors."); } else { LOG.info("Got jukeboxRoot from properties file: {}", jukeboxRoot); } } File f = new File(movieLibraryRoot); if (f.exists() && f.isDirectory() && jukeboxRoot == null) { jukeboxRoot = movieLibraryRoot; } if (movieLibraryRoot == null) { help(); return; } if (jukeboxRoot == null) { LOG.info("Wrong arguments specified: you must define the jukeboxRoot property (-o) !"); help(); return; } if (!f.exists()) { LOG.error("Directory or library configuration file '{}', not found.", movieLibraryRoot); return; } FileTools.initUnsafeChars(); FileTools.initSubtitleExtensions(); // make canonical names jukeboxRoot = FileTools.getCanonicalPath(jukeboxRoot); movieLibraryRoot = FileTools.getCanonicalPath(movieLibraryRoot); MovieJukebox ml = new MovieJukebox(movieLibraryRoot, jukeboxRoot); if (dumpLibraryStructure) { LOG.warn( "WARNING !!! A dump of your library directory structure will be generated for debug purpose. !!! Library won't be built or updated"); ml.makeDumpStructure(); } else { ml.generateLibrary(); } // Now rename the log files renameLogFile(); if (ScanningLimit.isLimitReached()) { LOG.warn("Scanning limit of {} was reached, please re-run to complete processing.", ScanningLimit.getLimit()); System.exit(EXIT_SCAN_LIMIT); } else { System.exit(EXIT_NORMAL); } }