List of usage examples for java.util List addAll
boolean addAll(Collection<? extends E> c);
From source file:de.citec.sc.matoll.process.Matoll_CreateMax.java
public static void main(String[] args) throws IOException, ParserConfigurationException, SAXException, InstantiationException, IllegalAccessException, ClassNotFoundException, Exception { String directory;// w w w . j a v a2 s. c o m String gold_standard_lexicon; String output_lexicon; String configFile; Language language; String output; Stopwords stopwords = new Stopwords(); HashMap<String, Double> maxima; maxima = new HashMap<String, Double>(); if (args.length < 3) { System.out.print("Usage: Matoll --mode=train/test <DIRECTORY> <CONFIG>\n"); return; } // Classifier classifier; directory = args[1]; configFile = args[2]; final Config config = new Config(); config.loadFromFile(configFile); gold_standard_lexicon = config.getGoldStandardLexicon(); String model_file = config.getModel(); output_lexicon = config.getOutputLexicon(); output = config.getOutput(); language = config.getLanguage(); LexiconLoader loader = new LexiconLoader(); Lexicon gold = loader.loadFromFile(gold_standard_lexicon); Set<String> uris = new HashSet<>(); // Map<Integer,String> sentence_list = new HashMap<>(); Map<Integer, Set<Integer>> mapping_words_sentences = new HashMap<>(); //consider only properties for (LexicalEntry entry : gold.getEntries()) { try { for (Sense sense : entry.getSenseBehaviours().keySet()) { String tmp_uri = sense.getReference().getURI().replace("http://dbpedia.org/ontology/", ""); if (!Character.isUpperCase(tmp_uri.charAt(0))) { uris.add(sense.getReference().getURI()); } } } catch (Exception e) { } ; } ModelPreprocessor preprocessor = new ModelPreprocessor(language); preprocessor.setCoreferenceResolution(false); Set<String> dep = new HashSet<>(); dep.add("prep"); dep.add("appos"); dep.add("nn"); dep.add("dobj"); dep.add("pobj"); dep.add("num"); preprocessor.setDEP(dep); List<File> list_files = new ArrayList<>(); if (config.getFiles().isEmpty()) { File folder = new File(directory); File[] files = folder.listFiles(); for (File file : files) { if (file.toString().contains(".ttl")) list_files.add(file); } } else { list_files.addAll(config.getFiles()); } System.out.println(list_files.size()); int sentence_counter = 0; Map<String, Set<Integer>> bag_words_uri = new HashMap<>(); Map<String, Integer> mapping_word_id = new HashMap<>(); for (File file : list_files) { Model model = RDFDataMgr.loadModel(file.toString()); for (Model sentence : getSentences(model)) { String reference = getReference(sentence); reference = reference.replace("http://dbpedia/", "http://dbpedia.org/"); if (uris.contains(reference)) { sentence_counter += 1; Set<Integer> words_ids = getBagOfWords(sentence, stopwords, mapping_word_id); //TODO: add sentence preprocessing String obj = getObject(sentence); String subj = getSubject(sentence); preprocessor.preprocess(sentence, subj, obj, language); //TODO: also return marker if object or subject of property (in SPARQL this has to be optional of course) String parsed_sentence = getParsedSentence(sentence); try (FileWriter fw = new FileWriter("mapping_sentences_to_ids_goldstandard.tsv", true); BufferedWriter bw = new BufferedWriter(fw); PrintWriter out = new PrintWriter(bw)) { out.println(sentence_counter + "\t" + parsed_sentence); } catch (IOException e) { e.printStackTrace(); } for (Integer word_id : words_ids) { if (mapping_words_sentences.containsKey(word_id)) { Set<Integer> tmp_set = mapping_words_sentences.get(word_id); tmp_set.add(sentence_counter); mapping_words_sentences.put(word_id, tmp_set); } else { Set<Integer> tmp_set = new HashSet<>(); tmp_set.add(sentence_counter); mapping_words_sentences.put(word_id, tmp_set); } } if (bag_words_uri.containsKey(reference)) { Set<Integer> tmp = bag_words_uri.get(reference); for (Integer w : words_ids) { tmp.add(w); } bag_words_uri.put(reference, tmp); } else { Set<Integer> tmp = new HashSet<>(); for (Integer w : words_ids) { tmp.add(w); } bag_words_uri.put(reference, tmp); } } } model.close(); } PrintWriter writer = new PrintWriter("bag_of_words_only_goldstandard.tsv"); StringBuilder string_builder = new StringBuilder(); for (String r : bag_words_uri.keySet()) { string_builder.append(r); for (Integer i : bag_words_uri.get(r)) { string_builder.append("\t"); string_builder.append(i); } string_builder.append("\n"); } writer.write(string_builder.toString()); writer.close(); writer = new PrintWriter("mapping_words_to_sentenceids_goldstandard.tsv"); string_builder = new StringBuilder(); for (Integer w : mapping_words_sentences.keySet()) { string_builder.append(w); for (int i : mapping_words_sentences.get(w)) { string_builder.append("\t"); string_builder.append(i); } string_builder.append("\n"); } writer.write(string_builder.toString()); writer.close(); }
From source file:io.cloudslang.lang.tools.build.SlangBuildMain.java
public static void main(String[] args) { loadUserProperties();/* w w w . j av a 2 s.c o m*/ configureLog4j(); ApplicationArgs appArgs = new ApplicationArgs(); parseArgs(args, appArgs); String projectPath = parseProjectPathArg(appArgs); final String contentPath = defaultIfEmpty(appArgs.getContentRoot(), projectPath + CONTENT_DIR); final String testsPath = defaultIfEmpty(appArgs.getTestRoot(), projectPath + TEST_DIR); List<String> testSuites = parseTestSuites(appArgs); boolean shouldPrintCoverageData = appArgs.shouldOutputCoverage(); boolean runTestsInParallel = appArgs.isParallel(); int threadCount = parseThreadCountArg(appArgs, runTestsInParallel); String testCaseTimeout = parseTestTimeout(appArgs); setProperty(TEST_CASE_TIMEOUT_IN_MINUTES_KEY, valueOf(testCaseTimeout)); final boolean shouldValidateDescription = appArgs.shouldValidateDescription(); final boolean shouldValidateCheckstyle = appArgs.shouldValidateCheckstyle(); String runConfigPath = FilenameUtils.normalize(appArgs.getRunConfigPath()); BuildMode buildMode = null; Set<String> changedFiles = null; try { String smartModePath = appArgs.getChangesOnlyConfigPath(); if (StringUtils.isEmpty(smartModePath)) { buildMode = BuildMode.BASIC; changedFiles = new HashSet<>(); printBuildModeInfo(buildMode); } else { buildMode = BuildMode.CHANGED; changedFiles = readChangedFilesFromSource(smartModePath); printBuildModeInfo(buildMode); } } catch (Exception ex) { log.error("Exception: " + ex.getMessage()); System.exit(1); } // Override with the values from the file if configured List<String> testSuitesParallel = new ArrayList<>(); List<String> testSuitesSequential = new ArrayList<>(); BulkRunMode bulkRunMode = runTestsInParallel ? ALL_PARALLEL : ALL_SEQUENTIAL; TestCaseRunMode unspecifiedTestSuiteRunMode = runTestsInParallel ? TestCaseRunMode.PARALLEL : TestCaseRunMode.SEQUENTIAL; if (get(runConfigPath).isAbsolute() && isRegularFile(get(runConfigPath), NOFOLLOW_LINKS) && equalsIgnoreCase(PROPERTIES_FILE_EXTENSION, FilenameUtils.getExtension(runConfigPath))) { Properties runConfigurationProperties = ArgumentProcessorUtils.getPropertiesFromFile(runConfigPath); shouldPrintCoverageData = getBooleanFromPropertiesWithDefault(TEST_COVERAGE, shouldPrintCoverageData, runConfigurationProperties); threadCount = getIntFromPropertiesWithDefaultAndRange(TEST_PARALLEL_THREAD_COUNT, Runtime.getRuntime().availableProcessors(), runConfigurationProperties, 1, MAX_THREADS_TEST_RUNNER + 1); testSuites = getTestSuitesForKey(runConfigurationProperties, TEST_SUITES_TO_RUN); testSuitesParallel = getTestSuitesForKey(runConfigurationProperties, TEST_SUITES_PARALLEL); testSuitesSequential = getTestSuitesForKey(runConfigurationProperties, TEST_SUITES_SEQUENTIAL); addErrorIfSameTestSuiteIsInBothParallelOrSequential(testSuitesParallel, testSuitesSequential); unspecifiedTestSuiteRunMode = getEnumInstanceFromPropertiesWithDefault(TEST_SUITES_RUN_UNSPECIFIED, unspecifiedTestSuiteRunMode, runConfigurationProperties); addWarningsForMisconfiguredTestSuites(unspecifiedTestSuiteRunMode, testSuites, testSuitesSequential, testSuitesParallel); bulkRunMode = POSSIBLY_MIXED; } else { // Warn when file is misconfigured, relative path, file does not exist or is not a properties file log.info(format(DID_NOT_DETECT_RUN_CONFIGURATION_PROPERTIES_FILE, runConfigPath)); } String testCaseReportLocation = getProperty(TEST_CASE_REPORT_LOCATION); if (StringUtils.isBlank(testCaseReportLocation)) { log.info("Test case report location property [" + TEST_CASE_REPORT_LOCATION + "] is not defined. Report will be skipped."); } // Setting thread count for visibility in ParallelTestCaseExecutorService setProperty(SLANG_TEST_RUNNER_THREAD_COUNT, valueOf(threadCount)); log.info(NEW_LINE + "------------------------------------------------------------"); log.info("Building project: " + projectPath); log.info("Content root is at: " + contentPath); log.info("Test root is at: " + testsPath); log.info("Active test suites are: " + getListForPrint(testSuites)); log.info("Parallel run mode is configured for test suites: " + getListForPrint(testSuitesParallel)); log.info("Sequential run mode is configured for test suites: " + getListForPrint(testSuitesSequential)); log.info("Default run mode '" + unspecifiedTestSuiteRunMode.name().toLowerCase() + "' is configured for test suites: " + getListForPrint( getDefaultRunModeTestSuites(testSuites, testSuitesParallel, testSuitesSequential))); log.info("Bulk run mode for tests: " + getBulkModeForPrint(bulkRunMode)); log.info("Print coverage data: " + valueOf(shouldPrintCoverageData)); log.info("Validate description: " + valueOf(shouldValidateDescription)); log.info("Validate checkstyle: " + valueOf(shouldValidateCheckstyle)); log.info("Thread count: " + threadCount); log.info("Test case timeout in minutes: " + (isEmpty(testCaseTimeout) ? valueOf(MAX_TIME_PER_TESTCASE_IN_MINUTES) : testCaseTimeout)); log.info(NEW_LINE + "Loading..."); ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext("spring/testRunnerContext.xml"); context.registerShutdownHook(); SlangBuilder slangBuilder = context.getBean(SlangBuilder.class); LoggingService loggingService = context.getBean(LoggingServiceImpl.class); Slang slang = context.getBean(Slang.class); try { updateTestSuiteMappings(context.getBean(TestRunInfoService.class), testSuitesParallel, testSuitesSequential, testSuites, unspecifiedTestSuiteRunMode); registerEventHandlers(slang); List<RuntimeException> exceptions = new ArrayList<>(); SlangBuildResults buildResults = slangBuilder.buildSlangContent(projectPath, contentPath, testsPath, testSuites, shouldValidateDescription, shouldValidateCheckstyle, bulkRunMode, buildMode, changedFiles); exceptions.addAll(buildResults.getCompilationExceptions()); if (exceptions.size() > 0) { logErrors(exceptions, projectPath, loggingService); } IRunTestResults runTestsResults = buildResults.getRunTestsResults(); Map<String, TestRun> skippedTests = runTestsResults.getSkippedTests(); if (isNotEmpty(skippedTests)) { printSkippedTestsSummary(skippedTests, loggingService); } printPassedTests(runTestsResults, loggingService); if (shouldPrintCoverageData) { printTestCoverageData(runTestsResults, loggingService); } if (isNotEmpty(runTestsResults.getFailedTests())) { printBuildFailureSummary(projectPath, runTestsResults, loggingService); } else { printBuildSuccessSummary(contentPath, buildResults, runTestsResults, loggingService); } loggingService.waitForAllLogTasksToFinish(); generateTestCaseReport(context.getBean(SlangTestCaseRunReportGeneratorService.class), runTestsResults, testCaseReportLocation); System.exit(isNotEmpty(runTestsResults.getFailedTests()) ? 1 : 0); } catch (Throwable e) { logErrorsPrefix(loggingService); loggingService.logEvent(Level.ERROR, "Exception: " + e.getMessage()); logErrorsSuffix(projectPath, loggingService); loggingService.waitForAllLogTasksToFinish(); System.exit(1); } }
From source file:com.xylocore.cassandra.query.TableScanQuery.java
public static void main(String[] args) { Cluster myCluster = null;//from ww w .j av a 2 s.c om try { myCluster = Cluster.builder().addContactPoint("127.0.0.1").withPort(9042).build(); Session mySession = myCluster.connect("cirdan"); List<Entity> myAllEntities = new ArrayList<Entity>(); BiConsumer<Row, Entity> myEntityExtractor = (myRow, myEntity) -> { myEntity.setA(myRow.getInt("a")); myEntity.setB(myRow.getInt("b")); myEntity.setC(myRow.getInt("c")); myEntity.setD(myRow.getInt("d")); }; Consumer<List<Entity>> myEntityProcessor = (myEntities) -> { myEntities.forEach(e -> { logger.debug("entity: {}", e); }); myAllEntities.addAll(myEntities); }; TableScanQueryExecutionContext<Entity> myExecutionContext = TableScanQueryExecutionContextBuilder .builder(Entity.class).entityCreator(() -> { return new Entity(); }).reuseEntity(false).entityExtractor(myEntityExtractor).entityProcessor(myEntityProcessor) .build(); TableScanQuery<Entity> myQuery = TableScanQueryBuilder.<Entity>builder().session(mySession) .keyspace("cirdan").tableName("b").columns("a", "b", "c", "d") // .column ( "a" ) // .column ( "b" ) // .column ( "c" ) // .column ( "d" ) .build(); CompletableFuture<Void> myFuture = myQuery.execute(myExecutionContext); myFuture.get(); myAllEntities.forEach((e) -> System.err.println(e.toString())); } catch (Exception myException) { myException.printStackTrace(); } finally { if (myCluster != null) { myCluster.close(); } } }
From source file:de.tudarmstadt.ukp.experiments.dip.wp1.documents.Step8GoldDataAggregator.java
public static void main(String[] args) throws Exception { String inputDir = args[0] + "/"; // output dir File outputDir = new File(args[1]); File turkersConfidence = new File(args[2]); if (outputDir.exists()) { outputDir.delete();// w w w . j a va 2 s .c o m } outputDir.mkdir(); List<String> annotatorsIDs = new ArrayList<>(); // for (File f : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { // QueryResultContainer queryResultContainer = QueryResultContainer // .fromXML(FileUtils.readFileToString(f, "utf-8")); // for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { // for (QueryResultContainer.MTurkRelevanceVote relevanceVote : rankedResults.mTurkRelevanceVotes) { // if (!annotatorsIDs.contains(relevanceVote.turkID)) // annotatorsIDs.add(relevanceVote.turkID); // } // } // } HashMap<String, Integer> countVotesForATurker = new HashMap<>(); // creates temporary file with format for mace // Hashmap annotations: key is the id of a document and a sentence // Value is an array votes[] of turkers decisions: true or false (relevant or not) // the length of this array equals the number of annotators in List<String> annotatorsIDs. // If an annotator worked on the task his decision is written in the array otherwise the value is NULL // key: queryID + clueWebID + sentenceID // value: true and false annotations TreeMap<String, Annotations> annotations = new TreeMap<>(); for (File f : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(f, "utf-8")); System.out.println("Reading " + f.getName()); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { String documentID = rankedResults.clueWebID; for (QueryResultContainer.MTurkRelevanceVote relevanceVote : rankedResults.mTurkRelevanceVotes) { Integer turkerID; if (!annotatorsIDs.contains(relevanceVote.turkID)) { annotatorsIDs.add(relevanceVote.turkID); turkerID = annotatorsIDs.size() - 1; } else { turkerID = annotatorsIDs.indexOf(relevanceVote.turkID); } Integer count = countVotesForATurker.get(relevanceVote.turkID); if (count == null) { count = 0; } count++; countVotesForATurker.put(relevanceVote.turkID, count); String id; List<Integer> trueVotes; List<Integer> falseVotes; for (QueryResultContainer.SingleSentenceRelevanceVote singleSentenceRelevanceVote : relevanceVote.singleSentenceRelevanceVotes) if (!"".equals(singleSentenceRelevanceVote.sentenceID)) { id = f.getName() + "_" + documentID + "_" + singleSentenceRelevanceVote.sentenceID; Annotations turkerVotes = annotations.get(id); if (turkerVotes == null) { trueVotes = new ArrayList<>(); falseVotes = new ArrayList<>(); turkerVotes = new Annotations(trueVotes, falseVotes); } trueVotes = turkerVotes.trueAnnotations; falseVotes = turkerVotes.falseAnnotations; if ("true".equals(singleSentenceRelevanceVote.relevant)) { // votes[turkerID] = true; trueVotes.add(turkerID); } else if ("false".equals(singleSentenceRelevanceVote.relevant)) { // votes[turkerID] = false; falseVotes.add(turkerID); } else { throw new IllegalStateException("Annotation value of sentence " + singleSentenceRelevanceVote.sentenceID + " in " + rankedResults.clueWebID + " equals " + singleSentenceRelevanceVote.relevant); } try { int allVotesCount = trueVotes.size() + falseVotes.size(); if (allVotesCount > 5) { System.err.println(id + " doesn't have 5 annotators: true: " + trueVotes.size() + " false: " + falseVotes.size()); // nasty hack, we're gonna strip some data; true votes first /* we can't do that, it breaks something down the line int toRemove = allVotesCount - 5; if (trueVotes.size() >= toRemove) { trueVotes = trueVotes .subList(0, trueVotes.size() - toRemove); } else if ( falseVotes.size() >= toRemove) { falseVotes = falseVotes .subList(0, trueVotes.size() - toRemove); } */ System.err.println("Adjusted: " + id + " doesn't have 5 annotators: true: " + trueVotes.size() + " false: " + falseVotes.size()); } } catch (IllegalStateException e) { e.printStackTrace(); } turkerVotes.trueAnnotations = trueVotes; turkerVotes.falseAnnotations = falseVotes; annotations.put(id, turkerVotes); } else { throw new IllegalStateException( "Empty Sentence ID in " + f.getName() + " for turker " + turkerID); } } } } File tmp = printHashMap(annotations, annotatorsIDs.size()); String file = TEMP_DIR + "/" + tmp.getName(); MACE.main(new String[] { "--prefix", file }); //gets the keys of the documents and sentences ArrayList<String> lines = (ArrayList<String>) FileUtils.readLines(new File(file + ".prediction")); int i = 0; TreeMap<String, TreeMap<String, ArrayList<HashMap<String, String>>>> ids = new TreeMap<>(); ArrayList<HashMap<String, String>> sentences; if (lines.size() != annotations.size()) { throw new IllegalStateException( "The size of prediction file is " + lines.size() + "but expected " + annotations.size()); } for (Map.Entry entry : annotations.entrySet()) { //1001.xml_clueweb12-1905wb-13-07360_8783 String key = (String) entry.getKey(); String[] IDs = key.split("_"); if (IDs.length > 2) { String queryID = IDs[0]; String clueWebID = IDs[1]; String sentenceID = IDs[2]; TreeMap<String, ArrayList<HashMap<String, String>>> clueWebIDs = ids.get(queryID); if (clueWebIDs == null) { clueWebIDs = new TreeMap<>(); } sentences = clueWebIDs.get(clueWebID); if (sentences == null) { sentences = new ArrayList<>(); } HashMap<String, String> sentence = new HashMap<>(); sentence.put(sentenceID, lines.get(i)); sentences.add(sentence); clueWebIDs.put(clueWebID, sentences); ids.put(queryID, clueWebIDs); } else { throw new IllegalStateException("Wrong ID " + key); } i++; } for (Map.Entry entry : ids.entrySet()) { TreeMap<Integer, String> value = (TreeMap<Integer, String>) entry.getValue(); String queryID = (String) entry.getKey(); QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(new File(inputDir, queryID), "utf-8")); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { for (Map.Entry val : value.entrySet()) { String clueWebID = (String) val.getKey(); if (clueWebID.equals(rankedResults.clueWebID)) { List<QueryResultContainer.SingleSentenceRelevanceVote> goldEstimatedLabels = new ArrayList<>(); List<QueryResultContainer.SingleSentenceRelevanceVote> turkersVotes = new ArrayList<>(); int size = 0; int hitSize = 0; String hitID = ""; for (QueryResultContainer.MTurkRelevanceVote vote : rankedResults.mTurkRelevanceVotes) { if (!hitID.equals(vote.hitID)) { hitID = vote.hitID; hitSize = vote.singleSentenceRelevanceVotes.size(); size = size + hitSize; turkersVotes.addAll(vote.singleSentenceRelevanceVotes); } else { if (vote.singleSentenceRelevanceVotes.size() != hitSize) { hitSize = vote.singleSentenceRelevanceVotes.size(); size = size + hitSize; turkersVotes.addAll(vote.singleSentenceRelevanceVotes); } } } ArrayList<HashMap<String, String>> sentenceList = (ArrayList<HashMap<String, String>>) val .getValue(); if (sentenceList.size() != turkersVotes.size()) { try { throw new IllegalStateException("Expected size of annotations is " + turkersVotes.size() + "but found " + sentenceList.size() + " for document " + rankedResults.clueWebID + " in " + queryID); } catch (IllegalStateException ex) { ex.printStackTrace(); } } for (QueryResultContainer.SingleSentenceRelevanceVote s : turkersVotes) { String valSentence = null; for (HashMap<String, String> anno : sentenceList) { if (anno.keySet().contains(s.sentenceID)) { valSentence = anno.get(s.sentenceID); } } QueryResultContainer.SingleSentenceRelevanceVote singleSentenceVote = new QueryResultContainer.SingleSentenceRelevanceVote(); singleSentenceVote.sentenceID = s.sentenceID; if (("false").equals(valSentence)) { singleSentenceVote.relevant = "false"; } else if (("true").equals(valSentence)) { singleSentenceVote.relevant = "true"; } else { throw new IllegalStateException("Annotation value of sentence " + singleSentenceVote.sentenceID + " equals " + val.getValue()); } goldEstimatedLabels.add(singleSentenceVote); } rankedResults.goldEstimatedLabels = goldEstimatedLabels; } } } File outputFile = new File(outputDir, queryID); FileUtils.writeStringToFile(outputFile, queryResultContainer.toXML(), "utf-8"); System.out.println("Finished " + outputFile); } ArrayList<String> annotators = (ArrayList<String>) FileUtils.readLines(new File(file + ".competence")); FileWriter fileWriter; StringBuilder sb = new StringBuilder(); for (int j = 0; j < annotatorsIDs.size(); j++) { String[] s = annotators.get(0).split("\t"); Float score = Float.parseFloat(s[j]); String turkerID = annotatorsIDs.get(j); System.out.println(turkerID + " " + score + " " + countVotesForATurker.get(turkerID)); sb.append(turkerID).append(" ").append(score).append(" ").append(countVotesForATurker.get(turkerID)) .append("\n"); } fileWriter = new FileWriter(turkersConfidence); fileWriter.append(sb.toString()); fileWriter.close(); }
From source file:it.units.malelab.ege.MappingPropertiesExperimenter.java
public static void main(String[] args) throws IOException, InterruptedException, ExecutionException { final int n = 10000; final int nDist = 10000; //prepare problems and methods List<String> problems = Lists.newArrayList("bool-parity5", "bool-mopm3", "sr-keijzer6", "sr-nguyen7", "sr-pagie1", "sr-vladislavleva4", "other-klandscapes3", "other-klandscapes7", "other-text"); List<String> mappers = new ArrayList<>(); for (int gs : new int[] { 64, 128, 256, 512, 1024 }) { mappers.add("ge-" + gs + "-2"); mappers.add("ge-" + gs + "-4"); mappers.add("ge-" + gs + "-8"); mappers.add("ge-" + gs + "-12"); mappers.add("pige-" + gs + "-4"); mappers.add("pige-" + gs + "-8"); mappers.add("pige-" + gs + "-16"); mappers.add("pige-" + gs + "-24"); mappers.add("hge-" + gs + "-0"); mappers.add("whge-" + gs + "-2"); mappers.add("whge-" + gs + "-3"); mappers.add("whge-" + gs + "-5"); }/*w w w .ja v a 2s. c o m*/ mappers.add("sge-0-5"); mappers.add("sge-0-6"); mappers.add("sge-0-7"); mappers.add("sge-0-8"); mappers.clear(); mappers.addAll(Lists.newArrayList("ge-1024-8", "pige-1024-16", "hge-1024-0", "whge-1024-3", "sge-0-6")); PrintStream filePrintStream = null; if (args.length > 0) { filePrintStream = new PrintStream(args[0]); } else { filePrintStream = System.out; } filePrintStream.printf("problem;mapper;genotypeSize;param;property;value%n"); //prepare distances Distance<Node<String>> phenotypeDistance = new CachedDistance<>(new LeavesEdit<String>()); Distance<Sequence> genotypeDistance = new CachedDistance<>(new Hamming()); //iterate for (String problemName : problems) { for (String mapperName : mappers) { System.out.printf("%20.20s, %20.20s", problemName, mapperName); //build problem Problem<String, NumericFitness> problem = null; if (problemName.equals("bool-parity5")) { problem = new Parity(5); } else if (problemName.equals("bool-mopm3")) { problem = new MultipleOutputParallelMultiplier(3); } else if (problemName.equals("sr-keijzer6")) { problem = new HarmonicCurve(); } else if (problemName.equals("sr-nguyen7")) { problem = new Nguyen7(1); } else if (problemName.equals("sr-pagie1")) { problem = new Pagie1(); } else if (problemName.equals("sr-vladislavleva4")) { problem = new Vladislavleva4(1); } else if (problemName.equals("other-klandscapes3")) { problem = new KLandscapes(3); } else if (problemName.equals("other-klandscapes7")) { problem = new KLandscapes(7); } else if (problemName.equals("other-text")) { problem = new Text(); } //build configuration and evolver Mapper mapper = null; int genotypeSize = Integer.parseInt(mapperName.split("-")[1]); int mapperMainParam = Integer.parseInt(mapperName.split("-")[2]); if (mapperName.split("-")[0].equals("ge")) { mapper = new StandardGEMapper<>(mapperMainParam, 1, problem.getGrammar()); } else if (mapperName.split("-")[0].equals("pige")) { mapper = new PiGEMapper<>(mapperMainParam, 1, problem.getGrammar()); } else if (mapperName.split("-")[0].equals("sge")) { mapper = new SGEMapper<>(mapperMainParam, problem.getGrammar()); } else if (mapperName.split("-")[0].equals("hge")) { mapper = new HierarchicalMapper<>(problem.getGrammar()); } else if (mapperName.split("-")[0].equals("whge")) { mapper = new WeightedHierarchicalMapper<>(mapperMainParam, false, true, problem.getGrammar()); } //prepare things Random random = new Random(1); Set<Sequence> genotypes = new LinkedHashSet<>(n); //build genotypes if (mapperName.split("-")[0].equals("sge")) { SGEGenotypeFactory<String> factory = new SGEGenotypeFactory<>((SGEMapper) mapper); while (genotypes.size() < n) { genotypes.add(factory.build(random)); } genotypeSize = factory.getBitSize(); } else { BitsGenotypeFactory factory = new BitsGenotypeFactory(genotypeSize); while (genotypes.size() < n) { genotypes.add(factory.build(random)); } } //build and fill map Multimap<Node<String>, Sequence> multimap = HashMultimap.create(); int progress = 0; for (Sequence genotype : genotypes) { Node<String> phenotype; try { if (mapperName.split("-")[0].equals("sge")) { phenotype = mapper.map((SGEGenotype<String>) genotype, new HashMap<>()); } else { phenotype = mapper.map((BitsGenotype) genotype, new HashMap<>()); } } catch (MappingException e) { phenotype = Node.EMPTY_TREE; } multimap.put(phenotype, genotype); progress = progress + 1; if (progress % Math.round(n / 10) == 0) { System.out.print("."); } } System.out.println(); //compute distances List<Pair<Double, Double>> allDistances = new ArrayList<>(); List<Pair<Double, Double>> allValidDistances = new ArrayList<>(); Multimap<Node<String>, Double> genotypeDistances = ArrayListMultimap.create(); for (Node<String> phenotype : multimap.keySet()) { for (Sequence genotype1 : multimap.get(phenotype)) { for (Sequence genotype2 : multimap.get(phenotype)) { double gDistance = genotypeDistance.d(genotype1, genotype2); genotypeDistances.put(phenotype, gDistance); if (genotypeDistances.get(phenotype).size() > nDist) { break; } } if (genotypeDistances.get(phenotype).size() > nDist) { break; } } } List<Map.Entry<Node<String>, Sequence>> entries = new ArrayList<>(multimap.entries()); Collections.shuffle(entries, random); for (Map.Entry<Node<String>, Sequence> entry1 : entries) { for (Map.Entry<Node<String>, Sequence> entry2 : entries) { double gDistance = genotypeDistance.d(entry1.getValue(), entry2.getValue()); double pDistance = phenotypeDistance.d(entry1.getKey(), entry2.getKey()); allDistances.add(new Pair<>(gDistance, pDistance)); if (!Node.EMPTY_TREE.equals(entry1.getKey()) && !Node.EMPTY_TREE.equals(entry2.getKey())) { allValidDistances.add(new Pair<>(gDistance, pDistance)); } if (allDistances.size() > nDist) { break; } } if (allDistances.size() > nDist) { break; } } //compute properties double invalidity = (double) multimap.get(Node.EMPTY_TREE).size() / (double) genotypes.size(); double redundancy = 1 - (double) multimap.keySet().size() / (double) genotypes.size(); double validRedundancy = redundancy; if (multimap.keySet().contains(Node.EMPTY_TREE)) { validRedundancy = 1 - ((double) multimap.keySet().size() - 1d) / (double) (genotypes.size() - multimap.get(Node.EMPTY_TREE).size()); } double locality = Utils.pearsonCorrelation(allDistances); double validLocality = Utils.pearsonCorrelation(allValidDistances); double[] sizes = new double[multimap.keySet().size()]; double[] meanGenotypeDistances = new double[multimap.keySet().size()]; int invalidIndex = -1; int c = 0; for (Node<String> phenotype : multimap.keySet()) { if (Node.EMPTY_TREE.equals(phenotype)) { invalidIndex = c; } sizes[c] = multimap.get(phenotype).size(); double[] distances = new double[genotypeDistances.get(phenotype).size()]; int k = 0; for (Double distance : genotypeDistances.get(phenotype)) { distances[k] = distance; k = k + 1; } meanGenotypeDistances[c] = StatUtils.mean(distances); c = c + 1; } double nonUniformity = Math.sqrt(StatUtils.variance(sizes)) / StatUtils.mean(sizes); double nonSynonymousity = StatUtils.mean(meanGenotypeDistances) / StatUtils.mean(firsts(allDistances)); double validNonUniformity = nonUniformity; double validNonSynonymousity = nonSynonymousity; if (invalidIndex != -1) { double[] validSizes = new double[multimap.keySet().size() - 1]; double[] validMeanGenotypeDistances = new double[multimap.keySet().size() - 1]; if (invalidIndex > 0) { System.arraycopy(sizes, 0, validSizes, 0, invalidIndex); System.arraycopy(meanGenotypeDistances, 0, validMeanGenotypeDistances, 0, invalidIndex); } System.arraycopy(sizes, invalidIndex + 1, validSizes, invalidIndex, sizes.length - invalidIndex - 1); System.arraycopy(meanGenotypeDistances, invalidIndex + 1, validMeanGenotypeDistances, invalidIndex, meanGenotypeDistances.length - invalidIndex - 1); validNonUniformity = Math.sqrt(StatUtils.variance(validSizes)) / StatUtils.mean(validSizes); validNonSynonymousity = StatUtils.mean(validMeanGenotypeDistances) / StatUtils.mean(firsts(allValidDistances)); } //compute locality filePrintStream.printf("%s;%s;%d;%d;invalidity;%f %n", problemName, mapperName.split("-")[0], genotypeSize, mapperMainParam, invalidity); filePrintStream.printf("%s;%s;%d;%d;redundancy;%f %n", problemName, mapperName.split("-")[0], genotypeSize, mapperMainParam, redundancy); filePrintStream.printf("%s;%s;%d;%d;validRedundancy;%f %n", problemName, mapperName.split("-")[0], genotypeSize, mapperMainParam, validRedundancy); filePrintStream.printf("%s;%s;%d;%d;locality;%f %n", problemName, mapperName.split("-")[0], genotypeSize, mapperMainParam, locality); filePrintStream.printf("%s;%s;%d;%d;validLLocality;%f %n", problemName, mapperName.split("-")[0], genotypeSize, mapperMainParam, validLocality); filePrintStream.printf("%s;%s;%d;%d;nonUniformity;%f %n", problemName, mapperName.split("-")[0], genotypeSize, mapperMainParam, nonUniformity); filePrintStream.printf("%s;%s;%d;%d;validNonUniformity;%f %n", problemName, mapperName.split("-")[0], genotypeSize, mapperMainParam, validNonUniformity); filePrintStream.printf("%s;%s;%d;%d;nonSynonymousity;%f %n", problemName, mapperName.split("-")[0], genotypeSize, mapperMainParam, nonSynonymousity); filePrintStream.printf("%s;%s;%d;%d;validNonSynonymousity;%f %n", problemName, mapperName.split("-")[0], genotypeSize, mapperMainParam, validNonSynonymousity); } } if (filePrintStream != null) { filePrintStream.close(); } }
From source file:com.act.lcms.db.io.ExportStandardIonResultsFromDB.java
public static void main(String[] args) throws Exception { Options opts = new Options(); for (Option.Builder b : OPTION_BUILDERS) { opts.addOption(b.build());//w ww. j a v a2s . c om } CommandLine cl = null; try { CommandLineParser parser = new DefaultParser(); cl = parser.parse(opts, args); } catch (ParseException e) { System.err.format("Argument parsing failed: %s\n", e.getMessage()); HELP_FORMATTER.printHelp(ExportStandardIonResultsFromDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); System.exit(1); } if (cl.hasOption("help")) { HELP_FORMATTER.printHelp(ExportStandardIonResultsFromDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); return; } try (DB db = DB.openDBFromCLI(cl)) { List<String> chemicalNames = new ArrayList<>(); if (cl.hasOption(OPTION_CONSTRUCT)) { // Extract the chemicals in the pathway and their product masses, then look up info on those chemicals List<Pair<ChemicalAssociatedWithPathway, Double>> productMasses = Utils .extractMassesForChemicalsAssociatedWithConstruct(db, cl.getOptionValue(OPTION_CONSTRUCT)); for (Pair<ChemicalAssociatedWithPathway, Double> pair : productMasses) { chemicalNames.add(pair.getLeft().getChemical()); } } if (cl.hasOption(OPTION_CHEMICALS)) { chemicalNames.addAll(Arrays.asList(cl.getOptionValues(OPTION_CHEMICALS))); } if (chemicalNames.size() == 0) { System.err.format("No chemicals can be found from the input query.\n"); System.exit(-1); } List<String> standardIonHeaderFields = new ArrayList<String>() { { add(STANDARD_ION_HEADER_FIELDS.CHEMICAL.name()); add(STANDARD_ION_HEADER_FIELDS.BEST_ION_FROM_ALGO.name()); add(STANDARD_ION_HEADER_FIELDS.MANUAL_PICK.name()); add(STANDARD_ION_HEADER_FIELDS.AUTHOR.name()); add(STANDARD_ION_HEADER_FIELDS.DIAGNOSTIC_PLOTS.name()); add(STANDARD_ION_HEADER_FIELDS.NOTE.name()); } }; String outAnalysis; if (cl.hasOption(OPTION_OUTPUT_PREFIX)) { outAnalysis = cl.getOptionValue(OPTION_OUTPUT_PREFIX) + "." + TSV_FORMAT; } else { outAnalysis = String.join("-", chemicalNames) + "." + TSV_FORMAT; } File lcmsDir = new File(cl.getOptionValue(OPTION_DIRECTORY)); if (!lcmsDir.isDirectory()) { System.err.format("File at %s is not a directory\n", lcmsDir.getAbsolutePath()); HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); System.exit(1); } String plottingDirectory = cl.getOptionValue(OPTION_PLOTTING_DIR); TSVWriter<String, String> resultsWriter = new TSVWriter<>(standardIonHeaderFields); resultsWriter.open(new File(outAnalysis)); // For each chemical, create a TSV row and a corresponding diagnostic plot for (String chemicalName : chemicalNames) { List<String> graphLabels = new ArrayList<>(); List<Double> yMaxList = new ArrayList<>(); String outData = plottingDirectory + "/" + chemicalName + ".data"; String outImg = plottingDirectory + "/" + chemicalName + ".pdf"; // For each diagnostic plot, open a new file stream. try (FileOutputStream fos = new FileOutputStream(outData)) { List<StandardIonResult> getResultByChemicalName = StandardIonResult.getByChemicalName(db, chemicalName); if (getResultByChemicalName != null && getResultByChemicalName.size() > 0) { // PART 1: Get the best metlin ion across all standard ion results for a given chemical String bestGlobalMetlinIon = AnalysisHelper .scoreAndReturnBestMetlinIonFromStandardIonResults(getResultByChemicalName, new HashMap<>(), true, true); // PART 2: Plot all the graphs related to the chemical. The plots are structured as follows: // // Page 1: All graphs (water, MeOH, Yeast) for Global ion picked (best ion among ALL standard ion runs for // the given chemical) by the algorithm // Page 2: All graphs for M+H // Page 3: All graphs for Local ions picked (best ion within a SINGLE standard ion run) + negative controls // for Yeast. // // Each page is demarcated by a blank graph. // Arrange results based on media Map<String, List<StandardIonResult>> categories = StandardIonResult .categorizeListOfStandardWellsByMedia(db, getResultByChemicalName); // This set contains all the best metlin ions corresponding to all the standard ion runs. Set<String> bestLocalIons = new HashSet<>(); bestLocalIons.add(bestGlobalMetlinIon); bestLocalIons.add(DEFAULT_ION); for (StandardIonResult result : getResultByChemicalName) { bestLocalIons.add(result.getBestMetlinIon()); } // We sort the best local ions are follows: // 1) Global best ion spectra 2) M+H spectra 3) Local best ion spectra List<String> bestLocalIonsArray = new ArrayList<>(bestLocalIons); Collections.sort(bestLocalIonsArray, new Comparator<String>() { @Override public int compare(String o1, String o2) { if (o1.equals(bestGlobalMetlinIon) && !o2.equals(bestGlobalMetlinIon)) { return -1; } else if (o1.equals(DEFAULT_ION) && !o2.equals(bestGlobalMetlinIon)) { return -1; } else { return 1; } } }); // This variable stores the index of the array at which all the remaining spectra are contained in one // page. This happens right after the M+H ion spectra. Integer combineAllSpectraIntoPageThreeFromIndex = 0; for (int i = 0; i < bestLocalIonsArray.size(); i++) { if (bestLocalIonsArray.get(i).equals(DEFAULT_ION)) { combineAllSpectraIntoPageThreeFromIndex = i + 1; } } for (int i = 0; i < bestLocalIonsArray.size(); i++) { String ion = bestLocalIonsArray.get(i); for (Map.Entry<String, List<StandardIonResult>> mediaToListOfIonResults : categories .entrySet()) { for (StandardIonResult result : mediaToListOfIonResults.getValue()) { // For every standard ion result, we plot the best global metlin ion and M+H. These plots are in the // pages 1 and 2. For all page 3 (aka miscellaneous spectra), we only plot the best local ion // corresponding to it's spectra and not some other graph's spectra. In the below condition, // we reach the page 3 case with not the same best ion as the spectra, in which case we just continue // and not draw anything on the page. if (i >= combineAllSpectraIntoPageThreeFromIndex && !(result.getBestMetlinIon().equals(ion))) { continue; } StandardWell positiveWell = StandardWell.getInstance().getById(db, result.getStandardWellId()); String positiveControlChemical = positiveWell.getChemical(); ScanData<StandardWell> encapsulatedDataForPositiveControl = AnalysisHelper .getScanDataForWell(db, lcmsDir, positiveWell, positiveControlChemical, positiveControlChemical); Set<String> singletonSet = Collections.singleton(ion); String additionalInfo = generateAdditionalLabelInformation(positiveWell, result, ion); List<String> labels = AnalysisHelper .writeScanData(fos, lcmsDir, MAX_INTENSITY, encapsulatedDataForPositiveControl, false, false, singletonSet) .stream().map(label -> label + additionalInfo) .collect(Collectors.toList()); yMaxList.add(encapsulatedDataForPositiveControl.getMs1ScanResults() .getMaxIntensityForIon(ion)); List<String> negativeLabels = null; // Only do the negative control in the miscellaneous page (page 3) and if the well is in yeast media. if (mediaToListOfIonResults.getKey() .equals(StandardWell.MEDIA_TYPE.YEAST.name()) && (i >= combineAllSpectraIntoPageThreeFromIndex && (result.getBestMetlinIon().equals(ion)))) { //TODO: Change the representative negative well to one that displays the highest noise in the future. // For now, we just use the first index among the negative wells. int representativeIndex = 0; StandardWell representativeNegativeControlWell = StandardWell.getInstance() .getById(db, result.getNegativeWellIds().get(representativeIndex)); ScanData encapsulatedDataForNegativeControl = AnalysisHelper .getScanDataForWell(db, lcmsDir, representativeNegativeControlWell, positiveWell.getChemical(), representativeNegativeControlWell.getChemical()); String negativePlateAdditionalInfo = generateAdditionalLabelInformation( representativeNegativeControlWell, null, null); negativeLabels = AnalysisHelper.writeScanData(fos, lcmsDir, MAX_INTENSITY, encapsulatedDataForNegativeControl, false, false, singletonSet) .stream().map(label -> label + negativePlateAdditionalInfo) .collect(Collectors.toList()); yMaxList.add(encapsulatedDataForNegativeControl.getMs1ScanResults() .getMaxIntensityForIon(ion)); } graphLabels.addAll(labels); if (negativeLabels != null) { graphLabels.addAll(negativeLabels); } } } // Add a blank graph to demarcate pages. if (i < combineAllSpectraIntoPageThreeFromIndex) { graphLabels.addAll(AnalysisHelper.writeScanData(fos, lcmsDir, 0.0, BLANK_SCAN, false, false, new HashSet<>())); yMaxList.add(0.0d); } } // We need to pass the yMax values as an array to the Gnuplotter. Double fontScale = null; if (cl.hasOption(FONT_SCALE)) { try { fontScale = Double.parseDouble(cl.getOptionValue(FONT_SCALE)); } catch (IllegalArgumentException e) { System.err.format("Argument for font-scale must be a floating point number.\n"); System.exit(1); } } Double[] yMaxes = yMaxList.toArray(new Double[yMaxList.size()]); Gnuplotter plotter = fontScale == null ? new Gnuplotter() : new Gnuplotter(fontScale); plotter.plot2D(outData, outImg, graphLabels.toArray(new String[graphLabels.size()]), "time", null, "intensity", "pdf", null, null, yMaxes, outImg + ".gnuplot"); Map<String, String> row = new HashMap<>(); row.put(STANDARD_ION_HEADER_FIELDS.CHEMICAL.name(), chemicalName); row.put(STANDARD_ION_HEADER_FIELDS.BEST_ION_FROM_ALGO.name(), bestGlobalMetlinIon); row.put(STANDARD_ION_HEADER_FIELDS.DIAGNOSTIC_PLOTS.name(), outImg); resultsWriter.append(row); resultsWriter.flush(); } } } resultsWriter.flush(); resultsWriter.close(); } }
From source file:com.techmahindra.vehicletelemetry.chart.CarEventServlet.java
public static void main(String[] args) throws TypeMismatchException { List<CarEvent> carData = new LinkedList<>(); /*carData.add(new CarEvent("city1","model1",41)); carData.add(new CarEvent("city1","model2",17)); /*www. j av a 2 s . com*/ carData.add(new CarEvent("city2","model1",31)); carData.add(new CarEvent("city2","model2",39)); carData.add(new CarEvent("Bellevue","model1",47));*/ /*carData.add(new CarEvent("Seattle","MediumSUV",1038)); carData.add(new CarEvent("Seattle","LargeSUV",2415)); carData.add(new CarEvent("Seattle","FamilySaloon",2388)); carData.add(new CarEvent("Seattle","SportsCar",1626)); //aggDrivers.add(new CarEvent("Seattle","sports car",276)); carData.add(new CarEvent("Seattle","Compactcar",204)); carData.add(new CarEvent("Seattle","SmallSUV",1133)); carData.add(new CarEvent("Seattle","StationWagon",1769)); carData.add(new CarEvent("Seattle","CompactCar",839)); carData.add(new CarEvent("Seattle","Hybrid",2603)); carData.add(new CarEvent("Seattle","Coupe",1081)); carData.add(new CarEvent("Seattle","Sedan",2603)); carData.add(new CarEvent("Seattle","Convertible",1608)); carData.add(new CarEvent("Redmond","MediumSUV",590)); carData.add(new CarEvent("Redmond","LargeSUV",1407)); carData.add(new CarEvent("Redmond","FamilySaloon",1535)); carData.add(new CarEvent("Redmond","SportsCar",1115)); //aggDrivers.add(new CarEvent("Redmond","sports car",102)); carData.add(new CarEvent("Redmond","Compactcar",102)); carData.add(new CarEvent("Redmond","SmallSUV",637)); carData.add(new CarEvent("Redmond","StationWagon",1079)); carData.add(new CarEvent("Redmond","CompactCar",606)); carData.add(new CarEvent("Redmond","Hybrid",1635)); carData.add(new CarEvent("Redmond","Coupe",605)); carData.add(new CarEvent("Redmond","Sedan",1568)); carData.add(new CarEvent("Redmond","Convertible",955)); carData.add(new CarEvent("ammamish","SportsCar",1)); carData.add(new CarEvent("ammamish","Sedan",21)); carData.add(new CarEvent("ellevue","MediumSUV",778)); carData.add(new CarEvent("ellevue","LargeSUV",2035)); carData.add(new CarEvent("ellevue","FamilySaloon",1952)); carData.add(new CarEvent("ellevue","SportsCar",1226)); carData.add(new CarEvent("ellevue","Compactcar",162)); //aggDrivers.add(new CarEvent("ellevue","sports car",192)); carData.add(new CarEvent("ellevue","SmallSUV",895)); carData.add(new CarEvent("ellevue","StationWagon",1469)); carData.add(new CarEvent("ellevue","CompactCar",629)); carData.add(new CarEvent("ellevue","Hybrid",1989)); carData.add(new CarEvent("ellevue","Coupe",811)); carData.add(new CarEvent("ellevue","Sedan",2004)); carData.add(new CarEvent("ellevue","Convertible",1122));*/ Map<String, List<Map<String, String>>> cityModelsMap = new LinkedHashMap<>(); for (CarEvent carEvent1 : carData) { Map<String, String> modelMap = new HashMap<>(); List<Map<String, String>> modelCountsList = new ArrayList<>(); if (!cityModelsMap.containsKey(carEvent1.getCity())) { modelMap.put(carEvent1.getModel(), carEvent1.getCount()); modelCountsList.add(modelMap); cityModelsMap.put(carEvent1.getCity(), modelCountsList); } else { List<Map<String, String>> existingModelCountsList = cityModelsMap.get(carEvent1.getCity()); modelMap.put(carEvent1.getModel(), carEvent1.getCount()); modelCountsList.add(modelMap); existingModelCountsList.addAll(modelCountsList); cityModelsMap.put(carEvent1.getCity(), existingModelCountsList); } } System.out.println("CityModelMap:" + cityModelsMap); //CityModelMap:{city1=[{model1=41}, {model11=17}, {model12=36}], city2=[{model2=31}, {model22=37}], city3=[{model3=47}, {model33=31}]} DataTable data = new DataTable(); data.addColumn(new ColumnDescription(CITY_COLUMN, ValueType.TEXT, "city")); for (String cityKey : cityModelsMap.keySet()) { List<Map<String, String>> existingModelCountsList = cityModelsMap.get(cityKey); for (Map existingModelCountMap : existingModelCountsList) { Set set = existingModelCountMap.keySet(); for (Object objModel : set) { String model = objModel.toString(); if (!(data.containsColumn(model))) { data.addColumn(new ColumnDescription(model, ValueType.NUMBER, model)); System.out.println("Column added:" + model); } } } } for (String cityKey : cityModelsMap.keySet()) { TableRow row = new TableRow(); for (ColumnDescription selectionColumn : data.getColumnDescriptions()) { String columnName = selectionColumn.getId(); if (columnName.equals(CITY_COLUMN)) { row.addCell(cityKey); continue; } List<Map<String, String>> existingModelCountsList = cityModelsMap.get(cityKey); for (Map existingModelCountMap : existingModelCountsList) { for (Object objModel : existingModelCountMap.keySet()) { String model = objModel.toString(); Integer count = Integer.parseInt(existingModelCountMap.get(objModel).toString()); System.out.println("Model :" + model + " Count:" + count); if (columnName.equals(model)) { row.addCell(count); continue; } } } } data.addRow(row); System.out.println("Adding row"); } System.out.println("Data is now:" + data.toString()); }
From source file:fr.eo.util.dumper.JSONDumper.java
/** * @param args main args//from w ww. j a v a2 s .com */ public static void main(String[] args) { String appName = args[0]; String jdbcConnectionType = args.length > 1 ? args[1] : "jtds"; System.out.println("Starting dumper ..."); try (Connection conn = getConnection(jdbcConnectionType)) { System.out.println("Getting database connection ..."); List<RequestDefinitionBean> requests = RequestDefinitionParser.getRequests(appName); baseFolder = RequestDefinitionParser.getAppBaseDir(appName) + "/"; System.out.println("Reading old table dumps..."); Map<String, JsonTableDump> oldTables = JSONDeltaDumper.readOldTables(baseFolder); List<JsonTableDump> newTables = new ArrayList<>(); for (RequestDefinitionBean request : requests) { try (Statement stmt = conn.createStatement()) { BufferedWriter bw = getWriter(request.name, baseFolder); if (!request.disabled) { System.out.println("Dumping " + request.name + "..."); ResultSet rs = stmt.executeQuery(request.sql); JsonTableWritable dump = new JsonTableWritable(); dump.name = request.table; while (rs.next()) { int pos = 0; for (String fieldName : request.fields) { Object obj = getFieldValue(request, pos, rs, fieldName); dump.addColumn(obj); pos++; } dump.commit(); } bw.append(dump.toJson()); newTables.add(dump); bw.flush(); bw.close(); } else { System.out.println("Skiping " + request.name + "..."); } } System.out.println("done."); } newTables.addAll(BlueprintDumper.dump(baseFolder)); newTables.addAll(TranslationsDumper.dump(baseFolder, jdbcConnectionType)); System.out.println("Computing delta..."); JSONDeltaDumper.computeDelta(oldTables, newTables, baseFolder); } catch (SQLException | ClassNotFoundException | IOException e) { e.printStackTrace(); } }
From source file:CourserankConnector.java
public static void main(String[] args) throws Exception { /////////////////////////////////////// //Tagger init //MaxentTagger tagger = new MaxentTagger("models/english-left3words-distsim.tagger"); ///// www .j a v a 2s .com //CLIENT INITIALIZATION ImportData importCourse = new ImportData(); HttpClient httpclient = new DefaultHttpClient(); httpclient = WebClientDevWrapper.wrapClient(httpclient); try { /* httpclient.getCredentialsProvider().setCredentials( new AuthScope(null, -1), new UsernamePasswordCredentials("eadrian", "eactresp1")); */ ////////////////////////////////////////////////// //Get Course Bulletin Departments page List<Course> courses = new ArrayList<Course>(); HttpGet httpget = new HttpGet("http://explorecourses.stanford.edu"); System.out.println("executing request" + httpget.getRequestLine()); HttpResponse response = httpclient.execute(httpget); HttpEntity entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); String bulletinpage = ""; //STORE RETURNED HTML TO BULLETINPAGE if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { bulletinpage += line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); /////////////////////////////////////////////////////////////////////////////// //Login to Courserank httpget = new HttpGet("https://courserank.com/stanford/main"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); String page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page += line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); //////////////////////////////////////////////////// //POST REQUEST LOGIN HttpPost post = new HttpPost("https://www.courserank.com/stanford/main"); List<NameValuePair> pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("RT", "")); pairs.add(new BasicNameValuePair("action", "login")); pairs.add(new BasicNameValuePair("password", "trespass")); pairs.add(new BasicNameValuePair("username", "eaconte@stanford.edu")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); HttpResponse resp = httpclient.execute(post); HttpEntity ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /////////////////////////////////////////////////// //THIS STEP MAY NOT BE NEEDED BUT GETS MAIN PROFILE PAGE HttpGet gethome = new HttpGet("https://www.courserank.com/stanford/home"); System.out.println("executing request" + gethome.getRequestLine()); HttpResponse gresp = httpclient.execute(gethome); HttpEntity gent = gresp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + gent.getContentLength()); InputStream i = gent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } ///////////////////////////////////////////////////////////////////////////////// //Parse Bulletin String results = getToken(bulletinpage, "RESULTS HEADER", "Additional Searches"); String[] depts = results.split("href"); //SPLIT FOR EACH DEPARTMENT LINK, ITERATE boolean ready = false; for (int i = 1; i < depts.length; i++) { //EXTRACT LINK, DEPARTMENT NAME AND ABBREVIATION String dept = new String(depts[i]); String abbr = getToken(dept, "(", ")"); String name = getToken(dept, ">", "("); name.trim(); //System.out.println(tagger.tagString(name)); String link = getToken(dept, "=\"", "\">"); System.out.println(name + " : " + abbr + " : " + link); System.out.println("======================================================================"); if (i <= 10 || i >= 127) //values to keep it to undergraduate courses. Excludes law, med, business, overseas continue; /*if (i<=46) continue; */ //Start at BIOHOP /*if (abbr.equals("INTNLREL")) ready = true; if (!ready) continue;*/ //Construct department course search URL //Then request page String URL = "http://explorecourses.stanford.edu/" + link + "&filter-term-Autumn=on&filter-term-Winter=on&filter-term-Spring=on"; httpget = new HttpGet(URL); //System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); //ystem.out.println("----------------------------------------"); //System.out.println(response.getStatusLine()); String rpage = ""; if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream in = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String line; while ((line = br.readLine()) != null) { rpage += line; //System.out.println(line); } br.close(); in.close(); } EntityUtils.consume(entity); //Process results page List<Course> deptCourses = new ArrayList<Course>(); List<Course> result = processResultPage(rpage); deptCourses.addAll(result); //While there are more result pages, keep going boolean more = (!(result.size() == 0) && (result.get((result.size() - 1)).courseNumber < 299)); boolean morepages = anotherPage(rpage); while (morepages && more) { URL = nextURL(URL); httpget = new HttpGet(URL); //System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); //System.out.println("----------------------------------------"); //System.out.println(response.getStatusLine()); rpage = ""; if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream in = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String line; while ((line = br.readLine()) != null) { rpage += line; //System.out.println(line); } br.close(); in.close(); } EntityUtils.consume(entity); morepages = anotherPage(rpage); result = processResultPage(rpage); deptCourses.addAll(result); more = (!(result.size() == 0) && (result.get((result.size() - 1)).courseNumber < 299)); /*String mores = more? "yes": "no"; String pagess = morepages?"yes":"no"; System.out.println("more: "+mores+" morepages: "+pagess); System.out.println("more");*/ } //Get course ratings for all department courses via courserank deptCourses = getRatings(httpclient, abbr, deptCourses); for (int j = 0; j < deptCourses.size(); j++) { Course c = deptCourses.get(j); System.out.println("" + c.title + " : " + c.rating); c.tags = name; c.code = c.code.trim(); c.department = name; c.deptAB = abbr; c.writeToDatabase(); //System.out.println(tagger.tagString(c.title)); } } if (!page.equals("")) return; /////////////////////////////////////////////////// //Get Course Bulletin Department courses /* httpget = new HttpGet("https://courserank.com/stanford/main"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page +=line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); //////////////////////////////////////////////////// //POST REQUEST LOGIN HttpPost post = new HttpPost("https://www.courserank.com/stanford/main"); List<NameValuePair> pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("RT", "")); pairs.add(new BasicNameValuePair("action", "login")); pairs.add(new BasicNameValuePair("password", "trespass")); pairs.add(new BasicNameValuePair("username", "eaconte@stanford.edu")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); HttpResponse resp = httpclient.execute(post); HttpEntity ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /////////////////////////////////////////////////// //THIS STEP MAY NOT BE NEEDED BUT GETS MAIN PROFILE PAGE HttpGet gethome = new HttpGet("https://www.courserank.com/stanford/home"); System.out.println("executing request" + gethome.getRequestLine()); HttpResponse gresp = httpclient.execute(gethome); HttpEntity gent = gresp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + gent.getContentLength()); InputStream i = gent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } //////////////////////////////////////// //GETS FIRST PAGE OF RESULTS EntityUtils.consume(gent); post = new HttpPost("https://www.courserank.com/stanford/search_results"); pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("filter_term_currentYear", "on")); pairs.add(new BasicNameValuePair("query", "")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); resp = httpclient.execute(post); ent = resp.getEntity(); System.out.println("----------------------------------------"); String rpage = ""; if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { rpage += line; System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); //////////////////////////////////////////////////// //PARSE FIRST PAGE OF RESULTS //int index = rpage.indexOf("div class=\"searchItem"); String []classSplit = rpage.split("div class=\"searchItem"); for (int i=1; i<classSplit.length; i++) { String str = classSplit[i]; //ID String CID = getToken(str, "course?id=","\">"); // CODE String CODE = getToken(str,"class=\"code\">" ,":</"); //TITLE String NAME = getToken(str, "class=\"title\">","</"); //DESCRIP String DES = getToken(str, "class=\"description\">","</"); //TERM String TERM = getToken(str, "Terms:", "|"); //UNITS String UNITS = getToken(str, "Units:", "<br/>"); //WORKLOAD String WLOAD = getToken(str, "Workload:", "|"); //GER String GER = getToken(str, "GERs:", "</d"); //RATING int searchIndex = 0; float rating = 0; while (true) { int ratingIndex = str.indexOf("large_Full", searchIndex); if (ratingIndex ==-1) { int halfratingIndex = str.indexOf("large_Half", searchIndex); if (halfratingIndex == -1) break; else rating += .5; break; } searchIndex = ratingIndex+1; rating++; } String RATING = ""+rating; //GRADE String GRADE = getToken(str, "div class=\"unofficialGrade\">", "</"); if (GRADE.equals("NOT FOUND")) { GRADE = getToken(str, "div class=\"officialGrade\">", "</"); } //REVIEWS String REVIEWS = getToken(str, "class=\"ratings\">", " ratings"); System.out.println(""+CODE+" : "+NAME + " : "+CID); System.out.println("----------------------------------------"); System.out.println("Term: "+TERM+" Units: "+UNITS+ " Workload: "+WLOAD + " Grade: "+ GRADE); System.out.println("Rating: "+RATING+ " Reviews: "+REVIEWS); System.out.println("=========================================="); System.out.println(DES); System.out.println("=========================================="); } /////////////////////////////////////////////////// //GETS SECOND PAGE OF RESULTS post = new HttpPost("https://www.courserank.com/stanford/search_results"); pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("filter_term_currentYear", "on")); pairs.add(new BasicNameValuePair("page", "2")); pairs.add(new BasicNameValuePair("query", "")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); resp = httpclient.execute(post); ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /* httpget = new HttpGet("https://github.com/"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page +=line; System.out.println(line); } br.close(); i.close(); }*/ EntityUtils.consume(entity); } finally { // When HttpClient instance is no longer needed, // shut down the connection manager to ensure // immediate deallocation of all system resources httpclient.getConnectionManager().shutdown(); } }
From source file:com.mgmtp.jfunk.core.JFunk.java
/** * Starts jFunk./*www .j av a2 s. co m*/ * * <pre> * -threadcount=<count> Optional Number of threads to be used. Allows for parallel * execution of test scripts. * -parallel Optional Allows a single script to be executed in parallel * depending on the number of threads specified. The * argument is ignored if multiple scripts are specified. * <script parameters> Optional Similar to Java system properties they can be provided * as key-value-pairs preceded by -S, e.g. -Skey=value. * These parameters are then available in the script as * Groovy variables. * <script(s)> Required At least one test script must be specified. * * Example: * java -cp <jFunkClasspath> com.mgmtp.jfunk.core.JFunk -Skey=value -threadcount=4 -parallel mytest.script * </pre> * * @param args * The program arguments. */ public static void main(final String[] args) { SLF4JBridgeHandler.install(); boolean exitWithError = true; StopWatch stopWatch = new StopWatch(); try { RESULT_LOG.info("jFunk started"); stopWatch.start(); int threadCount = 1; boolean parallel = false; Properties scriptProperties = new Properties(); List<File> scripts = Lists.newArrayList(); for (String arg : args) { if (arg.startsWith("-threadcount")) { String[] split = arg.split("="); Preconditions.checkArgument(split.length == 2, "The number of threads must be specified as follows: -threadcount=<value>"); threadCount = Integer.parseInt(split[1]); RESULT_LOG.info("Using " + threadCount + (threadCount == 1 ? " thread" : " threads")); } else if (arg.startsWith("-S")) { arg = arg.substring(2); String[] split = arg.split("="); Preconditions.checkArgument(split.length == 2, "Script parameters must be given in the form -S<name>=<value>"); scriptProperties.setProperty(split[0], normalizeScriptParameterValue(split[1])); RESULT_LOG.info("Using script parameter " + split[0] + " with value " + split[1]); } else if (arg.equals("-parallel")) { parallel = true; RESULT_LOG.info("Using parallel mode"); } else { scripts.add(new File(arg)); } } if (scripts.isEmpty()) { scripts.addAll(requestScriptsViaGui()); if (scripts.isEmpty()) { RESULT_LOG.info("Execution finished (took " + stopWatch + " H:mm:ss.SSS)"); System.exit(0); } } String propsFileName = System.getProperty("jfunk.props.file", "jfunk.properties"); Module module = ModulesLoader.loadModulesFromProperties(new JFunkDefaultModule(), propsFileName); Injector injector = Guice.createInjector(module); JFunkFactory factory = injector.getInstance(JFunkFactory.class); JFunkBase jFunk = factory.create(threadCount, parallel, scripts, scriptProperties); jFunk.execute(); exitWithError = false; } catch (JFunkExecutionException ex) { // no logging necessary } catch (Exception ex) { Logger.getLogger(JFunk.class).error("jFunk terminated unexpectedly.", ex); } finally { stopWatch.stop(); RESULT_LOG.info("Execution finished (took " + stopWatch + " H:mm:ss.SSS)"); } System.exit(exitWithError ? -1 : 0); }