List of usage examples for java.util List size
int size();
From source file:com.yahoo.pulsar.testclient.PerformanceProducer.java
public static void main(String[] args) throws Exception { final Arguments arguments = new Arguments(); JCommander jc = new JCommander(arguments); jc.setProgramName("pulsar-perf-producer"); try {/*from ww w. ja v a 2 s .c o m*/ jc.parse(args); } catch (ParameterException e) { System.out.println(e.getMessage()); jc.usage(); System.exit(-1); } if (arguments.help) { jc.usage(); System.exit(-1); } if (arguments.destinations.size() != 1) { System.out.println("Only one topic name is allowed"); jc.usage(); System.exit(-1); } if (arguments.confFile != null) { Properties prop = new Properties(System.getProperties()); prop.load(new FileInputStream(arguments.confFile)); if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("brokerServiceUrl"); } if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("webServiceUrl"); } // fallback to previous-version serviceUrl property to maintain backward-compatibility if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/"); } if (arguments.authPluginClassName == null) { arguments.authPluginClassName = prop.getProperty("authPlugin", null); } if (arguments.authParams == null) { arguments.authParams = prop.getProperty("authParams", null); } } arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime); // Dump config variables ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); log.info("Starting Pulsar perf producer with config: {}", w.writeValueAsString(arguments)); // Read payload data from file if needed byte payloadData[]; if (arguments.payloadFilename != null) { payloadData = Files.readAllBytes(Paths.get(arguments.payloadFilename)); } else { payloadData = new byte[arguments.msgSize]; } // Now processing command line arguments String prefixTopicName = arguments.destinations.get(0); List<Future<Producer>> futures = Lists.newArrayList(); EventLoopGroup eventLoopGroup; if (SystemUtils.IS_OS_LINUX) { eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer")); } else { eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer")); } ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setConnectionsPerBroker(arguments.maxConnections); clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS); if (isNotBlank(arguments.authPluginClassName)) { clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams); } PulsarClient client = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup); ProducerConfiguration producerConf = new ProducerConfiguration(); producerConf.setSendTimeout(0, TimeUnit.SECONDS); producerConf.setCompressionType(arguments.compression); // enable round robin message routing if it is a partitioned topic producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition); if (arguments.batchTime > 0) { producerConf.setBatchingMaxPublishDelay(arguments.batchTime, TimeUnit.MILLISECONDS); producerConf.setBatchingEnabled(true); producerConf.setMaxPendingMessages(arguments.msgRate); } for (int i = 0; i < arguments.numTopics; i++) { String topic = (arguments.numTopics == 1) ? prefixTopicName : String.format("%s-%d", prefixTopicName, i); log.info("Adding {} publishers on destination {}", arguments.numProducers, topic); for (int j = 0; j < arguments.numProducers; j++) { futures.add(client.createProducerAsync(topic, producerConf)); } } final List<Producer> producers = Lists.newArrayListWithCapacity(futures.size()); for (Future<Producer> future : futures) { producers.add(future.get()); } log.info("Created {} producers", producers.size()); Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { printAggregatedStats(); } }); Collections.shuffle(producers); AtomicBoolean isDone = new AtomicBoolean(); executor.submit(() -> { try { RateLimiter rateLimiter = RateLimiter.create(arguments.msgRate); long startTime = System.currentTimeMillis(); // Send messages on all topics/producers long totalSent = 0; while (true) { for (Producer producer : producers) { if (arguments.testTime > 0) { if (System.currentTimeMillis() - startTime > arguments.testTime) { log.info("------------------- DONE -----------------------"); printAggregatedStats(); isDone.set(true); Thread.sleep(5000); System.exit(0); } } if (arguments.numMessages > 0) { if (totalSent++ >= arguments.numMessages) { log.info("------------------- DONE -----------------------"); printAggregatedStats(); isDone.set(true); Thread.sleep(5000); System.exit(0); } } rateLimiter.acquire(); final long sendTime = System.nanoTime(); producer.sendAsync(payloadData).thenRun(() -> { messagesSent.increment(); bytesSent.add(payloadData.length); long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime); recorder.recordValue(latencyMicros); cumulativeRecorder.recordValue(latencyMicros); }).exceptionally(ex -> { log.warn("Write error on message", ex); System.exit(-1); return null; }); } } } catch (Throwable t) { log.error("Got error", t); } }); // Print report stats long oldTime = System.nanoTime(); Histogram reportHistogram = null; String statsFileName = "perf-producer-" + System.currentTimeMillis() + ".hgrm"; log.info("Dumping latency stats to {}", statsFileName); PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog); // Some log header bits histogramLogWriter.outputLogFormatVersion(); histogramLogWriter.outputLegend(); while (true) { try { Thread.sleep(10000); } catch (InterruptedException e) { break; } if (isDone.get()) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesSent.sumThenReset() / elapsed; double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8; reportHistogram = recorder.getIntervalHistogram(reportHistogram); log.info( "Throughput produced: {} msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0)); histogramLogWriter.outputIntervalHistogram(reportHistogram); reportHistogram.reset(); oldTime = now; } client.close(); }
From source file:edu.msu.cme.rdp.readseq.utils.QualityTrimmer.java
public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption("f", "fastq-out", false, "Write fastq instead of fasta file"); options.addOption("l", "less-than", false, "Trim at <= instead of strictly ="); options.addOption("i", "illumina", false, "Illumina trimming mode"); FastqWriter fastqOut = null;/* w w w .j ava 2s . c o m*/ FastaWriter fastaOut = null; byte qualTrim = -1; boolean writeFasta = true; boolean trimle = false; boolean illumina = false; List<SeqReader> readers = new ArrayList(); List<File> seqFiles = new ArrayList(); try { CommandLine line = new PosixParser().parse(options, args); if (line.hasOption("fastq-out")) { writeFasta = false; } if (line.hasOption("less-than")) { trimle = true; } if (line.hasOption("illumina")) { illumina = true; } args = line.getArgs(); if (args.length < 2) { throw new Exception("Unexpected number of arguments"); } if (args[0].length() != 1) { throw new Exception("Expected single character quality score"); } qualTrim = FastqCore.Phred33QualFunction.translate(args[0].charAt(0)); for (int index = 1; index < args.length; index++) { File seqFile = new File(args[index]); SeqReader reader; if (SeqUtils.guessFileFormat(seqFile) == SequenceFormat.FASTA) { if (index + 1 == args.length) { throw new Exception("Fasta files must be immediately followed by their quality file"); } File qualFile = new File(args[index + 1]); if (SeqUtils.guessFileFormat(qualFile) != SequenceFormat.FASTA) { throw new Exception(seqFile + " was not followed by a fasta quality file"); } reader = new QSeqReader(seqFile, qualFile); index++; } else { if (seqFile.getName().endsWith(".gz")) { reader = new SequenceReader(new GZIPInputStream(new FileInputStream(seqFile))); } else { reader = new SequenceReader(seqFile); } } readers.add(reader); seqFiles.add(seqFile); } } catch (Exception e) { new HelpFormatter().printHelp("USAGE: QualityTrimmer [options] <ascii_score> <seq_file> [qual_file]", options, true); System.err.println("Error: " + e.getMessage()); System.exit(1); } for (int readerIndex = 0; readerIndex < readers.size(); readerIndex++) { File seqFile = seqFiles.get(readerIndex); String outStem = "trimmed_" + seqFile.getName().substring(0, seqFile.getName().lastIndexOf(".")); if (writeFasta) { fastaOut = new FastaWriter(outStem + ".fasta"); } else { fastqOut = new FastqWriter(outStem + ".fastq", FastqCore.Phred33QualFunction); } int[] lengthHisto = new int[200]; SeqReader reader = readers.get(readerIndex); QSequence qseq; long totalLength = 0; int totalSeqs = 0; long trimmedLength = 0; int trimmedSeqs = 0; int zeroLengthAfterTrimming = 0; long startTime = System.currentTimeMillis(); while ((qseq = (QSequence) reader.readNextSequence()) != null) { char[] bases = qseq.getSeqString().toCharArray(); byte[] qual = qseq.getQuality(); if (bases.length != qual.length) { System.err.println(qseq.getSeqName() + ": Quality length doesn't match seq length for seq"); continue; } totalSeqs++; totalLength += bases.length; int trimIndex = -1; if (illumina && qual[bases.length - 1] == qualTrim) { trimIndex = bases.length - 1; while (trimIndex >= 0 && qual[trimIndex] == qualTrim) { trimIndex--; } trimIndex++; //Technically we're positioned over the first good base, move back to the last bad base } else if (!illumina) { for (int index = 0; index < bases.length; index++) { if (qual[index] == qualTrim || (trimle && qual[index] < qualTrim)) { trimIndex = index; break; } } } String outSeq; byte[] outQual; if (trimIndex == -1) { outSeq = qseq.getSeqString(); outQual = qseq.getQuality(); } else { outSeq = new String(bases, 0, trimIndex); outQual = Arrays.copyOfRange(qual, 0, trimIndex); trimmedSeqs++; } int len = outSeq.length(); trimmedLength += len; if (len >= lengthHisto.length) { lengthHisto = Arrays.copyOf(lengthHisto, len + 1); } lengthHisto[len]++; if (outSeq.length() == 0) { //System.err.println(qseq.getSeqName() + ": length 0 after trimming"); zeroLengthAfterTrimming++; continue; } if (writeFasta) { fastaOut.writeSeq(qseq.getSeqName(), qseq.getDesc(), outSeq); } else { fastqOut.writeSeq(qseq.getSeqName(), qseq.getDesc(), outSeq, outQual); } } reader.close(); if (writeFasta) { fastaOut.close(); } else { fastqOut.close(); } System.out.println( "Processed " + seqFile + " in " + (System.currentTimeMillis() - startTime) / 1000.0 + "s"); System.out.println("Before trimming:"); System.out.println("Total Sequences: " + totalSeqs); System.out.println("Total Sequence Data: " + totalLength); System.out.println("Average sequence length: " + ((float) totalLength / totalSeqs)); System.out.println(); System.out.println("After trimming:"); System.out.println("Total Sequences: " + (totalSeqs - zeroLengthAfterTrimming)); System.out.println("Sequences Trimmed: " + trimmedSeqs); System.out.println("Total Sequence Data: " + trimmedLength); System.out.println("Average sequence length: " + ((float) trimmedLength / (totalSeqs - zeroLengthAfterTrimming))); System.out.println(); System.out.println("Length\tCount"); for (int index = 0; index < lengthHisto.length; index++) { if (lengthHisto[index] == 0) { continue; } System.out.println(index + "\t" + lengthHisto[index]); } System.out.println(); System.out.println(); System.out.println(); } }
From source file:Manifest.java
public static void main(String[] args) throws Exception { // Set the default values of the command-line arguments boolean verify = false; // Verify manifest or create one? String manifestfile = "MANIFEST"; // Manifest file name String digestAlgorithm = "MD5"; // Algorithm for message digests String signername = null; // Signer. No sig. by default String signatureAlgorithm = "DSA"; // Algorithm for digital sig. String password = null; // Private keys are protected File keystoreFile = null; // Where are keys stored String keystoreType = null; // What kind of keystore String keystorePassword = null; // How to access keystore List filelist = new ArrayList(); // The files to digest // Parse the command-line arguments, overriding the defaults above for (int i = 0; i < args.length; i++) { if (args[i].equals("-v")) verify = true;//from ww w.j a va2s.c o m else if (args[i].equals("-m")) manifestfile = args[++i]; else if (args[i].equals("-da") && !verify) digestAlgorithm = args[++i]; else if (args[i].equals("-s") && !verify) signername = args[++i]; else if (args[i].equals("-sa") && !verify) signatureAlgorithm = args[++i]; else if (args[i].equals("-p")) password = args[++i]; else if (args[i].equals("-keystore")) keystoreFile = new File(args[++i]); else if (args[i].equals("-keystoreType")) keystoreType = args[++i]; else if (args[i].equals("-keystorePassword")) keystorePassword = args[++i]; else if (!verify) filelist.add(args[i]); else throw new IllegalArgumentException(args[i]); } // If certain arguments weren't supplied, get default values. if (keystoreFile == null) { File dir = new File(System.getProperty("user.home")); keystoreFile = new File(dir, ".keystore"); } if (keystoreType == null) keystoreType = KeyStore.getDefaultType(); if (keystorePassword == null) keystorePassword = password; if (!verify && signername != null && password == null) { System.out.println("Use -p to specify a password."); return; } // Get the keystore we'll use for signing or verifying signatures // If no password was provided, then assume we won't be dealing with // signatures, and skip the keystore. KeyStore keystore = null; if (keystorePassword != null) { keystore = KeyStore.getInstance(keystoreType); InputStream in = new BufferedInputStream(new FileInputStream(keystoreFile)); keystore.load(in, keystorePassword.toCharArray()); } // If -v was specified or no file were given, verify a manifest // Otherwise, create a new manifest for the specified files if (verify || (filelist.size() == 0)) verify(manifestfile, keystore); else create(manifestfile, digestAlgorithm, signername, signatureAlgorithm, keystore, password, filelist); }
From source file:de.tudarmstadt.ukp.experiments.dip.wp1.documents.Step8GoldDataAggregator.java
public static void main(String[] args) throws Exception { String inputDir = args[0] + "/"; // output dir File outputDir = new File(args[1]); File turkersConfidence = new File(args[2]); if (outputDir.exists()) { outputDir.delete();/*from w ww .ja v a 2 s .co m*/ } outputDir.mkdir(); List<String> annotatorsIDs = new ArrayList<>(); // for (File f : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { // QueryResultContainer queryResultContainer = QueryResultContainer // .fromXML(FileUtils.readFileToString(f, "utf-8")); // for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { // for (QueryResultContainer.MTurkRelevanceVote relevanceVote : rankedResults.mTurkRelevanceVotes) { // if (!annotatorsIDs.contains(relevanceVote.turkID)) // annotatorsIDs.add(relevanceVote.turkID); // } // } // } HashMap<String, Integer> countVotesForATurker = new HashMap<>(); // creates temporary file with format for mace // Hashmap annotations: key is the id of a document and a sentence // Value is an array votes[] of turkers decisions: true or false (relevant or not) // the length of this array equals the number of annotators in List<String> annotatorsIDs. // If an annotator worked on the task his decision is written in the array otherwise the value is NULL // key: queryID + clueWebID + sentenceID // value: true and false annotations TreeMap<String, Annotations> annotations = new TreeMap<>(); for (File f : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(f, "utf-8")); System.out.println("Reading " + f.getName()); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { String documentID = rankedResults.clueWebID; for (QueryResultContainer.MTurkRelevanceVote relevanceVote : rankedResults.mTurkRelevanceVotes) { Integer turkerID; if (!annotatorsIDs.contains(relevanceVote.turkID)) { annotatorsIDs.add(relevanceVote.turkID); turkerID = annotatorsIDs.size() - 1; } else { turkerID = annotatorsIDs.indexOf(relevanceVote.turkID); } Integer count = countVotesForATurker.get(relevanceVote.turkID); if (count == null) { count = 0; } count++; countVotesForATurker.put(relevanceVote.turkID, count); String id; List<Integer> trueVotes; List<Integer> falseVotes; for (QueryResultContainer.SingleSentenceRelevanceVote singleSentenceRelevanceVote : relevanceVote.singleSentenceRelevanceVotes) if (!"".equals(singleSentenceRelevanceVote.sentenceID)) { id = f.getName() + "_" + documentID + "_" + singleSentenceRelevanceVote.sentenceID; Annotations turkerVotes = annotations.get(id); if (turkerVotes == null) { trueVotes = new ArrayList<>(); falseVotes = new ArrayList<>(); turkerVotes = new Annotations(trueVotes, falseVotes); } trueVotes = turkerVotes.trueAnnotations; falseVotes = turkerVotes.falseAnnotations; if ("true".equals(singleSentenceRelevanceVote.relevant)) { // votes[turkerID] = true; trueVotes.add(turkerID); } else if ("false".equals(singleSentenceRelevanceVote.relevant)) { // votes[turkerID] = false; falseVotes.add(turkerID); } else { throw new IllegalStateException("Annotation value of sentence " + singleSentenceRelevanceVote.sentenceID + " in " + rankedResults.clueWebID + " equals " + singleSentenceRelevanceVote.relevant); } try { int allVotesCount = trueVotes.size() + falseVotes.size(); if (allVotesCount > 5) { System.err.println(id + " doesn't have 5 annotators: true: " + trueVotes.size() + " false: " + falseVotes.size()); // nasty hack, we're gonna strip some data; true votes first /* we can't do that, it breaks something down the line int toRemove = allVotesCount - 5; if (trueVotes.size() >= toRemove) { trueVotes = trueVotes .subList(0, trueVotes.size() - toRemove); } else if ( falseVotes.size() >= toRemove) { falseVotes = falseVotes .subList(0, trueVotes.size() - toRemove); } */ System.err.println("Adjusted: " + id + " doesn't have 5 annotators: true: " + trueVotes.size() + " false: " + falseVotes.size()); } } catch (IllegalStateException e) { e.printStackTrace(); } turkerVotes.trueAnnotations = trueVotes; turkerVotes.falseAnnotations = falseVotes; annotations.put(id, turkerVotes); } else { throw new IllegalStateException( "Empty Sentence ID in " + f.getName() + " for turker " + turkerID); } } } } File tmp = printHashMap(annotations, annotatorsIDs.size()); String file = TEMP_DIR + "/" + tmp.getName(); MACE.main(new String[] { "--prefix", file }); //gets the keys of the documents and sentences ArrayList<String> lines = (ArrayList<String>) FileUtils.readLines(new File(file + ".prediction")); int i = 0; TreeMap<String, TreeMap<String, ArrayList<HashMap<String, String>>>> ids = new TreeMap<>(); ArrayList<HashMap<String, String>> sentences; if (lines.size() != annotations.size()) { throw new IllegalStateException( "The size of prediction file is " + lines.size() + "but expected " + annotations.size()); } for (Map.Entry entry : annotations.entrySet()) { //1001.xml_clueweb12-1905wb-13-07360_8783 String key = (String) entry.getKey(); String[] IDs = key.split("_"); if (IDs.length > 2) { String queryID = IDs[0]; String clueWebID = IDs[1]; String sentenceID = IDs[2]; TreeMap<String, ArrayList<HashMap<String, String>>> clueWebIDs = ids.get(queryID); if (clueWebIDs == null) { clueWebIDs = new TreeMap<>(); } sentences = clueWebIDs.get(clueWebID); if (sentences == null) { sentences = new ArrayList<>(); } HashMap<String, String> sentence = new HashMap<>(); sentence.put(sentenceID, lines.get(i)); sentences.add(sentence); clueWebIDs.put(clueWebID, sentences); ids.put(queryID, clueWebIDs); } else { throw new IllegalStateException("Wrong ID " + key); } i++; } for (Map.Entry entry : ids.entrySet()) { TreeMap<Integer, String> value = (TreeMap<Integer, String>) entry.getValue(); String queryID = (String) entry.getKey(); QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(new File(inputDir, queryID), "utf-8")); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { for (Map.Entry val : value.entrySet()) { String clueWebID = (String) val.getKey(); if (clueWebID.equals(rankedResults.clueWebID)) { List<QueryResultContainer.SingleSentenceRelevanceVote> goldEstimatedLabels = new ArrayList<>(); List<QueryResultContainer.SingleSentenceRelevanceVote> turkersVotes = new ArrayList<>(); int size = 0; int hitSize = 0; String hitID = ""; for (QueryResultContainer.MTurkRelevanceVote vote : rankedResults.mTurkRelevanceVotes) { if (!hitID.equals(vote.hitID)) { hitID = vote.hitID; hitSize = vote.singleSentenceRelevanceVotes.size(); size = size + hitSize; turkersVotes.addAll(vote.singleSentenceRelevanceVotes); } else { if (vote.singleSentenceRelevanceVotes.size() != hitSize) { hitSize = vote.singleSentenceRelevanceVotes.size(); size = size + hitSize; turkersVotes.addAll(vote.singleSentenceRelevanceVotes); } } } ArrayList<HashMap<String, String>> sentenceList = (ArrayList<HashMap<String, String>>) val .getValue(); if (sentenceList.size() != turkersVotes.size()) { try { throw new IllegalStateException("Expected size of annotations is " + turkersVotes.size() + "but found " + sentenceList.size() + " for document " + rankedResults.clueWebID + " in " + queryID); } catch (IllegalStateException ex) { ex.printStackTrace(); } } for (QueryResultContainer.SingleSentenceRelevanceVote s : turkersVotes) { String valSentence = null; for (HashMap<String, String> anno : sentenceList) { if (anno.keySet().contains(s.sentenceID)) { valSentence = anno.get(s.sentenceID); } } QueryResultContainer.SingleSentenceRelevanceVote singleSentenceVote = new QueryResultContainer.SingleSentenceRelevanceVote(); singleSentenceVote.sentenceID = s.sentenceID; if (("false").equals(valSentence)) { singleSentenceVote.relevant = "false"; } else if (("true").equals(valSentence)) { singleSentenceVote.relevant = "true"; } else { throw new IllegalStateException("Annotation value of sentence " + singleSentenceVote.sentenceID + " equals " + val.getValue()); } goldEstimatedLabels.add(singleSentenceVote); } rankedResults.goldEstimatedLabels = goldEstimatedLabels; } } } File outputFile = new File(outputDir, queryID); FileUtils.writeStringToFile(outputFile, queryResultContainer.toXML(), "utf-8"); System.out.println("Finished " + outputFile); } ArrayList<String> annotators = (ArrayList<String>) FileUtils.readLines(new File(file + ".competence")); FileWriter fileWriter; StringBuilder sb = new StringBuilder(); for (int j = 0; j < annotatorsIDs.size(); j++) { String[] s = annotators.get(0).split("\t"); Float score = Float.parseFloat(s[j]); String turkerID = annotatorsIDs.get(j); System.out.println(turkerID + " " + score + " " + countVotesForATurker.get(turkerID)); sb.append(turkerID).append(" ").append(score).append(" ").append(countVotesForATurker.get(turkerID)) .append("\n"); } fileWriter = new FileWriter(turkersConfidence); fileWriter.append(sb.toString()); fileWriter.close(); }
From source file:squash.tools.FakeBookingCreator.java
public static void main(String[] args) throws IOException { int numberOfDays = 21; int numberOfCourts = 5; int maxCourtSpan = 5; int numberOfSlots = 16; int maxSlotSpan = 3; int minSurnameLength = 2; int maxSurnameLength = 20; int minBookingsPerDay = 0; int maxBookingsPerDay = 8; LocalDate startDate = LocalDate.of(2016, 7, 5); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); List<Booking> bookings = new ArrayList<>(); for (LocalDate date = startDate; date.isBefore(startDate.plusDays(numberOfDays)); date = date.plusDays(1)) { int numBookings = ThreadLocalRandom.current().nextInt(minBookingsPerDay, maxBookingsPerDay + 1); List<Booking> daysBookings = new ArrayList<>(); for (int bookingIndex = 0; bookingIndex < numBookings; bookingIndex++) { String player1 = RandomStringUtils.randomAlphabetic(1) + "." + RandomStringUtils.randomAlphabetic( ThreadLocalRandom.current().nextInt(minSurnameLength, maxSurnameLength + 1)); String player2 = RandomStringUtils.randomAlphabetic(1) + "." + RandomStringUtils.randomAlphabetic( ThreadLocalRandom.current().nextInt(minSurnameLength, maxSurnameLength + 1)); Set<ImmutablePair<Integer, Integer>> bookedCourts = new HashSet<>(); daysBookings.forEach((booking) -> { addBookingToSet(booking, bookedCourts); });/*from www.j ava 2 s. com*/ Booking booking; Set<ImmutablePair<Integer, Integer>> courtsToBook = new HashSet<>(); do { // Loop until we create a booking of free courts int court = ThreadLocalRandom.current().nextInt(1, numberOfCourts + 1); int courtSpan = ThreadLocalRandom.current().nextInt(1, Math.min(maxCourtSpan + 1, numberOfCourts - court + 2)); int slot = ThreadLocalRandom.current().nextInt(1, numberOfSlots + 1); int slotSpan = ThreadLocalRandom.current().nextInt(1, Math.min(maxSlotSpan + 1, numberOfSlots - slot + 2)); booking = new Booking(court, courtSpan, slot, slotSpan, player1 + "/" + player2); booking.setDate(date.format(formatter)); courtsToBook.clear(); addBookingToSet(booking, courtsToBook); } while (Boolean.valueOf(Sets.intersection(courtsToBook, bookedCourts).size() > 0)); daysBookings.add(booking); } bookings.addAll(daysBookings); } // Encode bookings as JSON // Create the node factory that gives us nodes. JsonNodeFactory factory = new JsonNodeFactory(false); // Create a json factory to write the treenode as json. JsonFactory jsonFactory = new JsonFactory(); ObjectNode rootNode = factory.objectNode(); ArrayNode bookingsNode = rootNode.putArray("bookings"); for (int i = 0; i < bookings.size(); i++) { Booking booking = bookings.get(i); ObjectNode bookingNode = factory.objectNode(); bookingNode.put("court", booking.getCourt()); bookingNode.put("courtSpan", booking.getCourtSpan()); bookingNode.put("slot", booking.getSlot()); bookingNode.put("slotSpan", booking.getSlotSpan()); bookingNode.put("name", booking.getName()); bookingNode.put("date", booking.getDate()); bookingsNode.add(bookingNode); } // Add empty booking rules array - just so restore works rootNode.putArray("bookingRules"); rootNode.put("clearBeforeRestore", true); try (JsonGenerator generator = jsonFactory.createGenerator(new File("FakeBookings.json"), JsonEncoding.UTF8)) { ObjectMapper mapper = new ObjectMapper(); mapper.setSerializationInclusion(Include.NON_EMPTY); mapper.setSerializationInclusion(Include.NON_NULL); mapper.writeTree(generator, rootNode); } }
From source file:com.inkubator.common.util.NewMain.java
/** * @param args the command line arguments */// ww w . j a v a 2 s .c om public static void main(String[] args) throws IOException { File file1 = new File( "C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\JSON_Ek\\Surabaya\\Page1.txt"); File file2 = new File( "C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\JSON_Ek\\Surabaya\\Page2.txt"); // File file3 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\json\\json\\menado\\page3.txt"); File file3 = new File( "C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\JSON_Ek\\Surabaya\\Page3.txt"); File file4 = new File( "C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\JSON_Ek\\Surabaya\\Page4.txt"); File file5 = new File( "C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\JSON_Ek\\Surabaya\\Page5.txt"); File file6 = new File( "C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\JSON_Ek\\Surabaya\\Page6.txt"); // File file7 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 7.txt"); // File file8 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 8.txt"); // File file9 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 9.txt"); // File file10 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 10.txt"); // File file11 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 11.txt"); // File file12 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 12.txt"); // File file13 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 13.txt"); // File file14 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 14.txt"); // File file15 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 15.txt"); // File file16 = new File("C:\\Users\\deni.fahri\\Downloads\\page16.txt"); // File file2 = new File("C:\\Users\\deni.fahri\\Documents\\hasil.txt"); String agoda = FilesUtil.getAsStringFromFile(file1); String agoda1 = FilesUtil.getAsStringFromFile(file2); String agoda2 = FilesUtil.getAsStringFromFile(file3); String agoda3 = FilesUtil.getAsStringFromFile(file4); String agoda4 = FilesUtil.getAsStringFromFile(file5); String agoda5 = FilesUtil.getAsStringFromFile(file6); // String agoda6 = FilesUtil.getAsStringFromFile(file7); // String agoda7 = FilesUtil.getAsStringFromFile(file8); // String agoda8 = FilesUtil.getAsStringFromFile(file9); // String agoda9 = FilesUtil.getAsStringFromFile(file10); // String agoda10 = FilesUtil.getAsStringFromFile(file11); // String agoda11 = FilesUtil.getAsStringFromFile(file12); // String agoda12 = FilesUtil.getAsStringFromFile(file13); // String agoda13 = FilesUtil.getAsStringFromFile(file14); // String agoda14 = FilesUtil.getAsStringFromFile(file15); // String agoda15 = FilesUtil.getAsStringFromFile(file16); //// System.out.println(" Test Nya adalah :" + agoda); //// String a=StringUtils.substringAfter("\"HotelTranslatedName\":", agoda); //// System.out.println(" hasil; "+a); //// // TODO code application logic here //// System.out.println("Nilai " + JsonConverter.getValueByKeyStatic(agoda, "HotelTranslatedName")); TypeToken<List<HotelModel>> token = new TypeToken<List<HotelModel>>() { }; Gson gson = new GsonBuilder().create(); // List<HotelModel> data = new ArrayList<>(); // HotelModel hotelModel = new HotelModel(); // hotelModel.setAddress("sdfsdffsfsdfsdfdsfdsf"); // hotelModel.setAccommodationName("Aku"); // HotelModel hotelModel1 = new HotelModel(); // hotelModel1.setAddress("sdfsdffsfsdfsdfdsfdsf"); // hotelModel1.setAccommodationName("Avvvku"); // HotelModel hotelModel2 = new HotelModel(); // hotelModel2.setAddress("sdfsdffsfsdfsdfdsfdsf"); // hotelModel2.setAccommodationName("Akvvvu"); // data.add(hotelModel); // data.add(hotelModel1); // data.add(hotelModel2); // String json = gson.toJson(data); List<HotelModel> total = new ArrayList<>(); List<HotelModel> data1 = new ArrayList<>(); List<HotelModel> data2 = new ArrayList<>(); List<HotelModel> data3 = new ArrayList<>(); List<HotelModel> data4 = new ArrayList<>(); List<HotelModel> data5 = new ArrayList<>(); List<HotelModel> data6 = new ArrayList<>(); List<HotelModel> data7 = new ArrayList<>(); List<HotelModel> data8 = new ArrayList<>(); List<HotelModel> data9 = new ArrayList<>(); List<HotelModel> data10 = new ArrayList<>(); List<HotelModel> data11 = new ArrayList<>(); List<HotelModel> data12 = new ArrayList<>(); List<HotelModel> data13 = new ArrayList<>(); List<HotelModel> data14 = new ArrayList<>(); List<HotelModel> data15 = new ArrayList<>(); List<HotelModel> data16 = new ArrayList<>(); data1 = gson.fromJson(agoda, token.getType()); data2 = gson.fromJson(agoda1, token.getType()); data3 = gson.fromJson(agoda2, token.getType()); data4 = gson.fromJson(agoda3, token.getType()); data5 = gson.fromJson(agoda4, token.getType()); data6 = gson.fromJson(agoda5, token.getType()); // data7 = gson.fromJson(agoda6, token.getType()); // data8 = gson.fromJson(agoda7, token.getType()); // data9 = gson.fromJson(agoda8, token.getType()); // data10 = gson.fromJson(agoda9, token.getType()); // data11 = gson.fromJson(agoda10, token.getType()); // data12 = gson.fromJson(agoda11, token.getType()); // data13 = gson.fromJson(agoda12, token.getType()); // data14 = gson.fromJson(agoda13, token.getType()); // data15 = gson.fromJson(agoda14, token.getType()); // data16 = gson.fromJson(agoda15, token.getType()); total.addAll(data1); total.addAll(data2); total.addAll(data3); total.addAll(data4); total.addAll(data5); total.addAll(data6); // total.addAll(data7); // total.addAll(data8); // total.addAll(data9); // total.addAll(data10); // total.addAll(data11); // total.addAll(data12); // total.addAll(data13); // total.addAll(data14); // total.addAll(data15); // total.addAll(data16); System.out.println(" Ukurannn nya " + total.size()); // System.out.println(" Ukurannya " + data2.size()); for (HotelModel mode : total) { System.out.println(mode); } // HotelModel hotelModel = gson.fromJson(agoda, HotelModel.class); // String Data = hotelModel.getHotelTranslatedName() + ";" + hotelModel.getStarRating() + ";" + hotelModel.getAddress() + ";" + hotelModel.getIsFreeWifi(); // FilesUtil.writeToFileFromString(file2, Data); // System.out.println(hotelModel); // HSSFWorkbook workbook = new HSSFWorkbook(); HSSFSheet sheet = workbook.createSheet("Agoda Data Hotel Surabaya"); //// TreeMap<String, Object[]> datatoExel = new TreeMap<>(); int i = 1; // datatoExel.put("1", new Object[]{"Hotel Agoda Jakarta"}); datatoExel.put("1", new Object[] { "Nama Hotel", "Arena", "Alamat", "Rating", "Apakah Gratis Wifi", "Harga Mulai Dari", "Longitude", "Latitude" }); for (HotelModel mode : total) { datatoExel.put(String.valueOf(i + 1), new Object[] { mode.getHotelTranslatedName(), mode.getAreaName(), mode.getAddress(), mode.getStarRating(), mode.getIsFreeWifi(), mode.getTextPrice() + " " + mode.getCurrencyCode(), mode.getCoordinate().getLongitude(), mode.getCoordinate().getLatitude() }); i++; } // //// int i=1; //// for (HotelModel mode : data2) { //// datatoExel.put(String.valueOf(i), new Object[]{1d, "John", 1500000d}); ////// } //// //// datatoExel.put("4", new Object[]{3d, "Dean", 700000d}); //// Set<String> keyset = datatoExel.keySet(); int rownum = 0; for (String key : keyset) { Row row = sheet.createRow(rownum++); Object[] objArr = datatoExel.get(key); int cellnum = 0; for (Object obj : objArr) { Cell cell = row.createCell(cellnum++); if (obj instanceof Date) { cell.setCellValue((Date) obj); } else if (obj instanceof Boolean) { cell.setCellValue((Boolean) obj); } else if (obj instanceof String) { cell.setCellValue((String) obj); } else if (obj instanceof Double) { cell.setCellValue((Double) obj); } } } try { FileOutputStream out = new FileOutputStream(new File("C:\\Users\\deni.fahri\\Documents\\Surabaya.xls")); workbook.write(out); out.close(); System.out.println("Excel written successfully.."); } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } }
From source file:com.act.biointerpretation.metadata.ProteinMetadataFactory.java
public static void main(String[] args) throws Exception { // TODO: This is referencing a temporary collection. Change it! // TODO: FIX THIS BEFORE MERGE! NoSQLAPI api = new NoSQLAPI("actv01_vijay_proteins", "actv01_vijay_proteins"); Iterator<Reaction> iterator = api.readRxnsFromInKnowledgeGraph(); //Create a single instance of the factory method to use for all json ProteinMetadataFactory factory = ProteinMetadataFactory.initiate(); //Run some tests try {/*from ww w.ja v a 2 s. c o m*/ if (factory.testHandlesubunits() == true) { System.out.println("Subunit test OK"); } } catch (Exception err) { System.err.println("Failed to test subunits"); } //Create a list to aggregate the results of the database scan List<ProteinMetadata> agg = new ArrayList<>(); //Scan the database and store ProteinMetadata objects while (iterator.hasNext()) { Reaction rxn = iterator.next(); Reaction.RxnDataSource source = rxn.getDataSource(); if (!source.equals(Reaction.RxnDataSource.BRENDA)) { continue; } Set<JSONObject> jsons = rxn.getProteinData(); for (JSONObject json : jsons) { ProteinMetadata meta = factory.create(json); agg.add(meta); } } //Write out any messages to file StringBuilder sb = new StringBuilder(); for (String aline : factory.dataList) { sb.append(aline).append("\n"); } File outfile = new File("output/ProteinMetadata/Factory_output.txt"); if (outfile.exists()) { outfile.delete(); } FileUtils.writeStringToFile(outfile, sb.toString()); sb = new StringBuilder(); for (String key : factory.dataMap.keySet()) { int value = factory.dataMap.get(key); sb.append(key + "\t" + value + "\n"); } outfile = new File("output/ProteinMetadata/Factory_output_map.txt"); if (outfile.exists()) { outfile.delete(); } FileUtils.writeStringToFile(outfile, sb.toString()); //Count up the results of modifications to get statistics int falsecount = 0; int truecount = 0; int nullcount = 0; for (ProteinMetadata datum : agg) { if (datum == null) { System.err.println("null datum"); continue; } if (datum.modifications == null) { nullcount++; } else if (datum.modifications == false) { falsecount++; } else if (datum.modifications == true) { truecount++; } } System.out.println("Total # protein metadata: " + agg.size()); System.out.println(); System.out.println("modification true count: " + truecount); System.out.println("modification false count: " + falsecount); System.out.println("modification null count: " + nullcount); System.out.println(); //Get some statistics for cloned nullcount = 0; int emptycount = 0; int colicount = 0; int humancount = 0; int bothcount = 0; for (ProteinMetadata datum : agg) { if (datum == null) { System.err.println("null datum"); continue; } if (datum.cloned == null) { nullcount++; continue; } if (datum.cloned.isEmpty()) { emptycount++; continue; } Integer human = datum.cloned.get(Host.Hsapiens); if (human != null && human > 0) { humancount++; } Integer coli = datum.cloned.get(Host.Ecoli); if (coli != null && coli > 0) { colicount++; if (human != null && human > 0) { bothcount++; } } } System.out.println("cloned null count: " + nullcount); System.out.println("cloned empty count: " + emptycount); System.out.println("cloned coli count: " + colicount); System.out.println("cloned human count: " + humancount); System.out.println("cloned both count: " + bothcount); System.out.println(); }
From source file:com.zimbra.kabuki.tools.img.ImageMerger.java
public static void main(String[] argv) throws Exception { // parse command-line CommandLineParser parser = new GnuParser(); CommandLine cl = null;// w w w .j a v a2s.c o m try { cl = parser.parse(OPTIONS, argv); } catch (ParseException pe) { System.err.println(pe); printUsageAndExit(); } if (cl == null) { printUsageAndExit(); } // handle options ImageMerger merger = new ImageMerger(); List<File> inputDirs = new LinkedList<File>(); if (cl.hasOption(O_INPUT)) { for (String dirname : cl.getOptionValue(O_INPUT).split("[;,]")) { File dir = new File(dirname); assertAndExit(dir.exists(), "directory %s doesn't exist", dir.toString()); assertAndExit(dir.isDirectory(), "%s is not a directory", dir.toString()); inputDirs.add(dir); } } if (cl.hasOption(O_OUTPUT)) { File dir = new File(cl.getOptionValue(O_OUTPUT)); assertAndExit(dir.exists(), "directory %s doesn't exist", dir.toString()); assertAndExit(dir.isDirectory(), "%s is not a directory", dir.toString()); merger.setOutputDirectory(dir); } if (cl.hasOption(O_CSS_PATH) && cl.hasOption(O_CSS_FILENAME)) { merger.setCssPath(cl.getOptionValue(O_CSS_PATH)); merger.setCssFilename(cl.getOptionValue(O_CSS_FILENAME)); String cacheFilename = cl.getOptionValue(O_CACHE_FILENAME); if (cacheFilename != null) { merger.setCacheFilename(cacheFilename); } } if (cl.hasOption(O_JS_FILENAME)) { merger.setJsFilename(cl.getOptionValue(O_JS_FILENAME)); } if (cl.hasOption(O_COPY)) { merger.setCopyFiles(true); } if (cl.hasOption(O_NO_APPEND)) { merger.setAppendOutput(false); } if (cl.hasOption(O_VERBOSE)) { merger.setVerbose(true); } if (cl.hasOption(O_DISABLED_IMAGES)) { printWarning("option -%s is deprecated", O_DISABLED_IMAGES); } if (cl.hasOption(O_SPACER_IMAGES)) { merger.setSpacerImagesPath(cl.getOptionValue(O_SPACER_IMAGES)); } // process assertAndExit(inputDirs.size() > 0, "must specify input directories"); merger.process(inputDirs); }
From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFTrainHybridBSP.java
public static void main(String[] args) throws Exception { // Defaults/* www . j a v a2 s . c om*/ int numBspTask = 1; // CPU + GPU tasks int numGpuBspTask = 1; // GPU tasks int blockSize = BLOCK_SIZE; int gridSize = GRID_SIZE; int maxIteration = 3; // 150; int matrixRank = 3; int skipCount = 1; double alpha = ALPHA; int userCount = 0; int itemCount = 0; int percentNonZeroValues = 0; int GPUPercentage = 20; boolean useTestExampleInput = true; boolean isDebugging = true; String inputFile = ""; String separator = "\\t"; Configuration conf = new HamaConfiguration(); FileSystem fs = FileSystem.get(conf); // Set numBspTask to maxTasks // BSPJobClient jobClient = new BSPJobClient(conf); // ClusterStatus cluster = jobClient.getClusterStatus(true); // numBspTask = cluster.getMaxTasks(); if (args.length > 0) { if (args.length >= 14) { numBspTask = Integer.parseInt(args[0]); numGpuBspTask = Integer.parseInt(args[1]); blockSize = Integer.parseInt(args[2]); gridSize = Integer.parseInt(args[3]); maxIteration = Integer.parseInt(args[4]); matrixRank = Integer.parseInt(args[5]); skipCount = Integer.parseInt(args[6]); alpha = Double.parseDouble(args[7]); userCount = Integer.parseInt(args[8]); itemCount = Integer.parseInt(args[9]); percentNonZeroValues = Integer.parseInt(args[10]); GPUPercentage = Integer.parseInt(args[11]); useTestExampleInput = Boolean.parseBoolean(args[12]); isDebugging = Boolean.parseBoolean(args[13]); // optional parameters if (args.length > 14) { inputFile = args[14]; } if (args.length > 15) { separator = args[15]; } } else { System.out.println("Wrong argument size!"); System.out.println(" Argument1=numBspTask"); System.out.println(" Argument2=numGpuBspTask"); System.out.println(" Argument3=blockSize"); System.out.println(" Argument4=gridSize"); System.out.println( " Argument5=maxIterations | Number of maximal iterations (" + maxIteration + ")"); System.out.println(" Argument6=matrixRank | matrixRank (" + matrixRank + ")"); System.out.println(" Argument7=skipCount | skipCount (" + skipCount + ")"); System.out.println(" Argument8=alpha | alpha (" + alpha + ")"); System.out.println(" Argument9=userCount | userCount (" + userCount + ")"); System.out.println(" Argument10=itemCount | itemCount (" + itemCount + ")"); System.out.println(" Argument11=percentNonZeroValues | percentNonZeroValues (" + percentNonZeroValues + ")"); System.out.println(" Argument12=GPUPercentage (percentage of input)"); System.out.println(" Argument13=testExample | Use testExample input (true|false=default)"); System.out.println(" Argument14=debug | Enable debugging (true|false=default)"); System.out.println(" Argument15=inputFile (optional) | MovieLens inputFile"); System.out.println(" Argument16=separator (optional) | default '" + separator + "' "); return; } } // Check if inputFile exists if ((!inputFile.isEmpty()) && (!new File(inputFile).exists())) { System.out.println("Error: inputFile: " + inputFile + " does not exist!"); return; } // Check parameters if ((inputFile.isEmpty()) && (!useTestExampleInput) && (userCount <= 0) && (itemCount <= 0) && (percentNonZeroValues <= 0)) { System.out.println("Invalid parameter: userCount: " + userCount + " itemCount: " + itemCount + " percentNonZeroValues: " + percentNonZeroValues); return; } // Check if blockSize < matrixRank when using GPU if ((numGpuBspTask > 0) && (blockSize < matrixRank)) { System.out.println("Error: BlockSize < matrixRank"); return; } // Check GPUPercentage if ((GPUPercentage < 0) && (GPUPercentage > 100)) { System.out.println("Error: GPUPercentage must be between 0 and 100 percent"); return; } // Set config variables conf.setBoolean(CONF_DEBUG, isDebugging); conf.setBoolean("hama.pipes.logging", isDebugging); // Set CPU tasks conf.setInt("bsp.peers.num", numBspTask); // Set GPU tasks conf.setInt("bsp.peers.gpu.num", numGpuBspTask); // Set GPU blockSize and gridSize conf.set(CONF_BLOCKSIZE, "" + blockSize); conf.set(CONF_GRIDSIZE, "" + gridSize); conf.setInt(OnlineCF.CONF_ITERATION_COUNT, maxIteration); conf.setInt(OnlineCF.CONF_MATRIX_RANK, matrixRank); conf.setInt(OnlineCF.CONF_SKIP_COUNT, skipCount); // Debug output LOG.info("NumBspTask: " + conf.getInt("bsp.peers.num", 0)); LOG.info("NumGpuBspTask: " + conf.getInt("bsp.peers.gpu.num", 0)); LOG.info("bsp.tasks.maximum: " + conf.get("bsp.tasks.maximum")); LOG.info("BlockSize: " + conf.get(CONF_BLOCKSIZE)); LOG.info("GridSize: " + conf.get(CONF_GRIDSIZE)); LOG.info("GPUPercentage: " + GPUPercentage); LOG.info("isDebugging: " + isDebugging); LOG.info("useTestExampleInput: " + useTestExampleInput); LOG.info("inputPath: " + CONF_INPUT_DIR); LOG.info("outputPath: " + CONF_OUTPUT_DIR); LOG.info("maxIteration: " + maxIteration); LOG.info("matrixRank: " + matrixRank); LOG.info("skipCount: " + skipCount); LOG.info("alpha: " + alpha); LOG.info("userCount: " + userCount); LOG.info("itemCount: " + itemCount); LOG.info("percentNonZeroValues: " + percentNonZeroValues); if (!inputFile.isEmpty()) { LOG.info("inputFile: " + inputFile); LOG.info("separator: " + separator); } // prepare Input int maxTestPrefs = 10; Path preferencesIn = new Path(CONF_INPUT_DIR, "preferences_in.seq"); List<Preference<Long, Long>> testPrefs = null; if (useTestExampleInput) { testPrefs = prepareTestInputData(conf, fs, CONF_INPUT_DIR, preferencesIn); } else if (inputFile.isEmpty()) { testPrefs = generateRandomInputData(conf, fs, CONF_INPUT_DIR, numBspTask, numGpuBspTask, userCount, itemCount, percentNonZeroValues, GPUPercentage, maxTestPrefs); } else if (!inputFile.isEmpty()) { // parse inputFile and return first entries for testing testPrefs = convertInputData(conf, fs, CONF_INPUT_DIR, preferencesIn, inputFile, separator, maxTestPrefs); } // Generate Job config BSPJob job = createOnlineCFTrainHybridBSPConf(conf, CONF_INPUT_DIR, CONF_OUTPUT_DIR); // Execute Job long startTime = System.currentTimeMillis(); if (job.waitForCompletion(true)) { LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds"); // Load Job results for testing OnlineCF recommender = new OnlineCF(); recommender.load(CONF_OUTPUT_DIR.toString(), false); // Test results int error = 0; double totalError = 0; for (Preference<Long, Long> test : testPrefs) { double expected = test.getValue().get(); double estimated = recommender.estimatePreference(test.getUserId(), test.getItemId()); if (testPrefs.size() <= 20) { LOG.info("(" + test.getUserId() + ", " + test.getItemId() + ", " + expected + "): " + estimated + " error: " + Math.abs(expected - estimated)); } totalError += Math.abs(expected - estimated); error += (Math.abs(expected - estimated) < 0.5) ? 1 : 0; } LOG.info("totalError: " + totalError); LOG.info("assertEquals(expected: " + (testPrefs.size() * 0.75) + " == " + error + " actual) with delta: 1"); if (isDebugging) { printOutput(conf, fs, ".log", new IntWritable(), new PipesVectorWritable()); } } }
From source file:org.opencastproject.remotetest.Main.java
public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption(new Option("help", false, "print this message")); options.addOption(new Option("withperf", false, "run the performance tests")); options.addOption(new Option("withserver", false, "run the tests for the server side components")); options.addOption(new Option("withcapture", false, "run the tests for the capture agent")); options.addOption(/*from ww w. ja v a2 s .co m*/ new Option("withff", false, "run the selenium user interface tests with the firefox browser")); options.addOption( new Option("withchrome", false, "run the selenium user interface tests with the chrome browser")); options.addOption( new Option("withsafari", false, "run the selenium user interface tests with the safari browser")); options.addOption(new Option("url", true, "run tests against the Matterhorn installation at this URL")); options.addOption( new Option("username", true, "the username to use when accessing the Matterhorn installation")); options.addOption( new Option("password", true, "the password to use when accessing the Matterhorn installation")); // create the parser CommandLineParser parser = new PosixParser(); List<Class<?>> testClasses = new ArrayList<Class<?>>(); CommandLine line = null; try { line = parser.parse(options, args); } catch (ParseException e) { System.err.println("Parsing commandline failed: " + e.getMessage()); System.exit(1); } if (line.hasOption("url")) { BASE_URL = line.getOptionValue("url"); } if (line.hasOption("username")) { USERNAME = line.getOptionValue("username"); } if (line.hasOption("password")) { PASSWORD = line.getOptionValue("password"); } if (line.hasOption("help")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("java -jar matterhorn-test-harness-<version>-jar-with-dependencies.jar>", options); System.exit(0); } // should we run the server-side tests if (line.hasOption("withserver")) { System.out.println("Running with the 'server' test suite enabled"); testClasses.add(ServerTests.class); if (line.hasOption("withperf")) { System.out.println("Running with the server performance test suite enabled"); testClasses.add(ServerPerformanceTests.class); } } if (line.hasOption("withcapture")) { System.out.println("Running 'capture' test suite"); testClasses.add(CaptureAgentTests.class); if (line.hasOption("withperf")) { // TODO: Add capture agent performance tests // System.out.println("Running with the 'capture' performance test suite enabled"); } } SeleniumServer seleniumServer = null; if (line.hasOption("withff") || line.hasOption("withchrome") || line.hasOption("withsafari")) { // Add the test suite testClasses.add(SeleniumTestSuite.class); // Assemble the browsers that the user wants to use in the tests if (line.hasOption("withff")) { runSeleneseTests(FIREFOX); BROWSERS.add(FIREFOX); } if (line.hasOption("withchrome")) { runSeleneseTests(CHROME); BROWSERS.add(CHROME); } if (line.hasOption("withsafari")) { runSeleneseTests(SAFARI); BROWSERS.add(SAFARI); } // Start the selenium server seleniumServer = new SeleniumServer(); seleniumServer.start(); } // if we don't have any test classes, add the server (not performance) tests... just so we have *something* to do if (testClasses.size() == 0) { System.out.println("No test suites specified... running server (not including performance) tests"); testClasses.add(ServerTests.class); } // run the tests System.out.println("Beginning matterhorn test suite on " + BASE_URL); Result result = JUnitCore.runClasses(testClasses.toArray(new Class<?>[testClasses.size()])); if (seleniumServer != null) { seleniumServer.stop(); } // print the results System.out.println(result.getRunCount() + " tests run, " + result.getIgnoreCount() + " tests ignored, " + result.getFailureCount() + " tests failed. Total time = " + result.getRunTime() + "ms"); if (result.getFailureCount() > 0) { for (Failure failure : result.getFailures()) { System.out.println(failure.getTrace()); } System.exit(1); } }