List of usage examples for java.util HashMap HashMap
public HashMap()
From source file:es.upm.oeg.tools.rdfshapes.utils.CadinalityResultGenerator.java
public static void main(String[] args) throws Exception { String endpoint = "http://3cixty.eurecom.fr/sparql"; List<String> classList = Files.readAllLines(Paths.get(classListPath), Charset.defaultCharset()); String classPropertyQueryString = readFile(classPropertyQueryPath, Charset.defaultCharset()); String propertyCardinalityQueryString = readFile(propertyCardinalityQueryPath, Charset.defaultCharset()); String individualCountQueryString = readFile(individualCountQueryPath, Charset.defaultCharset()); DecimalFormat df = new DecimalFormat("0.0000"); //Create the Excel workbook and sheet XSSFWorkbook wb = new XSSFWorkbook(); XSSFSheet sheet = wb.createSheet("Cardinality"); int currentExcelRow = 0; int classStartRow = 0; for (String clazz : classList) { Map<String, String> litMap = new HashMap<>(); Map<String, String> iriMap = ImmutableMap.of("class", clazz); String queryString = bindQueryString(individualCountQueryString, ImmutableMap.of(IRI_BINDINGS, iriMap, LITERAL_BINDINGS, litMap)); int individualCount; List<RDFNode> c = executeQueryForList(queryString, endpoint, "c"); if (c.size() == 1) { individualCount = c.get(0).asLiteral().getInt(); } else {//from www. java 2s .c o m continue; } // If there are zero individuals, continue if (individualCount == 0) { throw new IllegalStateException("Check whether " + classListPath + " and " + endpoint + " match."); } // System.out.println("***"); // System.out.println("### **" + clazz + "** (" + individualCount + ")"); // System.out.println("***"); // System.out.println(); classStartRow = currentExcelRow; XSSFRow row = sheet.createRow(currentExcelRow); XSSFCell cell = row.createCell(0); cell.setCellValue(clazz); cell.getCellStyle().setAlignment(CellStyle.ALIGN_CENTER); queryString = bindQueryString(classPropertyQueryString, ImmutableMap.of(IRI_BINDINGS, iriMap, LITERAL_BINDINGS, litMap)); List<RDFNode> nodeList = executeQueryForList(queryString, endpoint, "p"); for (RDFNode property : nodeList) { if (property.isURIResource()) { DescriptiveStatistics stats = new DescriptiveStatistics(); String propertyURI = property.asResource().getURI(); // System.out.println("* " + propertyURI); // System.out.println(); XSSFRow propertyRow = sheet.getRow(currentExcelRow); if (propertyRow == null) { propertyRow = sheet.createRow(currentExcelRow); } currentExcelRow++; XSSFCell propertyCell = propertyRow.createCell(1); propertyCell.setCellValue(propertyURI); Map<String, String> litMap2 = new HashMap<>(); Map<String, String> iriMap2 = ImmutableMap.of("class", clazz, "p", propertyURI); queryString = bindQueryString(propertyCardinalityQueryString, ImmutableMap.of(IRI_BINDINGS, iriMap2, LITERAL_BINDINGS, litMap2)); List<Map<String, RDFNode>> solnMaps = executeQueryForList(queryString, endpoint, ImmutableSet.of("card", "count")); int sum = 0; List<CardinalityCount> cardinalityList = new ArrayList<>(); if (solnMaps.size() > 0) { for (Map<String, RDFNode> soln : solnMaps) { int count = soln.get("count").asLiteral().getInt(); int card = soln.get("card").asLiteral().getInt(); for (int i = 0; i < count; i++) { stats.addValue(card); } CardinalityCount cardinalityCount = new CardinalityCount(card, count, (((double) count) / individualCount) * 100); cardinalityList.add(cardinalityCount); sum += count; } // Check for zero cardinality instances int count = individualCount - sum; if (count > 0) { for (int i = 0; i < count; i++) { stats.addValue(0); } CardinalityCount cardinalityCount = new CardinalityCount(0, count, (((double) count) / individualCount) * 100); cardinalityList.add(cardinalityCount); } } Map<Integer, Double> cardMap = new HashMap<>(); for (CardinalityCount count : cardinalityList) { cardMap.put(count.getCardinality(), count.getPrecentage()); } XSSFCell instanceCountCell = propertyRow.createCell(2); instanceCountCell.setCellValue(individualCount); XSSFCell minCell = propertyRow.createCell(3); minCell.setCellValue(stats.getMin()); XSSFCell maxCell = propertyRow.createCell(4); maxCell.setCellValue(stats.getMax()); XSSFCell p1 = propertyRow.createCell(5); p1.setCellValue(stats.getPercentile(1)); XSSFCell p99 = propertyRow.createCell(6); p99.setCellValue(stats.getPercentile(99)); XSSFCell mean = propertyRow.createCell(7); mean.setCellValue(df.format(stats.getMean())); for (int i = 0; i < 21; i++) { XSSFCell dataCell = propertyRow.createCell(8 + i); Double percentage = cardMap.get(i); if (percentage != null) { dataCell.setCellValue(df.format(percentage)); } else { dataCell.setCellValue(0); } } // System.out.println("| Min Card. |Max Card. |"); // System.out.println("|---|---|"); // System.out.println("| ? | ? |"); // System.out.println(); } } //System.out.println("class start: " + classStartRow + ", class end: " + (currentExcelRow -1)); //We have finished writting properties of one class, now it's time to merge the cells int classEndRow = currentExcelRow - 1; if (classStartRow < classEndRow) { sheet.addMergedRegion(new CellRangeAddress(classStartRow, classEndRow, 0, 0)); } } String filename = "3cixty.xls"; FileOutputStream fileOut = new FileOutputStream(filename); wb.write(fileOut); fileOut.close(); }
From source file:com.knowbout.epg.EPG.java
public static void main(String[] args) { String configFile = "/etc/flip.tv/epg.xml"; boolean sitemaponly = false; if (args.length > 0) { if (args.length == 2 && args[0].equals("-config")) { configFile = args[1];//from ww w.j a va2 s .c om } else if (args.length == 1 && args[0].equals("-sitemaponly")) { sitemaponly = true; } else { System.err.println("Usage: java " + EPG.class.getName() + " [-config <xmlfile>]"); System.exit(1); } } try { XMLConfiguration config = new XMLConfiguration(configFile); HashMap<String, String> hibernateProperties = new HashMap<String, String>(); Configuration database = config.subset("database"); hibernateProperties.put(Environment.DRIVER, database.getString("driver")); hibernateProperties.put(Environment.URL, database.getString("url")); hibernateProperties.put(Environment.USER, database.getString("user")); hibernateProperties.put(Environment.PASS, database.getString("password")); hibernateProperties.put(Environment.DATASOURCE, null); HibernateUtil.setProperties(hibernateProperties); if (!sitemaponly) { //test(config); // Get the server configuration to download the content Configuration provider = config.subset("provider"); InetAddress server = InetAddress.getByName(provider.getString("server")); File destinationFolder = new File(provider.getString("destinationFolder")); String username = provider.getString("username"); String password = provider.getString("password"); String remoteDirectory = provider.getString("remoteWorkingDirectory"); boolean forceDownload = provider.getBoolean("forceDownload"); Downloader downloader = new Downloader(server, username, password, destinationFolder, remoteDirectory, forceDownload); int count = downloader.downloadFiles(); log.info("Downloaded " + count + " files"); // int count = 14; if (count > 0) { log.info("Processing downloads now."); //Get the name of the files to process Configuration files = config.subset("files"); File headend = new File(destinationFolder, files.getString("headend")); File lineup = new File(destinationFolder, files.getString("lineup")); File stations = new File(destinationFolder, files.getString("stations")); File programs = new File(destinationFolder, files.getString("programs")); File schedules = new File(destinationFolder, files.getString("schedules")); Parser parser = new Parser(config, headend, lineup, stations, programs, schedules); parser.parse(); log.info("Finished parsing EPG Data. Invoking AlertQueue service now."); String alertUrl = config.getString("alertUrl"); HessianProxyFactory factory = new HessianProxyFactory(); AlertQueue alerts = (AlertQueue) factory.create(AlertQueue.class, alertUrl); alerts.checkAlerts(); log.info("Updating sitemap"); updateSitemap(config.getString("sitemap", "/usr/local/webapps/search/sitemap.xml.gz")); log.info("Exiting EPG now."); } else { log.info("No files were downloaded, so don't process the old files."); } } else { log.info("Updating sitemap"); updateSitemap(config.getString("sitemap", "/usr/local/webapps/search/sitemap.xml.gz")); log.info("Done updating sitemap"); } } catch (ConfigurationException e) { log.fatal("Configuration error in file " + configFile, e); e.printStackTrace(); } catch (UnknownHostException e) { log.fatal("Unable to connect to host", e); e.printStackTrace(); } catch (IOException e) { log.fatal("Error downloading or processing EPG information", e); e.printStackTrace(); } catch (Throwable e) { log.fatal("Unexpected Error", e); e.printStackTrace(); } }
From source file:com.github.benyzhous.springboot.web.core.gateway.sign.backend.Sign.java
/** * Demo/*from w ww. j av a 2 s . co m*/ * * @param args * @throws Exception */ public static void main(String[] args) throws Exception { String uri = "/list"; String httpMethod = "GET"; Map<String, String> headers = new HashMap<String, String>(); headers.put("x-ca-proxy-signature", "YTaCd7ZCm6wRYY/mc2Dj4lQOviuR4oSJA9YGxjYHYIo="); headers.put("x-ca-proxy-signature-headers", "RequestId,X-Ca-Proxy-Signature-Secret-Key"); headers.put("x-ca-proxy-signature-secret-key", "kaishu2099"); headers.put("requestid", "1609A1D2-2845-488E-B052-9BA2143D51B5"); //headers.put("HeaderKey2", "HeaderValue2"); Map<String, Object> paramsMap = new HashMap<String, Object>(); paramsMap.put("appid", "wxbb22392b3fefd76f"); paramsMap.put("AppID", "4206209"); /**paramsMap.put("FormKey1", "FormValue1"); paramsMap.put("FormKey2", "FormValue2");*/ byte[] inputStreamBytes = new byte[] {}; String gatewaySign = headers.get("x-ca-proxy-signature"); System.out.println("API??:" + gatewaySign); String serviceSign = serviceSign(uri, httpMethod, headers, paramsMap, inputStreamBytes); System.out.println("???:" + serviceSign); System.out.println("????:" + gatewaySign.equals(serviceSign)); }
From source file:com.ibm.watson.developer_cloud.conversation_tone_analyzer_integration.v1.ToneConversationIntegrationV1.java
public static void main(String[] args) throws Exception { // load the properties file Properties props = new Properties(); props.load(FileUtils.openInputStream(new File("tone_conversation_integration.properties"))); // instantiate the conversation service ConversationService conversationService = new ConversationService( ConversationService.VERSION_DATE_2016_07_11); conversationService.setUsernameAndPassword( props.getProperty("CONVERSATION_USERNAME", "<conversation_username>"), props.getProperty("CONVERSATION_PASSWORD", "<conversation_password>")); // instantiate the tone analyzer service ToneAnalyzer toneService = new ToneAnalyzer(ToneAnalyzer.VERSION_DATE_2016_05_19); toneService.setUsernameAndPassword(props.getProperty("TONE_ANALYZER_USERNAME", "<tone_analyzer_username>"), props.getProperty("TONE_ANALYZER_PASSWORD", "<tone_analyzer_password>")); // workspace id final String workspaceId = props.getProperty("WORKSPACE_ID", "<workspace_id>"); // maintain history in the context variable - will add a history variable to // each of the emotion, social and language tones final Boolean maintainHistory = false; /**/*from ww w. ja v a 2s. c o m*/ * Input for the conversation service: input (String): an input string (the user's conversation turn) and context * (Map<String,Object>: any context that needs to be maintained - either added by the client app or passed in the * response from the conversation service on the previous conversation turn. */ final String input = "I am happy"; final Map<String, Object> context = new HashMap<String, Object>(); // UPDATE CONTEXT HERE IF CONTINUING AN ONGOING CONVERSATION // set local context variable to the context from the last response from the // Conversation Service // (see the getContext() method of the MessageResponse class in // com.ibm.watson.developer_cloud.conversation.v1.model) // async call to Tone Analyzer toneService.getTone(input, null).enqueue(new ServiceCallback<ToneAnalysis>() { @Override public void onResponse(ToneAnalysis toneResponsePayload) { // update context with the tone data returned by the Tone Analyzer ToneDetection.updateUserTone(context, toneResponsePayload, maintainHistory); // call Conversation Service with the input and tone-aware context MessageRequest newMessage = new MessageRequest.Builder().inputText(input).context(context).build(); conversationService.message(workspaceId, newMessage) .enqueue(new ServiceCallback<MessageResponse>() { @Override public void onResponse(MessageResponse response) { System.out.println(response); } @Override public void onFailure(Exception e) { } }); } @Override public void onFailure(Exception e) { } }); }
From source file:com.nimbits.MainClass.java
public static void main(final String[] args) throws IOException, XMPPException, NimbitsException { final HashMap<String, String> argsMap = new HashMap<String, String>(); if (args == null || args.length == 0) { printUsage();//from w ww .ja v a 2 s. co m return; } else { processArgs(args, argsMap); } if (argsMap.containsKey(Parameters.i.getText())) { String[] fileArgs = KeyFile.processKeyFile(argsMap); processArgs(fileArgs, argsMap); } final boolean verbose = argsMap.containsKey(Parameters.verbose.getText()); final boolean listen = argsMap.containsKey(Parameters.listen.getText()); final String host = argsMap.containsKey(Parameters.host.getText()) ? argsMap.get(Parameters.host.getText()) : Path.PATH_NIMBITS_PUBLIC_SERVER; final String emailParam = argsMap.containsKey(Parameters.email.getText()) ? argsMap.get(Parameters.email.getText()) : null; final String key = argsMap.containsKey(Parameters.key.getText()) ? argsMap.get(Parameters.key.getText()) : null; final String appId = argsMap.containsKey(Parameters.appid.getText()) ? argsMap.get(Parameters.appid.getText()) : null; final String password = argsMap.containsKey(Parameters.password.getText()) ? argsMap.get(Parameters.password.getText()) : null; final String protocol = argsMap.containsKey(Parameters.protocol.getText()) ? argsMap.get(Parameters.protocol.getText()) : null; if (StringUtils.isEmpty(emailParam)) { throw new NimbitsException("you must specify an account i.e. email=test@example.com"); } if (StringUtils.isNotEmpty(host) && StringUtils.isNotEmpty(emailParam) & StringUtils.isNotEmpty(key)) { createClient(host, emailParam, key, password); } // // if (!loggedIn) { // out(true, "Access Denied."); // return; // } if (argsMap.containsKey(Parameters.action.getText())) { Action action = Action.valueOf(argsMap.get(Parameters.action.getText())); switch (action) { case read: case readValue: case readGps: case readJson: case readNote: readValue(argsMap, action); break; case record: case recordValue: recordValue(argsMap, verbose); break; case listen: listen(appId, protocol, emailParam); break; default: printUsage(); } } else if (argsMap.containsKey(Parameters.genkey.getText()) && argsMap.containsKey(Parameters.out.getText())) { out(true, KeyFile.genKey(argsMap)); } out(true, "exiting"); }
From source file:com.alibaba.jstorm.utils.FileAttribute.java
public static void main(String[] args) { Map<String, FileAttribute> map = new HashMap<String, FileAttribute>(); FileAttribute attribute = new FileAttribute(); attribute.setFileName("test"); attribute.setIsDir("true"); attribute.setModifyTime(new Date().toString()); attribute.setSize("4096"); map.put("test", attribute); System.out.println("Before:" + map); String jsonString = JStormUtils.to_json(map); Map<String, Map> map2 = (Map<String, Map>) JStormUtils.from_json(jsonString); Map jObject = map2.get("test"); FileAttribute attribute2 = FileAttribute.fromJSONObject(jObject); System.out.println("attribute2:" + attribute2); }
From source file:fr.inria.atlanmod.kyanos.benchmarks.XmiTraverser.java
public static void main(String[] args) { Options options = new Options(); Option inputOpt = OptionBuilder.create(IN); inputOpt.setArgName("INPUT"); inputOpt.setDescription("Input model"); inputOpt.setArgs(1);/*from w w w .j a va2 s. com*/ inputOpt.setRequired(true); Option inClassOpt = OptionBuilder.create(EPACKAGE_CLASS); inClassOpt.setArgName("CLASS"); inClassOpt.setDescription("FQN of EPackage implementation class"); inClassOpt.setArgs(1); inClassOpt.setRequired(true); options.addOption(inputOpt); options.addOption(inClassOpt); CommandLineParser parser = new PosixParser(); try { CommandLine commandLine = parser.parse(options, args); URI uri = URI.createFileURI(commandLine.getOptionValue(IN)); Class<?> inClazz = XmiTraverser.class.getClassLoader() .loadClass(commandLine.getOptionValue(EPACKAGE_CLASS)); inClazz.getMethod("init").invoke(null); ResourceSet resourceSet = new ResourceSetImpl(); resourceSet.getResourceFactoryRegistry().getExtensionToFactoryMap().put("xmi", new XMIResourceFactoryImpl()); resourceSet.getResourceFactoryRegistry().getExtensionToFactoryMap().put("zxmi", new XMIResourceFactoryImpl()); Resource resource = resourceSet.createResource(uri); Map<String, Object> loadOpts = new HashMap<String, Object>(); resource.load(loadOpts); LOG.log(Level.INFO, "Start counting"); int count = 0; long begin = System.currentTimeMillis(); for (Iterator<EObject> iterator = resource.getAllContents(); iterator.hasNext(); iterator .next(), count++) ; long end = System.currentTimeMillis(); LOG.log(Level.INFO, "End counting"); LOG.log(Level.INFO, MessageFormat.format("Resource {0} contains {1} elements", uri, count)); LOG.log(Level.INFO, MessageFormat.format("Time spent: {0}", MessageUtil.formatMillis(end - begin))); resource.unload(); } catch (ParseException e) { MessageUtil.showError(e.toString()); MessageUtil.showError("Current arguments: " + Arrays.toString(args)); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("java -jar <this-file.jar>", options, true); } catch (Throwable e) { MessageUtil.showError(e.toString()); } }
From source file:ch.epfl.lsir.xin.test.MostPopularTest.java
/** * @param args//w ww . j ava 2 s. c o m */ public static void main(String[] args) throws Exception { // TODO Auto-generated method stub PrintWriter logger = new PrintWriter(".//results//MostPopular"); PropertiesConfiguration config = new PropertiesConfiguration(); config.setFile(new File(".//conf//MostPopular.properties")); try { config.load(); } catch (ConfigurationException e) { // TODO Auto-generated catch block e.printStackTrace(); } logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + " Read rating data..."); DataLoaderFile loader = new DataLoaderFile(".//data//MoveLens100k.txt"); loader.readSimple(); DataSetNumeric dataset = loader.getDataset(); System.out.println("Number of ratings: " + dataset.getRatings().size() + " Number of users: " + dataset.getUserIDs().size() + " Number of items: " + dataset.getItemIDs().size()); logger.println("Number of ratings: " + dataset.getRatings().size() + ", Number of users: " + dataset.getUserIDs().size() + ", Number of items: " + dataset.getItemIDs().size()); logger.flush(); TrainTestSplitter splitter = new TrainTestSplitter(dataset); splitter.splitFraction(config.getDouble("TRAIN_FRACTION")); ArrayList<NumericRating> trainRatings = splitter.getTrain(); ArrayList<NumericRating> testRatings = splitter.getTest(); HashMap<String, Integer> userIDIndexMapping = new HashMap<String, Integer>(); HashMap<String, Integer> itemIDIndexMapping = new HashMap<String, Integer>(); //create rating matrix for (int i = 0; i < dataset.getUserIDs().size(); i++) { userIDIndexMapping.put(dataset.getUserIDs().get(i), i); } for (int i = 0; i < dataset.getItemIDs().size(); i++) { itemIDIndexMapping.put(dataset.getItemIDs().get(i), i); } RatingMatrix trainRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(), dataset.getItemIDs().size()); for (int i = 0; i < trainRatings.size(); i++) { trainRatingMatrix.set(userIDIndexMapping.get(trainRatings.get(i).getUserID()), itemIDIndexMapping.get(trainRatings.get(i).getItemID()), trainRatings.get(i).getValue()); } RatingMatrix testRatingMatrix = new RatingMatrix(dataset.getUserIDs().size(), dataset.getItemIDs().size()); for (int i = 0; i < testRatings.size(); i++) { //only consider 5-star rating in the test set // if( testRatings.get(i).getValue() < 5 ) // continue; testRatingMatrix.set(userIDIndexMapping.get(testRatings.get(i).getUserID()), itemIDIndexMapping.get(testRatings.get(i).getItemID()), testRatings.get(i).getValue()); } System.out.println("Training: " + trainRatingMatrix.getTotalRatingNumber() + " vs Test: " + testRatingMatrix.getTotalRatingNumber()); logger.println("Initialize a most popular based recommendation model."); MostPopular algo = new MostPopular(trainRatingMatrix); algo.setLogger(logger); algo.build(); algo.saveModel(".//localModels//" + config.getString("NAME")); logger.println("Save the model."); logger.flush(); HashMap<Integer, ArrayList<ResultUnit>> results = new HashMap<Integer, ArrayList<ResultUnit>>(); for (int i = 0; i < testRatingMatrix.getRow(); i++) { ArrayList<ResultUnit> rec = algo.getRecommendationList(i); if (rec == null) continue; int total = testRatingMatrix.getUserRatingNumber(i); if (total == 0)//this user is ignored continue; results.put(i, rec); } RankResultGenerator generator = new RankResultGenerator(results, algo.getTopN(), testRatingMatrix, trainRatingMatrix); System.out.println("Precision@N: " + generator.getPrecisionN()); System.out.println("Recall@N: " + generator.getRecallN()); System.out.println("MAP@N: " + generator.getMAPN()); System.out.println("MRR@N: " + generator.getMRRN()); System.out.println("NDCG@N: " + generator.getNDCGN()); System.out.println("AUC@N: " + generator.getAUC()); logger.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "\n" + "Precision@N: " + generator.getPrecisionN() + "\n" + "Recall@N: " + generator.getRecallN() + "\n" + "MAP@N: " + generator.getMAPN() + "\n" + "MRR@N: " + generator.getMRRN() + "\n" + "NDCG@N: " + generator.getNDCGN() + "\n" + "AUC@N: " + generator.getAUC()); logger.flush(); logger.close(); }
From source file:SparkStreaming.java
public static void main(String[] args) throws Exception { JSONObject jo = new JSONObject(); SparkConf sparkConf = new SparkConf().setAppName("mytestapp").setMaster("local[2]"); JavaStreamingContext jssc = new JavaStreamingContext(sparkConf, new Duration(2000)); int numThreads = Integer.parseInt("2"); Map<String, Integer> topicMap = new HashMap<>(); String[] topics = "testdemo".split(","); for (String topic : topics) { topicMap.put(topic, numThreads); }//from w w w . j ava 2s. c o m JavaPairReceiverInputDStream<String, String> messages = KafkaUtils.createStream(jssc, "localhost:2181", "testdemo", topicMap); JavaDStream<String> lines = messages.map(new Function<Tuple2<String, String>, String>() { @Override public String call(Tuple2<String, String> tuple2) { return tuple2._2(); } }); JavaDStream<String> words = lines.flatMap(new FlatMapFunction<String, String>() { @Override public Iterable<String> call(String x) throws IOException { String temperature = x.substring(x.indexOf(":") + 1); /* System.out.println("Temperature from spark++++++++++ "+temperature); URL url = new URL("http://localhost:8080/apachetest/Test?var1="+temperature); URLConnection conn = url.openConnection(); conn.setDoOutput(true);*/ final ThermometerDemo2 demo = new ThermometerDemo2("Thermometer Demo 2", temperature); demo.pack(); demo.setVisible(true); return Arrays.asList(x.split(" ")); } }); JavaPairDStream<String, Integer> wordCounts = words.mapToPair(new PairFunction<String, String, Integer>() { @Override public Tuple2<String, Integer> call(String s) { return new Tuple2<>(s, 1); } }).reduceByKey(new Function2<Integer, Integer, Integer>() { @Override public Integer call(Integer i1, Integer i2) { return i1 + i2; } }); wordCounts.print(); jssc.start(); jssc.awaitTermination(); }
From source file:de.tudarmstadt.ukp.experiments.dip.wp1.documents.Step8GoldDataAggregator.java
public static void main(String[] args) throws Exception { String inputDir = args[0] + "/"; // output dir File outputDir = new File(args[1]); File turkersConfidence = new File(args[2]); if (outputDir.exists()) { outputDir.delete();//w w w .j a v a2 s . co m } outputDir.mkdir(); List<String> annotatorsIDs = new ArrayList<>(); // for (File f : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { // QueryResultContainer queryResultContainer = QueryResultContainer // .fromXML(FileUtils.readFileToString(f, "utf-8")); // for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { // for (QueryResultContainer.MTurkRelevanceVote relevanceVote : rankedResults.mTurkRelevanceVotes) { // if (!annotatorsIDs.contains(relevanceVote.turkID)) // annotatorsIDs.add(relevanceVote.turkID); // } // } // } HashMap<String, Integer> countVotesForATurker = new HashMap<>(); // creates temporary file with format for mace // Hashmap annotations: key is the id of a document and a sentence // Value is an array votes[] of turkers decisions: true or false (relevant or not) // the length of this array equals the number of annotators in List<String> annotatorsIDs. // If an annotator worked on the task his decision is written in the array otherwise the value is NULL // key: queryID + clueWebID + sentenceID // value: true and false annotations TreeMap<String, Annotations> annotations = new TreeMap<>(); for (File f : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(f, "utf-8")); System.out.println("Reading " + f.getName()); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { String documentID = rankedResults.clueWebID; for (QueryResultContainer.MTurkRelevanceVote relevanceVote : rankedResults.mTurkRelevanceVotes) { Integer turkerID; if (!annotatorsIDs.contains(relevanceVote.turkID)) { annotatorsIDs.add(relevanceVote.turkID); turkerID = annotatorsIDs.size() - 1; } else { turkerID = annotatorsIDs.indexOf(relevanceVote.turkID); } Integer count = countVotesForATurker.get(relevanceVote.turkID); if (count == null) { count = 0; } count++; countVotesForATurker.put(relevanceVote.turkID, count); String id; List<Integer> trueVotes; List<Integer> falseVotes; for (QueryResultContainer.SingleSentenceRelevanceVote singleSentenceRelevanceVote : relevanceVote.singleSentenceRelevanceVotes) if (!"".equals(singleSentenceRelevanceVote.sentenceID)) { id = f.getName() + "_" + documentID + "_" + singleSentenceRelevanceVote.sentenceID; Annotations turkerVotes = annotations.get(id); if (turkerVotes == null) { trueVotes = new ArrayList<>(); falseVotes = new ArrayList<>(); turkerVotes = new Annotations(trueVotes, falseVotes); } trueVotes = turkerVotes.trueAnnotations; falseVotes = turkerVotes.falseAnnotations; if ("true".equals(singleSentenceRelevanceVote.relevant)) { // votes[turkerID] = true; trueVotes.add(turkerID); } else if ("false".equals(singleSentenceRelevanceVote.relevant)) { // votes[turkerID] = false; falseVotes.add(turkerID); } else { throw new IllegalStateException("Annotation value of sentence " + singleSentenceRelevanceVote.sentenceID + " in " + rankedResults.clueWebID + " equals " + singleSentenceRelevanceVote.relevant); } try { int allVotesCount = trueVotes.size() + falseVotes.size(); if (allVotesCount > 5) { System.err.println(id + " doesn't have 5 annotators: true: " + trueVotes.size() + " false: " + falseVotes.size()); // nasty hack, we're gonna strip some data; true votes first /* we can't do that, it breaks something down the line int toRemove = allVotesCount - 5; if (trueVotes.size() >= toRemove) { trueVotes = trueVotes .subList(0, trueVotes.size() - toRemove); } else if ( falseVotes.size() >= toRemove) { falseVotes = falseVotes .subList(0, trueVotes.size() - toRemove); } */ System.err.println("Adjusted: " + id + " doesn't have 5 annotators: true: " + trueVotes.size() + " false: " + falseVotes.size()); } } catch (IllegalStateException e) { e.printStackTrace(); } turkerVotes.trueAnnotations = trueVotes; turkerVotes.falseAnnotations = falseVotes; annotations.put(id, turkerVotes); } else { throw new IllegalStateException( "Empty Sentence ID in " + f.getName() + " for turker " + turkerID); } } } } File tmp = printHashMap(annotations, annotatorsIDs.size()); String file = TEMP_DIR + "/" + tmp.getName(); MACE.main(new String[] { "--prefix", file }); //gets the keys of the documents and sentences ArrayList<String> lines = (ArrayList<String>) FileUtils.readLines(new File(file + ".prediction")); int i = 0; TreeMap<String, TreeMap<String, ArrayList<HashMap<String, String>>>> ids = new TreeMap<>(); ArrayList<HashMap<String, String>> sentences; if (lines.size() != annotations.size()) { throw new IllegalStateException( "The size of prediction file is " + lines.size() + "but expected " + annotations.size()); } for (Map.Entry entry : annotations.entrySet()) { //1001.xml_clueweb12-1905wb-13-07360_8783 String key = (String) entry.getKey(); String[] IDs = key.split("_"); if (IDs.length > 2) { String queryID = IDs[0]; String clueWebID = IDs[1]; String sentenceID = IDs[2]; TreeMap<String, ArrayList<HashMap<String, String>>> clueWebIDs = ids.get(queryID); if (clueWebIDs == null) { clueWebIDs = new TreeMap<>(); } sentences = clueWebIDs.get(clueWebID); if (sentences == null) { sentences = new ArrayList<>(); } HashMap<String, String> sentence = new HashMap<>(); sentence.put(sentenceID, lines.get(i)); sentences.add(sentence); clueWebIDs.put(clueWebID, sentences); ids.put(queryID, clueWebIDs); } else { throw new IllegalStateException("Wrong ID " + key); } i++; } for (Map.Entry entry : ids.entrySet()) { TreeMap<Integer, String> value = (TreeMap<Integer, String>) entry.getValue(); String queryID = (String) entry.getKey(); QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(new File(inputDir, queryID), "utf-8")); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { for (Map.Entry val : value.entrySet()) { String clueWebID = (String) val.getKey(); if (clueWebID.equals(rankedResults.clueWebID)) { List<QueryResultContainer.SingleSentenceRelevanceVote> goldEstimatedLabels = new ArrayList<>(); List<QueryResultContainer.SingleSentenceRelevanceVote> turkersVotes = new ArrayList<>(); int size = 0; int hitSize = 0; String hitID = ""; for (QueryResultContainer.MTurkRelevanceVote vote : rankedResults.mTurkRelevanceVotes) { if (!hitID.equals(vote.hitID)) { hitID = vote.hitID; hitSize = vote.singleSentenceRelevanceVotes.size(); size = size + hitSize; turkersVotes.addAll(vote.singleSentenceRelevanceVotes); } else { if (vote.singleSentenceRelevanceVotes.size() != hitSize) { hitSize = vote.singleSentenceRelevanceVotes.size(); size = size + hitSize; turkersVotes.addAll(vote.singleSentenceRelevanceVotes); } } } ArrayList<HashMap<String, String>> sentenceList = (ArrayList<HashMap<String, String>>) val .getValue(); if (sentenceList.size() != turkersVotes.size()) { try { throw new IllegalStateException("Expected size of annotations is " + turkersVotes.size() + "but found " + sentenceList.size() + " for document " + rankedResults.clueWebID + " in " + queryID); } catch (IllegalStateException ex) { ex.printStackTrace(); } } for (QueryResultContainer.SingleSentenceRelevanceVote s : turkersVotes) { String valSentence = null; for (HashMap<String, String> anno : sentenceList) { if (anno.keySet().contains(s.sentenceID)) { valSentence = anno.get(s.sentenceID); } } QueryResultContainer.SingleSentenceRelevanceVote singleSentenceVote = new QueryResultContainer.SingleSentenceRelevanceVote(); singleSentenceVote.sentenceID = s.sentenceID; if (("false").equals(valSentence)) { singleSentenceVote.relevant = "false"; } else if (("true").equals(valSentence)) { singleSentenceVote.relevant = "true"; } else { throw new IllegalStateException("Annotation value of sentence " + singleSentenceVote.sentenceID + " equals " + val.getValue()); } goldEstimatedLabels.add(singleSentenceVote); } rankedResults.goldEstimatedLabels = goldEstimatedLabels; } } } File outputFile = new File(outputDir, queryID); FileUtils.writeStringToFile(outputFile, queryResultContainer.toXML(), "utf-8"); System.out.println("Finished " + outputFile); } ArrayList<String> annotators = (ArrayList<String>) FileUtils.readLines(new File(file + ".competence")); FileWriter fileWriter; StringBuilder sb = new StringBuilder(); for (int j = 0; j < annotatorsIDs.size(); j++) { String[] s = annotators.get(0).split("\t"); Float score = Float.parseFloat(s[j]); String turkerID = annotatorsIDs.get(j); System.out.println(turkerID + " " + score + " " + countVotesForATurker.get(turkerID)); sb.append(turkerID).append(" ").append(score).append(" ").append(countVotesForATurker.get(turkerID)) .append("\n"); } fileWriter = new FileWriter(turkersConfidence); fileWriter.append(sb.toString()); fileWriter.close(); }