List of usage examples for java.lang IllegalStateException IllegalStateException
public IllegalStateException(Throwable cause)
From source file:de.tudarmstadt.ukp.experiments.dip.wp1.documents.Step5LinguisticPreprocessing.java
public static void main(String[] args) throws Exception { // input dir - list of xml query containers // step4-boiler-plate/ File inputDir = new File(args[0]); // output dir File outputDir = new File(args[1]); if (!outputDir.exists()) { outputDir.mkdirs();/* ww w . ja va 2 s. co m*/ } // iterate over query containers for (File f : FileUtils.listFiles(inputDir, new String[] { "xml" }, false)) { QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(f, "utf-8")); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { // System.out.println(rankedResults.plainText); if (rankedResults.plainText != null) { String[] lines = StringUtils.split(rankedResults.plainText, "\n"); // collecting all cleaned lines List<String> cleanLines = new ArrayList<>(lines.length); // collecting line tags List<String> lineTags = new ArrayList<>(lines.length); for (String line : lines) { // get the tag String tag = null; Matcher m = OPENING_TAG_PATTERN.matcher(line); if (m.find()) { tag = m.group(1); } if (tag == null) { throw new IllegalArgumentException("No html tag found for line:\n" + line); } // replace the tag at the beginning and the end String noTagText = line.replaceAll("^<\\S+>", "").replaceAll("</\\S+>$", ""); // do some html cleaning noTagText = noTagText.replaceAll(" ", " "); noTagText = noTagText.trim(); // add to the output if (!noTagText.isEmpty()) { cleanLines.add(noTagText); lineTags.add(tag); } } if (cleanLines.isEmpty()) { // the document is empty System.err.println("Document " + rankedResults.clueWebID + " in query " + queryResultContainer.qID + " is empty"); } else { // now join them back to paragraphs String text = StringUtils.join(cleanLines, "\n"); // create JCas JCas jCas = JCasFactory.createJCas(); jCas.setDocumentText(text); jCas.setDocumentLanguage("en"); // annotate WebParagraph SimplePipeline.runPipeline(jCas, AnalysisEngineFactory.createEngineDescription(WebParagraphAnnotator.class)); // fill the original tag information List<WebParagraph> webParagraphs = new ArrayList<>( JCasUtil.select(jCas, WebParagraph.class)); // they must be the same size as original ones if (webParagraphs.size() != lineTags.size()) { throw new IllegalStateException( "Different size of annotated paragraphs and original lines"); } for (int i = 0; i < webParagraphs.size(); i++) { WebParagraph p = webParagraphs.get(i); // get tag String tag = lineTags.get(i); p.setOriginalHtmlTag(tag); } SimplePipeline.runPipeline(jCas, AnalysisEngineFactory.createEngineDescription(StanfordSegmenter.class, // only on existing WebParagraph annotations StanfordSegmenter.PARAM_ZONE_TYPES, WebParagraph.class.getCanonicalName())); // now convert to XMI ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(); XmiCasSerializer.serialize(jCas.getCas(), byteOutputStream); // encode to base64 String encoded = new BASE64Encoder().encode(byteOutputStream.toByteArray()); rankedResults.originalXmi = encoded; } } } // and save the query to output dir File outputFile = new File(outputDir, queryResultContainer.qID + ".xml"); FileUtils.writeStringToFile(outputFile, queryResultContainer.toXML(), "utf-8"); System.out.println("Finished " + outputFile); } }
From source file:eu.sendregning.oxalis.Main.java
public static void main(String[] args) throws Exception { OptionParser optionParser = getOptionParser(); if (args.length == 0) { System.out.println(""); optionParser.printHelpOn(System.out); System.out.println(""); return;//ww w .j a v a 2 s. c o m } OptionSet optionSet; try { optionSet = optionParser.parse(args); } catch (Exception e) { printErrorMessage(e.getMessage()); return; } File xmlInvoice = xmlDocument.value(optionSet); if (!xmlInvoice.exists()) { printErrorMessage("XML document " + xmlInvoice + " does not exist"); return; } String recipientId = recipient.value(optionSet); String senderId = sender.value(optionSet); try { System.out.println(""); System.out.println(""); // bootstraps the Oxalis outbound module OxalisOutboundModule oxalisOutboundModule = new OxalisOutboundModule(); // creates a transmission request builder and enable tracing TransmissionRequestBuilder requestBuilder = oxalisOutboundModule.getTransmissionRequestBuilder(); requestBuilder.trace(trace.value(optionSet)); System.out.println("Trace mode of RequestBuilder: " + requestBuilder.isTraceEnabled()); // add receiver participant if (recipientId != null) { requestBuilder.receiver(new ParticipantId(recipientId)); } // add sender participant if (senderId != null) { requestBuilder.sender((new ParticipantId(senderId))); } if (docType != null && docType.value(optionSet) != null) { requestBuilder.documentType(PeppolDocumentTypeId.valueOf(docType.value(optionSet))); } if (profileType != null && profileType.value(optionSet) != null) { requestBuilder.processType(PeppolProcessTypeId.valueOf(profileType.value(optionSet))); } // Supplies the payload requestBuilder.payLoad(new FileInputStream(xmlInvoice)); // Overrides the destination URL if so requested if (optionSet.has(destinationUrl)) { String destinationString = destinationUrl.value(optionSet); URL destination; try { destination = new URL(destinationString); } catch (MalformedURLException e) { printErrorMessage("Invalid destination URL " + destinationString); return; } // Fetches the transmission method, which was overridden on the command line BusDoxProtocol busDoxProtocol = BusDoxProtocol.instanceFrom(transmissionMethod.value(optionSet)); if (busDoxProtocol == BusDoxProtocol.AS2) { String accessPointSystemIdentifier = destinationSystemId.value(optionSet); if (accessPointSystemIdentifier == null) { throw new IllegalStateException("Must specify AS2 system identifier if using AS2 protocol"); } requestBuilder.overrideAs2Endpoint(destination, accessPointSystemIdentifier); } else { throw new IllegalStateException("Unknown busDoxProtocol : " + busDoxProtocol); } } // Specifying the details completed, creates the transmission request TransmissionRequest transmissionRequest = requestBuilder.build(); // Fetches a transmitter ... Transmitter transmitter = oxalisOutboundModule.getTransmitter(); // ... and performs the transmission TransmissionResponse transmissionResponse = transmitter.transmit(transmissionRequest); // Write the transmission id and where the message was delivered System.out.printf("Message using messageId %s sent to %s using %s was assigned transmissionId %s\n", transmissionResponse.getStandardBusinessHeader().getMessageId().stringValue(), transmissionResponse.getURL().toExternalForm(), transmissionResponse.getProtocol().toString(), transmissionResponse.getTransmissionId()); String evidenceFileName = transmissionResponse.getTransmissionId().toString() + "-evidence.dat"; IOUtils.copy(new ByteArrayInputStream(transmissionResponse.getEvidenceBytes()), new FileOutputStream(evidenceFileName)); System.out.printf("Wrote transmission receipt to " + evidenceFileName); } catch (Exception e) { System.out.println(""); System.out.println("Message failed : " + e.getMessage()); //e.printStackTrace(); System.out.println(""); } }
From source file:de.tudarmstadt.ukp.experiments.argumentation.convincingness.sampling.Step4MTurkOutputCollector.java
@SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { String inputDirWithArgumentPairs = args[0]; File[] resultFiles;/*from ww w . ja v a 2 s . c om*/ if (args[1].contains("*")) { File path = new File(args[1]); File directory = path.getParentFile(); String regex = path.getName().replaceAll("\\*", ""); List<File> files = new ArrayList<>(FileUtils.listFiles(directory, new String[] { regex }, false)); resultFiles = new File[files.size()]; for (int i = 0; i < files.size(); i++) { resultFiles[i] = files.get(i); } } else { // result file is a comma-separated list of CSV files from MTurk String[] split = args[1].split(","); resultFiles = new File[split.length]; for (int i = 0; i < split.length; i++) { resultFiles[i] = new File(split[i]); } } File outputDir = new File(args[2]); if (!outputDir.exists()) { if (!outputDir.mkdirs()) { throw new IOException("Cannot create directory " + outputDir); } } // error if output folder not empty to prevent any confusion by mixing files if (!FileUtils.listFiles(outputDir, null, false).isEmpty()) { throw new IllegalArgumentException("Output dir " + outputDir + " is not empty"); } // collected assignments with empty reason for rejections Set<String> assignmentsWithEmptyReason = new HashSet<>(); // parse with first line as header MTurkOutputReader mTurkOutputReader = new MTurkOutputReader(resultFiles); Collection<File> files = FileUtils.listFiles(new File(inputDirWithArgumentPairs), new String[] { "xml" }, false); if (files.isEmpty()) { throw new IOException("No xml files found in " + inputDirWithArgumentPairs); } // statistics: how many hits with how many assignments ; hit ID / assignments Map<String, Map<String, Integer>> assignmentsPerHits = new HashMap<>(); // collect accept/reject statistics for (Map<String, String> record : mTurkOutputReader) { boolean wasRejected = "Rejected".equals(record.get("assignmentstatus")); String hitID = record.get("hitid"); String hitTypeId = record.get("hittypeid"); if (!wasRejected) { // update statistics if (!assignmentsPerHits.containsKey(hitTypeId)) { assignmentsPerHits.put(hitTypeId, new HashMap<String, Integer>()); } if (!assignmentsPerHits.get(hitTypeId).containsKey(hitID)) { assignmentsPerHits.get(hitTypeId).put(hitID, 0); } assignmentsPerHits.get(hitTypeId).put(hitID, assignmentsPerHits.get(hitTypeId).get(hitID) + 1); } } // statistics: how many hits with how many assignments ; hit ID / assignments Map<String, Integer> approvedAssignmentsPerHit = new HashMap<>(); Map<String, Integer> rejectedAssignmentsPerHit = new HashMap<>(); // collect accept/reject statistics for (Map<String, String> record : mTurkOutputReader) { boolean approved = "Approved".equals(record.get("assignmentstatus")); boolean rejected = "Rejected".equals(record.get("assignmentstatus")); String hitID = record.get("hitid"); if (approved) { // update statistics if (!approvedAssignmentsPerHit.containsKey(hitID)) { approvedAssignmentsPerHit.put(hitID, 0); } approvedAssignmentsPerHit.put(hitID, approvedAssignmentsPerHit.get(hitID) + 1); } else if (rejected) { // update statistics if (!rejectedAssignmentsPerHit.containsKey(hitID)) { rejectedAssignmentsPerHit.put(hitID, 0); } rejectedAssignmentsPerHit.put(hitID, rejectedAssignmentsPerHit.get(hitID) + 1); } else { throw new IllegalStateException( "Unknown state: " + record.get("assignmentstatus") + " HITID: " + hitID); } } // System.out.println("Approved: " + approvedAssignmentsPerHit); // System.out.println("Rejected: " + rejectedAssignmentsPerHit); System.out.println("Approved (values): " + new HashSet<>(approvedAssignmentsPerHit.values())); System.out.println("Rejected (values): " + new HashSet<>(rejectedAssignmentsPerHit.values())); // rejection statistics int totalRejected = 0; for (Map.Entry<String, Integer> rejectionEntry : rejectedAssignmentsPerHit.entrySet()) { totalRejected += rejectionEntry.getValue(); } System.out.println("Total rejections: " + totalRejected); /* // generate .success files for adding more annotations for (File resultFile : resultFiles) { String hitTypeID = mTurkOutputReader.getHitTypeIdForFile().get(resultFile); // assignments for that hittypeid (= file) Map<String, Integer> assignments = assignmentsPerHits.get(hitTypeID); prepareUpdateHITsFiles(assignments, hitTypeID, resultFile); } */ int totalSavedPairs = 0; // load all previously prepared argument pairs for (File file : files) { List<ArgumentPair> argumentPairs = (List<ArgumentPair>) XStreamTools.getXStream().fromXML(file); List<AnnotatedArgumentPair> annotatedArgumentPairs = new ArrayList<>(); for (ArgumentPair argumentPair : argumentPairs) { AnnotatedArgumentPair annotatedArgumentPair = new AnnotatedArgumentPair(argumentPair); // is there such an answer? String key = "Answer." + argumentPair.getId(); // iterate only if there is such column to save time if (mTurkOutputReader.getColumnNames().contains(key)) { // now find the results for (Map<String, String> record : mTurkOutputReader) { if (record.containsKey(key)) { // extract the values AnnotatedArgumentPair.MTurkAssignment assignment = new AnnotatedArgumentPair.MTurkAssignment(); boolean wasRejected = "Rejected".equals(record.get("assignmentstatus")); // only non-rejected (if required) if (!wasRejected) { String hitID = record.get("hitid"); String workerID = record.get("workerid"); String assignmentId = record.get("assignmentid"); try { assignment.setAssignmentAcceptTime( DATE_FORMAT.parse(record.get("assignmentaccepttime"))); assignment.setAssignmentSubmitTime( DATE_FORMAT.parse(record.get("assignmentsubmittime"))); assignment.setHitComment(record.get("Answer.feedback")); assignment.setHitID(hitID); assignment.setTurkID(workerID); assignment.setAssignmentId(assignmentId); // and answer specific fields String valueRaw = record.get(key); // so far the label has had format aXXX_aYYY_a1, aXXX_aYYY_a2, or aXXX_aYYY_equal // strip now only true label String label = valueRaw.split("_")[2]; assignment.setValue(label); String reason = record.get(key + "_reason"); // missing reason if (reason == null) { assignmentsWithEmptyReason.add(assignmentId); } else { assignment.setReason(reason); // get worker's stance String stanceRaw = record.get(key + "_stance"); if (stanceRaw != null) { // parse stance String stance = stanceRaw.split("_stance_")[1]; assignment.setWorkerStance(stance); } // we take maximal 5 assignments Collections.sort(annotatedArgumentPair.mTurkAssignments, new Comparator<AnnotatedArgumentPair.MTurkAssignment>() { @Override public int compare(AnnotatedArgumentPair.MTurkAssignment o1, AnnotatedArgumentPair.MTurkAssignment o2) { return o1.getAssignmentAcceptTime() .compareTo(o2.getAssignmentAcceptTime()); } }); if (annotatedArgumentPair.mTurkAssignments .size() < MAXIMUM_ASSIGNMENTS_PER_HIT) { annotatedArgumentPair.mTurkAssignments.add(assignment); } } } catch (IllegalArgumentException | NullPointerException ex) { System.err.println("Malformed annotations for HIT " + hitID + ", worker " + workerID + ", assignment " + assignmentId + "; " + ex.getMessage() + ", full record: " + record); } } } } } // and if there are some annotations, add it to the result set if (!annotatedArgumentPair.mTurkAssignments.isEmpty()) { annotatedArgumentPairs.add(annotatedArgumentPair); } } if (!annotatedArgumentPairs.isEmpty()) { File outputFile = new File(outputDir, file.getName()); XStreamTools.toXML(annotatedArgumentPairs, outputFile); System.out.println("Saved " + annotatedArgumentPairs.size() + " annotated pairs to " + outputFile); totalSavedPairs += annotatedArgumentPairs.size(); } } System.out.println("Total saved " + totalSavedPairs + " pairs"); // print assignments with empty reasons if (!assignmentsWithEmptyReason.isEmpty()) { System.out.println( "== Assignments with empty reason:\nassignmentIdToReject\tassignmentIdToRejectComment"); for (String assignmentId : assignmentsWithEmptyReason) { System.out.println( assignmentId + "\t\"Dear worker, you did not fill the required field with a reason.\""); } } }
From source file:com.hortonworks.registries.storage.tool.sql.TablesInitializer.java
public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption(Option.builder("s").numberOfArgs(1).longOpt(OPTION_SCRIPT_ROOT_PATH) .desc("Root directory of script path").build()); options.addOption(Option.builder("c").numberOfArgs(1).longOpt(OPTION_CONFIG_FILE_PATH) .desc("Config file path").build()); options.addOption(Option.builder("m").numberOfArgs(1).longOpt(OPTION_MYSQL_JAR_URL_PATH) .desc("Mysql client jar url to download").build()); options.addOption(Option.builder().hasArg(false).longOpt(SchemaMigrationOption.CREATE.toString()) .desc("Run sql migrations from scatch").build()); options.addOption(Option.builder().hasArg(false).longOpt(SchemaMigrationOption.DROP.toString()) .desc("Drop all the tables in the target database").build()); options.addOption(Option.builder().hasArg(false).longOpt(SchemaMigrationOption.CHECK_CONNECTION.toString()) .desc("Check the connection for configured data source").build()); options.addOption(Option.builder().hasArg(false).longOpt(SchemaMigrationOption.MIGRATE.toString()) .desc("Execute schema migration from last check point").build()); options.addOption(Option.builder().hasArg(false).longOpt(SchemaMigrationOption.INFO.toString()) .desc("Show the status of the schema migration compared to the target database").build()); options.addOption(Option.builder().hasArg(false).longOpt(SchemaMigrationOption.VALIDATE.toString()) .desc("Validate the target database changes with the migration scripts").build()); options.addOption(Option.builder().hasArg(false).longOpt(SchemaMigrationOption.REPAIR.toString()).desc( "Repairs the DATABASE_CHANGE_LOG by removing failed migrations and correcting checksum of existing migration script") .build());//from www. j a v a 2s.c o m options.addOption(Option.builder().hasArg(false).longOpt(DISABLE_VALIDATE_ON_MIGRATE) .desc("Disable flyway validation checks while running migrate").build()); CommandLineParser parser = new BasicParser(); CommandLine commandLine = parser.parse(options, args); if (!commandLine.hasOption(OPTION_CONFIG_FILE_PATH) || !commandLine.hasOption(OPTION_SCRIPT_ROOT_PATH)) { usage(options); System.exit(1); } boolean isSchemaMigrationOptionSpecified = false; SchemaMigrationOption schemaMigrationOptionSpecified = null; for (SchemaMigrationOption schemaMigrationOption : SchemaMigrationOption.values()) { if (commandLine.hasOption(schemaMigrationOption.toString())) { if (isSchemaMigrationOptionSpecified) { System.out.println( "Only one operation can be execute at once, please select one of 'create', ',migrate', 'validate', 'info', 'drop', 'repair', 'check-connection'."); System.exit(1); } isSchemaMigrationOptionSpecified = true; schemaMigrationOptionSpecified = schemaMigrationOption; } } if (!isSchemaMigrationOptionSpecified) { System.out.println( "One of the option 'create', ',migrate', 'validate', 'info', 'drop', 'repair', 'check-connection' must be specified to execute."); System.exit(1); } String confFilePath = commandLine.getOptionValue(OPTION_CONFIG_FILE_PATH); String scriptRootPath = commandLine.getOptionValue(OPTION_SCRIPT_ROOT_PATH); String mysqlJarUrl = commandLine.getOptionValue(OPTION_MYSQL_JAR_URL_PATH); StorageProviderConfiguration storageProperties; Map<String, Object> conf; try { conf = Utils.readConfig(confFilePath); StorageProviderConfigurationReader confReader = new StorageProviderConfigurationReader(); storageProperties = confReader.readStorageConfig(conf); } catch (IOException e) { System.err.println("Error occurred while reading config file: " + confFilePath); System.exit(1); throw new IllegalStateException("Shouldn't reach here"); } String bootstrapDirPath = null; try { bootstrapDirPath = System.getProperty("bootstrap.dir"); Proxy proxy = Proxy.NO_PROXY; String httpProxyUrl = (String) conf.get(HTTP_PROXY_URL); String httpProxyUsername = (String) conf.get(HTTP_PROXY_USERNAME); String httpProxyPassword = (String) conf.get(HTTP_PROXY_PASSWORD); if ((httpProxyUrl != null) && !httpProxyUrl.isEmpty()) { URL url = new URL(httpProxyUrl); proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(url.getHost(), url.getPort())); if ((httpProxyUsername != null) && !httpProxyUsername.isEmpty()) { Authenticator.setDefault(getBasicAuthenticator(url.getHost(), url.getPort(), httpProxyUsername, httpProxyPassword)); } } MySqlDriverHelper.downloadMySQLJarIfNeeded(storageProperties, bootstrapDirPath, mysqlJarUrl, proxy); } catch (Exception e) { System.err.println("Error occurred while downloading MySQL jar. bootstrap dir: " + bootstrapDirPath); System.exit(1); throw new IllegalStateException("Shouldn't reach here"); } boolean disableValidateOnMigrate = commandLine.hasOption(DISABLE_VALIDATE_ON_MIGRATE); if (disableValidateOnMigrate) { System.out.println("Disabling validation on schema migrate"); } SchemaMigrationHelper schemaMigrationHelper = new SchemaMigrationHelper( SchemaFlywayFactory.get(storageProperties, scriptRootPath, !disableValidateOnMigrate)); try { schemaMigrationHelper.execute(schemaMigrationOptionSpecified); System.out .println(String.format("\"%s\" option successful", schemaMigrationOptionSpecified.toString())); } catch (Exception e) { System.err.println( String.format("\"%s\" option failed : %s", schemaMigrationOptionSpecified.toString(), e)); System.exit(1); } }
From source file:com.lightboxtechnologies.nsrl.OCSVTest.java
public static void main(String[] args) throws IOException { final String mfg_filename = args[0]; final String os_filename = args[1]; final String prod_filename = args[2]; final String hash_filename = args[3]; final LineTokenizer tok = new OpenCSVLineTokenizer(); final RecordLoader loader = new RecordLoader(); final ErrorConsumer err = new ErrorConsumer() { public void consume(BadDataException e, long linenum) { System.err.println("malformed record, line " + linenum); e.printStackTrace();/*from w w w . j a v a 2 s .com*/ } }; // read manufacturer, OS, product data final Map<String, MfgData> mfg = new HashMap<String, MfgData>(); final Map<String, OSData> os = new HashMap<String, OSData>(); final Map<Integer, List<ProdData>> prod = new HashMap<Integer, List<ProdData>>(); SmallTableLoader.load(mfg_filename, mfg, os_filename, os, prod_filename, prod, tok, err); // read hash data final RecordConsumer<HashData> hcon = new RecordConsumer<HashData>() { public void consume(HashData hd) { /* System.out.print(hd); for (ProdData pd : prod.get(hd.prod_code)) { System.out.print(pd); final MfgData pmd = mfg.get(pd.mfg_code); System.out.print(pmd); } final OSData osd = os.get(hd.os_code); System.out.print(osd); final MfgData osmd = mfg.get(osd.mfg_code); System.out.print(osmd); System.out.println(""); */ /* final Hex hex = new Hex(); String sha1 = null; String md5 = null; String crc32 = null; try { sha1 = (String) hex.encode((Object) hd.sha1); md5 = (String) hex.encode((Object) hd.md5); crc32 = (String) hex.encode((Object) hd.crc32); } catch (EncoderException e) { throw new RuntimeException(e); } */ final ObjectMapper mapper = new ObjectMapper(); final String sha1 = Hex.encodeHexString(hd.sha1); final String md5 = Hex.encodeHexString(hd.md5); final String crc32 = Hex.encodeHexString(hd.crc32); final Map<String, Object> hd_m = new HashMap<String, Object>(); hd_m.put("sha1", sha1); hd_m.put("md5", md5); hd_m.put("crc32", crc32); hd_m.put("name", hd.name); hd_m.put("size", hd.size); hd_m.put("special_code", hd.special_code); final OSData osd = os.get(hd.os_code); final MfgData osmfgd = mfg.get(osd.mfg_code); final Map<String, Object> os_m = new HashMap<String, Object>(); os_m.put("name", osd.name); os_m.put("version", osd.version); os_m.put("manufacturer", osmfgd.name); hd_m.put("os", os_m); final List<Map<String, Object>> pl_l = new ArrayList<Map<String, Object>>(); for (ProdData pd : prod.get(hd.prod_code)) { if (!osd.code.equals(pd.os_code)) { // os code mismatch /* System.err.println( "Hash record OS code == " + osd.code + " != " + pd.os_code + " == product record OS code" ); */ continue; } final Map<String, Object> prod_m = new HashMap<String, Object>(); prod_m.put("name", pd.name); prod_m.put("version", pd.version); prod_m.put("language", pd.language); prod_m.put("app_type", pd.app_type); prod_m.put("os_code", pd.os_code); final MfgData md = mfg.get(pd.mfg_code); prod_m.put("manufacturer", md.name); pl_l.add(prod_m); } if (pl_l.size() > 1) { System.err.println(hd.prod_code); } hd_m.put("products", pl_l); try { mapper.writeValue(System.out, hd_m); } catch (IOException e) { // should be impossible throw new IllegalStateException(e); } } }; final RecordProcessor<HashData> hproc = new HashRecordProcessor(); final LineHandler hlh = new DefaultLineHandler<HashData>(tok, hproc, hcon, err); InputStream zin = null; try { // zin = new GZIPInputStream(new FileInputStream(hash_filename)); zin = new FileInputStream(hash_filename); loader.load(zin, hlh); zin.close(); } finally { IOUtils.closeQuietly(zin); } System.out.println(); /* for (Map.Entry<String,String> e : mfg.entrySet()) { System.out.println(e.getKey() + " = " + e.getValue()); } for (Map.Entry<String,OSData> e : os.entrySet()) { System.out.println(e.getKey() + " = " + e.getValue()); } for (Map.Entry<Integer,List<ProdData>> e : prod.entrySet()) { System.out.println(e.getKey() + " = " + e.getValue()); } */ }
From source file:com.threadswarm.imagefeedarchiver.driver.CommandLineDriver.java
public static void main(String[] args) throws InterruptedException, ExecutionException, ParseException { // define available command-line options Options options = new Options(); options.addOption("h", "help", false, "display usage information"); options.addOption("u", "url", true, "RSS feed URL"); options.addOption("a", "user-agent", true, "User-Agent header value to use when making HTTP requests"); options.addOption("o", "output-directory", true, "output directory for downloaded images"); options.addOption("t", "thread-count", true, "number of worker threads, defaults to cpu-count + 1"); options.addOption("d", "delay", true, "delay between image downloads (in milliseconds)"); options.addOption("p", "notrack", false, "tell websites that you don't wish to be tracked (DNT)"); options.addOption("s", "https", false, "Rewrite image URLs to leverage SSL/TLS"); CommandLineParser commandLineParser = new BasicParser(); CommandLine commandLine = commandLineParser.parse(options, args); // print usage information if 'h'/'help' or no-args were given if (args.length == 0 || commandLine.hasOption("h")) { HelpFormatter helpFormatter = new HelpFormatter(); helpFormatter.printHelp("java -jar ImageFeedArchiver.jar", options); return; //abort execution }//from w ww . j av a2 s .com URI rssFeedUri = null; if (commandLine.hasOption("u")) { String rssFeedUrlString = commandLine.getOptionValue("u"); try { rssFeedUri = FeedUtils.getUriFromUrlString(rssFeedUrlString); } catch (MalformedURLException | URISyntaxException e) { LOGGER.error("The Feed URL you supplied was malformed or violated syntax rules.. exiting", e); System.exit(1); } LOGGER.info("Target RSS feed URL: {}", rssFeedUri); } else { throw new IllegalStateException("RSS feed URL was not specified!"); } File outputDirectory = null; if (commandLine.hasOption("o")) { outputDirectory = new File(commandLine.getOptionValue("o")); if (!outputDirectory.isDirectory()) throw new IllegalArgumentException("output directory must be a *directory*!"); LOGGER.info("Using output directory: '{}'", outputDirectory); } else { throw new IllegalStateException("output directory was not specified!"); } String userAgentString = null; if (commandLine.hasOption("a")) { userAgentString = commandLine.getOptionValue("a"); LOGGER.info("Setting 'User-Agent' header value to '{}'", userAgentString); } int threadCount; if (commandLine.hasOption("t")) { threadCount = Integer.parseInt(commandLine.getOptionValue("t")); } else { threadCount = Runtime.getRuntime().availableProcessors() + 1; } LOGGER.info("Using {} worker threads", threadCount); long downloadDelay = 0; if (commandLine.hasOption("d")) { String downloadDelayString = commandLine.getOptionValue("d"); downloadDelay = Long.parseLong(downloadDelayString); } LOGGER.info("Using a download-delay of {} milliseconds", downloadDelay); boolean doNotTrackRequested = commandLine.hasOption("p"); boolean forceHttps = commandLine.hasOption("s"); ApplicationContext context = new ClassPathXmlApplicationContext("META-INF/applicationContext.xml"); ((ConfigurableApplicationContext) context).registerShutdownHook(); HttpClient httpClient = (HttpClient) context.getBean("httpClient"); if (userAgentString != null) httpClient.getParams().setParameter(CoreProtocolPNames.USER_AGENT, userAgentString); ProcessedRssItemDAO processedRssItemDAO = (ProcessedRssItemDAO) context.getBean("processedRssItemDAO"); CommandLineDriver.Builder driverBuilder = new CommandLineDriver.Builder(rssFeedUri); driverBuilder.setDoNotTrackRequested(doNotTrackRequested).setOutputDirectory(outputDirectory) .setDownloadDelay(downloadDelay).setThreadCount(threadCount).setHttpClient(httpClient) .setForceHttps(forceHttps).setProcessedRssItemDAO(processedRssItemDAO); CommandLineDriver driver = driverBuilder.build(); driver.run(); }
From source file:de.tudarmstadt.ukp.experiments.dip.wp1.documents.Step8GoldDataAggregator.java
public static void main(String[] args) throws Exception { String inputDir = args[0] + "/"; // output dir File outputDir = new File(args[1]); File turkersConfidence = new File(args[2]); if (outputDir.exists()) { outputDir.delete();//from w ww. j av a 2 s.c o m } outputDir.mkdir(); List<String> annotatorsIDs = new ArrayList<>(); // for (File f : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { // QueryResultContainer queryResultContainer = QueryResultContainer // .fromXML(FileUtils.readFileToString(f, "utf-8")); // for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { // for (QueryResultContainer.MTurkRelevanceVote relevanceVote : rankedResults.mTurkRelevanceVotes) { // if (!annotatorsIDs.contains(relevanceVote.turkID)) // annotatorsIDs.add(relevanceVote.turkID); // } // } // } HashMap<String, Integer> countVotesForATurker = new HashMap<>(); // creates temporary file with format for mace // Hashmap annotations: key is the id of a document and a sentence // Value is an array votes[] of turkers decisions: true or false (relevant or not) // the length of this array equals the number of annotators in List<String> annotatorsIDs. // If an annotator worked on the task his decision is written in the array otherwise the value is NULL // key: queryID + clueWebID + sentenceID // value: true and false annotations TreeMap<String, Annotations> annotations = new TreeMap<>(); for (File f : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(f, "utf-8")); System.out.println("Reading " + f.getName()); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { String documentID = rankedResults.clueWebID; for (QueryResultContainer.MTurkRelevanceVote relevanceVote : rankedResults.mTurkRelevanceVotes) { Integer turkerID; if (!annotatorsIDs.contains(relevanceVote.turkID)) { annotatorsIDs.add(relevanceVote.turkID); turkerID = annotatorsIDs.size() - 1; } else { turkerID = annotatorsIDs.indexOf(relevanceVote.turkID); } Integer count = countVotesForATurker.get(relevanceVote.turkID); if (count == null) { count = 0; } count++; countVotesForATurker.put(relevanceVote.turkID, count); String id; List<Integer> trueVotes; List<Integer> falseVotes; for (QueryResultContainer.SingleSentenceRelevanceVote singleSentenceRelevanceVote : relevanceVote.singleSentenceRelevanceVotes) if (!"".equals(singleSentenceRelevanceVote.sentenceID)) { id = f.getName() + "_" + documentID + "_" + singleSentenceRelevanceVote.sentenceID; Annotations turkerVotes = annotations.get(id); if (turkerVotes == null) { trueVotes = new ArrayList<>(); falseVotes = new ArrayList<>(); turkerVotes = new Annotations(trueVotes, falseVotes); } trueVotes = turkerVotes.trueAnnotations; falseVotes = turkerVotes.falseAnnotations; if ("true".equals(singleSentenceRelevanceVote.relevant)) { // votes[turkerID] = true; trueVotes.add(turkerID); } else if ("false".equals(singleSentenceRelevanceVote.relevant)) { // votes[turkerID] = false; falseVotes.add(turkerID); } else { throw new IllegalStateException("Annotation value of sentence " + singleSentenceRelevanceVote.sentenceID + " in " + rankedResults.clueWebID + " equals " + singleSentenceRelevanceVote.relevant); } try { int allVotesCount = trueVotes.size() + falseVotes.size(); if (allVotesCount > 5) { System.err.println(id + " doesn't have 5 annotators: true: " + trueVotes.size() + " false: " + falseVotes.size()); // nasty hack, we're gonna strip some data; true votes first /* we can't do that, it breaks something down the line int toRemove = allVotesCount - 5; if (trueVotes.size() >= toRemove) { trueVotes = trueVotes .subList(0, trueVotes.size() - toRemove); } else if ( falseVotes.size() >= toRemove) { falseVotes = falseVotes .subList(0, trueVotes.size() - toRemove); } */ System.err.println("Adjusted: " + id + " doesn't have 5 annotators: true: " + trueVotes.size() + " false: " + falseVotes.size()); } } catch (IllegalStateException e) { e.printStackTrace(); } turkerVotes.trueAnnotations = trueVotes; turkerVotes.falseAnnotations = falseVotes; annotations.put(id, turkerVotes); } else { throw new IllegalStateException( "Empty Sentence ID in " + f.getName() + " for turker " + turkerID); } } } } File tmp = printHashMap(annotations, annotatorsIDs.size()); String file = TEMP_DIR + "/" + tmp.getName(); MACE.main(new String[] { "--prefix", file }); //gets the keys of the documents and sentences ArrayList<String> lines = (ArrayList<String>) FileUtils.readLines(new File(file + ".prediction")); int i = 0; TreeMap<String, TreeMap<String, ArrayList<HashMap<String, String>>>> ids = new TreeMap<>(); ArrayList<HashMap<String, String>> sentences; if (lines.size() != annotations.size()) { throw new IllegalStateException( "The size of prediction file is " + lines.size() + "but expected " + annotations.size()); } for (Map.Entry entry : annotations.entrySet()) { //1001.xml_clueweb12-1905wb-13-07360_8783 String key = (String) entry.getKey(); String[] IDs = key.split("_"); if (IDs.length > 2) { String queryID = IDs[0]; String clueWebID = IDs[1]; String sentenceID = IDs[2]; TreeMap<String, ArrayList<HashMap<String, String>>> clueWebIDs = ids.get(queryID); if (clueWebIDs == null) { clueWebIDs = new TreeMap<>(); } sentences = clueWebIDs.get(clueWebID); if (sentences == null) { sentences = new ArrayList<>(); } HashMap<String, String> sentence = new HashMap<>(); sentence.put(sentenceID, lines.get(i)); sentences.add(sentence); clueWebIDs.put(clueWebID, sentences); ids.put(queryID, clueWebIDs); } else { throw new IllegalStateException("Wrong ID " + key); } i++; } for (Map.Entry entry : ids.entrySet()) { TreeMap<Integer, String> value = (TreeMap<Integer, String>) entry.getValue(); String queryID = (String) entry.getKey(); QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(new File(inputDir, queryID), "utf-8")); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { for (Map.Entry val : value.entrySet()) { String clueWebID = (String) val.getKey(); if (clueWebID.equals(rankedResults.clueWebID)) { List<QueryResultContainer.SingleSentenceRelevanceVote> goldEstimatedLabels = new ArrayList<>(); List<QueryResultContainer.SingleSentenceRelevanceVote> turkersVotes = new ArrayList<>(); int size = 0; int hitSize = 0; String hitID = ""; for (QueryResultContainer.MTurkRelevanceVote vote : rankedResults.mTurkRelevanceVotes) { if (!hitID.equals(vote.hitID)) { hitID = vote.hitID; hitSize = vote.singleSentenceRelevanceVotes.size(); size = size + hitSize; turkersVotes.addAll(vote.singleSentenceRelevanceVotes); } else { if (vote.singleSentenceRelevanceVotes.size() != hitSize) { hitSize = vote.singleSentenceRelevanceVotes.size(); size = size + hitSize; turkersVotes.addAll(vote.singleSentenceRelevanceVotes); } } } ArrayList<HashMap<String, String>> sentenceList = (ArrayList<HashMap<String, String>>) val .getValue(); if (sentenceList.size() != turkersVotes.size()) { try { throw new IllegalStateException("Expected size of annotations is " + turkersVotes.size() + "but found " + sentenceList.size() + " for document " + rankedResults.clueWebID + " in " + queryID); } catch (IllegalStateException ex) { ex.printStackTrace(); } } for (QueryResultContainer.SingleSentenceRelevanceVote s : turkersVotes) { String valSentence = null; for (HashMap<String, String> anno : sentenceList) { if (anno.keySet().contains(s.sentenceID)) { valSentence = anno.get(s.sentenceID); } } QueryResultContainer.SingleSentenceRelevanceVote singleSentenceVote = new QueryResultContainer.SingleSentenceRelevanceVote(); singleSentenceVote.sentenceID = s.sentenceID; if (("false").equals(valSentence)) { singleSentenceVote.relevant = "false"; } else if (("true").equals(valSentence)) { singleSentenceVote.relevant = "true"; } else { throw new IllegalStateException("Annotation value of sentence " + singleSentenceVote.sentenceID + " equals " + val.getValue()); } goldEstimatedLabels.add(singleSentenceVote); } rankedResults.goldEstimatedLabels = goldEstimatedLabels; } } } File outputFile = new File(outputDir, queryID); FileUtils.writeStringToFile(outputFile, queryResultContainer.toXML(), "utf-8"); System.out.println("Finished " + outputFile); } ArrayList<String> annotators = (ArrayList<String>) FileUtils.readLines(new File(file + ".competence")); FileWriter fileWriter; StringBuilder sb = new StringBuilder(); for (int j = 0; j < annotatorsIDs.size(); j++) { String[] s = annotators.get(0).split("\t"); Float score = Float.parseFloat(s[j]); String turkerID = annotatorsIDs.get(j); System.out.println(turkerID + " " + score + " " + countVotesForATurker.get(turkerID)); sb.append(turkerID).append(" ").append(score).append(" ").append(countVotesForATurker.get(turkerID)) .append("\n"); } fileWriter = new FileWriter(turkersConfidence); fileWriter.append(sb.toString()); fileWriter.close(); }
From source file:com.github.ambry.store.DiskReformatter.java
public static void main(String[] args) throws Exception { VerifiableProperties properties = ToolUtils.getVerifiableProperties(args); DiskReformatterConfig config = new DiskReformatterConfig(properties); StoreConfig storeConfig = new StoreConfig(properties); ClusterMapConfig clusterMapConfig = new ClusterMapConfig(properties); ServerConfig serverConfig = new ServerConfig(properties); ClusterAgentsFactory clusterAgentsFactory = Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, config.hardwareLayoutFilePath, config.partitionLayoutFilePath); try (ClusterMap clusterMap = clusterAgentsFactory.getClusterMap()) { StoreKeyConverterFactory storeKeyConverterFactory = Utils.getObj( serverConfig.serverStoreKeyConverterFactory, properties, clusterMap.getMetricRegistry()); StoreKeyFactory storeKeyFactory = Utils.getObj(storeConfig.storeKeyFactory, clusterMap); DataNodeId dataNodeId = clusterMap.getDataNodeId(config.datanodeHostname, config.datanodePort); if (dataNodeId == null) { throw new IllegalArgumentException("Did not find node in clustermap with hostname:port - " + config.datanodeHostname + ":" + config.datanodePort); }//w w w. j ava2 s .c om DiskReformatter reformatter = new DiskReformatter(dataNodeId, Collections.EMPTY_LIST, config.fetchSizeInBytes, storeConfig, storeKeyFactory, clusterMap, SystemTime.getInstance(), storeKeyConverterFactory.getStoreKeyConverter()); AtomicInteger exitStatus = new AtomicInteger(0); CountDownLatch latch = new CountDownLatch(config.diskMountPaths.length); for (int i = 0; i < config.diskMountPaths.length; i++) { int finalI = i; Runnable runnable = () -> { try { reformatter.reformat(config.diskMountPaths[finalI], new File(config.scratchPaths[finalI])); latch.countDown(); } catch (Exception e) { throw new IllegalStateException(e); } }; Thread thread = Utils.newThread(config.diskMountPaths[finalI] + "-reformatter", runnable, true); thread.setUncaughtExceptionHandler((t, e) -> { exitStatus.set(1); logger.error("Reformatting {} failed", config.diskMountPaths[finalI], e); latch.countDown(); }); thread.start(); } latch.await(); System.exit(exitStatus.get()); } }
From source file:com.github.fritaly.svngraph.SvnGraph.java
public static void main(String[] args) throws Exception { if (args.length != 2) { System.out.println(String.format("%s <input-file> <output-file>", SvnGraph.class.getSimpleName())); System.exit(1);//from ww w . j a v a2 s .co m } final File input = new File(args[0]); if (!input.exists()) { throw new IllegalArgumentException( String.format("The given file '%s' doesn't exist", input.getAbsolutePath())); } final File output = new File(args[1]); final Document document = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(input); final History history = new History(document); final Set<String> rootPaths = history.getRootPaths(); System.out.println(rootPaths); for (String path : rootPaths) { System.out.println(path); System.out.println(history.getHistory(path).getRevisions()); System.out.println(); } int count = 0; FileWriter fileWriter = null; GraphMLWriter graphWriter = null; try { fileWriter = new FileWriter(output); graphWriter = new GraphMLWriter(fileWriter); final NodeStyle tagStyle = graphWriter.getNodeStyle(); tagStyle.setFillColor(Color.WHITE); graphWriter.graph(); // map associating node labels to their corresponding node id in the graph final Map<String, String> nodeIdsPerLabel = new TreeMap<>(); // the node style associated to each branch final Map<String, NodeStyle> nodeStyles = new TreeMap<>(); for (Revision revision : history.getSignificantRevisions()) { System.out.println(revision.getNumber() + " - " + revision.getMessage()); // TODO Render also the deletion of branches // there should be only 1 significant update per revision (the one with action ADD) for (Update update : revision.getSignificantUpdates()) { if (update.isCopy()) { // a merge is also considered a copy final RevisionPath source = update.getCopySource(); System.out.println(String.format(" > %s %s from %s@%d", update.getAction(), update.getPath(), source.getPath(), source.getRevision())); final String sourceRoot = Utils.getRootName(source.getPath()); if (sourceRoot == null) { // skip the revisions whose associated root is // null (happens whether a branch was created // outside the 'branches' directory for // instance) System.err.println(String.format("Skipped revision %d because of a null root", source.getRevision())); continue; } final String sourceLabel = computeNodeLabel(sourceRoot, source.getRevision()); // create a node for the source (path, revision) final String sourceId; if (nodeIdsPerLabel.containsKey(sourceLabel)) { // retrieve the id of the existing node sourceId = nodeIdsPerLabel.get(sourceLabel); } else { // create the new node if (Utils.isTagPath(source.getPath())) { graphWriter.setNodeStyle(tagStyle); } else { if (!nodeStyles.containsKey(sourceRoot)) { final NodeStyle style = new NodeStyle(); style.setFillColor(randomColor()); nodeStyles.put(sourceRoot, style); } graphWriter.setNodeStyle(nodeStyles.get(sourceRoot)); } sourceId = graphWriter.node(sourceLabel); nodeIdsPerLabel.put(sourceLabel, sourceId); } // and another for the newly created directory final String targetRoot = Utils.getRootName(update.getPath()); if (targetRoot == null) { System.err.println(String.format("Skipped revision %d because of a null root", revision.getNumber())); continue; } final String targetLabel = computeNodeLabel(targetRoot, revision.getNumber()); if (Utils.isTagPath(update.getPath())) { graphWriter.setNodeStyle(tagStyle); } else { if (!nodeStyles.containsKey(targetRoot)) { final NodeStyle style = new NodeStyle(); style.setFillColor(randomColor()); nodeStyles.put(targetRoot, style); } graphWriter.setNodeStyle(nodeStyles.get(targetRoot)); } final String targetId; if (nodeIdsPerLabel.containsKey(targetLabel)) { // retrieve the id of the existing node targetId = nodeIdsPerLabel.get(targetLabel); } else { // create the new node if (Utils.isTagPath(update.getPath())) { graphWriter.setNodeStyle(tagStyle); } else { if (!nodeStyles.containsKey(targetRoot)) { final NodeStyle style = new NodeStyle(); style.setFillColor(randomColor()); nodeStyles.put(targetRoot, style); } graphWriter.setNodeStyle(nodeStyles.get(targetRoot)); } targetId = graphWriter.node(targetLabel); nodeIdsPerLabel.put(targetLabel, targetId); } // create an edge between the 2 nodes graphWriter.edge(sourceId, targetId); } else { System.out.println(String.format(" > %s %s", update.getAction(), update.getPath())); } } System.out.println(); count++; } // Dispatch the revisions per corresponding branch final Map<String, Set<Long>> revisionsPerBranch = new TreeMap<>(); for (String nodeLabel : nodeIdsPerLabel.keySet()) { if (nodeLabel.contains("@")) { final String branchName = StringUtils.substringBefore(nodeLabel, "@"); final long revision = Long.parseLong(StringUtils.substringAfter(nodeLabel, "@")); if (!revisionsPerBranch.containsKey(branchName)) { revisionsPerBranch.put(branchName, new TreeSet<Long>()); } revisionsPerBranch.get(branchName).add(revision); } else { throw new IllegalStateException(nodeLabel); } } // Recreate the missing edges between revisions from a same branch for (String branchName : revisionsPerBranch.keySet()) { final List<Long> branchRevisions = new ArrayList<>(revisionsPerBranch.get(branchName)); for (int i = 0; i < branchRevisions.size() - 1; i++) { final String nodeLabel1 = String.format("%s@%d", branchName, branchRevisions.get(i)); final String nodeLabel2 = String.format("%s@%d", branchName, branchRevisions.get(i + 1)); graphWriter.edge(nodeIdsPerLabel.get(nodeLabel1), nodeIdsPerLabel.get(nodeLabel2)); } } graphWriter.closeGraph(); System.out.println(String.format("Found %d significant revisions", count)); } finally { if (graphWriter != null) { graphWriter.close(); } if (fileWriter != null) { fileWriter.close(); } } System.out.println("Done"); }
From source file:Main.java
public static void fail(String message) { throw new IllegalStateException(message); }