List of usage examples for org.apache.commons.io FileUtils listFiles
public static Collection listFiles(File directory, String[] extensions, boolean recursive)
From source file:de.tudarmstadt.ukp.argumentation.data.roomfordebate.DataFetcher.java
public static void main(String[] args) throws Exception { File crawledPagesFolder = new File(args[0]); if (!crawledPagesFolder.exists()) { crawledPagesFolder.mkdirs();//from ww w. j a v a 2s . co m } File outputFolder = new File(args[1]); if (!outputFolder.exists()) { outputFolder.mkdirs(); } // read links from text file final String urlsResourceName = "roomfordebate-urls.txt"; InputStream urlsStream = DataFetcher.class.getClassLoader().getResourceAsStream(urlsResourceName); if (urlsStream == null) { throw new IOException("Cannot find resource " + urlsResourceName + " on the classpath"); } // read list of urls List<String> urls = new ArrayList<>(); LineIterator iterator = IOUtils.lineIterator(urlsStream, "utf-8"); while (iterator.hasNext()) { // ignore commented url (line starts with #) String line = iterator.nextLine(); if (!line.startsWith("#") && !line.trim().isEmpty()) { urls.add(line.trim()); } } // download all crawlPages(urls, crawledPagesFolder); List<File> files = new ArrayList<>(FileUtils.listFiles(crawledPagesFolder, null, false)); Collections.sort(files, new Comparator<File>() { @Override public int compare(File o1, File o2) { return o1.getName().compareTo(o2.getName()); } }); int idCounter = 0; for (File file : files) { NYTimesCommentsScraper commentsScraper = new NYTimesCommentsScraper(); NYTimesArticleExtractor extractor = new NYTimesArticleExtractor(); String html = FileUtils.readFileToString(file, "utf-8"); idCounter++; File outputFileArticle = new File(outputFolder, String.format("Cx%03d.txt", idCounter)); File outputFileComments = new File(outputFolder, String.format("Dx%03d.txt", idCounter)); try { List<Comment> comments = commentsScraper.extractComments(html); Article article = extractor.extractArticle(html); saveArticleToText(article, outputFileArticle); System.out.println("Saved to " + outputFileArticle); saveCommentsToText(comments, outputFileComments, article); System.out.println("Saved to " + outputFileComments); } catch (IOException ex) { System.err.println(file.getName() + "\n" + ex.getMessage()); } } }
From source file:de.tudarmstadt.ukp.dkpro.argumentation.sequence.report.ConfusionMatrixTools.java
public static void main(String[] args) throws IOException { String path = args[0];// ww w . j av a 2 s . c om for (File file : FileUtils.listFiles(new File(path), new String[] { "csv" }, true)) { if (file.getName().startsWith("tokenLevelPredictions")) { generateNiceTable(file); } } }
From source file:de.tudarmstadt.ukp.experiments.dip.wp1.documents.Step8GoldDataAggregator.java
public static void main(String[] args) throws Exception { String inputDir = args[0] + "/"; // output dir File outputDir = new File(args[1]); File turkersConfidence = new File(args[2]); if (outputDir.exists()) { outputDir.delete();/* ww w. j a v a2 s . com*/ } outputDir.mkdir(); List<String> annotatorsIDs = new ArrayList<>(); // for (File f : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { // QueryResultContainer queryResultContainer = QueryResultContainer // .fromXML(FileUtils.readFileToString(f, "utf-8")); // for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { // for (QueryResultContainer.MTurkRelevanceVote relevanceVote : rankedResults.mTurkRelevanceVotes) { // if (!annotatorsIDs.contains(relevanceVote.turkID)) // annotatorsIDs.add(relevanceVote.turkID); // } // } // } HashMap<String, Integer> countVotesForATurker = new HashMap<>(); // creates temporary file with format for mace // Hashmap annotations: key is the id of a document and a sentence // Value is an array votes[] of turkers decisions: true or false (relevant or not) // the length of this array equals the number of annotators in List<String> annotatorsIDs. // If an annotator worked on the task his decision is written in the array otherwise the value is NULL // key: queryID + clueWebID + sentenceID // value: true and false annotations TreeMap<String, Annotations> annotations = new TreeMap<>(); for (File f : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(f, "utf-8")); System.out.println("Reading " + f.getName()); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { String documentID = rankedResults.clueWebID; for (QueryResultContainer.MTurkRelevanceVote relevanceVote : rankedResults.mTurkRelevanceVotes) { Integer turkerID; if (!annotatorsIDs.contains(relevanceVote.turkID)) { annotatorsIDs.add(relevanceVote.turkID); turkerID = annotatorsIDs.size() - 1; } else { turkerID = annotatorsIDs.indexOf(relevanceVote.turkID); } Integer count = countVotesForATurker.get(relevanceVote.turkID); if (count == null) { count = 0; } count++; countVotesForATurker.put(relevanceVote.turkID, count); String id; List<Integer> trueVotes; List<Integer> falseVotes; for (QueryResultContainer.SingleSentenceRelevanceVote singleSentenceRelevanceVote : relevanceVote.singleSentenceRelevanceVotes) if (!"".equals(singleSentenceRelevanceVote.sentenceID)) { id = f.getName() + "_" + documentID + "_" + singleSentenceRelevanceVote.sentenceID; Annotations turkerVotes = annotations.get(id); if (turkerVotes == null) { trueVotes = new ArrayList<>(); falseVotes = new ArrayList<>(); turkerVotes = new Annotations(trueVotes, falseVotes); } trueVotes = turkerVotes.trueAnnotations; falseVotes = turkerVotes.falseAnnotations; if ("true".equals(singleSentenceRelevanceVote.relevant)) { // votes[turkerID] = true; trueVotes.add(turkerID); } else if ("false".equals(singleSentenceRelevanceVote.relevant)) { // votes[turkerID] = false; falseVotes.add(turkerID); } else { throw new IllegalStateException("Annotation value of sentence " + singleSentenceRelevanceVote.sentenceID + " in " + rankedResults.clueWebID + " equals " + singleSentenceRelevanceVote.relevant); } try { int allVotesCount = trueVotes.size() + falseVotes.size(); if (allVotesCount > 5) { System.err.println(id + " doesn't have 5 annotators: true: " + trueVotes.size() + " false: " + falseVotes.size()); // nasty hack, we're gonna strip some data; true votes first /* we can't do that, it breaks something down the line int toRemove = allVotesCount - 5; if (trueVotes.size() >= toRemove) { trueVotes = trueVotes .subList(0, trueVotes.size() - toRemove); } else if ( falseVotes.size() >= toRemove) { falseVotes = falseVotes .subList(0, trueVotes.size() - toRemove); } */ System.err.println("Adjusted: " + id + " doesn't have 5 annotators: true: " + trueVotes.size() + " false: " + falseVotes.size()); } } catch (IllegalStateException e) { e.printStackTrace(); } turkerVotes.trueAnnotations = trueVotes; turkerVotes.falseAnnotations = falseVotes; annotations.put(id, turkerVotes); } else { throw new IllegalStateException( "Empty Sentence ID in " + f.getName() + " for turker " + turkerID); } } } } File tmp = printHashMap(annotations, annotatorsIDs.size()); String file = TEMP_DIR + "/" + tmp.getName(); MACE.main(new String[] { "--prefix", file }); //gets the keys of the documents and sentences ArrayList<String> lines = (ArrayList<String>) FileUtils.readLines(new File(file + ".prediction")); int i = 0; TreeMap<String, TreeMap<String, ArrayList<HashMap<String, String>>>> ids = new TreeMap<>(); ArrayList<HashMap<String, String>> sentences; if (lines.size() != annotations.size()) { throw new IllegalStateException( "The size of prediction file is " + lines.size() + "but expected " + annotations.size()); } for (Map.Entry entry : annotations.entrySet()) { //1001.xml_clueweb12-1905wb-13-07360_8783 String key = (String) entry.getKey(); String[] IDs = key.split("_"); if (IDs.length > 2) { String queryID = IDs[0]; String clueWebID = IDs[1]; String sentenceID = IDs[2]; TreeMap<String, ArrayList<HashMap<String, String>>> clueWebIDs = ids.get(queryID); if (clueWebIDs == null) { clueWebIDs = new TreeMap<>(); } sentences = clueWebIDs.get(clueWebID); if (sentences == null) { sentences = new ArrayList<>(); } HashMap<String, String> sentence = new HashMap<>(); sentence.put(sentenceID, lines.get(i)); sentences.add(sentence); clueWebIDs.put(clueWebID, sentences); ids.put(queryID, clueWebIDs); } else { throw new IllegalStateException("Wrong ID " + key); } i++; } for (Map.Entry entry : ids.entrySet()) { TreeMap<Integer, String> value = (TreeMap<Integer, String>) entry.getValue(); String queryID = (String) entry.getKey(); QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(new File(inputDir, queryID), "utf-8")); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { for (Map.Entry val : value.entrySet()) { String clueWebID = (String) val.getKey(); if (clueWebID.equals(rankedResults.clueWebID)) { List<QueryResultContainer.SingleSentenceRelevanceVote> goldEstimatedLabels = new ArrayList<>(); List<QueryResultContainer.SingleSentenceRelevanceVote> turkersVotes = new ArrayList<>(); int size = 0; int hitSize = 0; String hitID = ""; for (QueryResultContainer.MTurkRelevanceVote vote : rankedResults.mTurkRelevanceVotes) { if (!hitID.equals(vote.hitID)) { hitID = vote.hitID; hitSize = vote.singleSentenceRelevanceVotes.size(); size = size + hitSize; turkersVotes.addAll(vote.singleSentenceRelevanceVotes); } else { if (vote.singleSentenceRelevanceVotes.size() != hitSize) { hitSize = vote.singleSentenceRelevanceVotes.size(); size = size + hitSize; turkersVotes.addAll(vote.singleSentenceRelevanceVotes); } } } ArrayList<HashMap<String, String>> sentenceList = (ArrayList<HashMap<String, String>>) val .getValue(); if (sentenceList.size() != turkersVotes.size()) { try { throw new IllegalStateException("Expected size of annotations is " + turkersVotes.size() + "but found " + sentenceList.size() + " for document " + rankedResults.clueWebID + " in " + queryID); } catch (IllegalStateException ex) { ex.printStackTrace(); } } for (QueryResultContainer.SingleSentenceRelevanceVote s : turkersVotes) { String valSentence = null; for (HashMap<String, String> anno : sentenceList) { if (anno.keySet().contains(s.sentenceID)) { valSentence = anno.get(s.sentenceID); } } QueryResultContainer.SingleSentenceRelevanceVote singleSentenceVote = new QueryResultContainer.SingleSentenceRelevanceVote(); singleSentenceVote.sentenceID = s.sentenceID; if (("false").equals(valSentence)) { singleSentenceVote.relevant = "false"; } else if (("true").equals(valSentence)) { singleSentenceVote.relevant = "true"; } else { throw new IllegalStateException("Annotation value of sentence " + singleSentenceVote.sentenceID + " equals " + val.getValue()); } goldEstimatedLabels.add(singleSentenceVote); } rankedResults.goldEstimatedLabels = goldEstimatedLabels; } } } File outputFile = new File(outputDir, queryID); FileUtils.writeStringToFile(outputFile, queryResultContainer.toXML(), "utf-8"); System.out.println("Finished " + outputFile); } ArrayList<String> annotators = (ArrayList<String>) FileUtils.readLines(new File(file + ".competence")); FileWriter fileWriter; StringBuilder sb = new StringBuilder(); for (int j = 0; j < annotatorsIDs.size(); j++) { String[] s = annotators.get(0).split("\t"); Float score = Float.parseFloat(s[j]); String turkerID = annotatorsIDs.get(j); System.out.println(turkerID + " " + score + " " + countVotesForATurker.get(turkerID)); sb.append(turkerID).append(" ").append(score).append(" ").append(countVotesForATurker.get(turkerID)) .append("\n"); } fileWriter = new FileWriter(turkersConfidence); fileWriter.append(sb.toString()); fileWriter.close(); }
From source file:com.l2jserver.model.template.NPCTemplateConverter.java
public static void main(String[] args) throws SQLException, IOException, ClassNotFoundException, JAXBException { controllers.put("L2Teleporter", TeleporterController.class); controllers.put("L2CastleTeleporter", TeleporterController.class); controllers.put("L2Npc", BaseNPCController.class); controllers.put("L2Monster", MonsterController.class); controllers.put("L2FlyMonster", MonsterController.class); Class.forName("com.mysql.jdbc.Driver"); final File target = new File("generated/template/npc"); System.out.println("Scaning legacy HTML files..."); htmlScannedFiles = FileUtils.listFiles(L2J_HTML_FOLDER, new String[] { "html", "htm" }, true); final JAXBContext c = JAXBContext.newInstance(NPCTemplate.class, Teleports.class); final Connection conn = DriverManager.getConnection(JDBC_URL, JDBC_USERNAME, JDBC_PASSWORD); {// w ww . j av a2 s. c o m System.out.println("Converting teleport templates..."); teleportation.teleport = CollectionFactory.newList(); final Marshaller m = c.createMarshaller(); m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); final PreparedStatement st = conn.prepareStatement("SELECT * FROM teleport"); st.execute(); final ResultSet rs = st.getResultSet(); while (rs.next()) { final TeleportationTemplate template = new TeleportationTemplate(); template.id = new TeleportationTemplateID(rs.getInt("id"), null); template.name = rs.getString("Description"); TemplateCoordinate coord = new TemplateCoordinate(); coord.x = rs.getInt("loc_x"); coord.y = rs.getInt("loc_y"); coord.z = rs.getInt("loc_z"); template.point = coord; template.price = rs.getInt("price"); template.item = rs.getInt("itemId"); if (rs.getBoolean("fornoble")) { template.restrictions = new Restrictions(); template.restrictions.restriction = Arrays.asList("NOBLE"); } teleportation.teleport.add(template); } m.marshal(teleportation, getXMLSerializer(new FileOutputStream(new File(target, "../teleports.xml")))); // System.exit(0); } System.out.println("Generating template XML files..."); // c.generateSchema(new SchemaOutputResolver() { // @Override // public Result createOutput(String namespaceUri, // String suggestedFileName) throws IOException { // // System.out.println(new File(target, suggestedFileName)); // // return null; // return new StreamResult(new File(target, suggestedFileName)); // } // }); try { final Marshaller m = c.createMarshaller(); m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); final PreparedStatement st = conn.prepareStatement( "SELECT npc.*, npcskills.level AS race " + "FROM npc " + "LEFT JOIN npcskills " + "ON(npc.idTemplate = npcskills.npcid AND npcskills.skillid = ?)"); st.setInt(1, 4416); st.execute(); final ResultSet rs = st.getResultSet(); while (rs.next()) { Object[] result = fillNPC(rs); NPCTemplate t = (NPCTemplate) result[0]; String type = (String) result[1]; String folder = createFolder(type); if (folder.isEmpty()) { m.setProperty(Marshaller.JAXB_SCHEMA_LOCATION, "npc ../npc.xsd"); } else { m.setProperty(Marshaller.JAXB_SCHEMA_LOCATION, "npc ../../npc.xsd"); } final File file = new File(target, "npc/" + folder + "/" + t.getID().getID() + (t.getInfo().getName() != null ? "-" + camelCase(t.getInfo().getName().getValue()) : "") + ".xml"); file.getParentFile().mkdirs(); templates.add(t); try { m.marshal(t, getXMLSerializer(new FileOutputStream(file))); } catch (MarshalException e) { System.err.println("Could not generate XML template file for " + t.getInfo().getName().getValue() + " - " + t.getID()); file.delete(); } } System.out.println("Generated " + templates.size() + " templates"); System.gc(); System.out.println("Free: " + FileUtils.byteCountToDisplaySize(Runtime.getRuntime().freeMemory())); System.out.println("Total: " + FileUtils.byteCountToDisplaySize(Runtime.getRuntime().totalMemory())); System.out.println("Used: " + FileUtils.byteCountToDisplaySize( Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())); System.out.println("Max: " + FileUtils.byteCountToDisplaySize(Runtime.getRuntime().maxMemory())); } finally { conn.close(); } }
From source file:de.tudarmstadt.ukp.experiments.dip.wp1.documents.Step7CollectMTurkResults.java
public static void main(String[] args) throws Exception { // input dir - list of xml query containers // /home/user-ukp/research/data/dip/wp1-documents/step4-boiler-plate/ File inputDir = new File(args[0] + "/"); // MTurk result file // output dir File outputDir = new File(args[2]); if (!outputDir.exists()) { outputDir.mkdirs();/* www.ja v a 2 s . co m*/ } // Folder with success files File mturkSuccessDir = new File(args[1]); Collection<File> files = FileUtils.listFiles(mturkSuccessDir, new String[] { "result" }, false); if (files.isEmpty()) { throw new IllegalArgumentException("Input folder is empty. " + mturkSuccessDir); } HashMap<String, List<MTurkAnnotation>> mturkAnnotations = new HashMap<>(); // parsing all CSV files for (File mturkCSVResultFile : files) { System.out.println("Parsing " + mturkCSVResultFile.getName()); MTurkOutputReader outputReader = new MTurkOutputReader( new HashSet<>(Arrays.asList("annotation", "workerid")), mturkCSVResultFile); // for fixing broken data input: for each hit, collect all sentence IDs Map<String, SortedSet<String>> hitSentences = new HashMap<>(); // first iteration: collect the sentences for (Map<String, String> record : outputReader) { String hitID = record.get("hitid"); if (!hitSentences.containsKey(hitID)) { hitSentences.put(hitID, new TreeSet<>()); } String relevantSentences = record.get("Answer.relevant_sentences"); String irrelevantSentences = record.get("Answer.irrelevant_sentences"); if (relevantSentences != null) { hitSentences.get(hitID).addAll(Arrays.asList(relevantSentences.split(","))); } if (irrelevantSentences != null) { hitSentences.get(hitID).addAll(Arrays.asList(irrelevantSentences.split(","))); } } // and now second iteration for (Map<String, String> record : outputReader) { String hitID = record.get("hitid"); String annotatorID = record.get("workerid"); String acceptTime = record.get("assignmentaccepttime"); String submitTime = record.get("assignmentsubmittime"); String relevantSentences = record.get("Answer.relevant_sentences"); String irrelevantSentences = record.get("Answer.irrelevant_sentences"); String reject = record.get("reject"); String filename[]; String comment; String clueWeb; String[] relevant = {}; String[] irrelevant = {}; filename = record.get("annotation").split("_"); String fileXml = filename[0]; clueWeb = filename[1].trim(); comment = record.get("Answer.comment"); if (relevantSentences != null) { relevant = relevantSentences.split(","); } if (irrelevantSentences != null) { irrelevant = irrelevantSentences.split(","); } // sanitizing data: if both relevant and irrelevant are empty, that's a bug // we're gonna look up all sentences from this HIT and treat this assignment // as if there were only irrelevant ones if (relevant.length == 0 && irrelevant.length == 0) { SortedSet<String> strings = hitSentences.get(hitID); irrelevant = new String[strings.size()]; strings.toArray(irrelevant); } if (reject != null) { System.out.println(" HIT " + hitID + " annotated by " + annotatorID + " was rejected "); } else { /* // relevant sentences is a comma-delimited string, // this regular expression is rather strange // it must contain digits, it might be that there is only one space or a comma or some other char // digits are the sentence ids. if relevant sentences do not contain digits then it is wrong if (relevantSentences.matches("^\\D*$") && irrelevantSentences.matches("^\\D*$")) { try { throw new IllegalStateException( "No annotations found for HIT " + hitID + " in " + fileXml + " for document " + clueWeb); } catch (IllegalStateException ex) { ex.printStackTrace(); } } */ MTurkAnnotation mturkAnnotation; try { mturkAnnotation = new MTurkAnnotation(hitID, annotatorID, acceptTime, submitTime, comment, clueWeb, relevant, irrelevant); } catch (IllegalArgumentException ex) { throw new IllegalArgumentException("Record: " + record, ex); } List<MTurkAnnotation> listOfAnnotations = mturkAnnotations.get(fileXml); if (listOfAnnotations == null) { listOfAnnotations = new ArrayList<>(); } listOfAnnotations.add(mturkAnnotation); mturkAnnotations.put(fileXml, listOfAnnotations); } } // parser.close(); } // Debugging: output number of HITs of a query System.out.println("Accepted HITs for a query:"); for (Map.Entry e : mturkAnnotations.entrySet()) { ArrayList<MTurkAnnotation> a = (ArrayList<MTurkAnnotation>) e.getValue(); System.out.println(e.getKey() + " " + a.size()); } for (File f : FileUtils.listFiles(inputDir, new String[] { "xml" }, false)) { QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(f, "utf-8")); String fileName = f.getName(); List<MTurkAnnotation> listOfAnnotations = mturkAnnotations.get(fileName); if (listOfAnnotations == null || listOfAnnotations.isEmpty()) { throw new IllegalStateException("No annotations for " + f.getName()); } for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { for (MTurkAnnotation mtAnnotation : listOfAnnotations) { String clueWeb = mtAnnotation.clueWeb; if (rankedResults.clueWebID.equals(clueWeb)) { List<QueryResultContainer.MTurkRelevanceVote> mTurkRelevanceVotes = rankedResults.mTurkRelevanceVotes; QueryResultContainer.MTurkRelevanceVote relevanceVote = new QueryResultContainer.MTurkRelevanceVote(); String annotatorID = mtAnnotation.annotatorID; String hitID = mtAnnotation.hitID; String acceptTime = mtAnnotation.acceptTime; String submitTime = mtAnnotation.submitTime; String comment = mtAnnotation.comment; String[] relevant = mtAnnotation.relevant; String[] irrelevant = mtAnnotation.irrelevant; relevanceVote.turkID = annotatorID.trim(); relevanceVote.hitID = hitID.trim(); relevanceVote.acceptTime = acceptTime.trim(); relevanceVote.submitTime = submitTime.trim(); relevanceVote.comment = comment != null ? comment.trim() : null; if (relevant.length == 0 && irrelevant.length == 0) { try { throw new IllegalStateException("the length of the annotations is 0" + rankedResults.clueWebID + " for HIT " + relevanceVote.hitID); } catch (IllegalStateException e) { e.printStackTrace(); } } for (String r : relevant) { String sentenceId = r.trim(); if (!sentenceId.isEmpty() && sentenceId.matches("\\d+")) { QueryResultContainer.SingleSentenceRelevanceVote singleSentenceVote = new QueryResultContainer.SingleSentenceRelevanceVote(); singleSentenceVote.sentenceID = sentenceId; singleSentenceVote.relevant = "true"; relevanceVote.singleSentenceRelevanceVotes.add(singleSentenceVote); } } for (String r : irrelevant) { String sentenceId = r.trim(); if (!sentenceId.isEmpty() && sentenceId.matches("\\d+")) { QueryResultContainer.SingleSentenceRelevanceVote singleSentenceVote = new QueryResultContainer.SingleSentenceRelevanceVote(); singleSentenceVote.sentenceID = sentenceId; singleSentenceVote.relevant = "false"; relevanceVote.singleSentenceRelevanceVotes.add(singleSentenceVote); } } mTurkRelevanceVotes.add(relevanceVote); } } } File outputFile = new File(outputDir, f.getName()); FileUtils.writeStringToFile(outputFile, queryResultContainer.toXML(), "utf-8"); System.out.println("Finished " + outputFile); } }
From source file:com.google.gwt.resources.converter.Css2Gss.java
public static void main(String... args) { Options options = Options.parseOrQuit(args); if (options.singleFile) { try {//from ww w . ja va 2 s. c o m System.out.println( convertFile(options.resource, options.simpleBooleanConditions, options.scopeFiles)); System.exit(0); } catch (Exception e) { e.printStackTrace(); System.exit(-1); } } Collection<File> filesToConvert = FileUtils.listFiles(options.resource, new String[] { "css" }, options.recurse); for (File cssFile : filesToConvert) { try { if (doesCorrespondingGssFileExists(cssFile)) { System.out.println( "GSS file already exists - will not convert, file: " + cssFile.getAbsolutePath()); continue; } String gss = convertFile(cssFile, options.simpleBooleanConditions, options.scopeFiles); writeGss(gss, cssFile); System.out.println("Converted " + cssFile.getAbsolutePath()); } catch (Exception e) { System.err.println("Failed to convert " + cssFile.getAbsolutePath()); e.printStackTrace(); } } }
From source file:com.turn.ttorrent.cli.TorrentMain.java
/** * Torrent reader and creator./*from ww w .j a v a 2s . co m*/ * * <p> * You can use the {@code main()} function of this class to read or create * torrent files. See usage for details. * </p> * */ public static void main(String[] args) { BasicConfigurator.configure(new ConsoleAppender(new PatternLayout("%-5p: %m%n"))); CmdLineParser parser = new CmdLineParser(); CmdLineParser.Option help = parser.addBooleanOption('h', "help"); CmdLineParser.Option filename = parser.addStringOption('t', "torrent"); CmdLineParser.Option create = parser.addBooleanOption('c', "create"); CmdLineParser.Option pieceLength = parser.addIntegerOption('l', "length"); CmdLineParser.Option announce = parser.addStringOption('a', "announce"); try { parser.parse(args); } catch (CmdLineParser.OptionException oe) { System.err.println(oe.getMessage()); usage(System.err); System.exit(1); } // Display help and exit if requested if (Boolean.TRUE.equals((Boolean) parser.getOptionValue(help))) { usage(System.out); System.exit(0); } String filenameValue = (String) parser.getOptionValue(filename); if (filenameValue == null) { usage(System.err, "Torrent file must be provided!"); System.exit(1); } Integer pieceLengthVal = (Integer) parser.getOptionValue(pieceLength); if (pieceLengthVal == null) { pieceLengthVal = Torrent.DEFAULT_PIECE_LENGTH; } else { pieceLengthVal = pieceLengthVal * 1024; } logger.info("Using piece length of {} bytes.", pieceLengthVal); Boolean createFlag = (Boolean) parser.getOptionValue(create); //For repeated announce urls @SuppressWarnings("unchecked") Vector<String> announceURLs = (Vector<String>) parser.getOptionValues(announce); String[] otherArgs = parser.getRemainingArgs(); if (Boolean.TRUE.equals(createFlag) && (otherArgs.length != 1 || announceURLs.isEmpty())) { usage(System.err, "Announce URL and a file or directory must be " + "provided to create a torrent file!"); System.exit(1); } OutputStream fos = null; try { if (Boolean.TRUE.equals(createFlag)) { if (filenameValue != null) { fos = new FileOutputStream(filenameValue); } else { fos = System.out; } //Process the announce URLs into URIs List<URI> announceURIs = new ArrayList<URI>(); for (String url : announceURLs) { announceURIs.add(new URI(url)); } //Create the announce-list as a list of lists of URIs //Assume all the URI's are first tier trackers List<List<URI>> announceList = new ArrayList<List<URI>>(); announceList.add(announceURIs); File source = new File(otherArgs[0]); if (!source.exists() || !source.canRead()) { throw new IllegalArgumentException( "Cannot access source file or directory " + source.getName()); } String creator = String.format("%s (ttorrent)", System.getProperty("user.name")); Torrent torrent = null; if (source.isDirectory()) { List<File> files = new ArrayList<File>( FileUtils.listFiles(source, TrueFileFilter.TRUE, TrueFileFilter.TRUE)); Collections.sort(files); torrent = Torrent.create(source, files, pieceLengthVal, announceList, creator); } else { torrent = Torrent.create(source, pieceLengthVal, announceList, creator); } torrent.save(fos); } else { Torrent.load(new File(filenameValue), true); } } catch (Exception e) { logger.error("{}", e.getMessage(), e); System.exit(2); } finally { if (fos != System.out) { IOUtils.closeQuietly(fos); } } }
From source file:com.athena.dolly.websocket.client.test.WebSocketClient.java
public static void main(String[] args) throws Exception { URI uri;/*from w ww. j a va 2s . c o m*/ if (args.length > 0) { uri = new URI(args[0]); } else { uri = new URI("ws://localhost:7700/websocket"); } WebSocketClient client = new WebSocketClient(uri); client.initialize(); File directory = new File("C:/Private"); Collection<File> files = FileUtils.listFiles(directory, TrueFileFilter.TRUE, TrueFileFilter.TRUE); for (Iterator<File> iterator = files.iterator(); iterator.hasNext();) { File file = (File) iterator.next(); client.sendFile(file); } //client.sendFile(new File("C:/Temp/netty-master.zip")); client.shutdown(); }
From source file:com.utdallas.s3lab.smvhunter.monkey.MonkeyMe.java
/** * @param args// www . j av a 2s .c o m */ public static void main(String[] args) throws Exception { logger.info("start time ==== " + System.currentTimeMillis()); try { //load the adb location from the props file Properties props = new Properties(); props.load(new FileInputStream(new File("adb.props"))); //set the adb location WindowUpdate.adbLocation = props.getProperty("adb_location"); adbLocation = props.getProperty("adb_location"); //set root dir ROOT_DIRECTORY = props.getProperty("root_dir"); //completed txt completedFile = new File(ROOT_DIRECTORY + "tested.txt"); //db location for static analysis dbLocation = props.getProperty("db_location"); //location for smart input generation output smartInputLocation = props.getProperty("smart_input_location"); //udp dump location udpDumpLocation = props.getProperty("udp_dump_location"); //logcat dump location logCatLocation = props.getProperty("log_cat_location"); //strace output location straceOutputLocation = props.getProperty("strace_output_location"); //strace dump location straceDumpLocation = props.getProperty("strace_output_location"); //set x and y coords UIEnumerator.screenX = props.getProperty("x"); UIEnumerator.screenY = props.getProperty("y"); DeviceOfflineMonitor.START_EMULATOR = props.getProperty("restart"); //read output of static analysis readFromStaticAnalysisText(); logger.info("Read static analysis output"); readFromSmartInputText(); logger.info("Read smart input generation output"); //populate the queue with apps which are only present in the static analysis @SuppressWarnings("unchecked") final Set<? extends String> sslApps = new HashSet<String>( CollectionUtils.collect(FileUtils.readLines(new File(dbLocation)), new Transformer() { @Override public Object transform(Object input) { //get app file name String temp = StringUtils.substringBefore(input.toString(), " "); return StringUtils.substringAfterLast(temp, "/"); } })); Collection<File> fileList = FileUtils.listFiles(new File(ROOT_DIRECTORY), new String[] { "apk" }, false); CollectionUtils.filter(fileList, new Predicate() { @Override public boolean evaluate(Object object) { return sslApps.contains(StringUtils.substringAfterLast(object.toString(), "/")); } }); apkQueue = new LinkedBlockingDeque<File>(fileList); logger.info("finished listing files from the root directory"); try { //populate the tested apk list completedApps.addAll(FileUtils.readLines(completedFile)); } catch (Exception e) { //pass except logger.info("No tested.txt file found"); //create new file if (completedFile.createNewFile()) { logger.info("tested.txt created in root directory"); } else { logger.info("tested.txt file could not be created"); } } //get the executors for managing the emulators executors = Executors.newCachedThreadPool(); //set the devicemonitor exec DeviceOfflineMonitor.exec = executors; final List<Future<?>> futureList = new ArrayList<Future<?>>(); //start the offline device monitor (emulator management thread) logger.info("Starting Device Offline Monitor Thread"); executors.submit(new DeviceOfflineMonitor()); //get ADB backend object for device change listener AdbBackend adb = new AdbBackend(); //register for device change and wait for events //once event is received, start the MonkeyMe thread MonkeyDeviceChangeListener deviceChangeListener = new MonkeyDeviceChangeListener(executors, futureList); AndroidDebugBridge.addDeviceChangeListener(deviceChangeListener); logger.info("Listening to changes in devices (emulators)"); //wait for the latch to come down //this means that all the apks have been processed cdl.await(); logger.info("Finished testing all apps waiting for threads to join"); //now wait for every thread to finish for (Future<?> future : futureList) { future.get(); } logger.info("All threads terminated"); //stop listening for device update AndroidDebugBridge.removeDeviceChangeListener(deviceChangeListener); //stop the debug bridge AndroidDebugBridge.terminate(); //stop offline device monitor DeviceOfflineMonitor.stop = true; logger.info("adb and listeners terminated"); } finally { logger.info("Executing this finally"); executors.shutdownNow(); } logger.info("THE END!!"); }
From source file:gov.nasa.jpl.memex.pooledtimeseries.PoT.java
public static void main(String[] args) { System.loadLibrary(Core.NATIVE_LIBRARY_NAME); Option fileOpt = OptionBuilder.withArgName("file").hasArg().withLongOpt("file") .withDescription("Path to a single file").create('f'); Option dirOpt = OptionBuilder.withArgName("directory").hasArg().withLongOpt("dir") .withDescription("A directory with image files in it").create('d'); Option helpOpt = OptionBuilder.withLongOpt("help").withDescription("Print this message.").create('h'); Option pathFileOpt = OptionBuilder.withArgName("path file").hasArg().withLongOpt("pathfile") .withDescription(/*from ww w . j ava 2s.c o m*/ "A file containing full absolute paths to videos. Previous default was memex-index_temp.txt") .create('p'); Option outputFileOpt = OptionBuilder.withArgName("output file").withLongOpt("outputfile").hasArg() .withDescription("File containing similarity results. Defaults to ./similarity.txt").create('o'); Option jsonOutputFlag = OptionBuilder.withArgName("json output").withLongOpt("json") .withDescription("Set similarity output format to JSON. Defaults to .txt").create('j'); Option similarityFromFeatureVectorsOpt = OptionBuilder .withArgName("similarity from FeatureVectors directory") .withLongOpt("similarityFromFeatureVectorsDirectory").hasArg() .withDescription("calculate similarity matrix from given directory of feature vectors").create('s'); Options options = new Options(); options.addOption(dirOpt); options.addOption(pathFileOpt); options.addOption(fileOpt); options.addOption(helpOpt); options.addOption(outputFileOpt); options.addOption(jsonOutputFlag); options.addOption(similarityFromFeatureVectorsOpt); // create the parser CommandLineParser parser = new GnuParser(); try { // parse the command line arguments CommandLine line = parser.parse(options, args); String directoryPath = null; String pathFile = null; String singleFilePath = null; String similarityFromFeatureVectorsDirectory = null; ArrayList<Path> videoFiles = null; if (line.hasOption("dir")) { directoryPath = line.getOptionValue("dir"); } if (line.hasOption("pathfile")) { pathFile = line.getOptionValue("pathfile"); } if (line.hasOption("file")) { singleFilePath = line.getOptionValue("file"); } if (line.hasOption("outputfile")) { outputFile = line.getOptionValue("outputfile"); } if (line.hasOption("json")) { outputFormat = OUTPUT_FORMATS.JSON; } if (line.hasOption("similarityFromFeatureVectorsDirectory")) { similarityFromFeatureVectorsDirectory = line .getOptionValue("similarityFromFeatureVectorsDirectory"); } if (line.hasOption("help") || (line.getOptions() == null || (line.getOptions() != null && line.getOptions().length == 0)) || (directoryPath != null && pathFile != null && !directoryPath.equals("") && !pathFile.equals(""))) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("pooled_time_series", options); System.exit(1); } if (directoryPath != null) { File dir = new File(directoryPath); List<File> files = (List<File>) FileUtils.listFiles(dir, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE); videoFiles = new ArrayList<Path>(files.size()); for (File file : files) { String filePath = file.toString(); // When given a directory to load videos from we need to ensure that we // don't try to load the of.txt and hog.txt intermediate result files // that results from previous processing runs. if (!filePath.contains(".txt")) { videoFiles.add(file.toPath()); } } LOG.info("Added " + videoFiles.size() + " video files from " + directoryPath); } if (pathFile != null) { Path list_file = Paths.get(pathFile); videoFiles = loadFiles(list_file); LOG.info("Loaded " + videoFiles.size() + " video files from " + pathFile); } if (singleFilePath != null) { Path singleFile = Paths.get(singleFilePath); LOG.info("Loaded file: " + singleFile); videoFiles = new ArrayList<Path>(1); videoFiles.add(singleFile); } if (similarityFromFeatureVectorsDirectory != null) { File dir = new File(similarityFromFeatureVectorsDirectory); List<File> files = (List<File>) FileUtils.listFiles(dir, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE); videoFiles = new ArrayList<Path>(files.size()); for (File file : files) { String filePath = file.toString(); // We need to load only the *.of.txt and *.hog.txt values if (filePath.endsWith(".of.txt")) { videoFiles.add(file.toPath()); } if (filePath.endsWith(".hog.txt")) { videoFiles.add(file.toPath()); } } LOG.info("Added " + videoFiles.size() + " feature vectors from " + similarityFromFeatureVectorsDirectory); evaluateSimilarity(videoFiles, 1); } else { evaluateSimilarity(videoFiles, 1); } LOG.info("done."); } catch (ParseException exp) { // oops, something went wrong System.err.println("Parsing failed. Reason: " + exp.getMessage()); } }