List of usage examples for java.io BufferedReader readLine
public String readLine() throws IOException
From source file:edu.cmu.ark.QuestionTransducer.java
/** * main method for testing stage 2 in isolation. The QuestionAsker class's main method should be * used to generate questions from the end-to-end system. * //w ww . j a va 2s . co m * @param args */ public static void main(String[] args) { QuestionTransducer qt = new QuestionTransducer(); AnalysisUtilities.getInstance(); String buf; Tree inputTree; boolean printParse = false; boolean printOriginal = false; boolean treeInput = false; boolean printFeatures = false; Set<Question> inputTrees = new HashSet<Question>(); qt.setAvoidPronounsAndDemonstratives(true); for (int i = 0; i < args.length; i++) { if (args[i].equals("--debug")) { GlobalProperties.setDebug(true); } else if (args[i].equals("--print-parse")) { printParse = true; } else if (args[i].equals("--print-original")) { printOriginal = true; } else if (args[i].equals("--print-features")) { printFeatures = true; } else if (args[i].equals("--print-extracted-phrases")) { qt.setPrintExtractedPhrases(true); } else if (args[i].equals("--tree-input")) { treeInput = true; } else if (args[i].equals("--keep-pro")) { qt.setAvoidPronounsAndDemonstratives(false); } else if (args[i].equals("--properties")) { GlobalProperties.loadProperties(args[i + 1]); } } try { BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); //take input from the user on stdin if (GlobalProperties.getDebug()) System.err.println("\nInput Declarative Sentence:"); while ((buf = br.readLine()) != null) { if (treeInput) { buf = AnalysisUtilities.preprocessTreeString(buf); inputTree = AnalysisUtilities.getInstance().readTreeFromString(buf); AnalysisUtilities.getInstance().normalizeTree(inputTree); } else { if (AnalysisUtilities.filterOutSentenceByPunctuation(buf)) { continue; } buf = AnalysisUtilities.preprocess(buf); if (printOriginal) System.out.println("\n" + buf); ParseResult parseRes = AnalysisUtilities.getInstance().parseSentence(buf); inputTree = parseRes.parse; if (GlobalProperties.getDebug()) System.err.println("Parse Score: " + parseRes.score); } if (printParse) System.out.println(inputTree); inputTrees.clear(); Question tmp = new Question(); tmp.setIntermediateTree(inputTree.deepCopy()); tmp.setSourceTree(inputTree); inputTrees.add(tmp); //iterate over the trees given by the input List<Question> questions; for (Question q : inputTrees) { try { qt.generateQuestionsFromParse(q); questions = qt.getQuestions(); QuestionTransducer.removeDuplicateQuestions(questions); //iterate over the questions for each tree for (Question curQuestion : questions) { System.out.print(curQuestion.yield()); if (printFeatures) { System.out.print("\t"); int cnt = 0; for (Double val : curQuestion.featureValueList()) { if (cnt > 0) System.out.print(";"); System.out.print(NumberFormat.getInstance().format(val)); cnt++; } } System.out.println(); } } catch (Exception e) { e.printStackTrace(); } } if (GlobalProperties.getDebug()) System.err.println("\nInput Declarative Sentence:"); } } catch (IOException e) { e.printStackTrace(); } }
From source file:azkaban.webapp.AzkabanWebServer.java
/** * Azkaban using Jetty// w w w .j a v a2s . c om * * @param args */ public static void main(String[] args) throws Exception { logger.info("Starting Jetty Azkaban Web Server..."); Props azkabanSettings = AzkabanServer.loadProps(args); if (azkabanSettings == null) { logger.error("Azkaban Properties not loaded."); logger.error("Exiting Azkaban..."); return; } int maxThreads = azkabanSettings.getInt("jetty.maxThreads", DEFAULT_THREAD_NUMBER); boolean isStatsOn = azkabanSettings.getBoolean("jetty.connector.stats", true); logger.info("Setting up connector with stats on: " + isStatsOn); boolean ssl; int port; final Server server = new Server(); if (azkabanSettings.getBoolean("jetty.use.ssl", true)) { int sslPortNumber = azkabanSettings.getInt("jetty.ssl.port", DEFAULT_SSL_PORT_NUMBER); port = sslPortNumber; ssl = true; logger.info( "Setting up Jetty Https Server with port:" + sslPortNumber + " and numThreads:" + maxThreads); SslSocketConnector secureConnector = new SslSocketConnector(); secureConnector.setPort(sslPortNumber); secureConnector.setKeystore(azkabanSettings.getString("jetty.keystore")); secureConnector.setPassword(azkabanSettings.getString("jetty.password")); secureConnector.setKeyPassword(azkabanSettings.getString("jetty.keypassword")); secureConnector.setTruststore(azkabanSettings.getString("jetty.truststore")); secureConnector.setTrustPassword(azkabanSettings.getString("jetty.trustpassword")); secureConnector.setHeaderBufferSize(MAX_HEADER_BUFFER_SIZE); // set up vulnerable cipher suites to exclude List<String> cipherSuitesToExclude = azkabanSettings.getStringList("jetty.excludeCipherSuites"); logger.info("Excluded Cipher Suites: " + String.valueOf(cipherSuitesToExclude)); if (cipherSuitesToExclude != null && !cipherSuitesToExclude.isEmpty()) { secureConnector.setExcludeCipherSuites(cipherSuitesToExclude.toArray(new String[0])); } server.addConnector(secureConnector); } else { ssl = false; port = azkabanSettings.getInt("jetty.port", DEFAULT_PORT_NUMBER); SocketConnector connector = new SocketConnector(); connector.setPort(port); connector.setHeaderBufferSize(MAX_HEADER_BUFFER_SIZE); server.addConnector(connector); } // setting stats configuration for connectors for (Connector connector : server.getConnectors()) { connector.setStatsOn(isStatsOn); } String hostname = azkabanSettings.getString("jetty.hostname", "localhost"); azkabanSettings.put("server.hostname", hostname); azkabanSettings.put("server.port", port); azkabanSettings.put("server.useSSL", String.valueOf(ssl)); app = new AzkabanWebServer(server, azkabanSettings); boolean checkDB = azkabanSettings.getBoolean(AzkabanDatabaseSetup.DATABASE_CHECK_VERSION, false); if (checkDB) { AzkabanDatabaseSetup setup = new AzkabanDatabaseSetup(azkabanSettings); setup.loadTableInfo(); if (setup.needsUpdating()) { logger.error("Database is out of date."); setup.printUpgradePlan(); logger.error("Exiting with error."); System.exit(-1); } } try { JdbcExecutorLoader jdbcExecutorLoader = new JdbcExecutorLoader(azkabanSettings); jdbcExecutorLoader.updateExecutableJobsOnStartUp(); } catch (Exception e) { logger.warn("There could be some jobs KILLED by this server restart event," + "but their status still erroneously being shown as RUNNING." + "Please run the SQL " + "[update execution_jobs join execution_flows" + " on execution_jobs.exec_id = execution_flows.exec_id" + " set execution_jobs.status = 70 where" + " execution_jobs.status = 30 and execution_flows.status = 70]" + "on the database to rectify their status", e); } QueuedThreadPool httpThreadPool = new QueuedThreadPool(maxThreads); server.setThreadPool(httpThreadPool); String staticDir = azkabanSettings.getString("web.resource.dir", DEFAULT_STATIC_DIR); logger.info("Setting up web resource dir " + staticDir); Context root = new Context(server, "/", Context.SESSIONS); root.setMaxFormContentSize(MAX_FORM_CONTENT_SIZE); String defaultServletPath = azkabanSettings.getString("azkaban.default.servlet.path", "/index"); root.setResourceBase(staticDir); ServletHolder indexRedirect = new ServletHolder(new IndexRedirectServlet(defaultServletPath)); root.addServlet(indexRedirect, "/"); ServletHolder index = new ServletHolder(new ProjectServlet()); root.addServlet(index, "/index"); ServletHolder staticServlet = new ServletHolder(new DefaultServlet()); root.addServlet(staticServlet, "/css/*"); root.addServlet(staticServlet, "/js/*"); root.addServlet(staticServlet, "/images/*"); root.addServlet(staticServlet, "/fonts/*"); root.addServlet(staticServlet, "/favicon.ico"); root.addServlet(new ServletHolder(new ProjectManagerServlet()), "/manager"); root.addServlet(new ServletHolder(new ExecutorServlet()), "/executor"); root.addServlet(new ServletHolder(new HistoryServlet()), "/history"); root.addServlet(new ServletHolder(new ScheduleServlet()), "/schedule"); root.addServlet(new ServletHolder(new JMXHttpServlet()), "/jmx"); root.addServlet(new ServletHolder(new TriggerManagerServlet()), "/triggers"); root.addServlet(new ServletHolder(new StatsServlet()), "/stats"); root.addServlet(new ServletHolder(new AboutServlet()), "/about"); root.addServlet(new ServletHolder(new FileEditorServlet()), "/fileeditor"); ServletHolder restliHolder = new ServletHolder(new RestliServlet()); restliHolder.setInitParameter("resourcePackages", "azkaban.restli"); root.addServlet(restliHolder, "/restli/*"); String viewerPluginDir = azkabanSettings.getString("viewer.plugin.dir", "plugins/viewer"); loadViewerPlugins(root, viewerPluginDir, app.getVelocityEngine()); // triggerplugin String triggerPluginDir = azkabanSettings.getString("trigger.plugin.dir", "plugins/triggers"); Map<String, TriggerPlugin> triggerPlugins = loadTriggerPlugins(root, triggerPluginDir, app); app.setTriggerPlugins(triggerPlugins); // always have basic time trigger // TODO: find something else to do the job app.getTriggerManager().start(); root.setAttribute(ServerConstants.AZKABAN_SERVLET_CONTEXT_KEY, app); try { server.start(); } catch (Exception e) { logger.warn(e); Utils.croak(e.getMessage(), 1); } Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { try { logTopMemoryConsumers(); } catch (Exception e) { logger.info(("Exception when logging top memory consumers"), e); } logger.info("Shutting down http server..."); try { app.close(); server.stop(); server.destroy(); } catch (Exception e) { logger.error("Error while shutting down http server.", e); } logger.info("kk thx bye."); } public void logTopMemoryConsumers() throws Exception, IOException { if (new File("/bin/bash").exists() && new File("/bin/ps").exists() && new File("/usr/bin/head").exists()) { logger.info("logging top memeory consumer"); java.lang.ProcessBuilder processBuilder = new java.lang.ProcessBuilder("/bin/bash", "-c", "/bin/ps aux --sort -rss | /usr/bin/head"); Process p = processBuilder.start(); p.waitFor(); InputStream is = p.getInputStream(); java.io.BufferedReader reader = new java.io.BufferedReader(new InputStreamReader(is)); String line = null; while ((line = reader.readLine()) != null) { logger.info(line); } is.close(); } } }); logger.info("Server running on " + (ssl ? "ssl" : "") + " port " + port + "."); }
From source file:ci6226.buildindex.java
/** * @param args the command line arguments *//* w w w .ja v a 2 s . co m*/ public static void main(String[] args) throws FileNotFoundException, IOException, ParseException { String file = "/home/steven/Dropbox/workspace/ntu_coursework/ci6226/Assiment/yelpdata/yelp_training_set/yelp_training_set_review.json"; JSONParser parser = new JSONParser(); BufferedReader in = new BufferedReader(new FileReader(file)); // List<Document> jdocs = new LinkedList<Document>(); Date start = new Date(); String indexPath = "./myindex"; System.out.println("Indexing to directory '" + indexPath + "'..."); // Analyzer analyzer= new NGramAnalyzer(2,8); Analyzer analyzer = new myAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_47, analyzer); Directory dir = FSDirectory.open(new File(indexPath)); // :Post-Release-Update-Version.LUCENE_XY: // TODO: try different analyzer,stop words,words steming check size // Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_47); // Add new documents to an existing index: // iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); // Optional: for better indexing performance, if you // are indexing many documents, increase the RAM // buffer. But if you do this, increase the max heap // size to the JVM (eg add -Xmx512m or -Xmx1g): // // iwc.setRAMBufferSizeMB(256.0); IndexWriter writer = new IndexWriter(dir, iwc); // writer.addDocuments(jdocs); int line = 0; while (in.ready()) { String s = in.readLine(); Object obj = JSONValue.parse(s); JSONObject person = (JSONObject) obj; String text = (String) person.get("text"); String user_id = (String) person.get("user_id"); String business_id = (String) person.get("business_id"); String review_id = (String) person.get("review_id"); JSONObject votes = (JSONObject) person.get("votes"); long funny = (Long) votes.get("funny"); long cool = (Long) votes.get("cool"); long useful = (Long) votes.get("useful"); Document doc = new Document(); Field review_idf = new StringField("review_id", review_id, Field.Store.YES); doc.add(review_idf); Field business_idf = new StringField("business_id", business_id, Field.Store.YES); doc.add(business_idf); //http://qindongliang1922.iteye.com/blog/2030639 FieldType ft = new FieldType(); ft.setIndexed(true);// ft.setStored(true);// ft.setStoreTermVectors(true); ft.setTokenized(true); ft.setStoreTermVectorPositions(true);//? ft.setStoreTermVectorOffsets(true);//??? Field textf = new Field("text", text, ft); doc.add(textf); // Field user_idf = new StringField("user_id", user_id, Field.Store.YES); // doc.add(user_idf); // doc.add(new LongField("cool", cool, Field.Store.YES)); // doc.add(new LongField("funny", funny, Field.Store.YES)); // doc.add(new LongField("useful", useful, Field.Store.YES)); writer.addDocument(doc); System.out.println(line++); } writer.close(); Date end = new Date(); System.out.println(end.getTime() - start.getTime() + " total milliseconds"); // BufferedReader in = new BufferedReader(new FileReader(file)); //while (in.ready()) { // String s = in.readLine(); // //System.out.println(s); // JSONObject jsonObject = (JSONObject) ((Object)s); // String rtext = (String) jsonObject.get("text"); // System.out.println(rtext); // //long age = (Long) jsonObject.get("age"); // //System.out.println(age); //} //in.close(); }
From source file:CourserankConnector.java
public static void main(String[] args) throws Exception { /////////////////////////////////////// //Tagger init //MaxentTagger tagger = new MaxentTagger("models/english-left3words-distsim.tagger"); ////*from www . j ava 2 s .c om*/ //CLIENT INITIALIZATION ImportData importCourse = new ImportData(); HttpClient httpclient = new DefaultHttpClient(); httpclient = WebClientDevWrapper.wrapClient(httpclient); try { /* httpclient.getCredentialsProvider().setCredentials( new AuthScope(null, -1), new UsernamePasswordCredentials("eadrian", "eactresp1")); */ ////////////////////////////////////////////////// //Get Course Bulletin Departments page List<Course> courses = new ArrayList<Course>(); HttpGet httpget = new HttpGet("http://explorecourses.stanford.edu"); System.out.println("executing request" + httpget.getRequestLine()); HttpResponse response = httpclient.execute(httpget); HttpEntity entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); String bulletinpage = ""; //STORE RETURNED HTML TO BULLETINPAGE if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { bulletinpage += line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); /////////////////////////////////////////////////////////////////////////////// //Login to Courserank httpget = new HttpGet("https://courserank.com/stanford/main"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); String page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page += line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); //////////////////////////////////////////////////// //POST REQUEST LOGIN HttpPost post = new HttpPost("https://www.courserank.com/stanford/main"); List<NameValuePair> pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("RT", "")); pairs.add(new BasicNameValuePair("action", "login")); pairs.add(new BasicNameValuePair("password", "trespass")); pairs.add(new BasicNameValuePair("username", "eaconte@stanford.edu")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); HttpResponse resp = httpclient.execute(post); HttpEntity ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /////////////////////////////////////////////////// //THIS STEP MAY NOT BE NEEDED BUT GETS MAIN PROFILE PAGE HttpGet gethome = new HttpGet("https://www.courserank.com/stanford/home"); System.out.println("executing request" + gethome.getRequestLine()); HttpResponse gresp = httpclient.execute(gethome); HttpEntity gent = gresp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + gent.getContentLength()); InputStream i = gent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } ///////////////////////////////////////////////////////////////////////////////// //Parse Bulletin String results = getToken(bulletinpage, "RESULTS HEADER", "Additional Searches"); String[] depts = results.split("href"); //SPLIT FOR EACH DEPARTMENT LINK, ITERATE boolean ready = false; for (int i = 1; i < depts.length; i++) { //EXTRACT LINK, DEPARTMENT NAME AND ABBREVIATION String dept = new String(depts[i]); String abbr = getToken(dept, "(", ")"); String name = getToken(dept, ">", "("); name.trim(); //System.out.println(tagger.tagString(name)); String link = getToken(dept, "=\"", "\">"); System.out.println(name + " : " + abbr + " : " + link); System.out.println("======================================================================"); if (i <= 10 || i >= 127) //values to keep it to undergraduate courses. Excludes law, med, business, overseas continue; /*if (i<=46) continue; */ //Start at BIOHOP /*if (abbr.equals("INTNLREL")) ready = true; if (!ready) continue;*/ //Construct department course search URL //Then request page String URL = "http://explorecourses.stanford.edu/" + link + "&filter-term-Autumn=on&filter-term-Winter=on&filter-term-Spring=on"; httpget = new HttpGet(URL); //System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); //ystem.out.println("----------------------------------------"); //System.out.println(response.getStatusLine()); String rpage = ""; if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream in = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String line; while ((line = br.readLine()) != null) { rpage += line; //System.out.println(line); } br.close(); in.close(); } EntityUtils.consume(entity); //Process results page List<Course> deptCourses = new ArrayList<Course>(); List<Course> result = processResultPage(rpage); deptCourses.addAll(result); //While there are more result pages, keep going boolean more = (!(result.size() == 0) && (result.get((result.size() - 1)).courseNumber < 299)); boolean morepages = anotherPage(rpage); while (morepages && more) { URL = nextURL(URL); httpget = new HttpGet(URL); //System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); //System.out.println("----------------------------------------"); //System.out.println(response.getStatusLine()); rpage = ""; if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream in = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String line; while ((line = br.readLine()) != null) { rpage += line; //System.out.println(line); } br.close(); in.close(); } EntityUtils.consume(entity); morepages = anotherPage(rpage); result = processResultPage(rpage); deptCourses.addAll(result); more = (!(result.size() == 0) && (result.get((result.size() - 1)).courseNumber < 299)); /*String mores = more? "yes": "no"; String pagess = morepages?"yes":"no"; System.out.println("more: "+mores+" morepages: "+pagess); System.out.println("more");*/ } //Get course ratings for all department courses via courserank deptCourses = getRatings(httpclient, abbr, deptCourses); for (int j = 0; j < deptCourses.size(); j++) { Course c = deptCourses.get(j); System.out.println("" + c.title + " : " + c.rating); c.tags = name; c.code = c.code.trim(); c.department = name; c.deptAB = abbr; c.writeToDatabase(); //System.out.println(tagger.tagString(c.title)); } } if (!page.equals("")) return; /////////////////////////////////////////////////// //Get Course Bulletin Department courses /* httpget = new HttpGet("https://courserank.com/stanford/main"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page +=line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); //////////////////////////////////////////////////// //POST REQUEST LOGIN HttpPost post = new HttpPost("https://www.courserank.com/stanford/main"); List<NameValuePair> pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("RT", "")); pairs.add(new BasicNameValuePair("action", "login")); pairs.add(new BasicNameValuePair("password", "trespass")); pairs.add(new BasicNameValuePair("username", "eaconte@stanford.edu")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); HttpResponse resp = httpclient.execute(post); HttpEntity ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /////////////////////////////////////////////////// //THIS STEP MAY NOT BE NEEDED BUT GETS MAIN PROFILE PAGE HttpGet gethome = new HttpGet("https://www.courserank.com/stanford/home"); System.out.println("executing request" + gethome.getRequestLine()); HttpResponse gresp = httpclient.execute(gethome); HttpEntity gent = gresp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + gent.getContentLength()); InputStream i = gent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } //////////////////////////////////////// //GETS FIRST PAGE OF RESULTS EntityUtils.consume(gent); post = new HttpPost("https://www.courserank.com/stanford/search_results"); pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("filter_term_currentYear", "on")); pairs.add(new BasicNameValuePair("query", "")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); resp = httpclient.execute(post); ent = resp.getEntity(); System.out.println("----------------------------------------"); String rpage = ""; if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { rpage += line; System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); //////////////////////////////////////////////////// //PARSE FIRST PAGE OF RESULTS //int index = rpage.indexOf("div class=\"searchItem"); String []classSplit = rpage.split("div class=\"searchItem"); for (int i=1; i<classSplit.length; i++) { String str = classSplit[i]; //ID String CID = getToken(str, "course?id=","\">"); // CODE String CODE = getToken(str,"class=\"code\">" ,":</"); //TITLE String NAME = getToken(str, "class=\"title\">","</"); //DESCRIP String DES = getToken(str, "class=\"description\">","</"); //TERM String TERM = getToken(str, "Terms:", "|"); //UNITS String UNITS = getToken(str, "Units:", "<br/>"); //WORKLOAD String WLOAD = getToken(str, "Workload:", "|"); //GER String GER = getToken(str, "GERs:", "</d"); //RATING int searchIndex = 0; float rating = 0; while (true) { int ratingIndex = str.indexOf("large_Full", searchIndex); if (ratingIndex ==-1) { int halfratingIndex = str.indexOf("large_Half", searchIndex); if (halfratingIndex == -1) break; else rating += .5; break; } searchIndex = ratingIndex+1; rating++; } String RATING = ""+rating; //GRADE String GRADE = getToken(str, "div class=\"unofficialGrade\">", "</"); if (GRADE.equals("NOT FOUND")) { GRADE = getToken(str, "div class=\"officialGrade\">", "</"); } //REVIEWS String REVIEWS = getToken(str, "class=\"ratings\">", " ratings"); System.out.println(""+CODE+" : "+NAME + " : "+CID); System.out.println("----------------------------------------"); System.out.println("Term: "+TERM+" Units: "+UNITS+ " Workload: "+WLOAD + " Grade: "+ GRADE); System.out.println("Rating: "+RATING+ " Reviews: "+REVIEWS); System.out.println("=========================================="); System.out.println(DES); System.out.println("=========================================="); } /////////////////////////////////////////////////// //GETS SECOND PAGE OF RESULTS post = new HttpPost("https://www.courserank.com/stanford/search_results"); pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("filter_term_currentYear", "on")); pairs.add(new BasicNameValuePair("page", "2")); pairs.add(new BasicNameValuePair("query", "")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); resp = httpclient.execute(post); ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /* httpget = new HttpGet("https://github.com/"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page +=line; System.out.println(line); } br.close(); i.close(); }*/ EntityUtils.consume(entity); } finally { // When HttpClient instance is no longer needed, // shut down the connection manager to ensure // immediate deallocation of all system resources httpclient.getConnectionManager().shutdown(); } }
From source file:edu.umd.ujjwalgoel.AnalyzePMI.java
@SuppressWarnings({ "static-access" })
public static void main(String[] args) {
Options options = new Options();
options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INPUT));
CommandLine cmdline = null;/* w ww . ja v a2 s . c o m*/
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
System.exit(-1);
}
if (!cmdline.hasOption(INPUT)) {
System.out.println("args: " + Arrays.toString(args));
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(120);
formatter.printHelp(AnalyzePMI.class.getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
System.exit(-1);
}
String inputPath = cmdline.getOptionValue(INPUT);
System.out.println("input path: " + inputPath);
BufferedReader br = null;
int countPairs = 0;
List<PairOfWritables<PairOfStrings, FloatWritable>> pmis = new ArrayList<PairOfWritables<PairOfStrings, FloatWritable>>();
List<PairOfWritables<PairOfStrings, FloatWritable>> cloudPmis = new ArrayList<PairOfWritables<PairOfStrings, FloatWritable>>();
List<PairOfWritables<PairOfStrings, FloatWritable>> lovePmis = new ArrayList<PairOfWritables<PairOfStrings, FloatWritable>>();
PairOfWritables<PairOfStrings, FloatWritable> highestPMI = null;
PairOfWritables<PairOfStrings, FloatWritable> highestCloudPMI = null;
PairOfWritables<PairOfStrings, FloatWritable> highestCloudPMI2 = null;
PairOfWritables<PairOfStrings, FloatWritable> highestCloudPMI3 = null;
PairOfWritables<PairOfStrings, FloatWritable> highestLovePMI = null;
PairOfWritables<PairOfStrings, FloatWritable> highestLovePMI2 = null;
PairOfWritables<PairOfStrings, FloatWritable> highestLovePMI3 = null;
try {
FileSystem fs = FileSystem.get(new Configuration());
FileStatus[] status = fs.listStatus(new Path(inputPath));
//PairOfStrings pair = new PairOfStrings();
for (int i = 0; i < status.length; i++) {
br = new BufferedReader(new InputStreamReader(fs.open(status[i].getPath())));
String line = br.readLine();
while (line != null) {
String[] words = line.split("\\t");
float value = Float.parseFloat(words[1].trim());
String[] wordPair = words[0].replaceAll("\\(", "").replaceAll("\\)", "").split(",");
PairOfStrings pair = new PairOfStrings();
pair.set(wordPair[0].trim(), wordPair[1].trim());
if (wordPair[0].trim().equals("cloud")) {
PairOfWritables<PairOfStrings, FloatWritable> cloudPmi = new PairOfWritables<PairOfStrings, FloatWritable>();
cloudPmi.set(pair, new FloatWritable(value));
cloudPmis.add(cloudPmi);
if ((highestCloudPMI == null)
|| (highestCloudPMI.getRightElement().compareTo(cloudPmi.getRightElement()) < 0)) {
highestCloudPMI = cloudPmi;
} else if ((highestCloudPMI2 == null)
|| (highestCloudPMI2.getRightElement().compareTo(cloudPmi.getRightElement()) < 0)) {
highestCloudPMI2 = cloudPmi;
} else if ((highestCloudPMI3 == null)
|| (highestCloudPMI3.getRightElement().compareTo(cloudPmi.getRightElement()) < 0)) {
highestCloudPMI3 = cloudPmi;
}
}
if (wordPair[0].trim().equals("love")) {
PairOfWritables<PairOfStrings, FloatWritable> lovePmi = new PairOfWritables<PairOfStrings, FloatWritable>();
lovePmi.set(pair, new FloatWritable(value));
lovePmis.add(lovePmi);
if ((highestLovePMI == null)
|| (highestLovePMI.getRightElement().compareTo(lovePmi.getRightElement()) < 0)) {
highestLovePMI = lovePmi;
} else if ((highestLovePMI2 == null)
|| (highestLovePMI2.getRightElement().compareTo(lovePmi.getRightElement()) < 0)) {
highestLovePMI2 = lovePmi;
} else if ((highestLovePMI3 == null)
|| (highestLovePMI3.getRightElement().compareTo(lovePmi.getRightElement()) < 0)) {
highestLovePMI3 = lovePmi;
}
}
PairOfWritables<PairOfStrings, FloatWritable> pmi = new PairOfWritables<PairOfStrings, FloatWritable>();
pmi.set(pair, new FloatWritable(value));
pmis.add(pmi);
if (highestPMI == null) {
highestPMI = pmi;
} else if (highestPMI.getRightElement().compareTo(pmi.getRightElement()) < 0) {
highestPMI = pmi;
}
countPairs++;
line = br.readLine();
}
}
} catch (Exception ex) {
System.out.println("ERROR" + ex.getMessage());
}
/*Collections.sort(pmis, new Comparator<PairOfWritables<PairOfStrings, FloatWritable>>() {
public int compare(PairOfWritables<PairOfStrings, FloatWritable> e1,
PairOfWritables<PairOfStrings, FloatWritable> e2) {
/*if (e2.getRightElement().compareTo(e1.getRightElement()) == 0) {
return e1.getLeftElement().getLeftElement().compareTo(e2.getLeftElement().getLeftElement());
}
return e2.getRightElement().compareTo(e1.getRightElement());
}
});
Collections.sort(cloudPmis, new Comparator<PairOfWritables<PairOfStrings, FloatWritable>>() {
public int compare(PairOfWritables<PairOfStrings, FloatWritable> e1,
PairOfWritables<PairOfStrings, FloatWritable> e2) {
if (e2.getRightElement().compareTo(e1.getRightElement()) == 0) {
return e1.getLeftElement().getLeftElement().compareTo(e2.getLeftElement().getLeftElement());
}
return e2.getRightElement().compareTo(e1.getRightElement());
}
});
Collections.sort(lovePmis, new Comparator<PairOfWritables<PairOfStrings, FloatWritable>>() {
public int compare(PairOfWritables<PairOfStrings, FloatWritable> e1,
PairOfWritables<PairOfStrings, FloatWritable> e2) {
if (e2.getRightElement().compareTo(e1.getRightElement()) == 0) {
return e1.getLeftElement().getLeftElement().compareTo(e2.getLeftElement().getLeftElement());
}
return e2.getRightElement().compareTo(e1.getRightElement());
}
});
PairOfWritables<PairOfStrings, FloatWritable> highestPMI = pmis.get(0);
PairOfWritables<PairOfStrings, FloatWritable> highestCloudPMI = cloudPmis.get(0); PairOfWritables<PairOfStrings, FloatWritable> highestCloudPMI2 = cloudPmis.get(1);
PairOfWritables<PairOfStrings, FloatWritable> highestCloudPMI3 = cloudPmis.get(2);
PairOfWritables<PairOfStrings, FloatWritable> highestLovePMI = lovePmis.get(0); PairOfWritables<PairOfStrings, FloatWritable> highestLovePMI2 = lovePmis.get(1);
PairOfWritables<PairOfStrings, FloatWritable> highestLovePMI3 = lovePmis.get(2);*/
System.out.println("Total Distinct Pairs : " + countPairs);
System.out.println("Pair with highest PMI : (" + highestPMI.getLeftElement().getLeftElement() + ", "
+ highestPMI.getLeftElement().getRightElement());
System.out
.println("Word with highest PMI with Cloud : " + highestCloudPMI.getLeftElement().getRightElement()
+ " with value : " + highestCloudPMI.getRightElement().get());
System.out.println(
"Word with second highest PMI with Cloud : " + highestCloudPMI2.getLeftElement().getRightElement()
+ " with value : " + highestCloudPMI2.getRightElement().get());
System.out.println(
"Word with third highest PMI with Cloud : " + highestCloudPMI3.getLeftElement().getRightElement()
+ " with value : " + highestCloudPMI3.getRightElement().get());
System.out.println("Word with highest PMI with Love : " + highestLovePMI.getLeftElement().getRightElement()
+ " with value : " + highestLovePMI.getRightElement().get());
System.out.println(
"Word with second highest PMI with Love : " + highestLovePMI2.getLeftElement().getRightElement()
+ " with value : " + highestLovePMI2.getRightElement().get());
System.out.println(
"Word with third highest PMI with Love : " + highestLovePMI3.getLeftElement().getRightElement()
+ " with value : " + highestLovePMI3.getRightElement().get());
}
From source file:TwoFourTree.java
/** * The main method which the program executes from and which handles all * testing and input/output./*ww w . j a v a 2s .c o m*/ * * @param args String[] */ public static void main(String[] args) throws IOException { Comparator myComp = new IntegerComparator(); TwoFourTree myTree = new TwoFourTree(myComp); TwoFourTree firstTree = new TwoFourTree(myComp); TwoFourTree secondTree = new TwoFourTree(myComp); BufferedReader myReader = new BufferedReader(new InputStreamReader(System.in)); String input; boolean go = true; while (go == true) { System.out.print( "Format like this: -a 37 to add a 37 to the tree. -r 37 to remove a 37 from the tree: "); input = myReader.readLine(); if (input.substring(0, 2).equals("-a")) { Integer myInt = new Integer(input.substring(3, input.length())); myTree.insertElement(myInt, myInt); myTree.printAllElements(); } else if (input.substring(0, 2).equals("-r")) { Integer myInt = new Integer(input.substring(3, input.length())); myTree.removeElement(myInt); myTree.printAllElements(); } else System.out.print("\nYou're bad!"); System.out.print("\n"); } /*System.out.println("Loading firstTree with values ..."); Integer firstInt1 = new Integer(20); firstTree.insertElement(firstInt1, firstInt1); Integer firstInt2 = new Integer(50); firstTree.insertElement(firstInt2, firstInt2); Integer firstInt3 = new Integer(30); firstTree.insertElement(firstInt3, firstInt3); Integer firstInt4 = new Integer(40); firstTree.insertElement(firstInt4, firstInt4); Integer firstInt5 = new Integer(15); firstTree.insertElement(firstInt5, firstInt5); Integer firstInt6 = new Integer(35); firstTree.insertElement(firstInt6, firstInt6); Integer firstInt7 = new Integer(55); firstTree.insertElement(firstInt7, firstInt7); Integer firstInt8 = new Integer(38); firstTree.insertElement(firstInt8, firstInt8); Integer firstInt9 = new Integer(17); firstTree.insertElement(firstInt9, firstInt9); Integer firstInt10 = new Integer(37); firstTree.insertElement(firstInt10, firstInt10); Integer firstInt11 = new Integer(16); firstTree.insertElement(firstInt11, firstInt11); Integer firstInt12 = new Integer(36); firstTree.insertElement(firstInt12, firstInt12); Integer firstInt13 = new Integer(15); firstTree.insertElement(firstInt13, firstInt13); Integer firstInt14 = new Integer(36); firstTree.insertElement(firstInt14, firstInt14); Integer firstInt15 = new Integer(30); firstTree.insertElement(firstInt15, firstInt15); Integer firstInt16 = new Integer(34); firstTree.insertElement(firstInt16, firstInt16); Integer firstInt17 = new Integer(34); firstTree.insertElement(firstInt17, firstInt17); Integer firstInt18 = new Integer(30); firstTree.insertElement(firstInt18, firstInt18); Integer firstInt19 = new Integer(30); firstTree.insertElement(firstInt18, firstInt18); System.out.println("firstTree is fully loaded as follows:"); firstTree.printAllElements(); System.out.println("Checking parent/child connections ..."); firstTree.checkTree(firstTree.root()); System.out.println("All parent/child connections valid."); System.out.println("\nRemove 30"); firstTree.removeElement(30); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 30"); firstTree.removeElement(30); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 30"); firstTree.removeElement(30); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 30"); firstTree.removeElement(30); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 15"); firstTree.removeElement(15); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 55"); firstTree.removeElement(55); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 35"); firstTree.removeElement(35); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 38"); firstTree.removeElement(38); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 40"); firstTree.removeElement(40); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 36"); firstTree.removeElement(36); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 17"); firstTree.removeElement(17); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 34"); firstTree.removeElement(34); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 34"); firstTree.removeElement(34); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 36"); firstTree.removeElement(36); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 20"); firstTree.removeElement(20); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 37"); firstTree.removeElement(37); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 16"); firstTree.removeElement(16); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 50"); firstTree.removeElement(50); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println(""); System.out.println("Remove 15"); firstTree.removeElement(15); System.out.println("Removed successfully. Print new tree:"); firstTree.printAllElements(); System.out.println("done"); System.out.println("\n\nLoading secondTree with values ..."); Integer secondInt1 = new Integer(47); secondTree.insertElement(secondInt1, secondInt1); Integer secondInt2 = new Integer(83); secondTree.insertElement(secondInt2, secondInt2); Integer secondInt3 = new Integer(22); secondTree.insertElement(secondInt3, secondInt3); Integer secondInt4 = new Integer(16); secondTree.insertElement(secondInt4, secondInt4); Integer secondInt5 = new Integer(49); secondTree.insertElement(secondInt5, secondInt5); Integer secondInt6 = new Integer(100); secondTree.insertElement(secondInt6, secondInt6); Integer secondInt7 = new Integer(38); secondTree.insertElement(secondInt7, secondInt7); Integer secondInt8 = new Integer(3); secondTree.insertElement(secondInt8, secondInt8); Integer secondInt9 = new Integer(53); secondTree.insertElement(secondInt9, secondInt9); Integer secondInt10 = new Integer(66); secondTree.insertElement(secondInt10, secondInt10); Integer secondInt11 = new Integer(19); secondTree.insertElement(secondInt11, secondInt11); Integer secondInt12 = new Integer(23); secondTree.insertElement(secondInt12, secondInt12); Integer secondInt13 = new Integer(24); secondTree.insertElement(secondInt13, secondInt13); Integer secondInt14 = new Integer(88); secondTree.insertElement(secondInt14, secondInt14); Integer secondInt15 = new Integer(1); secondTree.insertElement(secondInt15, secondInt15); Integer secondInt16 = new Integer(97); secondTree.insertElement(secondInt16, secondInt16); Integer secondInt17 = new Integer(94); secondTree.insertElement(secondInt17, secondInt17); Integer secondInt18 = new Integer(35); secondTree.insertElement(secondInt18, secondInt18); Integer secondInt19 = new Integer(51); secondTree.insertElement(secondInt19, secondInt19); System.out.println("firstTree is fully loaded as follows:"); secondTree.printAllElements(); System.out.println("Checking parent/child connections ..."); secondTree.checkTree(secondTree.root()); System.out.println("All parent/child connections valid."); System.out.println("\nRemove 19"); secondTree.removeElement(19); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 3"); secondTree.removeElement(3); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 66"); secondTree.removeElement(66); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 88"); secondTree.removeElement(88); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 94"); secondTree.removeElement(94); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 100"); secondTree.removeElement(100); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 97"); secondTree.removeElement(97); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 47"); secondTree.removeElement(47); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 53"); secondTree.removeElement(53); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 38"); secondTree.removeElement(38); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 24"); secondTree.removeElement(24); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 83"); secondTree.removeElement(83); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 49"); secondTree.removeElement(49); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 23"); secondTree.removeElement(23); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 51"); secondTree.removeElement(51); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 1"); secondTree.removeElement(1); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 22"); secondTree.removeElement(22); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 16"); secondTree.removeElement(16); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println(""); System.out.println("Remove 35"); secondTree.removeElement(35); System.out.println("Removed successfully. Print new tree:"); secondTree.printAllElements(); System.out.println("done");*/ }
From source file:enrichment.Disambiguate.java
/**prerequisites: * cd silk_2.5.3/*_links/* w w w . ja v a 2s . c om*/ * cat *.nt|sort -t' ' -k3 > $filename * * @param args $filename * @throws IOException * @throws URISyntaxException */ public static void main(String[] args) { File file = new File(args[0]); if (file.isDirectory()) { args = file.list(new OnlyExtFilenameFilter("nt")); } BufferedReader in; for (int q = 0; q < args.length; q++) { String filename = null; if (file.isDirectory()) { filename = file.getPath() + File.separator + args[q]; } else { filename = args[q]; } try { FileWriter output = new FileWriter(filename + "_disambiguated.nt"); String prefix = "@prefix rdrel: <http://rdvocab.info/RDARelationshipsWEMI/> .\n" + "@prefix dbpedia: <http://de.dbpedia.org/resource/> .\n" + "@prefix frbr: <http://purl.org/vocab/frbr/core#> .\n" + "@prefix lobid: <http://lobid.org/resource/> .\n" + "@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .\n" + "@prefix foaf: <http://xmlns.com/foaf/0.1/> .\n" + "@prefix mo: <http://purl.org/ontology/mo/> .\n" + "@prefix wikipedia: <https://de.wikipedia.org/wiki/> ."; output.append(prefix + "\n\n"); in = new BufferedReader(new InputStreamReader(new FileInputStream(filename))); HashMap<String, HashMap<String, ArrayList<String>>> hm = new HashMap<String, HashMap<String, ArrayList<String>>>(); String s; HashMap<String, ArrayList<String>> hmLobid = new HashMap<String, ArrayList<String>>(); Stack<String> old_object = new Stack<String>(); while ((s = in.readLine()) != null) { String[] triples = s.split(" "); String object = triples[2].substring(1, triples[2].length() - 1); if (old_object.size() > 0 && !old_object.firstElement().equals(object)) { hmLobid = new HashMap<String, ArrayList<String>>(); old_object = new Stack<String>(); } old_object.push(object); String subject = triples[0].substring(1, triples[0].length() - 1); System.out.print("\nSubject=" + object); System.out.print("\ntriples[2]=" + triples[2]); hmLobid.put(subject, getAllCreators(new URI(subject))); hm.put(object, hmLobid); } // get all dbpedia resources for (String key_one : hm.keySet()) { System.out.print("\n==============\n==== " + key_one + "\n==============="); int resources_cnt = hm.get(key_one).keySet().size(); ArrayList<String>[] creators = new ArrayList[resources_cnt]; HashMap<String, Integer> creators_backed = new HashMap<String, Integer>(); int x = 0; // get all lobid_resources subsumed under the dbpedia resource for (String subject_uri : hm.get(key_one).keySet()) { creators[x] = new ArrayList<String>(); System.out.print("\n subject_uri=" + subject_uri); Iterator<String> ite = hm.get(key_one).get(subject_uri).iterator(); int y = 0; // get all creators of the lobid resource while (ite.hasNext()) { String creator = ite.next(); System.out.print("\n " + creator); if (creators_backed.containsKey(creator)) { y = creators_backed.get(creator); } else { y = creators_backed.size(); creators_backed.put(creator, y); } while (creators[x].size() <= y) { creators[x].add("-"); } creators[x].set(y, creator); y++; } x++; } if (creators_backed.size() == 1) { System.out .println("\n" + "Every resource pointing to " + key_one + " has the same creator!"); for (String key_two : hm.get(key_one).keySet()) { output.append("<" + key_two + "> rdrel:workManifested <" + key_one + "> .\n"); output.append("<" + key_two + "> mo:wikipedia <" + key_one.replaceAll("dbpedia\\.org/resource", "wikipedia\\.org/wiki") + "> .\n"); } } /*else { for (int a = 0; a < creators.length; a++) { System.out.print(creators[a].toString()+","); } }*/ } output.flush(); if (output != null) { output.close(); } } catch (Exception e) { System.out.print("Exception while working on " + filename + ": \n"); e.printStackTrace(System.out); } } }
From source file:DIA_Umpire_Quant.DIA_Umpire_ProtQuant.java
/** * @param args the command line arguments *//*from w w w .j a v a2 s . co m*/ public static void main(String[] args) throws FileNotFoundException, IOException, Exception { System.out.println( "================================================================================================="); System.out.println( "DIA-Umpire protein quantitation module (version: " + UmpireInfo.GetInstance().Version + ")"); if (args.length != 1) { System.out.println( "command format error, the correct format should be: java -jar -Xmx10G DIA_Umpire_PortQuant.jar diaumpire_module.params"); return; } try { ConsoleLogger.SetConsoleLogger(Level.INFO); ConsoleLogger.SetFileLogger(Level.DEBUG, FilenameUtils.getFullPath(args[0]) + "diaumpire_orotquant.log"); } catch (Exception e) { } Logger.getRootLogger().info("Version: " + UmpireInfo.GetInstance().Version); Logger.getRootLogger().info("Parameter file:" + args[0]); BufferedReader reader = new BufferedReader(new FileReader(args[0])); String line = ""; String WorkFolder = ""; int NoCPUs = 2; String Combined_Prot = ""; boolean DefaultProtFiltering = true; float Freq = 0f; int TopNPep = 6; int TopNFrag = 6; String FilterWeight = "GW"; float MinWeight = 0.9f; TandemParam tandemPara = new TandemParam(DBSearchParam.SearchInstrumentType.TOF5600); HashMap<String, File> AssignFiles = new HashMap<>(); boolean ExportSaint = false; boolean SAINT_MS1 = false; boolean SAINT_MS2 = true; HashMap<String, String[]> BaitList = new HashMap<>(); HashMap<String, String> BaitName = new HashMap<>(); HashMap<String, String[]> ControlList = new HashMap<>(); HashMap<String, String> ControlName = new HashMap<>(); //<editor-fold defaultstate="collapsed" desc="Reading parameter file"> while ((line = reader.readLine()) != null) { line = line.trim(); Logger.getRootLogger().info(line); if (!"".equals(line) && !line.startsWith("#")) { //System.out.println(line); if (line.equals("==File list begin")) { do { line = reader.readLine(); line = line.trim(); if (line.equals("==File list end")) { continue; } else if (!"".equals(line)) { File newfile = new File(line); if (newfile.exists()) { AssignFiles.put(newfile.getAbsolutePath(), newfile); } else { Logger.getRootLogger().info("File: " + newfile + " does not exist."); } } } while (!line.equals("==File list end")); } if (line.split("=").length < 2) { continue; } String type = line.split("=")[0].trim(); String value = line.split("=")[1].trim(); switch (type) { case "Path": { WorkFolder = value; break; } case "path": { WorkFolder = value; break; } case "Thread": { NoCPUs = Integer.parseInt(value); break; } case "Fasta": { tandemPara.FastaPath = value; break; } case "Combined_Prot": { Combined_Prot = value; break; } case "DefaultProtFiltering": { DefaultProtFiltering = Boolean.parseBoolean(value); break; } case "DecoyPrefix": { if (!"".equals(value)) { tandemPara.DecoyPrefix = value; } break; } case "ProteinFDR": { tandemPara.ProtFDR = Float.parseFloat(value); break; } case "FilterWeight": { FilterWeight = value; break; } case "MinWeight": { MinWeight = Float.parseFloat(value); break; } case "TopNFrag": { TopNFrag = Integer.parseInt(value); break; } case "TopNPep": { TopNPep = Integer.parseInt(value); break; } case "Freq": { Freq = Float.parseFloat(value); break; } //<editor-fold defaultstate="collapsed" desc="SaintOutput"> case "ExportSaintInput": { ExportSaint = Boolean.parseBoolean(value); break; } case "QuantitationType": { switch (value) { case "MS1": { SAINT_MS1 = true; SAINT_MS2 = false; break; } case "MS2": { SAINT_MS1 = false; SAINT_MS2 = true; break; } case "BOTH": { SAINT_MS1 = true; SAINT_MS2 = true; break; } } break; } // case "BaitInputFile": { // SaintBaitFile = value; // break; // } // case "PreyInputFile": { // SaintPreyFile = value; // break; // } // case "InterationInputFile": { // SaintInteractionFile = value; // break; // } default: { if (type.startsWith("BaitName_")) { BaitName.put(type.substring(9), value); } if (type.startsWith("BaitFile_")) { BaitList.put(type.substring(9), value.split("\t")); } if (type.startsWith("ControlName_")) { ControlName.put(type.substring(12), value); } if (type.startsWith("ControlFile_")) { ControlList.put(type.substring(12), value.split("\t")); } break; } //</editor-fold> } } } //</editor-fold> //Initialize PTM manager using compomics library PTMManager.GetInstance(); //Check if the fasta file can be found if (!new File(tandemPara.FastaPath).exists()) { Logger.getRootLogger().info("Fasta file :" + tandemPara.FastaPath + " cannot be found, the process will be terminated, please check."); System.exit(1); } //Check if the prot.xml file can be found if (!new File(Combined_Prot).exists()) { Logger.getRootLogger().info("ProtXML file: " + Combined_Prot + " cannot be found, the export protein summary table will be empty."); } LCMSID protID = null; //Parse prot.xml and generate protein master list given an FDR if (Combined_Prot != null && !Combined_Prot.equals("")) { protID = LCMSID.ReadLCMSIDSerialization(Combined_Prot); if (!"".equals(Combined_Prot) && protID == null) { protID = new LCMSID(Combined_Prot, tandemPara.DecoyPrefix, tandemPara.FastaPath); ProtXMLParser protxmlparser = new ProtXMLParser(protID, Combined_Prot, 0f); //Use DIA-Umpire default protein FDR calculation if (DefaultProtFiltering) { protID.RemoveLowLocalPWProtein(0.8f); protID.RemoveLowMaxIniProbProtein(0.9f); protID.FilterByProteinDecoyFDRUsingMaxIniProb(tandemPara.DecoyPrefix, tandemPara.ProtFDR); } //Get protein FDR calculation without other filtering else { protID.FilterByProteinDecoyFDRUsingLocalPW(tandemPara.DecoyPrefix, tandemPara.ProtFDR); } protID.LoadSequence(); protID.WriteLCMSIDSerialization(Combined_Prot); } Logger.getRootLogger().info("Protein No.:" + protID.ProteinList.size()); } HashMap<String, HashMap<String, FragmentPeak>> IDSummaryFragments = new HashMap<>(); //Generate DIA file list ArrayList<DIAPack> FileList = new ArrayList<>(); try { File folder = new File(WorkFolder); if (!folder.exists()) { Logger.getRootLogger().info("The path : " + WorkFolder + " cannot be found."); System.exit(1); } for (final File fileEntry : folder.listFiles()) { if (fileEntry.isFile() && (fileEntry.getAbsolutePath().toLowerCase().endsWith(".mzxml") | fileEntry.getAbsolutePath().toLowerCase().endsWith(".mzml")) && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q1.mzxml") && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q2.mzxml") && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q3.mzxml")) { AssignFiles.put(fileEntry.getAbsolutePath(), fileEntry); } if (fileEntry.isDirectory()) { for (final File fileEntry2 : fileEntry.listFiles()) { if (fileEntry2.isFile() && (fileEntry2.getAbsolutePath().toLowerCase().endsWith(".mzxml") | fileEntry2.getAbsolutePath().toLowerCase().endsWith(".mzml")) && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q1.mzxml") && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q2.mzxml") && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q3.mzxml")) { AssignFiles.put(fileEntry2.getAbsolutePath(), fileEntry2); } } } } Logger.getRootLogger().info("No. of files assigned :" + AssignFiles.size()); for (File fileEntry : AssignFiles.values()) { Logger.getRootLogger().info(fileEntry.getAbsolutePath()); } for (File fileEntry : AssignFiles.values()) { String mzXMLFile = fileEntry.getAbsolutePath(); if (mzXMLFile.toLowerCase().endsWith(".mzxml") | mzXMLFile.toLowerCase().endsWith(".mzml")) { DIAPack DiaFile = new DIAPack(mzXMLFile, NoCPUs); Logger.getRootLogger().info( "================================================================================================="); Logger.getRootLogger().info("Processing " + mzXMLFile); if (!DiaFile.LoadDIASetting()) { Logger.getRootLogger().info("Loading DIA setting failed, job is incomplete"); System.exit(1); } if (!DiaFile.LoadParams()) { Logger.getRootLogger().info("Loading parameters failed, job is incomplete"); System.exit(1); } Logger.getRootLogger().info("Loading identification results " + mzXMLFile + "...."); //If the serialization file for ID file existed if (DiaFile.ReadSerializedLCMSID()) { DiaFile.IDsummary.ReduceMemoryUsage(); DiaFile.IDsummary.ClearAssignPeakCluster(); FileList.add(DiaFile); HashMap<String, FragmentPeak> FragMap = new HashMap<>(); IDSummaryFragments.put(FilenameUtils.getBaseName(mzXMLFile), FragMap); } } } //<editor-fold defaultstate="collapsed" desc="Peptide and fragment selection"> Logger.getRootLogger().info("Peptide and fragment selection across the whole dataset"); ArrayList<LCMSID> SummaryList = new ArrayList<>(); for (DIAPack diafile : FileList) { if (protID != null) { //Generate protein list according to mapping of peptide ions for each DIA file to the master protein list diafile.IDsummary.GenerateProteinByRefIDByPepSeq(protID, true); diafile.IDsummary.ReMapProPep(); } if ("GW".equals(FilterWeight)) { diafile.IDsummary.SetFilterByGroupWeight(); } else if ("PepW".equals(FilterWeight)) { diafile.IDsummary.SetFilterByWeight(); } SummaryList.add(diafile.IDsummary); } FragmentSelection fragselection = new FragmentSelection(SummaryList); fragselection.freqPercent = Freq; fragselection.GeneratePepFragScoreMap(); fragselection.GenerateTopFragMap(TopNFrag); fragselection.GenerateProtPepScoreMap(MinWeight); fragselection.GenerateTopPepMap(TopNPep); //</editor-fold> //<editor-fold defaultstate="collapsed" desc="Writing general reports"> ExportTable export = new ExportTable(WorkFolder, SummaryList, IDSummaryFragments, protID, fragselection); export.Export(TopNPep, TopNFrag, Freq); //</editor-fold> //<editor-fold defaultstate="collapsed" desc="//<editor-fold defaultstate="collapsed" desc="Generate SAINT input files"> if (ExportSaint && protID != null) { HashMap<String, DIAPack> Filemap = new HashMap<>(); for (DIAPack DIAfile : FileList) { Filemap.put(DIAfile.GetBaseName(), DIAfile); } FileWriter baitfile = new FileWriter(WorkFolder + "SAINT_Bait_" + DateTimeTag.GetTag() + ".txt"); FileWriter preyfile = new FileWriter(WorkFolder + "SAINT_Prey_" + DateTimeTag.GetTag() + ".txt"); FileWriter interactionfileMS1 = null; FileWriter interactionfileMS2 = null; if (SAINT_MS1) { interactionfileMS1 = new FileWriter( WorkFolder + "SAINT_Interaction_MS1_" + DateTimeTag.GetTag() + ".txt"); } if (SAINT_MS2) { interactionfileMS2 = new FileWriter( WorkFolder + "SAINT_Interaction_MS2_" + DateTimeTag.GetTag() + ".txt"); } HashMap<String, String> PreyID = new HashMap<>(); for (String samplekey : ControlName.keySet()) { String name = ControlName.get(samplekey); for (String file : ControlList.get(samplekey)) { baitfile.write(FilenameUtils.getBaseName(file) + "\t" + name + "\t" + "C\n"); LCMSID IDsummary = Filemap.get(FilenameUtils.getBaseName(file)).IDsummary; if (SAINT_MS1) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS1, file, name, PreyID, 1); } if (SAINT_MS2) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS2, file, name, PreyID, 2); } } } for (String samplekey : BaitName.keySet()) { String name = BaitName.get(samplekey); for (String file : BaitList.get(samplekey)) { baitfile.write(FilenameUtils.getBaseName(file) + "\t" + name + "\t" + "T\n"); LCMSID IDsummary = Filemap.get(FilenameUtils.getBaseName(file)).IDsummary; if (SAINT_MS1) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS1, file, name, PreyID, 1); } if (SAINT_MS2) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS2, file, name, PreyID, 2); } } } baitfile.close(); if (SAINT_MS1) { interactionfileMS1.close(); } if (SAINT_MS2) { interactionfileMS2.close(); } for (String AccNo : PreyID.keySet()) { preyfile.write(AccNo + "\t" + PreyID.get(AccNo) + "\n"); } preyfile.close(); } //</editor-fold> Logger.getRootLogger().info("Job done"); Logger.getRootLogger().info( "================================================================================================="); } catch (Exception e) { Logger.getRootLogger().error(ExceptionUtils.getStackTrace(e)); throw e; } }
From source file:com.medicaid.mmis.util.DataLoader.java
/** * The main function, imports the files given as arguments. * /*w w w . j ava 2 s .c o m*/ * @param args the file names * @throws IOException for read/write errors * @throws PortalServiceException for any other errors */ public static void main(String[] args) throws IOException, PortalServiceException { if (args.length != 2) { System.out.println("2 file path arguments are required."); return; } PropertyConfigurator.configure("log4j.properties"); logger = Logger.getLogger(DataLoader.class); LookupServiceBean lookupBean = new LookupServiceBean(); EntityManagerFactory emf = Persistence.createEntityManagerFactory("cms-data-load"); EntityManager em = emf.createEntityManager(); lookupBean.setEm(em); DataLoader loader = new DataLoader(); loader.setLookup(lookupBean); SequenceGeneratorBean sequence = new SequenceGeneratorBean(); sequence.setEm(em); ProviderEnrollmentServiceBean enrollmentBean = new ProviderEnrollmentServiceBean(); enrollmentBean.setEm(em); enrollmentBean.setSequence(sequence); enrollmentBean.setLookupService(lookupBean); loader.setEnrollmentService(enrollmentBean); long processId = sequence.getNextValue("PROCESS_ID"); System.out.println("Started process id " + processId); BufferedReader br = null; PrintWriter accepted = null; PrintWriter rejected = null; try { System.out.println("Processing file 1..."); File success = new File("accepted_1_" + processId + ".txt"); File failure = new File("rejected_1_" + processId + ".txt"); success.createNewFile(); failure.createNewFile(); accepted = new PrintWriter(success); rejected = new PrintWriter(failure); br = new BufferedReader(new FileReader(args[0])); String line = null; int total = 0; int errors = 0; while ((line = br.readLine()) != null) { total++; try { em.getTransaction().begin(); loader.readProviderFile(new ByteArrayInputStream(line.getBytes())); em.getTransaction().commit(); accepted.println(line); logger.info("Commit row " + total); } catch (PortalServiceException e) { rejected.println(line); em.getTransaction().rollback(); errors++; logger.error("Rollback row " + total + " :" + e.getMessage()); } } accepted.flush(); accepted.close(); rejected.flush(); rejected.close(); br.close(); System.out.println("Total records read: " + total); System.out.println("Total rejected: " + errors); System.out.println("Processing file 2..."); success = new File("accepted_2_" + processId + ".txt"); failure = new File("rejected_2_" + processId + ".txt"); success.createNewFile(); failure.createNewFile(); accepted = new PrintWriter(success); rejected = new PrintWriter(failure); br = new BufferedReader(new FileReader(args[1])); line = null; total = 0; errors = 0; while ((line = br.readLine()) != null) { total++; try { em.getTransaction().begin(); Map<String, OwnershipInformation> owners = loader .readWS000EXT2OWNBEN(new ByteArrayInputStream(line.getBytes())); for (Map.Entry<String, OwnershipInformation> entry : owners.entrySet()) { enrollmentBean.addBeneficialOwners(entry.getKey(), entry.getValue()); } em.getTransaction().commit(); accepted.println(line); logger.info("Commit row " + total); } catch (PortalServiceException e) { rejected.println(line); em.getTransaction().rollback(); errors++; logger.error("Rollback row " + total + " :" + e.getMessage()); } } accepted.flush(); rejected.flush(); System.out.println("Total records read: " + total); System.out.println("Total rejected: " + errors); } finally { if (br != null) { br.close(); } if (accepted != null) { accepted.close(); } if (rejected != null) { rejected.close(); } } }
From source file:it.codestudio.callbyj.CallByJ.java
/** * The main method./*from ww w.j a v a 2 s . c om*/ * * @param args the arguments */ public static void main(String[] args) { options.addOption("aCom", "audio_com", true, "Specify serial COM port for audio streaming (3G APPLICATION ...)"); options.addOption("cCom", "command_com", true, "Specify serial COM port for modem AT command (PC UI INTERFACE ...)"); options.addOption("p", "play_message", false, "Play recorded message instead to respond with audio from mic"); options.addOption("h", "help", false, "Print help for this application"); try { BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); String audioCOM = ""; String commandCOM = ""; Boolean playMessage = false; args = Utils.fitlerNullString(args); CommandLine cmd = parser.parse(options, args); if (cmd.hasOption("aCom")) { audioCOM = cmd.getOptionValue("aCom"); ; } if (cmd.hasOption("cCom")) { commandCOM = cmd.getOptionValue("cCom"); ; } if (cmd.hasOption("p")) { playMessage = true; } if (audioCOM != null && commandCOM != null && !audioCOM.isEmpty() && !commandCOM.isEmpty()) { comManager = ComManager.getInstance(commandCOM, audioCOM, playMessage); } else { HelpFormatter f = new HelpFormatter(); f.printHelp("\r Exaple: CallByJ -aCom COM11 -cCom COM10 \r OptionsTip", options); return; } options = new Options(); options.addOption("h", "help", false, "Print help for this application"); options.addOption("p", "pin", true, "Specify pin of device, if present"); options.addOption("c", "call", true, "Start call to specified number"); options.addOption("e", "end", false, "End all active call"); options.addOption("t", "terminate", false, "Terminate application"); options.addOption("r", "respond", false, "Respond to incoming call"); comManager.startModemCommandManager(); while (true) { try { String[] commands = { br.readLine() }; commands = Utils.fitlerNullString(commands); cmd = parser.parse(options, commands); if (cmd.hasOption('h')) { HelpFormatter f = new HelpFormatter(); f.printHelp("OptionsTip", options); } if (cmd.hasOption('p')) { String devicePin = cmd.getOptionValue("p"); comManager.insertPin(devicePin); } if (cmd.hasOption('c')) { String numberToCall = cmd.getOptionValue("c"); comManager.sendCommandToModem(ATCommands.CALL, numberToCall); } if (cmd.hasOption('r')) { comManager.respondToIncomingCall(); } if (cmd.hasOption('e')) { comManager.sendCommandToModem(ATCommands.END_CALL); } if (cmd.hasOption('t')) { comManager.sendCommandToModem(ATCommands.END_CALL); comManager.terminate(); System.out.println("CallByJ closed!"); break; //System.exit(0); } } catch (Exception e) { logger.error(e.getMessage(), e); } } } catch (Exception e) { logger.error(e.getMessage(), e); } finally { if (comManager != null) comManager.terminate(); } }