List of usage examples for com.mongodb MongoClient close
public void close()
From source file:ch.hslu.dmg.InitDB.java
public void initVorlesungen() { MongoClient mongo = new MongoClient("localhost", 27017); MongoDatabase database = mongo.getDatabase("unidb"); MongoCollection<Document> vorlesung = database.getCollection("vorlesungen"); vorlesung.drop();/*from w w w.j a v a 2s .c o m*/ ArrayList<Document> vorlesungen = new ArrayList<>(); vorlesungen.add(new Document("VorlNr", 5001).append("Titel", "Grundzge").append("SWS", 4) .append("GelesenVon", 2137)); vorlesungen.add( new Document("VorlNr", 5041).append("Titel", "Ethik").append("SWS", 4).append("GelesenVon", 2125)); vorlesungen.add(new Document("VorlNr", 5043).append("Titel", "Erkentnistheorie").append("SWS", 3) .append("GelesenVon", 2126)); vorlesungen.add(new Document("VorlNr", 5049).append("Titel", "Maeeutik").append("SWS", 2) .append("GelesenVon", 2125)); vorlesungen.add( new Document("VorlNr", 4052).append("Titel", "Logik").append("SWS", 4).append("GelesenVon", 2125)); vorlesungen.add(new Document("VorlNr", 5052).append("Titel", "Wissenschaftstheorie").append("SWS", 3) .append("GelesenVon", 2126)); vorlesungen.add(new Document("VorlNr", 5216).append("Titel", "Bioethik").append("SWS", 2) .append("GelesenVon", 2126)); vorlesungen.add(new Document("VorlNr", 5259).append("Titel", "Der Wiener Kreis").append("SWS", 2) .append("GelesenVon", 2133)); vorlesungen.add(new Document("VorlNr", 5022).append("Titel", "Glaube und Wissen").append("SWS", 2) .append("GelesenVon", 2134)); vorlesungen.add(new Document("VorlNr", 4630).append("Titel", "Die 3 Kritiken").append("SWS", 4) .append("GelesenVon", 2137)); vorlesung.insertMany(vorlesungen); vorlesung.updateOne(eq("VorlNr", 5041), new Document("$set", new Document("Voraussetzung", vorlesung.find(eq("VorlNr", 5001)).iterator().next().get("_id")))); vorlesung.updateOne(eq("VorlNr", 5043), new Document("$set", new Document("Voraussetzung", vorlesung.find(eq("VorlNr", 5001)).iterator().next().get("_id")))); vorlesung.updateOne(eq("VorlNr", 5049), new Document("$set", new Document("Voraussetzung", vorlesung.find(eq("VorlNr", 5001)).iterator().next().get("_id")))); vorlesung.updateOne(eq("VorlNr", 5216), new Document("$set", new Document("Voraussetzung", vorlesung.find(eq("VorlNr", 5041)).iterator().next().get("_id")))); vorlesung.updateOne(eq("VorlNr", 5259), new Document("$set", new Document("Voraussetzung", vorlesung.find(eq("VorlNr", 5052)).iterator().next().get("_id")))); BasicDBList voraussetzungen = new BasicDBList(); voraussetzungen.add(vorlesung.find(eq("VorlNr", 5043)).iterator().next().get("_id")); voraussetzungen.add(vorlesung.find(eq("VorlNr", 5041)).iterator().next().get("_id")); vorlesung.updateOne(eq("VorlNr", 5052), new Document("$set", new Document("Voraussetzung", voraussetzungen))); mongo.close(); }
From source file:ch.hslu.dmg.InitDB.java
public void initAssistenten() { MongoClient mongo = new MongoClient("localhost", 27017); MongoDatabase database = mongo.getDatabase("unidb"); MongoCollection<Document> assistent = database.getCollection("assistenten"); assistent.drop();// w w w. ja v a 2s . com ArrayList<Document> assistenten = new ArrayList<>(); assistenten.add(new Document("PersNr", 3002).append("Name", "Platon").append("Fachgebiet", "Ideenlehre") .append("Boss", 2125)); assistenten.add(new Document("PersNr", 3003).append("Name", "Aristoteles") .append("Fachgebiet", "Syllogistik").append("Boss", 2125)); assistenten.add(new Document("PersNr", 3004).append("Name", "Wittgenstein") .append("Fachgebiet", "Sprachtheorie").append("Boss", 2126)); assistenten.add(new Document("PersNr", 3005).append("Name", "Rhetikus") .append("Fachgebiet", "Planetenbewegung").append("Boss", 2127)); assistenten.add(new Document("PersNr", 3006).append("Name", "Newton") .append("Fachgebiet", "Keplersche Gesetze").append("Boss", 2127)); assistenten.add(new Document("PersNr", 3007).append("Name", "Spinoza") .append("Fachgebiet", "Gott und Natur").append("Boss", 2134)); assistent.insertMany(assistenten); mongo.close(); }
From source file:ch.hslu.dmg.InitDB.java
public void initPruefen() { MongoClient mongo = new MongoClient("localhost", 27017); MongoDatabase database = mongo.getDatabase("unidb"); MongoCollection<Document> pruefungen = database.getCollection("pruefungen"); pruefungen.drop();/*w ww . ja v a 2 s .co m*/ ArrayList<Document> pruefung = new ArrayList<>(); pruefung.add(new Document("Legi", 28106).append("VorlNr", 5001).append("PersNr", 2126).append("Note", 1)); pruefung.add(new Document("Legi", 25403).append("VorlNr", 5041).append("PersNr", 2125).append("Note", 2)); pruefung.add(new Document("Legi", 27550).append("VorlNr", 4630).append("PersNr", 2137).append("Note", 2)); pruefungen.insertMany(pruefung); mongo.close(); }
From source file:cn.edu.hfut.dmic.webcollector.example.DemoSelenium.java
License:Open Source License
public static void main(String[] args) throws Exception { Executor executor = new Executor() { @Override//from w w w. j av a 2 s. c o m public void execute(CrawlDatum datum, CrawlDatums next) throws Exception { MongoClient mongoClient = new MongoClient("localhost", 27017); // ? // DBCollection dbCollection = mongoClient.getDB("maoyan_crawler").getCollection("rankings_am"); DB db = mongoClient.getDB("maoyan_crawler"); // ????? Set<String> colls = db.getCollectionNames(); for (String s : colls) { // Collection(?"") if (s.equals("attend_rate")) { db.getCollection(s).drop(); } } DBCollection dbCollection = db.getCollection("attend_rate"); HtmlUnitDriver driver = new HtmlUnitDriver(); driver.setJavascriptEnabled(false); driver.get(datum.getUrl()); System.out.println(driver.getPageSource()); WebElement click_view = driver.findElement(By.xpath("//div[@id='seatContent']//span[1]")); click_view.click(); String gold_seat = driver.getWindowHandle(); driver.switchTo().window(gold_seat); System.out.println(driver.getPageSource()); WebElement city_name = driver.findElement(By.xpath("//*[@id='all-citys']/div[1]/ul/li[1]/a")); System.out.println(city_name.getText()); WebElement element = driver.findElementByCssSelector("div#seat_table"); List<WebElement> movie_name = element.findElements(By.className("c1 lineDot")); List<WebElement> boxoffice_rate = element.findElements(By.className("c2 red")); List<WebElement> visit_pershow = element.findElements(By.className("c3 gray")); WebElement cityarea = driver.findElementByCssSelector("span[class='today']"); System.out.println(cityarea.getText()); for (int i = 0; i < movie_name.size(); i++) { System.out.println(movie_name.get(i).getText()); System.out.println(boxoffice_rate.get(i).getText()); System.out.println(visit_pershow.get(i).getText()); BasicDBObject dbObject = new BasicDBObject(); dbObject.append("title", cityarea.getText()).append("movie_name", movie_name.get(i).getText()) .append("boxoffice_rate", boxoffice_rate.get(i).getText()) .append("visit_pershow", visit_pershow.get(i).getText()); dbCollection.insert(dbObject); } mongoClient.close(); } }; //DBDBManager DBManager manager = new BerkeleyDBManager("crawl"); //Crawler?DBManagerExecutor Crawler crawler = new Crawler(manager, executor); crawler.addSeed("http://pf.maoyan.com/attend/rate"); crawler.start(1); }
From source file:cn.edu.hfut.dmic.webcollector.example.FirefoxSelenium3.java
License:Open Source License
public static void main(String[] args) throws Exception { Executor executor = new Executor() { @Override/*from w ww . j a va 2s . co m*/ public void execute(CrawlDatum datum, CrawlDatums next) throws Exception { MongoClient mongoClient = new MongoClient("localhost", 27017); // ? // DBCollection dbCollection = mongoClient.getDB("maoyan_crawler").getCollection("rankings_am"); DB db = mongoClient.getDB("maoyan_crawler"); // ????? Set<String> colls = db.getCollectionNames(); for (String s : colls) { // Collection(?"") if (s.equals("attend_rate")) { db.getCollection(s).drop(); } } DBCollection dbCollection = db.getCollection("attend_rate"); ProfilesIni pi = new ProfilesIni(); FirefoxProfile profile = pi.getProfile("default"); WebDriver driver = new FirefoxDriver(profile); driver.manage().window().maximize(); driver.manage().timeouts().pageLoadTimeout(3, TimeUnit.SECONDS); // driver.setJavascriptEnabled(false); driver.get(datum.getUrl()); // System.out.println(driver.getPageSource()); driver.findElement(By.xpath("//*[@id='seat_city']")).click(); driver.switchTo().window(driver.getWindowHandle()); int city_num = driver.findElements(By.xpath("//div[@id='all-citys']/div/ul/li/a")).size(); for (int i = 0; i < city_num; i++) { System.out.println("A city chosen" + i); System.out.println( driver.findElements(By.xpath("//div[@id='all-citys']/div/ul/li/a")).get(i).getText()); String city = driver.findElements(By.xpath("//div[@id='all-citys']/div/ul/li/a")).get(i) .getText(); ((JavascriptExecutor) driver).executeScript("arguments[0].scrollIntoView(true);", driver.findElements(By.xpath("//div[@id='all-citys']/div/ul/li/a")).get(i)); ((JavascriptExecutor) driver).executeScript("window.scrollBy(0, -250)", ""); Thread.sleep(1000); new Actions(driver) .moveToElement( driver.findElements(By.xpath("//div[@id='all-citys']/div/ul/li/a")).get(i)) .click().perform(); driver.switchTo().window(driver.getWindowHandle()); // System.out.println(driver.findElement(By.xpath("//span[@class='today']/em")).getText()); System.out.println(driver.findElement(By.xpath("//span[@class='today']")).getText()); for (int j = 0; j < driver .findElements(By.xpath("//div[@id='seat_table']//ul//li[@class='c1 lineDot']")) .size(); j++) { System.out.println(driver .findElements(By.xpath("//div[@id='seat_table']//ul//li[@class='c1 lineDot']")) .get(j).getText()); System.out.println( driver.findElements(By.xpath("//div[@id='seat_table']//ul//li[@class='c2 red']")) .get(j).getText()); System.out.println( driver.findElements(By.xpath("//div[@id='seat_table']//ul//li[@class='c3 gray']")) .get(j).getText()); BasicDBObject dbObject = new BasicDBObject(); dbObject.append("title", driver.findElement(By.xpath("//span[@class='today']")).getText()) .append("city", city) .append("mov_cnname", driver.findElements( By.xpath("//div[@id='seat_table']//ul//li[@class='c1 lineDot']")) .get(j).getText()) .append("boxoffice_rate", driver .findElements(By.xpath("//div[@id='seat_table']//ul//li[@class='c2 red']")) .get(j).getText()) .append("visit_pershow", driver .findElements(By.xpath("//div[@id='seat_table']//ul//li[@class='c3 gray']")) .get(j).getText()); dbCollection.insert(dbObject); } System.out.println("new city list to choose"); new Actions(driver).moveToElement(driver.findElement(By.xpath("//*[@id='seat_city']"))).click() .perform(); driver.switchTo().window(driver.getWindowHandle()); Thread.sleep(500); } driver.close(); driver.quit(); mongoClient.close(); } }; //DBDBManager DBManager manager = new BerkeleyDBManager("crawl"); //Crawler?DBManagerExecutor Crawler crawler = new Crawler(manager, executor); crawler.addSeed("http://pf.maoyan.com/attend/rate"); crawler.start(1); }
From source file:cn.edu.hfut.dmic.webcollector.example.TutorialCrawler.java
License:Open Source License
@Override public void visit(Page page, CrawlDatums next) { if (page.matchUrl("http://pf.maoyan.com/rankings/america.*")) { MongoClient mongoClient = new MongoClient("localhost", 27017); // ?//from ww w.jav a 2 s . co m // DBCollection dbCollection = mongoClient.getDB("maoyan_crawler").getCollection("rankings_am"); DB db = mongoClient.getDB("maoyan_crawler"); // ????? Set<String> colls = db.getCollectionNames(); for (String s : colls) { // Collection(?"") if (s.equals("rankings_am")) { db.getCollection(s).drop(); } } DBCollection dbCollection = db.getCollection("rankings_am"); String title = page.select("span[id=year-box]").text(); Elements table = page.select("table[id=na-list]"); Elements data_set = table.select("tr"); List amList = new ArrayList(); for (Element id : data_set) { Elements tds = id.select("td"); for (Element td : tds) { amList.add(td.text()); } String en_name = tds.select("p[class=first-line]").text(); String cn_name = tds.select("p[class=second-line]").text(); if (amList.size() > 0) { System.out.println(amList); BasicDBObject dbObject = new BasicDBObject(); dbObject.append("title", title).append("rank", amList.get(0)).append("mov_cnname", cn_name) .append("mov_enname", en_name).append("toweek_rev", amList.get(2)) .append("total_rev", amList.get(3)).append("val_week", amList.get(4)); amList.removeAll(amList); dbCollection.insert(dbObject); } } mongoClient.close(); } else if (page.matchUrl("http://pf.maoyan.com/rankings/day.*")) { MongoClient mongoClient2 = new MongoClient("localhost", 27017); // ? DB db = mongoClient2.getDB("maoyan_crawler"); // ????? Set<String> colls = db.getCollectionNames(); for (String s : colls) { // Collection(?"") if (s.equals("rankings_day")) { db.getCollection(s).drop(); } } DBCollection dbCollection2 = db.getCollection("rankings_day"); String title = page.select("span[id=year-box]").text(); String update_time = page.select("span[id=update-time]").text(); title = title + update_time; System.out.println(title); Elements data_set = page.select("tr"); List dayList = new ArrayList(); for (Element id : data_set) { Elements tds = id.select("td"); for (Element td : tds) { dayList.add(td.text()); } if (dayList.size() > 0) { System.out.println(dayList); BasicDBObject dbObject = new BasicDBObject(); dbObject.append("title", title).append("rank", dayList.get(0)) .append("mov_name", dayList.get(1)).append("today_rev", dayList.get(2)) .append("date", dayList.get(3)).append("val_week", dayList.get(4)); dayList.removeAll(dayList); dbCollection2.insert(dbObject); } } mongoClient2.close(); } else if (page.matchUrl("http://pf.maoyan.com/rankings/market.*")) { MongoClient mongoClient3 = new MongoClient("localhost", 27017); // ? DB db = mongoClient3.getDB("maoyan_crawler"); // ????? Set<String> colls = db.getCollectionNames(); for (String s : colls) { // Collection(?"") if (s.equals("rankings_market")) { db.getCollection(s).drop(); } } DBCollection dbCollection3 = db.getCollection("rankings_market"); String title = page.select("span[id=year-box]").text(); String update_time = page.select("span[id=update-time]").text(); title = title + update_time; System.out.println(title); Elements data_set = page.select("tr"); List dayList = new ArrayList(); for (Element id : data_set) { Elements tds = id.select("td"); for (Element td : tds) { dayList.add(td.text()); } if (dayList.size() > 0) { System.out.println(dayList); BasicDBObject dbObject = new BasicDBObject(); dbObject.append("title", title).append("rank", dayList.get(0)).append("date", dayList.get(1)) .append("today_rev", dayList.get(2)).append("total_sessions", dayList.get(3)) .append("total_visit_count", dayList.get(4)); dayList.removeAll(dayList); dbCollection3.insert(dbObject); } } mongoClient3.close(); } else if (page.matchUrl("http://pf.maoyan.com/rankings/year.*")) { MongoClient mongoClient4 = new MongoClient("localhost", 27017); // ? DB db = mongoClient4.getDB("maoyan_crawler"); // ????? Set<String> colls = db.getCollectionNames(); for (String s : colls) { // Collection(?"") if (s.equals("rankings_year")) { db.getCollection(s).drop(); } } DBCollection dbCollection4 = db.getCollection("rankings_year"); String title = page.select("span[id=year-box]").text(); String update_time = page.select("span[id=update-time]").text(); title = title + update_time; System.out.println(title); Elements table = page.select("div[id=ranks-list]"); // System.out.println(table); Elements data_set = table.select("ul[class=row]"); // System.out.println(data_set); List dayList = new ArrayList(); for (Element id : data_set) { Elements lis = id.select("li"); for (Element li : lis) { dayList.add(li.text()); } String cn_name = lis.select("p[class=first-line]").text(); String release_date = lis.select("p[class=second-line]").text(); if (dayList.size() > 0) { System.out.println(dayList); BasicDBObject dbObject = new BasicDBObject(); dbObject.append("title", title).append("rank", dayList.get(0)).append("name", cn_name) .append("release date", release_date).append("year_rev", dayList.get(2)) .append("avg_price", dayList.get(3)).append("avg_visit_count", dayList.get(4)); dayList.removeAll(dayList); dbCollection4.insert(dbObject); } } mongoClient4.close(); } else if (page.matchUrl("http://pf.maoyan.com/")) { MongoClient mongoClient5 = new MongoClient("localhost", 27017); // ? DB db = mongoClient5.getDB("maoyan_crawler"); // ????? Set<String> colls = db.getCollectionNames(); for (String s : colls) { // Collection(?"") if (s.equals("main_page")) { db.getCollection(s).drop(); } } DBCollection dbCollection5 = db.getCollection("main_page"); String title = page.select("span[id=dayStr]").text(); String box_type = page.select("span[id=box-type]").text(); String ticket_count = page.select("span[id=ticket_count]").text(); box_type = box_type + ticket_count; System.out.println(title + "\n" + box_type); Elements table = page.select("div[id=ticket_tbody]"); //System.out.println(table); Elements data_set = table.select("ul"); //System.out.println(data_set); List dayList = new ArrayList(); for (Element id : data_set) { Elements lis = id.select("li"); for (Element li : lis) { dayList.add(li.text()); } // System.out.println(lis); String cn_name = lis.select("b").first().text(); String comment = lis.select("em").text(); // System.out.println(cn_name+ "\n" +comment); if (dayList.size() > 0) { System.out.println(dayList); BasicDBObject dbObject = new BasicDBObject(); dbObject.append("title", title).append("box_type", box_type).append("name", cn_name) .append("comment", comment).append("realtime_rev", dayList.get(1)) .append("rev_percent", dayList.get(2)).append("schedule_percent", dayList.get(3)) .append("total_rev", dayList.get(4)); dayList.removeAll(dayList); dbCollection5.insert(dbObject); } } mongoClient5.close(); } }
From source file:cn.edu.hfut.dmic.webcollector.example.WeiboCrawler.java
License:Open Source License
@Override public void visit(Page page, CrawlDatums next) { int pageNum = Integer.valueOf(page.getMetaData("pageNum")); /*??*///from w ww . j av a2 s .c o m Elements weibos = page.select("div.c"); try { MongoClient mongoClient = new MongoClient("localhost", 27017); // ? MongoDatabase mongoDatabase = mongoClient.getDatabase("weibo_crawler"); System.out.println("Connect to database successfully"); MongoCollection<Document> collection = mongoDatabase.getCollection("webpage"); //? /** * 1. org.bson.Document ?key-value? * 2. ?List<Document> * 3. ???? mongoCollection.insertMany(List<Document>) ??? mongoCollection.insertOne(Document) * */ for (Element weibo : weibos) { Document document = new Document("content", "" + pageNum + "" + ":" + weibo.text()); List<Document> documents = new ArrayList<Document>(); documents.add(document); collection.insertMany(documents); } System.out.println("??"); mongoClient.close(); } catch (Exception e) { System.err.println(e.getClass().getName() + ": " + e.getMessage()); } }
From source file:cn.weibo.webcollector.spider.WeiboCrawler.java
License:Open Source License
@Override public void visit(Page page, CrawlDatums next) { String inlink = page.meta("inlink"); String title = page.select("title").text(); String url = page.getUrl();/*from w w w.jav a 2 s. c o m*/ /*??*/ Elements weibos = page.select("div[id].c"); try { MongoClient mongoClient = new MongoClient("localhost", 27017); Logger mongoLogger = Logger.getLogger("org.mongodb.driver"); mongoLogger.setLevel(Level.ERROR); // ? MongoDatabase mongoDatabase = mongoClient.getDatabase("weibo_crawler"); System.out.println("Connect to database successfully"); MongoCollection<Document> collection = mongoDatabase.getCollection("weibo_page"); //? /** * 1. org.bson.Document ?key-value? * 2. ?List<Document> * 3. ???? mongoCollection.insertMany(List<Document>) ??? mongoCollection.insertOne(Document) * */ for (Element weibo : weibos) { if (weibo.text().length() != 0) { Document document = new Document("url", url).append("title", title) .append("content", weibo.text()).append("inlink", inlink); List<Document> documents = new ArrayList<Document>(); documents.add(document); collection.insertMany(documents); } } System.out.println("??"); mongoClient.close(); } catch (Exception e) { System.err.println(e.getClass().getName() + ": " + e.getMessage()); } }
From source file:co.aurasphere.mongodb.university.classes.m101j.homework23.MainClass.java
License:Open Source License
/** * The main method.//from w w w . ja v a2 s .co m * * @param args * the arguments */ public static void main(String[] args) { MongoClient client = new MongoClient(); MongoDatabase numbersDB = client.getDatabase("students"); MongoCollection<Document> grades = numbersDB.getCollection("grades"); // Gets all the grades. MongoCursor<Document> cursor = grades.find(Filters.eq("type", "homework")) .sort(Sorts.ascending("student_id", "score")).iterator(); Object previousStudentId = null; try { // Finds the lowest homework score. while (cursor.hasNext()) { Document entry = cursor.next(); // If the student_id is different from the previous one, this // means that this is the student's lowest graded homework // (they are sorted by score for each student). if (!entry.get("student_id").equals(previousStudentId)) { Object id = entry.get("_id"); grades.deleteOne(Filters.eq("_id", id)); } // The current document ID becomes the new previous one. previousStudentId = entry.get("student_id"); } // Gets the student with the highest average in the class. AggregateIterable<Document> results = grades .aggregate(Arrays.asList(Aggregates.group("$student_id", Accumulators.avg("average", "$score")), Aggregates.sort(Sorts.descending("average")), Aggregates.limit(1))); // There should be only one result. Prints it. System.out.println("Solution : " + results.iterator().next().toJson()); } finally { cursor.close(); } client.close(); }
From source file:co.aurasphere.mongodb.university.classes.m101j.homework31.MainClass.java
License:Open Source License
/** * The main method.//from w w w .j av a2 s . com * * @param args * the arguments */ @SuppressWarnings("unchecked") public static void main(String[] args) { MongoClient client = new MongoClient(); MongoDatabase db = client.getDatabase("school"); MongoCollection<Document> students = db.getCollection("students"); // Gets all the students. MongoCursor<Document> cursor = students.find().iterator(); try { while (cursor.hasNext()) { Document student = cursor.next(); List<Document> scores = (List<Document>) student.get("scores"); // Finds the lowest homework score. Document minScoreObj = null; double minScore = Double.MAX_VALUE; for (Document scoreDocument : scores) { double score = scoreDocument.getDouble("score"); String type = scoreDocument.getString("type"); // Swaps the scores. if (type.equals("homework") && score < minScore) { minScore = score; minScoreObj = scoreDocument; } } // Removes the lowest score. if (minScoreObj != null) { scores.remove(minScoreObj); } // Updates the record. students.updateOne(Filters.eq("_id", student.get("_id")), new Document("$set", new Document("scores", scores))); } // Gets the student with the highest average in the class. AggregateIterable<Document> results = students.aggregate(Arrays.asList(Aggregates.unwind("$scores"), Aggregates.group("$_id", Accumulators.avg("average", "$scores.score")), Aggregates.sort(Sorts.descending("average")), Aggregates.limit(1))); // There should be only one result. Prints it. System.out.println("Solution : " + results.iterator().next().toJson()); } finally { cursor.close(); } client.close(); }