List of usage examples for com.mongodb MongoClient getDB
@Deprecated public DB getDB(final String dbName)
From source file:cn.edu.hfut.dmic.webcollector.example.TutorialCrawler.java
License:Open Source License
@Override public void visit(Page page, CrawlDatums next) { if (page.matchUrl("http://pf.maoyan.com/rankings/america.*")) { MongoClient mongoClient = new MongoClient("localhost", 27017); // ?// w ww .j ava 2s. c om // DBCollection dbCollection = mongoClient.getDB("maoyan_crawler").getCollection("rankings_am"); DB db = mongoClient.getDB("maoyan_crawler"); // ????? Set<String> colls = db.getCollectionNames(); for (String s : colls) { // Collection(?"") if (s.equals("rankings_am")) { db.getCollection(s).drop(); } } DBCollection dbCollection = db.getCollection("rankings_am"); String title = page.select("span[id=year-box]").text(); Elements table = page.select("table[id=na-list]"); Elements data_set = table.select("tr"); List amList = new ArrayList(); for (Element id : data_set) { Elements tds = id.select("td"); for (Element td : tds) { amList.add(td.text()); } String en_name = tds.select("p[class=first-line]").text(); String cn_name = tds.select("p[class=second-line]").text(); if (amList.size() > 0) { System.out.println(amList); BasicDBObject dbObject = new BasicDBObject(); dbObject.append("title", title).append("rank", amList.get(0)).append("mov_cnname", cn_name) .append("mov_enname", en_name).append("toweek_rev", amList.get(2)) .append("total_rev", amList.get(3)).append("val_week", amList.get(4)); amList.removeAll(amList); dbCollection.insert(dbObject); } } mongoClient.close(); } else if (page.matchUrl("http://pf.maoyan.com/rankings/day.*")) { MongoClient mongoClient2 = new MongoClient("localhost", 27017); // ? DB db = mongoClient2.getDB("maoyan_crawler"); // ????? Set<String> colls = db.getCollectionNames(); for (String s : colls) { // Collection(?"") if (s.equals("rankings_day")) { db.getCollection(s).drop(); } } DBCollection dbCollection2 = db.getCollection("rankings_day"); String title = page.select("span[id=year-box]").text(); String update_time = page.select("span[id=update-time]").text(); title = title + update_time; System.out.println(title); Elements data_set = page.select("tr"); List dayList = new ArrayList(); for (Element id : data_set) { Elements tds = id.select("td"); for (Element td : tds) { dayList.add(td.text()); } if (dayList.size() > 0) { System.out.println(dayList); BasicDBObject dbObject = new BasicDBObject(); dbObject.append("title", title).append("rank", dayList.get(0)) .append("mov_name", dayList.get(1)).append("today_rev", dayList.get(2)) .append("date", dayList.get(3)).append("val_week", dayList.get(4)); dayList.removeAll(dayList); dbCollection2.insert(dbObject); } } mongoClient2.close(); } else if (page.matchUrl("http://pf.maoyan.com/rankings/market.*")) { MongoClient mongoClient3 = new MongoClient("localhost", 27017); // ? DB db = mongoClient3.getDB("maoyan_crawler"); // ????? Set<String> colls = db.getCollectionNames(); for (String s : colls) { // Collection(?"") if (s.equals("rankings_market")) { db.getCollection(s).drop(); } } DBCollection dbCollection3 = db.getCollection("rankings_market"); String title = page.select("span[id=year-box]").text(); String update_time = page.select("span[id=update-time]").text(); title = title + update_time; System.out.println(title); Elements data_set = page.select("tr"); List dayList = new ArrayList(); for (Element id : data_set) { Elements tds = id.select("td"); for (Element td : tds) { dayList.add(td.text()); } if (dayList.size() > 0) { System.out.println(dayList); BasicDBObject dbObject = new BasicDBObject(); dbObject.append("title", title).append("rank", dayList.get(0)).append("date", dayList.get(1)) .append("today_rev", dayList.get(2)).append("total_sessions", dayList.get(3)) .append("total_visit_count", dayList.get(4)); dayList.removeAll(dayList); dbCollection3.insert(dbObject); } } mongoClient3.close(); } else if (page.matchUrl("http://pf.maoyan.com/rankings/year.*")) { MongoClient mongoClient4 = new MongoClient("localhost", 27017); // ? DB db = mongoClient4.getDB("maoyan_crawler"); // ????? Set<String> colls = db.getCollectionNames(); for (String s : colls) { // Collection(?"") if (s.equals("rankings_year")) { db.getCollection(s).drop(); } } DBCollection dbCollection4 = db.getCollection("rankings_year"); String title = page.select("span[id=year-box]").text(); String update_time = page.select("span[id=update-time]").text(); title = title + update_time; System.out.println(title); Elements table = page.select("div[id=ranks-list]"); // System.out.println(table); Elements data_set = table.select("ul[class=row]"); // System.out.println(data_set); List dayList = new ArrayList(); for (Element id : data_set) { Elements lis = id.select("li"); for (Element li : lis) { dayList.add(li.text()); } String cn_name = lis.select("p[class=first-line]").text(); String release_date = lis.select("p[class=second-line]").text(); if (dayList.size() > 0) { System.out.println(dayList); BasicDBObject dbObject = new BasicDBObject(); dbObject.append("title", title).append("rank", dayList.get(0)).append("name", cn_name) .append("release date", release_date).append("year_rev", dayList.get(2)) .append("avg_price", dayList.get(3)).append("avg_visit_count", dayList.get(4)); dayList.removeAll(dayList); dbCollection4.insert(dbObject); } } mongoClient4.close(); } else if (page.matchUrl("http://pf.maoyan.com/")) { MongoClient mongoClient5 = new MongoClient("localhost", 27017); // ? DB db = mongoClient5.getDB("maoyan_crawler"); // ????? Set<String> colls = db.getCollectionNames(); for (String s : colls) { // Collection(?"") if (s.equals("main_page")) { db.getCollection(s).drop(); } } DBCollection dbCollection5 = db.getCollection("main_page"); String title = page.select("span[id=dayStr]").text(); String box_type = page.select("span[id=box-type]").text(); String ticket_count = page.select("span[id=ticket_count]").text(); box_type = box_type + ticket_count; System.out.println(title + "\n" + box_type); Elements table = page.select("div[id=ticket_tbody]"); //System.out.println(table); Elements data_set = table.select("ul"); //System.out.println(data_set); List dayList = new ArrayList(); for (Element id : data_set) { Elements lis = id.select("li"); for (Element li : lis) { dayList.add(li.text()); } // System.out.println(lis); String cn_name = lis.select("b").first().text(); String comment = lis.select("em").text(); // System.out.println(cn_name+ "\n" +comment); if (dayList.size() > 0) { System.out.println(dayList); BasicDBObject dbObject = new BasicDBObject(); dbObject.append("title", title).append("box_type", box_type).append("name", cn_name) .append("comment", comment).append("realtime_rev", dayList.get(1)) .append("rev_percent", dayList.get(2)).append("schedule_percent", dayList.get(3)) .append("total_rev", dayList.get(4)); dayList.removeAll(dayList); dbCollection5.insert(dbObject); } } mongoClient5.close(); } }
From source file:cn.weibo.webcollector.spider.WeiboCrawler.java
License:Open Source License
public static void main(String[] args) throws Exception { MongoClient mongoClient = new MongoClient("127.0.0.1", 27017); Logger mongoLogger = Logger.getLogger("org.mongodb.driver"); mongoLogger.setLevel(Level.ERROR); // ?//from ww w . jav a 2 s . c o m // DBCollection dbCollection = // mongoClient.getDB("maoyan_crawler").getCollection("rankings_am"); DB db = mongoClient.getDB("weibo_crawler"); // ????? Set<String> colls = db.getCollectionNames(); for (String s : colls) { // Collection(?"") if (s.equals("weibo_page")) { db.getCollection(s).drop(); } } WeiboCrawler crawler = new WeiboCrawler("weibo_crawler", true); crawler.setThreads(3); /*???10?*/ for (int i = 1; i <= 10; i++) { crawler.addSeed(new CrawlDatum("http://weibo.cn/entpaparazzi?vt=4&page=" + i) //? .meta("inlink", "seed").meta("depth", "1")); } for (int i = 1; i <= 10; i++) { crawler.addSeed(new CrawlDatum("http://weibo.cn/dianyingpiaofangba?vt=4&page=" + i) //? .meta("inlink", "seed").meta("depth", "1")); } for (int i = 1; i <= 10; i++) { crawler.addSeed(new CrawlDatum("http://weibo.cn/houson100037?vt=4&page=" + i) //Houson .meta("inlink", "seed").meta("depth", "1")); } for (int i = 1; i <= 10; i++) { crawler.addSeed(new CrawlDatum("http://weibo.cn/kaopuyingping?vt=4&page=" + i) //? .meta("inlink", "seed").meta("depth", "1")); } for (int i = 1; i <= 10; i++) { crawler.addSeed(new CrawlDatum("http://weibo.cn/rottentomato?vt=4&page=" + i) // .meta("inlink", "seed").meta("depth", "1")); } for (int i = 1; i <= 10; i++) { crawler.addSeed(new CrawlDatum("http://weibo.cn/cfcu?vt=4&page=" + i) //? .meta("inlink", "seed").meta("depth", "1")); } for (int i = 1; i <= 10; i++) { crawler.addSeed(new CrawlDatum("http://weibo.cn/moviefactory?vt=4&page=" + i) // .meta("inlink", "seed").meta("depth", "1")); } for (int i = 1; i <= 10; i++) { crawler.addSeed(new CrawlDatum("http://weibo.cn/wodianying?vt=4&page=" + i) //Mr .meta("inlink", "seed").meta("depth", "1")); } for (int i = 1; i <= 10; i++) { crawler.addSeed(new CrawlDatum("http://weibo.cn/movietheworld?vt=4&page=" + i) //? .meta("inlink", "seed").meta("depth", "1")); } for (int i = 1; i <= 10; i++) { crawler.addSeed(new CrawlDatum("http://weibo.cn/badmovie?vt=4&page=" + i) // .meta("inlink", "seed").meta("depth", "1")); } crawler.addRegex("-.*\\.(jpg|png|gif).*"); crawler.addRegex("-.*top.*"); crawler.addRegex("http://weibo.cn/badmovie.*"); crawler.addRegex("http://weibo.cn/movietheworld.*"); crawler.addRegex("http://weibo.cn/wodianying.*"); crawler.addRegex("http://weibo.cn/moviefactory.*"); crawler.addRegex("http://weibo.cn/cfcu.*"); crawler.addRegex("http://weibo.cn/rottentomato.*"); crawler.addRegex("http://weibo.cn/kaopuyingping.*"); crawler.addRegex("http://weibo.cn/houson100037.*"); crawler.addRegex("http://weibo.cn/dianyingpiaofangba.*"); crawler.addRegex("http://weibo.cn/entpaparazzi.*"); crawler.addRegex("http://weibo.cn/comment/.*"); crawler.start(1); }
From source file:com.aankam.servlet.MongoCrudServlet.java
/** * Processes requests for both HTTP <code>GET</code> and <code>POST</code> * methods.//from w w w . jav a 2 s.c o m * * @param request servlet request * @param response servlet response * @throws ServletException if a servlet-specific error occurs * @throws IOException if an I/O error occurs */ protected void processRequest(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { MongoClientURI uri = new MongoClientURI("mongodb://Username:Pasword@ds061721.mongolab.com:61721/cs612"); MongoClient client = new MongoClient(uri); DB db = client.getDB(uri.getDatabase()); response.setContentType("text/html;charset=UTF-8"); String serialNo = request.getParameter("serialNo"); String name = request.getParameter("name"); String emailId = request.getParameter("emailId"); String operation = request.getParameter("operation"); BasicDBObject customer; BasicDBObject updateQuery; BasicDBObject deleteQuery; BasicDBObject searchQuery; DBCollection customersCollection; customersCollection = db.getCollection("Customers"); DBObject searchResult = null; boolean flag = false; DBCursor cursor; switch (operation) { case "create": customer = new BasicDBObject(); customer.put("serialNo", serialNo); customer.put("name", name); customer.put("emailId", emailId); customersCollection.insert(customer); break; case "update": updateQuery = new BasicDBObject(); updateQuery.append("serialNo", serialNo); cursor = customersCollection.find(updateQuery); customer = new BasicDBObject(); customer.put("serialNo", serialNo); customer.put("name", name); customer.put("emailId", emailId); if (cursor.hasNext()) { customersCollection.update(updateQuery, customer); flag = true; } break; case "delete": deleteQuery = new BasicDBObject(); deleteQuery.append("serialNo", serialNo); customersCollection.remove(deleteQuery); break; case "search": searchQuery = new BasicDBObject(); searchQuery.append("serialNo", serialNo); cursor = customersCollection.find(searchQuery); while (cursor.hasNext()) { searchResult = cursor.next(); } break; default: break; } try (PrintWriter out = response.getWriter()) { /* TODO output your page here. You may use following sample code. */ out.println("<!DOCTYPE html>"); out.println("<html>"); out.println("<head>"); out.println("<title>Servlet MongoCrudServlet</title>"); out.println("</head>"); out.println("<body>"); switch (operation) { case "create": out.println("<h3>Customer was successfully created.</h3>"); break; case "update": if (flag == true) { out.println("<h3>Customer was successfully updated.</h3>"); } else { out.println("<h3>Customer was not found to update.</h3>"); } break; case "delete": out.println("<h3>Customer was successfully deleted.</h3>"); break; case "search": if (searchResult != null) { out.println("<h3>The following customer was found:</h3>"); out.println("<h4>Serial No :" + searchResult.get("serialNo") + "</h4>"); out.println("<h4>Name :" + searchResult.get("name") + "</h4>"); out.println("<h4>Email Id :" + searchResult.get("emailId") + "</h4>"); } else { out.println("<h3>Customer not found.</h3>"); } break; default: break; } out.println("<a href=\"index.html\">Back to Main Form</a>"); out.println("</body>"); out.println("</html>"); } }
From source file:com.Aleksandar.Zoric.MongoMain.java
public String mapReduceFunction() { MongoClient mongoClient = new MongoClient(new ServerAddress("localhost", 27017)); DB db = mongoClient.getDB("amarokforumdb"); DBCollection collection = db.getCollection("comments"); long count = db.getCollection("comments").count(); System.out.println("Current amount of documents: " + count); String map = "function() { " + "var category; " + "var numOfDocuments = " + count + ";" + "for(i = 0; i < numOfDocuments; i++){ " + "if (numOfDocuments <= 100) {" + "category = 'New Comments'; }" + "else if(numOfDocuments > 100){" + "category = 'Old Comments'; }}" + "emit(category,1);};"; String reduce = "function(key, values) { " + "var sum = 0; " + "values.forEach(function(doc) { " + "sum += 1; " + "}); " + "return {comments: sum};} "; MapReduceCommand cmd = new MapReduceCommand(collection, map, reduce, null, MapReduceCommand.OutputType.INLINE, null); MapReduceOutput out = collection.mapReduce(cmd); System.out.println("Mapreduce results"); String result = null;//from w w w . ja v a2 s . com for (DBObject o : out.results()) { result += o; } return result; }
From source file:com.almende.eve.state.mongo.MongoStateFactory.java
License:Apache License
/** * constructor which uses readily available mongo client instance and * database name./* ww w . j a v a 2 s . co m*/ * * @param mongoClient * the mongo client * @param databaseName * the database name * @param collectionName * the collection name */ public MongoStateFactory(MongoClient mongoClient, String databaseName, String collectionName) { this(new Jongo(mongoClient.getDB(databaseName)), collectionName); }
From source file:com.almende.eve.state.mongo.MongoStateFactory.java
License:Apache License
/** * constructor using configuration mapping provided through the YAML file. * //from w ww .j a v a2s . co m * @param params * the params * @throws UnknownHostException * the unknown host exception */ public MongoStateFactory(Map<String, Object> params) throws UnknownHostException { // initialization of client & jongo MongoClient client = createClient( ((params.containsKey("uriHost")) ? (String) params.get("uriHost") : "localhost"), // parse URI ((params.containsKey("port")) ? (Integer) params.get("port") : 27017) // parse port ); String databaseName = (params != null && params.containsKey("database")) ? (String) params.get("database") : "eve"; this.jongo = new Jongo(client.getDB(databaseName)); this.collectionName = (params != null && params.containsKey("collection")) ? (String) params.get("collection") : "agents"; this.jongo.runCommand("{collMod: '" + this.collectionName + "', usePowerOf2Sizes : true }"); }
From source file:com.aw.util.MongoDbUtil.java
/** * * @param dbName/*w w w. j av a 2 s. co m*/ * @return */ public static DB getDB(String dbName) { if (db != null) { return db; } MongoClient client = getMongoClient(); if (client != null) { db = client.getDB(dbName); return db; } else { return null; } }
From source file:com.ayu.filter.DbListner.java
License:Open Source License
/** * @see ServletContextListener#contextInitialized(ServletContextEvent) *//*w w w . j a va 2 s.co m*/ public void contextInitialized(ServletContextEvent arg0) { MongoClient mongoClient = null; try { mongoClient = new MongoClient(); } catch (UnknownHostException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } DB db = mongoClient.getDB("test"); DBCollection coll; coll = db.getCollection("test"); DBCursor cursor = coll.find(); while (cursor.hasNext()) { String ip = (String) cursor.next().get("Ip-Address"); if (arg0.getServletContext().getAttribute(ip) == null) { arg0.getServletContext().setAttribute(ip, ip); } } }
From source file:com.ayu.filter.RegularService.java
License:Open Source License
@Async public void registerUser(String ip, String date, String type, String document) { //System.out.println(" Attack from "+ip +" captured at"+ date+" "+"type of attack is"+" "+type); //System.out.println(" Database Insertion "); try {/*from w w w. j a v a2 s. c om*/ Thread.sleep(5000); } catch (InterruptedException e) { e.printStackTrace(); } MongoClient mongoClient = null; try { mongoClient = new MongoClient(); } catch (UnknownHostException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } // or, to connect to a replica set, supply a seed list of members //MongoClient mongoClient = new MongoClient(Arrays.asList(new ServerAddress("localhost", 27017), // new ServerAddress("localhost", 27018), // new ServerAddress("localhost", 27019))); DB db = mongoClient.getDB(document); DBCollection coll; coll = db.getCollection(document); BasicDBObject doc = new BasicDBObject("Ip-Address", ip).append("Date", date).append("Attack-Type", type); //System.out.println("Data Display"); coll.insert(doc); mongoClient.close(); //System.out.println(" Asynchronous method call of database Complete "); }
From source file:com.azure.api.MongoDAO.java
public static LinkedList<Pojoo> MongoConnection() throws IOException { MongoClient mongo = new MongoClient("localhost", 27017); DB db = mongo.getDB("test"); DBCollection table = db.getCollection("DataFinal"); List<String> uniId = table.distinct("Asin"); //BasicDBObject searchQuery = new BasicDBObject(); //searchQuery.put("Asin", "000100039X"); DBObject groupFields = new BasicDBObject("_id", "$Asin"); groupFields.put("count", new BasicDBObject("$sum", 1)); //DBObject group = new BasicDBObject("$group", groupFields ); DBObject sortFields = new BasicDBObject("count", -1); //AggregationOutput output = table.aggregate(group, new BasicDBObject("$sort", sortFields )); //System.out.println(output.results());//Top reviews groupFields.put("average", new BasicDBObject("$avg", "$Ratings")); DBObject group2 = new BasicDBObject("$group", groupFields); AggregationOutput output2 = table.aggregate(group2, new BasicDBObject("$sort", sortFields)); Iterable<DBObject> iterable = output2.results(); //LinkedList<String>stringResult = new LinkedList<String>(); //LinkedList<String>stringrating = new LinkedList<String>(); //if(awsResult.size()!=0){awsResult.remove();} //if(stringResult.size()!=0){stringResult.remove();} //System.out.println("fffffffffffffffff"+stringResult.size()); int count = 0; LinkedList<Pojoo> uiList = new LinkedList<Pojoo>(); for (DBObject res : output2.results()) { //System.out.println("TTTTTTTT "+res.get("_id").toString()); //System.out.println("VVVVVVVV "+res.get("average").toString()); String awsResult = awsApi.getDetails(res.get("_id").toString()); //stringResult.add(res.get("_id").toString()); //stringrating.add(res.get("average").toString()); Pojoo polo = new Pojoo(); polo.setStringResult(awsResult); polo.setStringrating(res.get("average").toString()); uiList.add(polo);/*from w ww . ja v a 2s . c o m*/ count++; if (count == 8) { res = null; break; } } count = 0; //awsResult = awsApi.getDetails(stringResult); //for(String htp: awsResult){ //Pojoo polo = new Pojoo(); //polo.setStringResult(htp); //polo.setStringrating(res.get("average").toString()); //uiList.add(polo); //} // System.out.println("WWWWWW"+awsResult); //Pojoo img = new Pojoo(); //img.setStringrating(stringrating); //img.setStringResult(awsResult); //stringResult.remove(); //System.out.println("eeeeeeeeeeeeeee"+awsResult.size()); //for(String ret:awsResult) //{ // System.out.println(ret); //} return uiList; }