List of usage examples for com.mongodb DBCursor next
@Override
public DBObject next()
From source file:com.ciphertool.sherlock.dao.NGramDao.java
License:Open Source License
/** * Returns a list of top N NGrams. We have to use the low-level MongoDB API because otherwise the query takes * forever due to the limitation of Spring Data not providing cursor functionality. *//*w w w . ja va 2s . co m*/ public List<NGram> findTopMostFrequentByNumWords(int numWordsQueryParam, int top) { DBCollection collection = mongoOperations.getCollection(DatabaseConstants.NGRAM_COLLECTION); DBCursor cursor; if (top > 0) { cursor = collection.find(new BasicDBObject("numWords", numWordsQueryParam)) .sort(new BasicDBObject("frequencyWeight", -1)).limit(top); } else { cursor = collection.find(new BasicDBObject("numWords", numWordsQueryParam)) .sort(new BasicDBObject("frequencyWeight", -1)); } List<NGram> results = new ArrayList<NGram>(); while (cursor.hasNext()) { DBObject next = cursor.next(); String nGram = (String) next.get("nGram"); Integer numWords = (Integer) next.get("numWords"); Long frequencyWeight = (Long) next.get("frequencyWeight"); results.add(new NGram(nGram, numWords, frequencyWeight)); } return results; }
From source file:com.conventus.mongodb.dao.MongoDBGenericDAO.java
public List<T> readAll() { List<T> data = new ArrayList<T>(); DBCursor cursor = this.col.find(); while (cursor.hasNext()) { DBObject doc = cursor.next(); T obj = this.converter.toObject(doc); data.add(obj);/* w w w . j a v a2s . com*/ } return data; }
From source file:com.crawler.controller.ParserFromIZManga.java
public static void getChapterByStoriesIZManga(List<String> story) { try {//from w ww.j av a 2 s . c o m String title = ""; String author = ""; String status = ""; String source = ""; String type = ""; String image = ""; String description = ""; List<String> chapterList = new ArrayList<>(); List<String> data = new ArrayList<>(); int i = 0; for (i = 0; i < story.size(); i++) { String homelink = "http://izmanga.com/chapter_id-" + story.get(i); Document linkget = Jsoup.connect(homelink).timeout(60 * 1000).get(); Elements chapter_list = linkget.getElementsByClass("row"); Elements info_topic = linkget.getElementsByClass("manga-info-text"); Elements info_pic = linkget.getElementsByClass("manga-info-pic"); Elements info_content = linkget.getElementsByClass("manga-info-content"); Elements info = info_topic.get(0).getElementsByTag("li"); source = "http://www.izmanga.com"; title = info.get(0).getElementsByTag("h1").text(); title = title + " - " + source.substring(11); author = info.get(1).text(); author = author.substring(author.indexOf(": ") + 2).trim(); status = info.get(2).text(); status = status.substring(status.indexOf(": ") + 2).trim(); description = info_content.get(0).text().trim(); description = description.substring(description.indexOf(": ") + 2).trim(); image = info_pic.get(0).getElementsByTag("img").get(0).attr("src"); if (!image.startsWith("http")) { image = source + image; } type = info.get(7).getElementsByTag("a").text().replace(" ", ","); DBCursor cursor = dao.checkNewestChap(title.trim()); String newest_chap = ""; Map<String, String> result = new LinkedHashMap<>(); if (cursor != null) { newest_chap = (String) cursor.next().get("newest_chap"); } result = getDataIZManga(chapter_list, story, i, newest_chap); System.out.println( title + " - " + image + " - " + author + " - " + status + " - " + source + " - " + type); if (newest_chap == null || newest_chap.isEmpty()) { dao.insertStory(title, author, status, source, type, image, description, result); } else { dao.updateStory(title, author, status, source, type, image, description, result); } } } catch (Exception e) { e.printStackTrace(); } }
From source file:com.crawler.controller.ParserFromKissManga.java
public static void getChapterByStoriesKissManga(String href) { try {//from w w w . j ava 2 s . com String title = ""; String author = ""; String status = ""; String source = "http://www.kissmanga.com"; String type = ""; String image = ""; String chapter_name = ""; String description = ""; String homelink = "http://kissmanga.com" + href; Document linkget = Jsoup.connect(homelink).timeout(60 * 1000).get(); Elements info = linkget.getElementsByClass("barContent").first().getElementsByTag("p"); if (info != null) { title = linkget.getElementsByClass("bigChar").text(); image = linkget.getElementsByClass("barContent").get(3).getElementsByTag("img").attr("src"); if (info.get(0).getElementsByTag("span").text().equalsIgnoreCase("Other name:")) { type = info.get(1).getElementsByTag("a").text().replace(" ", ","); author = info.get(2).getElementsByTag("a").text(); status = info.get(3).text(); description = info.get(5).text(); } else { type = info.get(0).getElementsByTag("a").text().replace(" ", ","); author = info.get(1).getElementsByTag("a").text(); status = info.get(2).text(); description = info.get(4).text(); } title = title + " - " + source.substring(11); status = status.substring(0, status.indexOf("Views")).trim(); int start = status.indexOf("Status:"); status = status.substring(start + 8).trim(); System.out.println(title + " - " + image + " - " + type + " - " + author + " - " + status + " - " + description); Elements chapter_list = linkget.getElementsByClass("listing"); Elements chapter = chapter_list.first().getElementsByTag("a"); DBCursor cursor = dao.checkNewestChap(title.trim()); String newest_chap = ""; Map<String, String> result = new LinkedHashMap<>(); if (cursor != null) { newest_chap = (String) cursor.next().get("newest_chap"); } result = getDataKissManga(chapter, newest_chap); //System.out.println(title + " - " + image + " - " + author + " - " + status + " - " + source + " - " + type); if (newest_chap == null || newest_chap.isEmpty()) { dao.insertStory(title, author, status, source, type, image, description, result); } else { dao.updateStory(title, author, status, source, type, image, description, result); } } } catch (Exception e) { e.printStackTrace(); } }
From source file:com.crawler.controller.ParserFromMangaHead.java
public static void getChapterByStoriesMangaHead(List<String> href) { try {// w w w. j ava 2 s . c o m String title = ""; String author = ""; String status = ""; String source = ""; String type = ""; String thumb = ""; String description = ""; String updating = "?ang cp nht.."; String chapter = ""; for (int i = 0; i < href.size(); i++) { author = status = type = thumb = description = updating; source = "http://mangahead.com"; String homelink = "http://mangahead.com" + href.get(i); Document linkget = Jsoup.connect(homelink).timeout(60 * 1000).get(); Elements images = linkget.getElementsByClass("mangahead_thumbnail_cell"); Elements info = linkget.getElementsByClass("mangaviewer_toppest_navig").get(0) .getElementsByTag("a"); title = info.get(2).text().trim() + " Raw"; //System.out.println(title); chapter = linkget.getElementsByClass("mangaviewer_toppest_navig").get(0).text(); chapter = chapter.substring(chapter.lastIndexOf("/") + 2).trim(); String data = ""; for (Element image : images) { data += image.getElementsByTag("a").attr("href") .replace("?action=big&size=original&fromthumbnail=true", "\u0020") .replace("/index.php/", "http://s9.mangahead.com/mangas/") + "|"; } DBCursor cursor = dao.checkNewestChap(title.trim()); // 32000 khang String newest_chap = ""; Map<String, String> result = new LinkedHashMap<>(); if (cursor != null) { newest_chap = (String) cursor.next().get("newest_chap"); } result.put(chapter, data); System.out.println( title + " - " + thumb + " - " + author + " - " + status + " - " + source + " - " + type); if (newest_chap == null || newest_chap.isEmpty()) { dao.insertStory(title, author, status, source, type, thumb, description, result); } else { dao.updateStory(title, author, status, source, type, thumb, description, result); } } } catch (Exception e) { } }
From source file:com.cyslab.craftvm.rest.mongo.AggregateServlet.java
License:GNU General Public License
@Override protected void doPost(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException { log.trace("doPost()"); if (!can_read(req)) { res.sendError(SC_UNAUTHORIZED);//from w w w. j a va 2 s . c o m return; } InputStream is = req.getInputStream(); String db_name = req.getParameter("dbname"); String col_name = req.getParameter("colname"); if (db_name == null || col_name == null) { error(res, SC_BAD_REQUEST, Status.get("param name missing")); return; } String skip = req.getParameter("skip"); String limit = req.getParameter("limit"); DB db = mongo.getDB(db_name); DBCollection col = db.getCollection(col_name); BufferedReader r = null; DBObject q = null, sort = null; try { r = new BufferedReader(new InputStreamReader(is)); String data = r.readLine(); if (data == null) { error(res, SC_BAD_REQUEST, Status.get("no data")); return; } try { q = (DBObject) JSON.parse(data); } catch (JSONParseException e) { error(res, SC_BAD_REQUEST, Status.get("can not parse data")); return; } // sort param data = r.readLine(); if (data != null) { try { sort = (DBObject) JSON.parse(data); } catch (JSONParseException e) { error(res, SC_BAD_REQUEST, Status.get("can not parse sort arg")); return; } } } finally { if (r != null) r.close(); } DBCursor c; if (sort == null) c = col.find(q); else c = col.find(q).sort(sort); if (c == null) { error(res, SC_NOT_FOUND, Status.get("no documents found")); return; } res.setIntHeader("X-Documents-Count", c.count()); if (limit != null) { try { c.limit(Math.min(Integer.parseInt(limit), MAX_FIELDS_TO_RETURN)); } catch (NumberFormatException e) { error(res, SC_BAD_REQUEST, Status.get("can not parse limit")); c.close(); return; } } else c.limit(MAX_FIELDS_TO_RETURN); if (skip != null) { try { c.skip(Integer.parseInt(skip)); } catch (NumberFormatException e) { error(res, SC_BAD_REQUEST, Status.get("can not parse skip")); c.close(); return; } } StringBuilder buf = tl.get(); // reset buf buf.setLength(0); int no = 0; buf.append("["); while (c.hasNext()) { DBObject o = c.next(); JSON.serialize(o, buf); buf.append(","); no++; } if (no > 0) buf.setCharAt(buf.length() - 1, ']'); else buf.append(']'); res.setIntHeader("X-Documents-Returned", no); out_str(req, buf.toString(), "application/json"); }
From source file:com.cyslab.craftvm.rest.mongo.AggregateServlet.java
License:GNU General Public License
@Override protected void doGet(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException { log.trace("doGet()"); if (!can_read(req)) { res.sendError(SC_UNAUTHORIZED);/*from w w w . j a v a 2 s . c om*/ return; } String db_name = req.getParameter("dbname"); String col_name = req.getParameter("colname"); if (db_name == null || col_name == null) { error(res, SC_BAD_REQUEST, Status.get("param name missing")); return; } String skip = req.getParameter("skip"); String limit = req.getParameter("limit"); DB db = mongo.getDB(db_name); DBCollection col = db.getCollection(col_name); DBCursor c = col.find(); if (c == null) { error(res, SC_NOT_FOUND, Status.get("no documents found")); return; } res.setIntHeader("X-Documents-Count", c.count()); if (limit != null) { try { c.limit(Math.min(Integer.parseInt(limit), MAX_FIELDS_TO_RETURN)); } catch (NumberFormatException e) { error(res, SC_BAD_REQUEST, Status.get("can not parse limit")); c.close(); return; } } else c.limit(MAX_FIELDS_TO_RETURN); if (skip != null) { try { c.skip(Integer.parseInt(skip)); } catch (NumberFormatException e) { error(res, SC_BAD_REQUEST, Status.get("can not parse skip")); c.close(); return; } } StringBuilder buf = tl.get(); buf.setLength(0); int no = 0; buf.append("["); while (c.hasNext()) { DBObject o = c.next(); JSON.serialize(o, buf); buf.append(","); no++; } if (no > 0) buf.setCharAt(buf.length() - 1, ']'); else buf.append(']'); res.setIntHeader("X-Documents-Returned", no); out_str(req, buf.toString(), "application/json"); }
From source file:com.cyslab.craftvm.rest.mongo.GridfsServlet.java
License:GNU General Public License
@Override protected void doGet(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException { log.trace("doGet()"); if (!can_read(req)) { res.sendError(SC_UNAUTHORIZED);//from ww w . j a va 2 s . c om return; } String db_name = req.getParameter("dbname"); String bucket_name = req.getParameter("bucketname"); if (db_name == null || bucket_name == null) { String names[] = req2mongonames(req); if (names != null) { db_name = names[0]; bucket_name = names[1]; } if (db_name == null) { error(res, SC_BAD_REQUEST, Status.get("param name missing")); return; } } if (bucket_name == null) bucket_name = "fs"; DB db = mongo.getDB(db_name); String fs_cache_key = db_name + bucket_name; GridFS fs = fs_cache.get(fs_cache_key); if (fs == null) { fs = new GridFS(db, bucket_name); fs_cache.put(fs_cache_key, fs); } // mongo auth String user = req.getParameter("user"); String passwd = req.getParameter("passwd"); if (user != null && passwd != null && (!db.isAuthenticated())) { boolean auth = db.authenticate(user, passwd.toCharArray()); if (!auth) { res.sendError(SC_UNAUTHORIZED); return; } } String op = req.getParameter("op"); if (op == null) op = "get"; StringBuilder buf = tl.get(); // reset buf buf.setLength(0); // list if ("get".equals(op)) { String file_name = req.getParameter("filename"); if (file_name == null) { error(res, SC_BAD_REQUEST, Status.get("param name missing")); return; } GridFSDBFile db_file = fs.findOne(file_name); if (db_file == null) { error(res, SC_NOT_FOUND, Status.get("file does not exists")); return; } res.setContentLength((int) db_file.getLength()); String ct = db_file.getContentType(); if (ct != null) res.setContentType(ct); OutputStream os = res.getOutputStream(); long l; while ((l = db_file.writeTo(os)) > 0) ; os.flush(); os.close(); } // list else if ("list".equals(op)) { DBCursor c = fs.getFileList(); if (c == null) { error(res, SC_NOT_FOUND, Status.get("no documents found")); return; } int no = 0; buf.append("["); while (c.hasNext()) { DBObject o = c.next(); JSON.serialize(o, buf); buf.append(","); no++; } if (no > 0) buf.setCharAt(buf.length() - 1, ']'); else buf.append(']'); out_str(req, buf.toString(), "application/json"); } // info else if ("info".equals(op)) { String file_name = req.getParameter("filename"); if (file_name == null) { error(res, SC_BAD_REQUEST, Status.get("param name missing")); return; } GridFSDBFile db_file = fs.findOne(file_name); if (db_file == null) { error(res, SC_NOT_FOUND, Status.get("no documents found")); return; } buf.append("{"); buf.append(String.format("\"ContentType\":%s,", db_file.getContentType())); buf.append(String.format("\"Length\":%d,", db_file.getLength())); buf.append(String.format("\"MD5\":%s", db_file.getMD5())); buf.append("}"); out_str(req, buf.toString(), "application/json"); } else res.sendError(SC_BAD_REQUEST); }
From source file:com.cyslab.craftvm.rest.mongo.QueryServlet.java
License:GNU General Public License
@Override protected void doPost(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException { log.trace("doPost()"); // server auth if (!can_read(req)) { res.sendError(SC_UNAUTHORIZED);//from w w w .j a va 2 s. c o m return; } String ret = null; InputStream is = req.getInputStream(); String db_name = req.getParameter("dbname"); String col_name = req.getParameter("colname"); if (db_name == null || col_name == null) { String names[] = req2mongonames(req); if (names != null) { db_name = names[0]; col_name = names[1]; } if (db_name == null || col_name == null) { error(res, SC_BAD_REQUEST, Status.get("param name missing")); return; } } String skip = req.getParameter("skip"); String limit = req.getParameter("limit"); String _fields = req.getParameter("fields"); String fields[] = null; if (_fields != null) fields = _fields.split("[,]"); DB db = mongo.getDB(db_name); // mongo auth String user = req.getParameter("user"); String passwd = req.getParameter("passwd"); if (user != null && passwd != null && (!db.isAuthenticated())) { boolean auth = db.authenticate(user, passwd.toCharArray()); if (!auth) { res.sendError(SC_UNAUTHORIZED); return; } } DBCollection col = db.getCollection(col_name); StringBuilder buf = tl.get(); // reset buf buf.setLength(0); BufferedReader r = null; DBObject q = null, sort = null; try { r = new BufferedReader(new InputStreamReader(is)); String data = r.readLine(); if (data == null) { error(res, SC_BAD_REQUEST, Status.get("no data")); return; } try { q = (DBObject) JSON.parse(data); if (cache != null) { buf.append(db_name); buf.append(col_name); buf.append(data); } } catch (JSONParseException e) { error(res, SC_BAD_REQUEST, Status.get("can not parse data")); return; } // sort param data = r.readLine(); if (data != null) { try { sort = (DBObject) JSON.parse(data); } catch (JSONParseException e) { error(res, SC_BAD_REQUEST, Status.get("can not parse sort arg")); return; } if (cache != null) buf.append(data); } } finally { if (r != null) r.close(); } // limit int lim; if (limit != null) { try { lim = Math.min(Integer.parseInt(limit), MAX_FIELDS_TO_RETURN); } catch (NumberFormatException e) { error(res, SC_BAD_REQUEST, Status.get("can not parse limit")); return; } } else { lim = MAX_FIELDS_TO_RETURN; } if (cache != null) { buf.append(";limit="); buf.append(lim); } // skip int sk = -1; if (skip != null) { try { sk = Integer.parseInt(skip); if (cache != null) { buf.append(";skip="); buf.append(sk); } } catch (NumberFormatException e) { error(res, SC_BAD_REQUEST, Status.get("can not parse skip")); return; } } // fields if (cache != null && _fields != null) { buf.append(";fields="); buf.append(_fields); } // from cache String cache_key = null; if (cache != null) { cache_key = buf.toString(); try { ret = (String) cache.get(cache_key); } // some wrong char in key catch (IllegalArgumentException e) { int l = buf.length(); for (int i = 0; i < l; i++) { if (buf.charAt(i) == ' ') buf.setCharAt(i, '*'); } cache_key = buf.toString(); ret = (String) cache.get(cache_key); } if (ret != null) { out_str(req, ret, "application/json"); return; } } // cursor DBCursor c; if (fields != null) { StringBuilder sb = new StringBuilder(); sb.append("{"); int len = fields.length; for (int i = 0; i < len; i++) { String s = fields[i]; sb.append('"'); sb.append(s); sb.append('"'); sb.append(":1"); if (i != (len - 1)) sb.append(","); } sb.append("}"); c = col.find(q, (DBObject) JSON.parse(sb.toString())); } else c = col.find(q); if (c == null || c.count() == 0) { error(res, SC_NOT_FOUND, Status.get("no documents found")); return; } if (sort != null) c.sort(sort); res.setIntHeader("X-Documents-Count", c.count()); c.limit(lim); if (sk != -1) c.skip(sk); // reset buf buf.setLength(0); int no = 0; buf.append("["); while (c.hasNext()) { DBObject o = c.next(); if (rm_id) o.removeField("_id"); JSON.serialize(o, buf); buf.append(","); no++; } c.close(); if (no > 0) buf.setCharAt(buf.length() - 1, ']'); else buf.append(']'); res.setIntHeader("X-Documents-Returned", no); ret = buf.toString(); if (cache != null) cache.set(cache_key, ret); out_str(req, ret, "application/json"); }
From source file:com.cyslab.craftvm.rest.mongo.QueryServlet.java
License:GNU General Public License
@Override protected void doGet(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException { log.trace("doGet()"); if (!can_read(req)) { res.sendError(SC_UNAUTHORIZED);//from w w w . j a va2 s.c o m return; } String db_name = req.getParameter("dbname"); String col_name = req.getParameter("colname"); if (db_name == null || col_name == null) { String names[] = req2mongonames(req); if (names != null) { db_name = names[0]; col_name = names[1]; } if (db_name == null || col_name == null) { error(res, SC_BAD_REQUEST, Status.get("param name missing")); return; } } String skip = req.getParameter("skip"); String limit = req.getParameter("limit"); DB db = mongo.getDB(db_name); // mongo auth String user = req.getParameter("user"); String passwd = req.getParameter("passwd"); if (user != null && passwd != null && (!db.isAuthenticated())) { boolean auth = db.authenticate(user, passwd.toCharArray()); if (!auth) { res.sendError(SC_UNAUTHORIZED); return; } } DBCollection col = db.getCollection(col_name); DBCursor c = col.find(); if (c == null || c.count() == 0) { error(res, SC_NOT_FOUND, Status.get("no documents found")); return; } res.setIntHeader("X-Documents-Count", c.count()); if (limit != null) { try { c.limit(Math.min(Integer.parseInt(limit), MAX_FIELDS_TO_RETURN)); } catch (NumberFormatException e) { error(res, SC_BAD_REQUEST, Status.get("can not parse limit")); c.close(); return; } } else c.limit(MAX_FIELDS_TO_RETURN); if (skip != null) { try { c.skip(Integer.parseInt(skip)); } catch (NumberFormatException e) { error(res, SC_BAD_REQUEST, Status.get("can not parse skip")); c.close(); return; } } StringBuilder buf = tl.get(); buf.setLength(0); int no = 0; buf.append("["); while (c.hasNext()) { DBObject o = c.next(); if (rm_id) o.removeField("_id"); JSON.serialize(o, buf); buf.append(","); no++; } c.close(); if (no > 0) buf.setCharAt(buf.length() - 1, ']'); else buf.append(']'); res.setIntHeader("X-Documents-Returned", no); out_str(req, buf.toString(), "application/json"); }