List of usage examples for com.mongodb DBCursor count
public int count()
From source file:com.hangum.tadpole.mongodb.core.editors.main.MongoDBTableEditor.java
License:Open Source License
/** * ? ? ./*from w ww . ja v a 2 s. co m*/ * * @param basicFields * @param basicWhere * @param basicSort */ private void find(BasicDBObject basicFields, DBObject basicWhere, BasicDBObject basicSort, int cntSkip, int cntLimit) throws Exception { if ((cntLimit - cntSkip) >= defaultMaxCount) { // " " + defaultMaxCount + " ? . Prefernece? ? ." // Search can not exceed the number 5. Set in Perference. throw new Exception(String.format(Messages.MongoDBTableEditor_0, "" + defaultMaxCount)); //$NON-NLS-2$ } mapColumns = new HashMap<Integer, String>(); sourceDataList = new ArrayList<HashMap<Integer, Object>>(); DB mongoDB = MongoConnectionManager.getInstance(userDB); DBCollection dbCollection = mongoDB.getCollection(tableName); // ?? DBCursor dbCursor = null; try { if (cntSkip > 0 && cntLimit > 0) { sbConsoleMsg.append("############[query]#####################").append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Fields]" + basicFields.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Where]" + basicWhere.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Sort] " + basicSort.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Skip]" + cntSkip).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Limit]" + cntLimit).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("############[query]#####################"); //$NON-NLS-1$ dbCursor = dbCollection.find(basicWhere, basicFields).sort(basicSort).skip(cntSkip).limit(cntLimit); } else if (cntSkip == 0 && cntLimit > 0) { sbConsoleMsg.append("############[query]#####################").append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Fields]" + basicFields.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Where]" + basicWhere.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Sort] " + basicSort.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Limit]" + cntLimit).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("############[query]#####################").append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ dbCursor = dbCollection.find(basicWhere, basicFields).sort(basicSort).limit(cntLimit); } else { sbConsoleMsg.append("############[query]#####################").append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Fields]" + basicFields.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Where]" + basicWhere.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Sort] " + basicSort.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("############[query]#####################").append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ dbCursor = dbCollection.find(basicWhere, basicFields).sort(basicSort); } DBObject explainDBObject = dbCursor.explain(); sbConsoleMsg.append("############[result]#####################").append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[query explain]\r\n" + JSONUtil.getPretty(explainDBObject.toString())) //$NON-NLS-1$ .append("\r\n"); //$NON-NLS-1$ sbConsoleMsg.append("[error]\r\n" + JSONUtil.getPretty(mongoDB.getLastError().toString())) //$NON-NLS-1$ .append("\r\n"); //$NON-NLS-1$ sbConsoleMsg.append("############[result]#####################").append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ mongoDB.forceError(); mongoDB.resetError(); if (logger.isDebugEnabled()) logger.debug(sbConsoleMsg); // ?? . int totCnt = 0; listTrees = new ArrayList<MongodbTreeViewDTO>(); for (DBObject dbObject : dbCursor) { // ? ? if (mapColumns.size() == 0) mapColumns = MongoDBTableColumn.getTabelColumnView(dbObject); // append tree text columnInfo.get(key) MongodbTreeViewDTO treeDto = new MongodbTreeViewDTO(dbObject, "(" + totCnt + ") {..}", "", //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$ "Document"); //$NON-NLS-1$ parserTreeObject(dbObject, treeDto, dbObject); listTrees.add(treeDto); // append table text HashMap<Integer, Object> dataMap = new HashMap<Integer, Object>(); for (int i = 0; i < mapColumns.size(); i++) { Object keyVal = dbObject.get(mapColumns.get(i)); if (keyVal == null) dataMap.put(i, ""); //$NON-NLS-1$ else dataMap.put(i, keyVal.toString()); } // ?? ? ? id dataMap.put(MongoDBDefine.PRIMARY_ID_KEY, dbObject); sourceDataList.add(dataMap); // append row text totCnt++; } txtCnt = dbCursor.count() + "/" + totCnt + Messages.MongoDBTableEditor_69; //$NON-NLS-1$ } finally { if (dbCursor != null) dbCursor.close(); } }
From source file:com.hangum.tadpole.mongodb.core.test.MongoTestAndORComplexStmt.java
License:Open Source License
/** * @param args/*from w ww .ja v a 2 s .co m*/ */ public static void main(String[] args) throws Exception { ConAndAuthentication testMongoCls = new ConAndAuthentication(); Mongo mongo = testMongoCls.connection(ConAndAuthentication.serverurl, ConAndAuthentication.port); DB db = mongo.getDB("test"); DBCollection myColl = db.getCollection("rental"); BasicDBObject mainQuery = new BasicDBObject(); // tmp and BasicDBObject tmpAndQuery = new BasicDBObject(); tmpAndQuery.append("inventory_id", 100); tmpAndQuery.append("rental_id", new BasicDBObject("$ne", 1)); mainQuery.put("$and", tmpAndQuery); // tmp or ArrayList<BasicDBObject> myList = new ArrayList<BasicDBObject>(); myList.add(new BasicDBObject("customer_id", 3)); mainQuery.put("$or", myList); System.out.println(mainQuery.toString()); DBCursor myCursor = myColl.find(mainQuery); System.out.println("[result cursor size is " + myCursor.count()); while (myCursor.hasNext()) { System.out.println(myCursor.next()); } mongo.close(); }
From source file:com.ibm.ws.lars.rest.PersistenceBean.java
License:Apache License
private int queryCount(DBObject filterObject) { if (logger.isLoggable(Level.FINE)) { logger.fine("queryCount: Querying database with query object " + filterObject); }/* ww w . j ava 2s . c om*/ DBCursor cursor = getAssetCollection().find(filterObject); int count = cursor.count(); if (logger.isLoggable(Level.FINE)) { logger.fine("queryCount: found " + count + " assets."); } return count; }
From source file:com.icoin.trading.webui.init.SystemController.java
License:Apache License
@RequestMapping(value = "/collection/{id}", method = RequestMethod.GET) public String collection(@PathVariable("id") String collectionName, @RequestParam(value = "page", defaultValue = "1") int pageNumber, @RequestParam(value = "itemsperpage", defaultValue = "5") int itemsPerPage, Model model) { DBCursor dbCursor = springTemplate.getCollection(collectionName).find(); List<DBObject> dbObjects = dbCursor.skip((pageNumber - 1) * itemsPerPage).limit(itemsPerPage).toArray(); List<Map> items = new ArrayList<Map>(dbCursor.length()); for (DBObject dbObject : dbObjects) { items.add(dbObject.toMap());/*from ww w .j a va 2 s .c o m*/ } model.addAttribute("items", items); int totalItems = dbCursor.count(); int numPages = ((int) Math.floor(totalItems / itemsPerPage)) + 1; model.addAttribute("numPages", numPages); model.addAttribute("page", pageNumber); model.addAttribute("itemsPerPage", itemsPerPage); model.addAttribute("collectionName", collectionName); return "data/collection"; }
From source file:com.ifactory.service.weather.photo.PhotoService.java
License:Apache License
public ArrayList<Photo> get() { DB db = mongoClient.getDB(this.dbName); DBCollection coll = db.getCollection(PHOTO_COLLECTION); BasicDBObject query = null;//from www . j a va 2 s . c om DBCursor cursor = null; ArrayList<Photo> photoList = new ArrayList(); int weatherClassMin = -1; int weatherClassMax = -1; double rad = this.radius; while (true) { // If latitude and longitude were given, append geo search query if (this.lat != UNAVAILABLE_LATITUDE && this.lng != UNAVAILABLE_LONGITUDE) { query = setGeoCoord(this.lat, this.lng, rad); } else { query = new BasicDBObject(); } // It the weather Id has given, append weather search query if (this.weatherId > 0) { if (weatherClassMin == -1 && weatherClassMax == -1) { query.append("weather", this.weatherId); } else { System.out.println("query with weatherClassMin(" + weatherClassMin + ") and weatherClassMax(" + weatherClassMax + ")"); query.append("weather", new BasicDBObject("$gte", weatherClassMin).append("$lte", weatherClassMax)); System.out.println(query.toString()); } } try { cursor = coll.find(query).limit(this.limit); if (cursor.count() > 0) { System.out.println( "photo found(lat: " + this.lat + ", lng: " + this.lng + ", radius: " + rad + ")"); break; } else if (this.growable == false || rad >= UNAVAILABLE_LATITUDE) { if (rad >= UNAVAILABLE_LATITUDE) { rad = this.radius / 2; if (weatherClassMin == -1 && weatherClassMax == -1) { // In this case, there is no proper photos by the given weather. // Let's find any photos bound for same weather class. weatherClassMin = ((int) this.weatherId / 100) * 100; weatherClassMax = (((int) this.weatherId / 100) + 1) * 100; System.out.println("weatherClassMin and weatherClassMax exist"); } else if (this.weatherId > 0) { this.weatherId = 0; System.out.println("weatherid goes to zero"); continue; } else { break; } } else { break; } } } catch (CommandFailureException e) { cursor = null; break; } rad = rad * 2; } try { while (cursor != null && cursor.hasNext()) { DBObject obj = cursor.next(); Photo.Builder b = new Photo.Builder((String) obj.get("name"), ((Number) obj.get("weather")).intValue()); ArrayList<Double> coord = ((ArrayList<Double>) ((DBObject) obj.get("geo")).get("coordinates")); b.geoCoord(coord.get(0), coord.get(1)).day(((Boolean) obj.get("day")).booleanValue()) .timestamp(((Number) obj.get("timestamp")).longValue()).origin(this.origin); photoList.add(b.build()); } } finally { if (cursor != null) { cursor.close(); } } return photoList; }
From source file:com.ikanow.infinit.e.api.custom.mapreduce.CustomHandler.java
License:Open Source License
/** * Returns all jobs that you have access to by checking for matching communities * /* w ww .j a v a 2 s .c o m*/ * @param userid * @return */ public ResponsePojo getJobOrJobs(String userid, String jobIdOrTitle, String commIds) { ResponsePojo rp = new ResponsePojo(); try { DBObject dbo = DbManager.getSocial().getPerson() .findOne(new BasicDBObject(CustomMapReduceJobPojo._id_, new ObjectId(userid))); if (dbo != null) { PersonPojo pp = PersonPojo.fromDb(dbo, PersonPojo.class); HashSet<ObjectId> communities = new HashSet<ObjectId>(); HashSet<ObjectId> savedCommunities = communities; // (change it if we do additional filtering) for (PersonCommunityPojo pcp : pp.getCommunities()) { communities.add(pcp.get_id()); } if (null != commIds) { // Filter out communities based on command line String[] commIdStrs = SocialUtils.getCommunityIds(pp.get_id().toString(), commIds); HashSet<ObjectId> filterCommunities = new HashSet<ObjectId>(); for (String commId : commIdStrs) { filterCommunities.add(new ObjectId(commId)); } savedCommunities = new HashSet<ObjectId>(communities); communities.retainAll(filterCommunities); } //TESTED (by hand) BasicDBObject commquery = new BasicDBObject(CustomMapReduceJobPojo.communityIds_, new BasicDBObject(MongoDbManager.in_, communities)); if (null != jobIdOrTitle) { try { if (jobIdOrTitle.contains(",")) { String[] jobidstrs = jobIdOrTitle.split("\\s*,\\s*"); ObjectId[] jobids = new ObjectId[jobidstrs.length]; for (int i = 0; i < jobidstrs.length; ++i) { jobids[i] = new ObjectId(jobidstrs[i]); } commquery.put(CustomMapReduceJobPojo._id_, new BasicDBObject(MongoDbManager.in_, jobids)); } else { ObjectId jobid = new ObjectId(jobIdOrTitle); commquery.put(CustomMapReduceJobPojo._id_, jobid); } } catch (Exception e) { // Must be a jobtitle if (jobIdOrTitle.contains(",")) { String[] jobtitles = jobIdOrTitle.split("\\s*,\\s*"); commquery.put(CustomMapReduceJobPojo.jobtitle_, new BasicDBObject(MongoDbManager.in_, jobtitles)); } else { commquery.put(CustomMapReduceJobPojo.jobtitle_, jobIdOrTitle); } } } DBCursor dbc = DbManager.getCustom().getLookup().find(commquery); if ((0 == dbc.count()) && (null != jobIdOrTitle)) { rp.setResponse(new ResponseObject("Custom Map Reduce Get Jobs", false, "No jobs to find")); } else { rp.setResponse( new ResponseObject("Custom Map Reduce Get Jobs", true, "succesfully returned jobs")); List<CustomMapReduceJobPojo> jobs = CustomMapReduceJobPojo.listFromDb(dbc, CustomMapReduceJobPojo.listType()); // Extra bit of code, need to eliminate partial community matches, eg if job belongs to A,B and we only belong to A // then obviously we can't see the job since it contains data from B Iterator<CustomMapReduceJobPojo> jobsIt = jobs.iterator(); while (jobsIt.hasNext()) { CustomMapReduceJobPojo job = jobsIt.next(); if (!savedCommunities.containsAll(job.communityIds)) { jobsIt.remove(); } } //TESTED rp.setData(jobs, new CustomMapReduceJobPojoApiMap(savedCommunities)); } } else { rp.setResponse(new ResponseObject("Custom Map Reduce Get Jobs", false, "error retrieving users communities")); } } catch (Exception e) { // If an exception occurs log the error logger.error("Exception Message: " + e.getMessage(), e); rp.setResponse(new ResponseObject("Custom Map Reduce Get Jobs", false, "error retrieving jobs")); } return rp; }
From source file:com.ikanow.infinit.e.api.knowledge.aliases.AliasManager.java
License:Open Source License
public synchronized AliasLookupTable getAliasLookupTable(String communityListStr, String[] communityStrArray, List<ObjectId> communityList, String userIdStr) { // 1. Normalize input args if ((null == communityListStr) || (communityListStr.startsWith("*"))) { StringBuffer sb = new StringBuffer(); if (null != communityStrArray) { for (String sid : communityStrArray) { sb.append(sid);//from w w w. j av a 2 s. c o m sb.append(","); } } else { // communityList != null for (ObjectId oid : communityList) { sb.append(oid.toString()); sb.append(","); } } if (sb.length() > 0) { sb.setLength(sb.length() - 1); } communityListStr = sb.toString(); } //TESTED if (null == communityList) { if (null == communityStrArray) { communityStrArray = communityListStr.split("\\."); } communityList = new ArrayList<ObjectId>(communityStrArray.length); for (String s : communityStrArray) { try { communityList.add(new ObjectId(s)); } catch (Exception e) { } // just carry on } } //TOTEST // 2. Get alias table AliasLookupTable table = _aliasLookups.get(communityListStr); Date now = new Date(); // 3. Check alias table refresh boolean bRefresh = true; if (null != table) { // If it exists, check if it needs to be refreshed (check share modified times if >1 minute say) Date tableLastMod = table.getLastModified(); if (null != tableLastMod) { if (now.getTime() - tableLastMod.getTime() <= 60000) { // less than a minute ago bRefresh = false; } } if (bRefresh) { // Check shares to see if we really need to refresh logger.debug("Alias table exists, checking for refresh: " + communityListStr); BasicDBObject query = new BasicDBObject(SharePojo.type_, "infinite-entity-alias"); query.put(ShareCommunityPojo.shareQuery_id_, new BasicDBObject(MongoDbManager.in_, communityList)); query.put(SharePojo.endorsed_, new BasicDBObject(MongoDbManager.in_, communityList)); BasicDBObject fields = new BasicDBObject(SharePojo.modified_, 1); DBCursor dbc = MongoDbManager.getSocial().getShare().find(query, fields); if (dbc.count() == table.getNumAliasShares()) { // easy answer is out! bRefresh = false; for (DBObject dbo : dbc) { Date date = (Date) dbo.get(SharePojo.modified_); if ((date != null) && (date.getTime() > tableLastMod.getTime())) { logger.debug("Alias: change in date for " + dbo.get(SharePojo._id_) + ": " + date); bRefresh = true; break; } } } //TOTEST else { logger.debug("Alias: change in #shares, now: " + dbc.count() + " vs prev: " + table.getNumAliasShares()); } } } //TESTED else { table = new AliasLookupTable(); _aliasLookups.put(communityListStr, table); } // 4. Perform alias table refresh if (bRefresh) { BasicDBObject query = new BasicDBObject(SharePojo.type_, "infinite-entity-alias"); query.put(ShareCommunityPojo.shareQuery_id_, new BasicDBObject(MongoDbManager.in_, communityList)); query.put(SharePojo.endorsed_, new BasicDBObject(MongoDbManager.in_, communityList)); List<SharePojo> aliasShares = SharePojo.listFromDb(MongoDbManager.getSocial().getShare().find(query), SharePojo.listType()); if (null != aliasShares) { logger.debug("Refresh/build alias table " + communityListStr + ": " + aliasShares.size()); } else { logger.debug("Clear alias table " + communityListStr); aliasShares = new ArrayList<SharePojo>(); } table.buildOrUpdateAliasTable(aliasShares, userIdStr); } //TESTED return table.isEmpty() ? null : table; }
From source file:com.ikanow.infinit.e.api.knowledge.QueryHandler.java
License:Open Source License
private ResponsePojo getSavedQueryInstead(String storedQueryNameOrId, String[] communityIdStrs, AdvancedQueryPojo query) {/*from w ww. j a v a 2 s . c om*/ ResponsePojo rp = null; ObjectId oid = null; BasicDBObject jobQuery = null; try { oid = new ObjectId(storedQueryNameOrId); jobQuery = new BasicDBObject(CustomMapReduceJobPojo._id_, oid); } catch (Exception e) { jobQuery = new BasicDBObject(CustomMapReduceJobPojo.jobtitle_, storedQueryNameOrId); } CustomMapReduceJobPojo savedJob = CustomMapReduceJobPojo .fromDb(DbManager.getCustom().getLookup().findOne(jobQuery), CustomMapReduceJobPojo.class); if (null != savedJob) { // Is this even a saved job? if (null != savedJob.jarURL) { savedJob = null; } } if (null != savedJob) { // Authorization boolean auth = false; String communityIdStrList = Arrays.toString(communityIdStrs); for (ObjectId commId : savedJob.communityIds) { if (communityIdStrList.contains(commId.toString())) { auth = true; break; } } if (!auth) { savedJob = null; } if (null == savedJob) { throw new RuntimeException( "Can't find saved query, or is a custom job not a query, or authorization error"); } // OK go get the results of the job DBCollection coll = DbManager.getCollection(savedJob.getOutputDatabase(), savedJob.outputCollection); BasicDBObject result = (BasicDBObject) coll.findOne(); // (at some point support multiple saved queries) if (null == result) { throw new RuntimeException("Saved query is empty"); } BasicDBObject apiResultToConvert = (BasicDBObject) result.get("value"); if (null == apiResultToConvert) { throw new RuntimeException("Saved query has invalid format"); } rp = ResponsePojo.fromDb(apiResultToConvert); } else if (null != oid) { // Support new user/doc queues SharePojo share = SharePojo.fromDb(DbManager.getSocial().getShare().findOne(jobQuery), SharePojo.class); if ((null == share) || (null == share.getShare()) || (!share.getType().equals(DocumentQueueControlPojo.UserQueue) && !share.getType().equals(DocumentQueueControlPojo.SavedQueryQueue))) { throw new RuntimeException( "Can't find saved query, or is a custom job not a query, or authorization error"); } else { // share.share is a DocumentQueueControlPojo DocumentQueueControlPojo queue = DocumentQueueControlPojo.fromApi(share.getShare(), DocumentQueueControlPojo.class); BasicDBObject docQuery1 = new BasicDBObject(DocumentPojo._id_, new BasicDBObject(DbManager.in_, queue.getQueueList())); BasicDBObject docQuery2 = new BasicDBObject(DocumentPojo.updateId_, new BasicDBObject(DbManager.in_, queue.getQueueList())); BasicDBObject docQuery = new BasicDBObject(DbManager.or_, Arrays.asList(docQuery1, docQuery2)); DBCursor dbc = DbManager.getDocument().getMetadata().find(docQuery).limit(query.score.numAnalyze); ScoringUtils scoreStats = new ScoringUtils(); List<BasicDBObject> docs = null; StatisticsPojo stats = new StatisticsPojo(); stats.setSavedScores(query.output.docs.skip, dbc.count()); try { boolean lockAcquired = true; try { lockAcquired = this.acquireConcurrentAccessLock(); } catch (InterruptedException e) { //(that's fine just carry on) lockAcquired = false; } if (!lockAcquired) { rp.setResponse( new ResponseObject("Query", false, "Query engine busy, please try again later.")); return rp; } scoreStats.setAliasLookupTable(_aliasLookup); docs = scoreStats.calcTFIDFAndFilter(DbManager.getDocument().getMetadata(), dbc, query.score, query.output, stats, false, query.output.docs.skip, query.output.docs.numReturn, communityIdStrs, null, null, null, null, null, null, null, null); } finally { scoreStats.clearAsMuchMemoryAsPossible(); this.releaseConcurrentAccessLock(); } rp = new ResponsePojo(); rp.setResponse(new ResponseObject("Query", true, "Saved Query: " + share.getTitle())); rp.setStats(stats); if ((null != docs) && (docs.size() > 0)) { rp.setData(docs, (BasePojoApiMap<BasicDBObject>) null); } else { // (ensure there's always an empty list) docs = new ArrayList<BasicDBObject>(0); rp.setData(docs, (BasePojoApiMap<BasicDBObject>) null); } } //end if user or saved query queue } return rp; }
From source file:com.ikanow.infinit.e.core.mapreduce.HadoopJobRunner.java
License:Open Source License
private void createConfigXML(Writer out, String title, String input, String fields, boolean isCustomTable, String outputDatabase, String output, String tempOutputCollection, String mapper, String reducer, String combiner, String query, List<ObjectId> communityIds, String outputKey, String outputValue, String arguments) throws IOException { String dbserver = prop_general.getDatabaseServer(); output = outputDatabase + "." + tempOutputCollection; int nSplits = 8; int nDocsPerSplit = 12500; //add communities to query if this is not a custom table if (!isCustomTable) { // Start with the old query: BasicDBObject oldQueryObj = null; if (query.startsWith("{")) { oldQueryObj = (BasicDBObject) com.mongodb.util.JSON.parse(query); } else {// ww w. j a v a 2 s .c o m oldQueryObj = new BasicDBObject(); } // Community Ids aren't indexed in the metadata collection, but source keys are, so we need to transform to that BasicDBObject keyQuery = new BasicDBObject(SourcePojo.communityIds_, new BasicDBObject(DbManager.in_, communityIds)); boolean bAdminOverride = false; if (oldQueryObj.containsField("admin")) { // For testing only... if (1 == communityIds.size()) { ObjectId communityId = communityIds.get(0); if (RESTTools.adminLookup(communityId.toString())) { bAdminOverride = true; if (oldQueryObj.containsField("max.splits")) { nSplits = oldQueryObj.getInt("max.splits"); } if (oldQueryObj.containsField("max.docs.per.split")) { nDocsPerSplit = oldQueryObj.getInt("max.docs.per.split"); } } } } //(end diagnostic/benchmarking/test code for admins only part 1) if (bAdminOverride) { oldQueryObj = (BasicDBObject) oldQueryObj.get("admin"); //(end diagnostic/benchmarking/test code for admins only part 2) } else if (oldQueryObj.containsField(DocumentPojo.sourceKey_) || input.startsWith("feature.")) { // Source Key specified by user, stick communityIds check in for security oldQueryObj.put(DocumentPojo.communityId_, new BasicDBObject(DbManager.in_, communityIds)); } else { // Source key not specified by user, transform communities->sourcekeys BasicDBObject keyFields = new BasicDBObject(SourcePojo.key_, 1); DBCursor dbc = MongoDbManager.getIngest().getSource().find(keyQuery, keyFields); if (dbc.count() > 500) { // (too many source keys let's keep the query size sensible...) oldQueryObj.put(DocumentPojo.communityId_, new BasicDBObject(DbManager.in_, communityIds)); } else { HashSet<String> sourceKeys = new HashSet<String>(); while (dbc.hasNext()) { DBObject dbo = dbc.next(); String sourceKey = (String) dbo.get(SourcePojo.key_); if (null != sourceKey) { sourceKeys.add(sourceKey); } } if (sourceKeys.isEmpty()) { // query returns empty throw new RuntimeException("Communities contain no sources"); } BasicDBObject newQueryClauseObj = new BasicDBObject(DbManager.in_, sourceKeys); // Now combine the queries... oldQueryObj.put(DocumentPojo.sourceKey_, newQueryClauseObj); } // (end if too many source keys across the communities) } //(end if need to break source keys down into communities) query = oldQueryObj.toString(); } else { //get the custom table (and database) input = getCustomDbAndCollection(input); } if (arguments == null) arguments = ""; // Generic configuration out.write("<?xml version=\"1.0\"?>\n<configuration>"); // Mongo specific configuration out.write("\n\t<property><!-- name of job shown in jobtracker --><name>mongo.job.name</name><value>" + title + "</value></property>" + "\n\t<property><!-- run the job verbosely ? --><name>mongo.job.verbose</name><value>true</value></property>" + "\n\t<property><!-- Run the job in the foreground and wait for response, or background it? --><name>mongo.job.background</name><value>false</value></property>" + "\n\t<property><!-- If you are reading from mongo, the URI --><name>mongo.input.uri</name><value>mongodb://" + dbserver + "/" + input + "</value></property>" + "\n\t<property><!-- If you are writing to mongo, the URI --><name>mongo.output.uri</name><value>mongodb://" + dbserver + "/" + output + "</value> </property>" + "\n\t<property><!-- The query, in JSON, to execute [OPTIONAL] --><name>mongo.input.query</name><value>" + query + "</value></property>" + "\n\t<property><!-- The fields, in JSON, to read [OPTIONAL] --><name>mongo.input.fields</name><value>" + ((fields == null) ? ("") : fields) + "</value></property>" + "\n\t<property><!-- A JSON sort specification for read [OPTIONAL] --><name>mongo.input.sort</name><value></value></property>" + "\n\t<property><!-- The number of documents to limit to for read [OPTIONAL] --><name>mongo.input.limit</name><value>0</value><!-- 0 == no limit --></property>" + "\n\t<property><!-- The number of documents to skip in read [OPTIONAL] --><!-- TODO - Are we running limit() or skip() first? --><name>mongo.input.skip</name><value>0</value> <!-- 0 == no skip --></property>" + "\n\t<property><!-- Class for the mapper --><name>mongo.job.mapper</name><value>" + mapper + "</value></property>" + "\n\t<property><!-- Reducer class --><name>mongo.job.reducer</name><value>" + reducer + "</value></property>" + "\n\t<property><!-- InputFormat Class --><name>mongo.job.input.format</name><value>com.ikanow.infinit.e.data_model.custom.InfiniteMongoInputFormat</value></property>" + "\n\t<property><!-- OutputFormat Class --><name>mongo.job.output.format</name><value>com.mongodb.hadoop.MongoOutputFormat</value></property>" + "\n\t<property><!-- Output key class for the output format --><name>mongo.job.output.key</name><value>" + outputKey + "</value></property>" + "\n\t<property><!-- Output value class for the output format --><name>mongo.job.output.value</name><value>" + outputValue + "</value></property>" + "\n\t<property><!-- Output key class for the mapper [optional] --><name>mongo.job.mapper.output.key</name><value></value></property>" + "\n\t<property><!-- Output value class for the mapper [optional] --><name>mongo.job.mapper.output.value</name><value></value></property>" + "\n\t<property><!-- Class for the combiner [optional] --><name>mongo.job.combiner</name><value>" + combiner + "</value></property>" + "\n\t<property><!-- Partitioner class [optional] --><name>mongo.job.partitioner</name><value></value></property>" + "\n\t<property><!-- Sort Comparator class [optional] --><name>mongo.job.sort_comparator</name><value></value></property>" + "\n\t<property><!-- Split Size [optional] --><name>mongo.input.split_size</name><value>32</value></property>"); // Infinit.e specific configuration out.write("\n\t<property><!-- User Arguments [optional] --><name>arguments</name><value>" + StringEscapeUtils.escapeXml(arguments) + "</value></property>" + "\n\t<property><!-- Maximum number of splits [optional] --><name>max.splits</name><value>" + nSplits + "</value></property>" + "\n\t<property><!-- Maximum number of docs per split [optional] --><name>max.docs.per.split</name><value>" + nDocsPerSplit + "</value></property>"); // Closing thoughts: out.write("\n</configuration>"); out.flush(); out.close(); }
From source file:com.ikanow.infinit.e.core.utils.SourceUtils.java
License:Open Source License
private static int pruneSource(SourcePojo source, int nToPrune, int ttl_days) { int nTotalDocsDeleted = 0; int nDocsDeleted = 0; // (code taken mostly from SourceHandler.deleteSource) if (null != source.getKey()) { // or may delete everything! BasicDBObject docQuery = new BasicDBObject(DocumentPojo.sourceKey_, source.getDistributedKeyQueryTerm()); if (ttl_days > 0) { Date ageOut = new Date(new Date().getTime() - ttl_days * 24L * 3600L * 1000L); ObjectId oldestAllowedId = new ObjectId(ageOut); docQuery.put(DocumentPojo._id_, new BasicDBObject(DbManager.lt_, oldestAllowedId)); } //TODO: TOTEST docQuery.put(DocumentPojo.index_, new BasicDBObject(DbManager.ne_, "?DEL?")); // (robustness) BasicDBObject sortField = new BasicDBObject(DocumentPojo._id_, 1); BasicDBObject docFields = new BasicDBObject(); docFields.append(DocumentPojo.url_, 1); docFields.append(DocumentPojo.sourceUrl_, 1); docFields.append(DocumentPojo.index_, 1); docFields.append(DocumentPojo.sourceKey_, 1); StoreAndIndexManager dataStore = new StoreAndIndexManager(); ObjectId nextId = null;/*from www . j av a 2 s .c o m*/ while (nToPrune > 0) { int nToDelete = nToPrune; if (nToDelete > 10000) { nToDelete = 10000; } if (null != nextId) { docQuery.put(DocumentPojo._id_, new BasicDBObject(DbManager.gt_, nextId)); } //TESTED (by hand) DBCursor dbc = DbManager.getDocument().getMetadata().find(docQuery, docFields).sort(sortField) .limit(nToDelete); // (ie batches of 10K, ascending ordered by _id) nToPrune -= nToDelete; if (0 == nDocsDeleted) { nDocsDeleted = dbc.count(); nTotalDocsDeleted += nDocsDeleted; } if (0 == dbc.size()) { break; } List<DocumentPojo> docs = DocumentPojo.listFromDb(dbc, DocumentPojo.listType()); nextId = dataStore.removeFromDatastore_byURL(docs, source); } } // No need to do anything related to soft deletion, this is all handled when the harvest ends return nTotalDocsDeleted; }