List of usage examples for com.mongodb BasicDBObject toString
@SuppressWarnings("deprecation") public String toString()
Returns a JSON serialization of this object
The output will look like: {"a":1, "b":["x","y","z"]} }
From source file:com.emuneee.camerasyncmanager.util.DatabaseUtil.java
License:Apache License
/** * Inserts a list of cameras/*from ww w . j a va 2s. com*/ * @param cameras * @return */ public boolean insertCameras(List<Camera> cameras) { sLogger.info("Inserting " + cameras.size() + " cameras"); boolean result = false; DB db = getDatabase(); try { DBCollection collection = db.getCollection("camera"); for (Camera camera : cameras) { BasicDBObject dbObj = cameraToDBObject(camera); sLogger.debug("Inserting: " + dbObj.toString()); collection.insert(dbObj); result = true; } } catch (Exception e) { sLogger.error("Exception inserting cameras"); sLogger.error(e); } return result; }
From source file:com.hangum.tadpole.mongodb.core.editors.main.MongoDBTableEditor.java
License:Open Source License
/** * ? ? .//w w w .ja va 2 s .c o m * * @param basicFields * @param basicWhere * @param basicSort */ private void find(BasicDBObject basicFields, DBObject basicWhere, BasicDBObject basicSort, int cntSkip, int cntLimit) throws Exception { if ((cntLimit - cntSkip) >= defaultMaxCount) { // " " + defaultMaxCount + " ? . Prefernece? ? ." // Search can not exceed the number 5. Set in Perference. throw new Exception(String.format(Messages.MongoDBTableEditor_0, "" + defaultMaxCount)); //$NON-NLS-2$ } mapColumns = new HashMap<Integer, String>(); sourceDataList = new ArrayList<HashMap<Integer, Object>>(); DB mongoDB = MongoConnectionManager.getInstance(userDB); DBCollection dbCollection = mongoDB.getCollection(tableName); // ?? DBCursor dbCursor = null; try { if (cntSkip > 0 && cntLimit > 0) { sbConsoleMsg.append("############[query]#####################").append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Fields]" + basicFields.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Where]" + basicWhere.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Sort] " + basicSort.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Skip]" + cntSkip).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Limit]" + cntLimit).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("############[query]#####################"); //$NON-NLS-1$ dbCursor = dbCollection.find(basicWhere, basicFields).sort(basicSort).skip(cntSkip).limit(cntLimit); } else if (cntSkip == 0 && cntLimit > 0) { sbConsoleMsg.append("############[query]#####################").append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Fields]" + basicFields.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Where]" + basicWhere.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Sort] " + basicSort.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Limit]" + cntLimit).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("############[query]#####################").append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ dbCursor = dbCollection.find(basicWhere, basicFields).sort(basicSort).limit(cntLimit); } else { sbConsoleMsg.append("############[query]#####################").append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Fields]" + basicFields.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Where]" + basicWhere.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[Sort] " + basicSort.toString()).append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("############[query]#####################").append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ dbCursor = dbCollection.find(basicWhere, basicFields).sort(basicSort); } DBObject explainDBObject = dbCursor.explain(); sbConsoleMsg.append("############[result]#####################").append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ sbConsoleMsg.append("[query explain]\r\n" + JSONUtil.getPretty(explainDBObject.toString())) //$NON-NLS-1$ .append("\r\n"); //$NON-NLS-1$ sbConsoleMsg.append("[error]\r\n" + JSONUtil.getPretty(mongoDB.getLastError().toString())) //$NON-NLS-1$ .append("\r\n"); //$NON-NLS-1$ sbConsoleMsg.append("############[result]#####################").append("\r\n"); //$NON-NLS-1$ //$NON-NLS-2$ mongoDB.forceError(); mongoDB.resetError(); if (logger.isDebugEnabled()) logger.debug(sbConsoleMsg); // ?? . int totCnt = 0; listTrees = new ArrayList<MongodbTreeViewDTO>(); for (DBObject dbObject : dbCursor) { // ? ? if (mapColumns.size() == 0) mapColumns = MongoDBTableColumn.getTabelColumnView(dbObject); // append tree text columnInfo.get(key) MongodbTreeViewDTO treeDto = new MongodbTreeViewDTO(dbObject, "(" + totCnt + ") {..}", "", //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$ "Document"); //$NON-NLS-1$ parserTreeObject(dbObject, treeDto, dbObject); listTrees.add(treeDto); // append table text HashMap<Integer, Object> dataMap = new HashMap<Integer, Object>(); for (int i = 0; i < mapColumns.size(); i++) { Object keyVal = dbObject.get(mapColumns.get(i)); if (keyVal == null) dataMap.put(i, ""); //$NON-NLS-1$ else dataMap.put(i, keyVal.toString()); } // ?? ? ? id dataMap.put(MongoDBDefine.PRIMARY_ID_KEY, dbObject); sourceDataList.add(dataMap); // append row text totCnt++; } txtCnt = dbCursor.count() + "/" + totCnt + Messages.MongoDBTableEditor_69; //$NON-NLS-1$ } finally { if (dbCursor != null) dbCursor.close(); } }
From source file:com.hangum.tadpole.mongodb.core.test.MongoTestAndORComplexStmt.java
License:Open Source License
/** * @param args/*from w ww . j a v a 2s .c o m*/ */ public static void main(String[] args) throws Exception { ConAndAuthentication testMongoCls = new ConAndAuthentication(); Mongo mongo = testMongoCls.connection(ConAndAuthentication.serverurl, ConAndAuthentication.port); DB db = mongo.getDB("test"); DBCollection myColl = db.getCollection("rental"); BasicDBObject mainQuery = new BasicDBObject(); // tmp and BasicDBObject tmpAndQuery = new BasicDBObject(); tmpAndQuery.append("inventory_id", 100); tmpAndQuery.append("rental_id", new BasicDBObject("$ne", 1)); mainQuery.put("$and", tmpAndQuery); // tmp or ArrayList<BasicDBObject> myList = new ArrayList<BasicDBObject>(); myList.add(new BasicDBObject("customer_id", 3)); mainQuery.put("$or", myList); System.out.println(mainQuery.toString()); DBCursor myCursor = myColl.find(mainQuery); System.out.println("[result cursor size is " + myCursor.count()); while (myCursor.hasNext()) { System.out.println(myCursor.next()); } mongo.close(); }
From source file:com.ifactory.service.weather.photo.PhotoService.java
License:Apache License
public ArrayList<Photo> get() { DB db = mongoClient.getDB(this.dbName); DBCollection coll = db.getCollection(PHOTO_COLLECTION); BasicDBObject query = null; DBCursor cursor = null;//from www .j a v a 2 s . co m ArrayList<Photo> photoList = new ArrayList(); int weatherClassMin = -1; int weatherClassMax = -1; double rad = this.radius; while (true) { // If latitude and longitude were given, append geo search query if (this.lat != UNAVAILABLE_LATITUDE && this.lng != UNAVAILABLE_LONGITUDE) { query = setGeoCoord(this.lat, this.lng, rad); } else { query = new BasicDBObject(); } // It the weather Id has given, append weather search query if (this.weatherId > 0) { if (weatherClassMin == -1 && weatherClassMax == -1) { query.append("weather", this.weatherId); } else { System.out.println("query with weatherClassMin(" + weatherClassMin + ") and weatherClassMax(" + weatherClassMax + ")"); query.append("weather", new BasicDBObject("$gte", weatherClassMin).append("$lte", weatherClassMax)); System.out.println(query.toString()); } } try { cursor = coll.find(query).limit(this.limit); if (cursor.count() > 0) { System.out.println( "photo found(lat: " + this.lat + ", lng: " + this.lng + ", radius: " + rad + ")"); break; } else if (this.growable == false || rad >= UNAVAILABLE_LATITUDE) { if (rad >= UNAVAILABLE_LATITUDE) { rad = this.radius / 2; if (weatherClassMin == -1 && weatherClassMax == -1) { // In this case, there is no proper photos by the given weather. // Let's find any photos bound for same weather class. weatherClassMin = ((int) this.weatherId / 100) * 100; weatherClassMax = (((int) this.weatherId / 100) + 1) * 100; System.out.println("weatherClassMin and weatherClassMax exist"); } else if (this.weatherId > 0) { this.weatherId = 0; System.out.println("weatherid goes to zero"); continue; } else { break; } } else { break; } } } catch (CommandFailureException e) { cursor = null; break; } rad = rad * 2; } try { while (cursor != null && cursor.hasNext()) { DBObject obj = cursor.next(); Photo.Builder b = new Photo.Builder((String) obj.get("name"), ((Number) obj.get("weather")).intValue()); ArrayList<Double> coord = ((ArrayList<Double>) ((DBObject) obj.get("geo")).get("coordinates")); b.geoCoord(coord.get(0), coord.get(1)).day(((Boolean) obj.get("day")).booleanValue()) .timestamp(((Number) obj.get("timestamp")).longValue()).origin(this.origin); photoList.add(b.build()); } } finally { if (cursor != null) { cursor.close(); } } return photoList; }
From source file:com.ikanow.aleph2.shared.crud.mongodb.services.MongoDbCrudService.java
License:Apache License
@Override public final boolean deregisterOptimizedQuery(final List<String> ordered_field_list) { try {//from w w w . j a va 2 s . co m final BasicDBObject index_keys = new BasicDBObject(ordered_field_list.stream() .collect(Collectors.toMap(f -> f, f -> 1, (v1, v2) -> 1, LinkedHashMap::new))); if (_state.orig_coll instanceof FongoDBCollection) { // (doesn't do the exception, so check by hand) final String index_keys_str = index_keys.toString(); final List<DBObject> matching_indexes = _state.orig_coll.getIndexInfo().stream() .filter(dbo -> index_keys_str.equals(dbo.get("key").toString())) .collect(Collectors.toList()); if (matching_indexes.isEmpty()) { throw new MongoException(ErrorUtils.get(ErrorUtils.MISSING_MONGODB_INDEX_KEY, index_keys_str)); } } _state.orig_coll.dropIndex(index_keys); return true; } catch (MongoException ex) { return false; } }
From source file:com.ikanow.aleph2.v1.document_db.utils.LegacyV1HadoopUtils.java
License:Open Source License
/** parse the V1 query string * @param query// w w w . java 2s . c o m * @return the required objects embedded in various tuples */ public static Tuple4<String, Tuple2<Integer, Integer>, BasicDBObject, DBObject> parseQueryObject( final String query, final List<String> community_ids) { // Some fixed variables just to avoid changing the guts of the (tested in v1) code final boolean isCustomTable = false; @SuppressWarnings("unused") Integer nDebugLimit = null; final boolean bLocalMode = false; @SuppressWarnings("unused") final Boolean incrementalMode = null; final String input = "doc_metadata.metadata"; // Output objects final String out_query; int nSplits = 8; int nDocsPerSplit = 12500; List<ObjectId> communityIds = community_ids.stream().map(s -> new ObjectId(s)).collect(Collectors.toList()); //C/P code: //add communities to query if this is not a custom table BasicDBObject oldQueryObj = null; BasicDBObject srcTags = null; // Start with the old query: if (query.startsWith("{")) { oldQueryObj = (BasicDBObject) com.mongodb.util.JSON.parse(query); } else { oldQueryObj = new BasicDBObject(); } boolean elasticsearchQuery = oldQueryObj.containsField("qt") && !isCustomTable; @SuppressWarnings("unused") int nLimit = 0; if (oldQueryObj.containsField(":limit")) { nLimit = oldQueryObj.getInt(":limit"); oldQueryObj.remove(":limit"); } if (oldQueryObj.containsField(":splits")) { nSplits = oldQueryObj.getInt(":splits"); oldQueryObj.remove(":splits"); } if (oldQueryObj.containsField(":srctags")) { srcTags = new BasicDBObject(SourcePojo.tags_, oldQueryObj.get(":srctags")); oldQueryObj.remove(":srctags"); } if (bLocalMode) { // If in local mode, then set this to a large number so we always run inside our limit/split version // (since for some reason MongoInputFormat seems to fail on large collections) nSplits = InfiniteMongoSplitter.MAX_SPLITS; } if (oldQueryObj.containsField(":docsPerSplit")) { nDocsPerSplit = oldQueryObj.getInt(":docsPerSplit"); oldQueryObj.remove(":docsPerSplit"); } final DBObject fields = (DBObject) oldQueryObj.remove(":fields"); oldQueryObj.remove(":output"); oldQueryObj.remove(":reducers"); @SuppressWarnings("unused") String mapperKeyClass = oldQueryObj.getString(":mapper_key_class", ""); @SuppressWarnings("unused") String mapperValueClass = oldQueryObj.getString(":mapper_value_class", ""); oldQueryObj.remove(":mapper_key_class"); oldQueryObj.remove(":mapper_value_class"); String cacheList = null; Object cacheObj = oldQueryObj.get(":caches"); if (null != cacheObj) { cacheList = cacheObj.toString(); // (either array of strings, or single string) if (!cacheList.startsWith("[")) { cacheList = "[" + cacheList + "]"; // ("must" now be valid array) } oldQueryObj.remove(":caches"); } //TESTED // if (null != nDebugLimit) { // (debug mode override) // nLimit = nDebugLimit; // } // boolean tmpIncMode = ( null != incrementalMode) && incrementalMode; @SuppressWarnings("unused") String otherCollections = null; Date fromOverride = null; Date toOverride = null; Object fromOverrideObj = oldQueryObj.remove(":tmin"); Object toOverrideObj = oldQueryObj.remove(":tmax"); if (null != fromOverrideObj) { fromOverride = dateStringFromObject(fromOverrideObj, true); } if (null != toOverrideObj) { toOverride = dateStringFromObject(toOverrideObj, false); } if (!isCustomTable) { if (elasticsearchQuery) { oldQueryObj.put("communityIds", communityIds); //tmin/tmax not supported - already have that capability as part of the query } else { if (input.equals("feature.temporal")) { if ((null != fromOverride) || (null != toOverride)) { oldQueryObj.put("value.maxTime", createDateRange(fromOverride, toOverride, true)); } //TESTED oldQueryObj.put("_id.c", new BasicDBObject(DbManager.in_, communityIds)); } else { oldQueryObj.put(DocumentPojo.communityId_, new BasicDBObject(DbManager.in_, communityIds)); if ((null != fromOverride) || (null != toOverride)) { oldQueryObj.put(JsonUtils._ID, createDateRange(fromOverride, toOverride, false)); } //TESTED if (input.equals("doc_metadata.metadata")) { oldQueryObj.put(DocumentPojo.index_, new BasicDBObject(DbManager.ne_, "?DEL?")); // (ensures not soft-deleted) } } } } else { throw new RuntimeException("Custom Tables not currently supported (no plans to)"); // if ((null != fromOverride) || (null != toOverride)) { // oldQueryObj.put(JsonUtils._ID, createDateRange(fromOverride, toOverride, false)); // }//TESTED // //get the custom table (and database) // // String[] candidateInputs = input.split("\\s*,\\s*"); // input = CustomOutputManager.getCustomDbAndCollection(candidateInputs[0]); // if (candidateInputs.length > 1) { // otherCollections = Arrays.stream(candidateInputs) // .skip(1L) // .map(i -> CustomOutputManager.getCustomDbAndCollection(i)) // .map(i -> "mongodb://"+dbserver+"/"+i).collect(Collectors.joining("|")); // } } out_query = oldQueryObj.toString(); return Tuples._4T(out_query, Tuples._2T(nSplits, nDocsPerSplit), srcTags, fields); }
From source file:com.ikanow.infinit.e.api.knowledge.federated.SimpleFederatedQueryEngine.java
License:Open Source License
public BasicDBObject createDocFromJson(BasicDBList jsonList, String url, FederatedRequest request, SourceFederatedQueryConfigPojo endpointInfo) { BasicDBObject doc = null; // (don't create unless needed) BasicDBList ents = null;//from w w w. ja v a2 s . c o m StringBuffer entVals = null; HashSet<String> entDedup = null; if (_testMode) { // In test mode, need to return the JSON even if no entities are specified doc = new BasicDBObject(); } if (null != endpointInfo.docConversionMap) { for (Map.Entry<String, String> docInfo : endpointInfo.docConversionMap.entrySet()) { for (Object jsonObj : jsonList) { BasicDBObject json = (BasicDBObject) jsonObj; try { String key = docInfo.getKey(); // (allow user to not prepend array: if they don't want to) if ((1 == json.size()) && json.containsKey((Object) "array")) { if (!key.startsWith("array:") && !key.startsWith(":array") && !key.startsWith("$:array") && !key.startsWith("::") && !key.startsWith("$::")) { if (key.startsWith(":")) { // jpath key = ":array" + key; } else if (key.startsWith("$:")) { // jpath key = "$:array" + key.substring(1); } else { key = "array:" + key; } } } //TESTED (by hand) if (key.startsWith(":")) { // jpath key = "$" + key; } // NOTE: *not* org.json.JSONArray JSONArray candidateEntities = null; if (key.startsWith("$")) { JSONArray candidateEntities_tmp = JsonPath.read(json.toString(), key.replace(':', '.')); if (null != candidateEntities_tmp) { candidateEntities = new JSONArray(); for (Object o : candidateEntities_tmp) { if (o instanceof String) { candidateEntities.add(o); } else if (o instanceof JSONArray) { candidateEntities.addAll((JSONArray) o); } } //TESTED (displayUrl vs entities, 3.2) } //DEBUG //System.out.println(candidateEntities); } //(TESTED (permutations above by hand)) else { String s = (String) MongoDbUtil.getProperty(json, key.replace(':', '.')); if (null != s) { candidateEntities = new JSONArray(); candidateEntities.add(s); } } //TESTED (3.1) if (null != candidateEntities) for (int i = 0; i < candidateEntities.size(); ++i) { Object o = candidateEntities.get(i); if (!(o instanceof String)) { continue; } String s = o.toString(); if (null == doc) { doc = new BasicDBObject(); //(various fields added below) } if (docInfo.getValue().equalsIgnoreCase(DocumentPojo.displayUrl_)) { doc.put(DocumentPojo.displayUrl_, s); } //TESTED (3.1, 4.*) else { // Entities! if (null == ents) { ents = new BasicDBList(); } String index = s.toLowerCase() + "/" + docInfo.getValue().toLowerCase(); if (null == entDedup) { entDedup = new HashSet<String>(); } else if (entDedup.contains(index)) { // Entity deduplication continue; } //TESTED (3.2) entDedup.add(index); if (null == entVals) { entVals = new StringBuffer(": "); } else { entVals.append(", "); } entVals.append(s); String dimension = null; if (null != endpointInfo.typeToDimensionMap) { try { dimension = EntityPojo.Dimension .valueOf( endpointInfo.typeToDimensionMap.get(docInfo.getValue())) .toString(); } catch (Exception e) { } } if (null == dimension) { dimension = EntityPojo.Dimension.What.toString(); } //TESTED (by hand) // (alternative to "made up" values would be to go looking in the existing docs/ents?) // (we'll try to avoid that for now...) BasicDBObject ent = new BasicDBObject(); ent.put(EntityPojo.disambiguated_name_, s); ent.put(EntityPojo.type_, docInfo.getValue()); ent.put(EntityPojo.dimension_, dimension); ent.put(EntityPojo.relevance_, 1.0); ent.put(EntityPojo.doccount_, 1L); // (ie relative to this query) ent.put(EntityPojo.averageFreq_, 1.0); ent.put(EntityPojo.datasetSignificance_, 10.0); // (ie relative to this query) ent.put(EntityPojo.significance_, 10.0); // (ie relative to this query) ent.put(EntityPojo.frequency_, 1.0); ent.put(EntityPojo.index_, index); ent.put(EntityPojo.queryCoverage_, 100.0); // (ie relative to this query) ent.put(EntityPojo.totalfrequency_, 1.0); // (ie relative to this query) ents.add(ent); } //TESTED (3.1, 4.*) } } catch (Exception e) { //(do nothing? null or the wrong type) //e.printStackTrace(); } } //end loop over various JSON objects retrieved } //(End loop over doc conversion elements) } //TESTED (3.*, 4.*) if ((null == ents) && !_testMode) { // don't return unless there are any entities return null; } else if (null != doc) { // Insert mandatory fields: // (Note the query format is a little bit different, the following fields are converted to arrays: // sourceKey, source, communityId, mediaType) doc.put(DocumentPojo._id_, new ObjectId()); doc.put(DocumentPojo.url_, url); doc.put(DocumentPojo.created_, new Date()); doc.put(DocumentPojo.modified_, new Date()); doc.put(DocumentPojo.publishedDate_, new Date()); doc.put(DocumentPojo.sourceKey_, endpointInfo.parentSource.getKey()); doc.put(DocumentPojo.source_, endpointInfo.parentSource.getTitle()); doc.put(DocumentPojo.communityId_, new ObjectId(request.communityIdStrs[0])); doc.put(DocumentPojo.mediaType_, endpointInfo.parentSource.getMediaType()); doc.put(DocumentPojo.metadata_, new BasicDBObject("json", jsonList.toArray())); if ((null != entVals) && (entVals.length() > 165)) { // (arbitrary length) entVals.setLength(165); entVals.append("..."); } doc.put(DocumentPojo.title_, new StringBuffer(endpointInfo.titlePrefix).append(": ") .append(request.requestParameter).append(entVals).toString()); doc.put(DocumentPojo.entities_, ents); Gson gson = new GsonBuilder().setPrettyPrinting().create(); JsonParser jp = new JsonParser(); JsonElement je = jp.parse(jsonList.toString()); doc.put(DocumentPojo.description_, gson.toJson(je)); // (prettified JSON) } //TESTED (3.*, 4.*) return doc; }
From source file:com.ikanow.infinit.e.api.knowledge.federated.SimpleFederatedQueryEngine.java
License:Open Source License
public static void simpleDocCache(FederatedRequest request, BasicDBObject doc) { if (null != request.cachedDoc_expired) { ObjectId updateId = request.cachedDoc_expired.getObjectId(DocumentPojo.updateId_); if (null != updateId) { doc.put(DocumentPojo.updateId_, updateId); } else {// w ww .ja v a2s.c o m doc.put(DocumentPojo.updateId_, request.cachedDoc_expired.getObjectId(DocumentPojo._id_)); } BasicDBObject docUpdate = new BasicDBObject(DocumentPojo.url_, doc.getString(DocumentPojo.url_)); docUpdate.put(DocumentPojo.sourceKey_, doc.getString(DocumentPojo.sourceKey_)); DbManager.getDocument().getMetadata().remove(docUpdate); //DEBUG if (_DEBUG) _logger.debug("DEB: postQA4a: re-cached ... " + docUpdate.toString() + ": " + doc.getObjectId(DocumentPojo.updateId_)); } else if (null == request.cachedDoc) { // if no currently cached doc, simply save what we have //DEBUG if (_DEBUG) _logger.debug("DEB: postQA4b: cached ... " + doc); DbManager.getDocument().getMetadata().save(doc); } // (else already have a valid cached doc so nothing to do) }
From source file:com.ikanow.infinit.e.api.knowledge.federated.SimpleFederatedQueryEngine.java
License:Open Source License
public static void handleComplexDocCaching(FederatedRequest request, boolean cacheMode, ScoringUtils scoreStats) {//from w ww. java2 s. com List<DocumentPojo> docWrapper = request.complexSourceProcResults; //In non-test mode .. Go through the list of docs and work out what the deal is with caching, ie remove docs + add update ids //Also go through and set default scores for any entities that haven't been scored based on existing docs BasicDBObject query = new BasicDBObject(DocumentPojo.sourceKey_, request.endpointInfo.parentSource.getKey()); BasicDBObject fields = new BasicDBObject(DocumentPojo.updateId_, 1); // (ie _id and updateId only) String srcUrl = null; for (DocumentPojo outDoc : docWrapper) { if (null == srcUrl) { srcUrl = outDoc.getSourceUrl(); } // Always make the text non-transient, so gets stored outDoc.makeFullTextNonTransient(); if (null == outDoc.getId()) { outDoc.setId(new ObjectId()); } if (cacheMode && (null == request.cachedResult)) { // (if result not previously cached) // Step 1: deduplication query.put(DocumentPojo.url_, outDoc.getUrl()); BasicDBObject outVal = (BasicDBObject) DbManager.getDocument().getMetadata().findOne(query, fields); if (null != outVal) { //DEBUG if (_DEBUG) _logger.debug("DEB: HCDC1: update cache from : " + outVal + " for " + outDoc.getUrl()); // Use updateId if it exists, otherwise _id ObjectId updateId = outVal.getObjectId(DocumentPojo.updateId_); if (null == updateId) { updateId = outVal.getObjectId(DocumentPojo._id_); } outDoc.setUpdateId(updateId); } } //TESTED (by hand - single and multiple docs mode) // Step 2: add fake scores to all the entities that didn't get scores from the aggregation manager if (null != outDoc.getEntities()) for (EntityPojo ent : outDoc.getEntities()) { boolean fakeStats = true; if (null != scoreStats) { if (scoreStats.fillInEntityStatistics(ent)) { fakeStats = false; } } if (fakeStats) { ent.setDoccount(1L); ent.setTotalfrequency(1L); ent.setDatasetSignificance(10.0); ent.setSignificance(10.0); ent.setQueryCoverage(100.0); } //DEBUG if (_DEBUG) _logger.debug( "DEB: HCDC2: entity: " + ent.getIndex() + " , sig=" + ent.getDatasetSignificance()); } //TESTED if (null != outDoc.getAssociations()) for (AssociationPojo assoc : outDoc.getAssociations()) { assoc.setAssoc_sig(10.0); assoc.setDoccount(1L); } } //TESTED (by hand - overlapping and non-overlapping case) if (cacheMode && (null == request.cachedResult)) { // (if result not previously cached) //Remove old docs now we have new ones DbManager.getDocument().getMetadata().remove(query); // remove everything with this specific URL (ie simple source) query.remove(DocumentPojo.url_); query.put(DocumentPojo.sourceUrl_, srcUrl); DbManager.getDocument().getMetadata().remove(query); // remove everything with this specific _source_ URL (ie docs generated from this URL) // Now cache all the existing docs: @SuppressWarnings("unchecked") ArrayList<Object> tmpDocList = (ArrayList<Object>) DocumentPojo.listToDb(docWrapper, DocumentPojo.listType()); DbManager.getDocument().getMetadata().insert(tmpDocList.toArray(new BasicDBObject[0])); //DEBUG if (_DEBUG) _logger.debug("DEB: HCDC3: remove/insert cache: " + query.toString()); } //TESTED (by hand - single and multiple docs) }
From source file:com.ikanow.infinit.e.api.knowledge.federated.SimpleFederatedQueryEngine.java
License:Open Source License
public void test_CheckIfDocAdded(String testName, List<BasicDBObject> docs) { // 1) Did a doc get added? if (2 != docs.size()) { System.out.println("*** " + testName + ": didn't add doc? " + docs.size()); System.exit(-1);/*from www . j a va 2s . c om*/ } // 2) Was it the right doc? BasicDBObject doc = docs.get(0); doc.remove(DocumentPojo._id_); doc.remove(DocumentPojo.created_); doc.remove(DocumentPojo.modified_); doc.remove(DocumentPojo.publishedDate_); String docToCheck = "{ \"displayUrl\" : \"http://test3_1\" , \"url\" : \"inf://federated/fakeendpoint.123/test3_1/testentityin\" , \"sourceKey\" : [ \"fakeendpoint.123\"] , \"source\" : [ \"fakeendpoint\"] , \"communityId\" : [ \"4c927585d591d31d7b37097a\"] , \"mediaType\" : [ \"Report\"] , \"metadata\" : { \"json\" : [ { \"test\" : { \"field\" : [ \"test3_1\" , \"test3_1\"] , \"field2\" : \"http://test3_1\"}}]} , \"title\" : \"fake endpoint: : test3_1: test3_1\" , \"entities\" : [ { \"disambiguated_name\" : \"test3_1\" , \"type\" : \"TestEntityOut\" , \"dimension\" : \"What\" , \"relevance\" : 1.0 , \"doccount\" : 1 , \"averageFreq\" : 1.0 , \"datasetSignificance\" : 10.0 , \"significance\" : 10.0 , \"frequency\" : 1.0 , \"index\" : \"test3_1/testentityout\" , \"queryCoverage\" : 100.0 , \"totalfrequency\" : 1.0}] , \"description\" : \"[\\n {\\n \\\"test\\\": {\\n \\\"field\\\": [\\n \\\"test3_1\\\",\\n \\\"test3_1\\\"\\n ],\\n \\\"field2\\\": \\\"http://test3_1\\\"\\n }\\n }\\n]\" , \"aggregateSignif\" : 115.0 , \"queryRelevance\" : 105.0 , \"score\" : 110.0}"; if (!docToCheck.equals(doc.toString())) { System.out.println( "*** " + testName + ": document incorrect:\n" + docToCheck + "\nVS\n" + doc.toString()); System.exit(-1); } // 3) Did the doc get cached? DBCollection endpointCacheCollection = getCacheCollection(); String hardwiredCacheId = "http://localhost:8186/test/test3_1/?param1=test3_1"; BasicDBObject cachedVal = (BasicDBObject) endpointCacheCollection .findOne(new BasicDBObject(SimpleFederatedCache._id_, hardwiredCacheId)); if (null == cachedVal) { System.out.println( "*** " + testName + ": no cache for: " + doc.get(DocumentPojo.url_) + " / " + hardwiredCacheId); System.exit(-1); } else { Date expiryDate = cachedVal.getDate(SimpleFederatedCache.expiryDate_, null); if ((null == expiryDate) || ((new Date().getTime() - expiryDate.getTime()) > 10 * 1000L)) { System.out.println("*** " + testName + ": expiry date for: " + doc.get(DocumentPojo.url_) + ": " + expiryDate + " vs now= " + new Date()); System.exit(-1); } BasicDBObject cachedJson = (BasicDBObject) cachedVal.get(SimpleFederatedCache.cachedJson_); Object docVal = null; try { Object[] docVals = (Object[]) ((BasicDBObject) doc.get(DocumentPojo.metadata_)).get("json"); docVal = docVals[0]; } catch (Exception e) { @SuppressWarnings("rawtypes") Collection docVals = (Collection) ((BasicDBObject) doc.get(DocumentPojo.metadata_)).get("json"); docVal = docVals.iterator().next(); } if ((null == cachedJson) || !cachedJson.equals(docVal)) { System.out.println("*** " + testName + ": cache: " + doc.get(DocumentPojo.url_) + ": cached val incorrect:\n" + docVal + "\nVS\n" + cachedJson); System.exit(-1); } } }