List of usage examples for com.mongodb BasicDBObject removeField
public Object removeField(final String key)
From source file:com.ikanow.infinit.e.api.knowledge.processing.ScoringUtils.java
License:Open Source License
private void stage4_prepareDocsForOutput(AdvancedQueryPojo.QueryScorePojo scoreParams, StatisticsPojo scores, long nToClientLimit, LinkedList<BasicDBObject> returnList) { // Get the documents long nDocs = 0; double dBestScore = 0.0; double dAvgScore = 0.0; double dSigFactor = 100.0 / (_s3_dSigScalingFactor * _s2_dApproxAverageDocumentSig); double dRelFactor = 100.0 / (_s3_dLuceneScalingFactor * _s0_avgLuceneScore); // Start at the bottom of the list, so don't need to worry about skipping documents, just count out from the bottom // The call to stage3_calculateTFTerms with nStart+nToClientLimit handles the rest Iterator<TempDocBucket> pqIt = _s3_pqDocs.iterator(); while (pqIt.hasNext() && (nDocs < nToClientLimit)) { TempDocBucket qsf = pqIt.next(); nDocs++;//from w w w . j ava2s . c o m if (!_s0_sortingByDate) { dBestScore = qsf.totalScore; } dAvgScore += dBestScore; BasicDBObject f = qsf.dbo; // Phase "0" - these are the highest prio events boolean bNeedToFilterAndAliasAssoc_event = true; boolean bNeedToFilterAndAliasAssoc_fact = true; boolean bNeedToFilterAndAliasAssoc_summary = true; if (null != _s0_standaloneEventAggregator) { ScoringUtils_Associations.addStandaloneEvents(qsf.dbo, qsf.aggSignificance, 0, _s0_standaloneEventAggregator, _s0_bEntityTypeFilterPositive, _s0_bAssocVerbFilterPositive, _s0_entityTypeFilter, _s0_assocVerbFilter, _s0_bEvents, _s0_bSummaries, _s0_bFacts); bNeedToFilterAndAliasAssoc_event = false; bNeedToFilterAndAliasAssoc_fact = false; bNeedToFilterAndAliasAssoc_summary = false; } //TESTED if (null != _s0_lowAccuracyAssociationAggregator_events) { ScoringUtils_Associations.addStandaloneEvents(qsf.dbo, qsf.aggSignificance, 0, _s0_lowAccuracyAssociationAggregator_events, _s0_bEntityTypeFilterPositive, _s0_bAssocVerbFilterPositive, _s0_entityTypeFilter, _s0_assocVerbFilter, true, false, false); bNeedToFilterAndAliasAssoc_event = false; } //TESTED if (null != _s0_lowAccuracyAssociationAggregator_facts) { ScoringUtils_Associations.addStandaloneEvents(qsf.dbo, qsf.aggSignificance, 0, _s0_lowAccuracyAssociationAggregator_facts, _s0_bEntityTypeFilterPositive, _s0_bAssocVerbFilterPositive, _s0_entityTypeFilter, _s0_assocVerbFilter, false, false, true); bNeedToFilterAndAliasAssoc_fact = false; } //TESTED try { DocumentPojoApiMap.mapToApi(f); // Handle deduplication/multi-community code: if (null != qsf.dupList) { try { ScoringUtils_MultiCommunity.community_combineDuplicateDocs(qsf); } catch (Exception e) { // Do nothing, just carry on with minimal damage! } } // Scoring: double d = qsf.aggSignificance * dSigFactor; if (Double.isNaN(d)) { f.put(DocumentPojo.aggregateSignif_, 0.0); } else { f.put(DocumentPojo.aggregateSignif_, d); } d = qsf.luceneScore * dRelFactor; if (Double.isNaN(d)) { f.put(DocumentPojo.queryRelevance_, 0.0); } else { f.put(DocumentPojo.queryRelevance_, d); } if (!_s0_sortingByDate) { f.put(DocumentPojo.score_, qsf.totalScore); } BasicDBList l = (BasicDBList) (f.get(DocumentPojo.entities_)); // Handle update ids vs normal ids: ObjectId updateId = (ObjectId) f.get(DocumentPojo.updateId_); if (null != updateId) { // swap the 2... f.put(DocumentPojo.updateId_, f.get(DocumentPojo._id_)); f.put(DocumentPojo._id_, updateId); } // Check if entities enabled if ((null != l) && (!_s0_bGeoEnts && !_s0_bNonGeoEnts)) { f.removeField(DocumentPojo.entities_); l = null; } //TESTED // Check if events etc enabled if ((!_s0_bEvents && !_s0_bFacts && !_s0_bSummaries)) { f.removeField(DocumentPojo.associations_); } //TESTED else if (!_s0_bEvents || !_s0_bFacts || !_s0_bSummaries || (null != _s0_assocVerbFilter)) { // Keep only specified event_types BasicDBList lev = (BasicDBList) (f.get(DocumentPojo.associations_)); if (null != lev) { for (Iterator<?> e0 = lev.iterator(); e0.hasNext();) { BasicDBObject e = (BasicDBObject) e0.next(); // Type filter boolean bNeedToFilterAndAliasAssoc = true; String sEvType = e.getString(AssociationPojo.assoc_type_); boolean bKeep = true; if (null == sEvType) { bKeep = false; } else if (sEvType.equalsIgnoreCase("event")) { if (!_s0_bEvents) bKeep = false; bNeedToFilterAndAliasAssoc = bNeedToFilterAndAliasAssoc_event; } else if (sEvType.equalsIgnoreCase("fact")) { if (!_s0_bFacts) bKeep = false; bNeedToFilterAndAliasAssoc = bNeedToFilterAndAliasAssoc_fact; } else if (sEvType.equalsIgnoreCase("summary")) { if (!_s0_bSummaries) bKeep = false; bNeedToFilterAndAliasAssoc = bNeedToFilterAndAliasAssoc_summary; } if (!bKeep) { e0.remove(); } else { // Type matches, now for some more complex logic.... if (bNeedToFilterAndAliasAssoc) { // (otherwise done already) bKeep = ScoringUtils_Associations.filterAndAliasAssociation(e, _s1_aliasLookup, true, _s0_bEntityTypeFilterPositive, _s0_bAssocVerbFilterPositive, _s0_entityTypeFilter, _s0_assocVerbFilter); if (!bKeep) { e0.remove(); } } //TESTED } //(end output filter logic) } // (end loop over events) } // (end if this doc has events) } //TESTED // Check if metadata is enabled if (!_s0_bMetadata) { f.removeField(DocumentPojo.metadata_); } //TESTED if (null != l) { for (Iterator<?> e0 = l.iterator(); e0.hasNext();) { BasicDBObject e = (BasicDBObject) e0.next(); if (!_s0_bNonGeoEnts) { // then must only be getting geo (else wouldn't be in this loop) if (null == e.get(EntityPojo.geotag_)) { e0.remove(); continue; } } String entity_index = e.getString(EntityPojo.index_); if (null == entity_index) continue; EntSigHolder shp = (EntSigHolder) _s1_entitiesInDataset.get(entity_index); if (null != shp) { // Stage 4x: alias processing, just overwrite // (note don't delete "duplicate entities", hard-to-be-globally-consistent // and will potentially throw data away which might be undesirable) if (null != shp.masterAliasSH) { shp = shp.masterAliasSH; // (already has all the aggregated values used below) if (!entity_index.equals(shp.aliasInfo.getIndex())) { e.put(EntityPojo.index_, shp.aliasInfo.getIndex()); e.put(EntityPojo.disambiguated_name_, shp.aliasInfo.getDisambiguatedName()); e.put(EntityPojo.type_, shp.aliasInfo.getType()); e.put(EntityPojo.dimension_, shp.aliasInfo.getDimension()); if (null != shp.aliasInfo.getGeotag()) { BasicDBObject aliasedGeoTag = new BasicDBObject(); aliasedGeoTag.put(GeoPojo.lat_, shp.aliasInfo.getGeotag().lat); aliasedGeoTag.put(GeoPojo.lon_, shp.aliasInfo.getGeotag().lon); e.put(EntityPojo.geotag_, aliasedGeoTag); if (null != shp.aliasInfo.getOntology_type()) { e.put(EntityPojo.ontology_type_, shp.aliasInfo.getOntology_type()); } } //TESTED } } //TESTED // end Stage 4x of alias processing double dataSig = shp.datasetSignificance; if (Double.isNaN(dataSig)) { e.put(EntityPojo.datasetSignificance_, 0.0); } else { e.put(EntityPojo.datasetSignificance_, dataSig); } e.put(EntityPojo.queryCoverage_, shp.queryCoverage); e.put(EntityPojo.averageFreq_, shp.avgFreqOverQuerySubset); if (shp.nTotalSentimentValues > 0) { e.put(EntityPojo.positiveSentiment_, shp.positiveSentiment); e.put(EntityPojo.negativeSentiment_, shp.negativeSentiment); e.put(EntityPojo.sentimentCount_, shp.nTotalSentimentValues); } } else { // (most likely to occur if the entity is discarded (alias/filter) or is corrupt in some way) e0.remove(); continue; } } //(end loop over entities) } // (end if feed has entities) //TESTED // Explain if enabled if (null != qsf.explain) { f.put(DocumentPojo.explain_, qsf.explain); } // Add to the end of the list (so will come back from API call in natural order, highest first) returnList.addFirst(f); // (add elements to the front of the list so that the top of the list is ordered by priority) } catch (Exception e) { // Probably a JSON error, just carry on String title = f.getString(DocumentPojo.title_); logger.error(title + ": " + e.getMessage()); } } // (end loop over feeds) //TESTED // Update the scores: scores.maxScore = (float) dBestScore; if (nDocs > 0) { scores.avgScore = (float) dAvgScore / nDocs; } }
From source file:com.ikanow.infinit.e.api.knowledge.processing.ScoringUtils.java
License:Open Source License
private void stage4_prepareEntsForOutput(LinkedList<BasicDBObject> entityReturn) { if (_s0_nNumEntsReturn > 0) { // (else entities not enabled) for (EntSigHolder qsf = _s3_pqEnt.poll(); null != qsf; qsf = _s3_pqEnt.poll()) // (start with lowest ranking) {//from ww w . jav a 2 s . c o m BasicDBObject ent = qsf.unusedDbo; if (null == ent) { int nTries = 0; if (null != qsf.entityInstances) { // (should never be null but just to be on the safe side... for (TempEntityInDocBucket tefb : qsf.entityInstances) { // (Try to find an entity that wasn't promoted ie can now be re-used // if we can't find one quite quickly then bail out and we'll pay the cost of cloning it) if (!tefb.doc.bPromoted) { ent = tefb.dbo; break; } else if (++nTries > 10) { break; } } if (null == ent) { ent = qsf.entityInstances.get(0).dbo; } } else { // (no entityInstances, something alias-related has gone wrong, just skip) continue; } } //TESTED qsf.entityInstances = null; // (don't need this any more, can be gc'd) try { if (null != qsf.aliasInfo) { if (!qsf.index.equals(qsf.aliasInfo.getIndex())) { ent.put(EntityPojo.index_, qsf.aliasInfo.getIndex()); ent.put(EntityPojo.disambiguated_name_, qsf.aliasInfo.getDisambiguatedName()); ent.put(EntityPojo.type_, qsf.aliasInfo.getType()); ent.put(EntityPojo.dimension_, qsf.aliasInfo.getDimension()); if (null != qsf.aliasInfo.getGeotag()) { BasicDBObject aliasedGeoTag = new BasicDBObject(); aliasedGeoTag.put(GeoPojo.lat_, qsf.aliasInfo.getGeotag().lat); aliasedGeoTag.put(GeoPojo.lon_, qsf.aliasInfo.getGeotag().lon); ent.put(EntityPojo.geotag_, aliasedGeoTag); if (null != qsf.aliasInfo.getOntology_type()) { ent.put(EntityPojo.ontology_type_, qsf.aliasInfo.getOntology_type()); } } //TESTED } } //TESTED if (null == ent.get(EntityPojo.datasetSignificance_)) { // Not getting promoted so need to add fields... if (Double.isNaN(qsf.datasetSignificance)) { ent.put("datasetSignificance", 0.0); } else { ent.put(EntityPojo.datasetSignificance_, qsf.datasetSignificance); } ent.put(EntityPojo.queryCoverage_, qsf.queryCoverage); ent.put(EntityPojo.averageFreq_, qsf.avgFreqOverQuerySubset); if (qsf.nTotalSentimentValues > 0) { ent.put(EntityPojo.positiveSentiment_, qsf.positiveSentiment); ent.put(EntityPojo.negativeSentiment_, qsf.negativeSentiment); ent.put(EntityPojo.sentimentCount_, qsf.nTotalSentimentValues); } } else { // (... but can just use it without cloning) BasicDBObject ent2 = new BasicDBObject(); for (Map.Entry<String, Object> kv : ent.entrySet()) { ent2.append(kv.getKey(), kv.getValue()); } ent = ent2; } ent.removeField(EntityPojo.relevance_); if (Double.isNaN(qsf.maxDocSig)) { ent.put(EntityPojo.significance_, 0.0); } else { ent.put(EntityPojo.significance_, qsf.maxDocSig); } ent.put(EntityPojo.frequency_, (long) qsf.maxFreq); entityReturn.addFirst(ent); } catch (Exception e) { // Probably a JSON error, just carry on String title = ent.getString(EntityPojo.index_); logger.error(title + ": " + e.getMessage()); } //TESTED } } //TESTED }
From source file:com.streamreduce.rest.resource.api.AbstractOwnableResource.java
License:Apache License
protected InventoryItemResponseDTO toFullDTO(InventoryItem inventoryItem) { InventoryItemResponseDTO dto = new InventoryItemResponseDTO(); Connection connection = inventoryItem.getConnection(); BasicDBObject payload = applicationManager.getInventoryService().getInventoryItemPayload(inventoryItem); super.toBaseDTO(inventoryItem, dto); dto.setOwner(inventoryItem.getUser().getId().equals(securityService.getCurrentUser().getId())); dto.setConnectionAlias(connection.getAlias()); dto.setConnectionId(connection.getId()); dto.setConnectionType(connection.getType()); dto.setConnectionProviderId(connection.getProviderId()); dto.setExternalId(inventoryItem.getExternalId()); dto.setType(inventoryItem.getType()); // Prune any sensitive information from the payload if (!dto.isOwner()) { if (payload.containsField("adminPassword")) { payload.removeField("adminPassword"); }// w ww . j a va 2 s . co m if (payload.containsField("credentials")) { payload.removeField("credentials"); } } dto.setPayload(payload); return dto; }
From source file:com.tengen.Final8.java
License:Apache License
public static void main(String[] args) throws UnknownHostException { MongoClient client = new MongoClient(); DB db = client.getDB("test"); DBCollection animals = db.getCollection("animals"); BasicDBObject animal = new BasicDBObject("animal", "monkey"); animals.insert(animal);/*from w ww. ja va 2 s. c om*/ animal.removeField("animal"); animal.append("animal", "cat"); animals.insert(animal); animal.removeField("animal"); animal.append("animal", "lion"); animals.insert(animal); }
From source file:controllers.DadosBean.java
public void guardarDadosPC() { contratacao.setPid(Integer.parseInt(mySessionBean.getPid())); String muser = mySessionBean.getUser(); //verificar se o registo j existe BasicDBObject res = (BasicDBObject) conn.getDBCollection() .findOne(new BasicDBObject("pid", contratacao.getPid())); BasicDBObject o = CPParsers.contratacaoToJson(contratacao); if (res == null) { //faz insert String user = mySessionBean.getUser(); o.put("user", muser); o.put("updated", new Date()); conn.getDBCollection().insert(o); } else {/*w w w . jav a2s. c o m*/ //faz backup conn.setDBCollection("collProcContratacao_Backup"); res.put("_idOld", res.getObjectId("_id")); res.removeField("_id"); conn.getDBCollection().insert(res); //faz update conn.setDBCollection("collProcContratacao"); o.put("user", muser); o.put("updated", new Date()); o.removeField("_id"); conn.getDBCollection().update(new BasicDBObject("pid", res.getInt("pid")), o); } APPEUtils.printMessage("Dados guardados com sucesso!"); }
From source file:fr.gouv.vitam.cases.ElasticSearchAccess.java
License:Open Source License
private static final BasicDBObject getFiltered(final BSONObject bson) { BasicDBObject maip = new BasicDBObject(); maip.putAll(bson);//from ww w. j ava 2 s. co m maip.removeField(VitamLinks.DAip2DAip.field1to2); // Keep it maip.removeField(VitamLinks.DAip2DAip.field2to1); //maip.removeField(VitamLinks.Domain2DAip.field2to1); //maip.removeField(VitamLinks.DAip2Dua.field1to2); //maip.removeField(VitamLinks.DAip2PAip.field1to2); // maip.removeField(ParserIngest.REFID); // DOMDEPTH already ok but duplicate it @SuppressWarnings("unchecked") final HashMap<String, Integer> map = (HashMap<String, Integer>) maip.get(DAip.DAIPDEPTHS); //final List<String> list = new ArrayList<>(map.keySet()); maip.append(DAip.DAIPPARENTS, map.keySet());// was list); return maip; }
From source file:fr.gouv.vitam.cases.ElasticSearchAccess.java
License:Open Source License
/** * /*from www.jav a 2 s . c o m*/ * @param dbvitam * @param model * @param bson * @return True if inserted in ES */ public static final boolean addEsIndex(final CassandraAccess dbvitam, final String model, final BSONObject bson) { BasicDBObject maip = getFiltered(bson); final String id = maip.getString(VitamType.ID); maip.removeField(VitamType.ID); return dbvitam.addEsEntryIndex(model, id, maip.toString()); }
From source file:fr.gouv.vitam.cases.ElasticSearchAccess.java
License:Open Source License
/** * Should be called only once saved (last time), but for the moment let the object as it is, next should remove not indexable * entries//from ww w. j a v a2 s . c om * * @param dbvitam * @param model * @param indexes * @param bson * @return the number of DAip incorporated (0 if none) */ public static final int addEsIndex(final CassandraAccess dbvitam, final String model, final Map<String, String> indexes, final BSONObject bson) { BasicDBObject maip = getFiltered(bson); final String id = maip.getString(VitamType.ID); maip.removeField(VitamType.ID); // System.err.println(maip); // System.err.println(this); indexes.put(id, maip.toString()); int nb = 0; if (indexes.size() > GlobalDatas.LIMIT_ES_NEW_INDEX) { nb = indexes.size(); dbvitam.addEsEntryIndex(indexes, model); // dbvitam.flushOnDisk(); indexes.clear(); System.out.print("o"); } maip.clear(); maip = null; return nb; }
From source file:fr.gouv.vitam.mdbes.CopyOfResultMongodb.java
License:Open Source License
/** * Save the document if new, update it (keeping non set fields, replacing set fields) * * @param collection// w w w .ja v a 2 s . c o m */ protected final void updateOrSave(final VitamCollection collection) { final String id = (String) obj.get(ID); if (id != null) { final BasicDBObject upd = new BasicDBObject(obj); upd.removeField(ID); collection.collection.update(new BasicDBObject(ID, id), new BasicDBObject("$set", upd)); } }
From source file:fr.gouv.vitam.mdbes.ElasticSearchAccess.java
License:Open Source License
/** * //w w w . j av a 2 s . c o m * @param dbvitam * @param model * @param bson * @return True if inserted in ES */ public static final boolean addEsIndex(final MongoDbAccess dbvitam, final String model, final BSONObject bson) { BasicDBObject maip = getFiltered(bson); final String id = maip.getString(VitamType.ID); maip.removeField(VitamType.ID); return dbvitam.addEsEntryIndex(model, id, maip.toString()); }