List of usage examples for com.mongodb DBCollection findOne
@Nullable public DBObject findOne(final Object id)
From source file:com.ikanow.infinit.e.api.knowledge.federated.SimpleFederatedQueryEngine.java
License:Open Source License
private BasicDBObject getCache(String url, SourceFederatedQueryConfigPojo endpoint) { if ((null != endpoint.cacheTime_days) && (endpoint.cacheTime_days <= 0)) { // cache disabled return null; }/* ww w . ja va2s .co m*/ DBCollection endpointCacheCollection = getCacheCollection(); if (!_staticInitializer) { _staticInitializer = true; endpointCacheCollection.ensureIndex(new BasicDBObject(SimpleFederatedCache.expiryDate_, 1)); } BasicDBObject cacheObj = (BasicDBObject) endpointCacheCollection .findOne(new BasicDBObject(SimpleFederatedCache._id_, url)); if (null == cacheObj) { return null; } // else found something, means there's stuff in the DB // so check it's not too big: Date now = new Date(); if ((-1 == _lastChecked) || (now.getTime() > (_lastChecked + (600L * 1000L)))) { // (only check every 10 minutes) if (endpointCacheCollection.count() > SimpleFederatedCache.QUERY_FEDERATION_CACHE_CLEANSE_SIZE) { _lastChecked = now.getTime(); // Remove everything with expiry date older than now endpointCacheCollection.remove(new BasicDBObject(SimpleFederatedCache.expiryDate_, new BasicDBObject(DbManager.lt_, new Date()))); } } //TESTED (4.3) Date expiryDate = cacheObj.getDate(SimpleFederatedCache.expiryDate_, now); if (now.getTime() < expiryDate.getTime()) { return cacheObj; } else { return null; } //TESTED (4.2) }
From source file:com.ikanow.infinit.e.api.knowledge.federated.SimpleFederatedQueryEngine.java
License:Open Source License
public void test_CheckIfDocAdded(String testName, List<BasicDBObject> docs) { // 1) Did a doc get added? if (2 != docs.size()) { System.out.println("*** " + testName + ": didn't add doc? " + docs.size()); System.exit(-1);/* w w w . j a v a 2 s . c o m*/ } // 2) Was it the right doc? BasicDBObject doc = docs.get(0); doc.remove(DocumentPojo._id_); doc.remove(DocumentPojo.created_); doc.remove(DocumentPojo.modified_); doc.remove(DocumentPojo.publishedDate_); String docToCheck = "{ \"displayUrl\" : \"http://test3_1\" , \"url\" : \"inf://federated/fakeendpoint.123/test3_1/testentityin\" , \"sourceKey\" : [ \"fakeendpoint.123\"] , \"source\" : [ \"fakeendpoint\"] , \"communityId\" : [ \"4c927585d591d31d7b37097a\"] , \"mediaType\" : [ \"Report\"] , \"metadata\" : { \"json\" : [ { \"test\" : { \"field\" : [ \"test3_1\" , \"test3_1\"] , \"field2\" : \"http://test3_1\"}}]} , \"title\" : \"fake endpoint: : test3_1: test3_1\" , \"entities\" : [ { \"disambiguated_name\" : \"test3_1\" , \"type\" : \"TestEntityOut\" , \"dimension\" : \"What\" , \"relevance\" : 1.0 , \"doccount\" : 1 , \"averageFreq\" : 1.0 , \"datasetSignificance\" : 10.0 , \"significance\" : 10.0 , \"frequency\" : 1.0 , \"index\" : \"test3_1/testentityout\" , \"queryCoverage\" : 100.0 , \"totalfrequency\" : 1.0}] , \"description\" : \"[\\n {\\n \\\"test\\\": {\\n \\\"field\\\": [\\n \\\"test3_1\\\",\\n \\\"test3_1\\\"\\n ],\\n \\\"field2\\\": \\\"http://test3_1\\\"\\n }\\n }\\n]\" , \"aggregateSignif\" : 115.0 , \"queryRelevance\" : 105.0 , \"score\" : 110.0}"; if (!docToCheck.equals(doc.toString())) { System.out.println( "*** " + testName + ": document incorrect:\n" + docToCheck + "\nVS\n" + doc.toString()); System.exit(-1); } // 3) Did the doc get cached? DBCollection endpointCacheCollection = getCacheCollection(); String hardwiredCacheId = "http://localhost:8186/test/test3_1/?param1=test3_1"; BasicDBObject cachedVal = (BasicDBObject) endpointCacheCollection .findOne(new BasicDBObject(SimpleFederatedCache._id_, hardwiredCacheId)); if (null == cachedVal) { System.out.println( "*** " + testName + ": no cache for: " + doc.get(DocumentPojo.url_) + " / " + hardwiredCacheId); System.exit(-1); } else { Date expiryDate = cachedVal.getDate(SimpleFederatedCache.expiryDate_, null); if ((null == expiryDate) || ((new Date().getTime() - expiryDate.getTime()) > 10 * 1000L)) { System.out.println("*** " + testName + ": expiry date for: " + doc.get(DocumentPojo.url_) + ": " + expiryDate + " vs now= " + new Date()); System.exit(-1); } BasicDBObject cachedJson = (BasicDBObject) cachedVal.get(SimpleFederatedCache.cachedJson_); Object docVal = null; try { Object[] docVals = (Object[]) ((BasicDBObject) doc.get(DocumentPojo.metadata_)).get("json"); docVal = docVals[0]; } catch (Exception e) { @SuppressWarnings("rawtypes") Collection docVals = (Collection) ((BasicDBObject) doc.get(DocumentPojo.metadata_)).get("json"); docVal = docVals.iterator().next(); } if ((null == cachedJson) || !cachedJson.equals(docVal)) { System.out.println("*** " + testName + ": cache: " + doc.get(DocumentPojo.url_) + ": cached val incorrect:\n" + docVal + "\nVS\n" + cachedJson); System.exit(-1); } } }
From source file:com.ikanow.infinit.e.processing.generic.store_and_index.StoreAndIndexManager.java
License:Open Source License
/** * Remove a doc from the data store/* w w w. j a v a 2s .c om*/ * @param col * @param doc - assumes _id set * @param fields - fields to retrieve (set in outside the doc loop for performance, url, index, sourceKey) * * CALLED FROM: removeFromDataStore_byId(col, List<doc>, bDeleteContent) * removeFromDataStore_byId(List<doc>, bDeleteContent) * resizeDB() <- _ID, SOURCEKEY, INDEX, SOURCEURL */ private void removeFromDatastore_byId(DBCollection col, DocumentPojo doc) { boolean bDeleteContent = docHasExternalContent(doc.getUrl(), doc.getSourceUrl()); if (bDeleteContent) { // Remove its content also: if (!_diagnosticMode) { BasicDBObject contentQuery = new BasicDBObject(DocumentPojo.url_, doc.getUrl()); contentQuery.put(DocumentPojo.sourceKey_, doc.getSourceKey()); DbManager.getDocument().getContent().remove(contentQuery); } else { System.out.println("StoreAndIndexManager.removeFromDatastore_byId, delete content: " + doc.getSourceKey() + "/" + doc.getUrl()); } } // Update Mongodb with the data BasicDBObject query = new BasicDBObject(); query.put(DocumentPojo.sourceKey_, SourcePojo.getDistributedKeyQueryTerm(doc.getSourceKey())); // (needed because on newer machines this is the shard key) query.put(DocumentPojo._id_, doc.getId()); if (!_diagnosticMode) { BasicDBObject softDelete = getSoftDeleteUpdate(); col.update(query, softDelete); // (can do this on sharded collections because it uses sourceKey+_id, the shard key) } else { // (diagnostic mode) if (null != col.findOne(query)) { System.out .println("StoreAndIndexManager.removeFromDatastore_byId, delete: " + doc.toDb().toString()); } else { System.out.println("StoreAndIndexManager.removeFromDatastore_byId, delete: DOC NOT FOUND"); } } }
From source file:com.ikanow.infinit.e.utility.MongoDocumentTxfer.java
License:Apache License
private void doTransfer(BasicDBObject query, int nSkip, int nLimit, boolean bAggregate, BasicDBObject chunk) throws IOException { PropertiesManager pm = new PropertiesManager(); int nMaxContentSize_bytes = pm.getMaxContentSize(); // Initialize the DB: DBCollection docsDB = DbManager.getDocument().getMetadata(); DBCollection contentDB = DbManager.getDocument().getContent(); DBCollection sourcesDB = DbManager.getIngest().getSource(); ElasticSearchManager.setDefaultClusterName("infinite-aws"); // 1. Get the documents from the DB (combining data + metadata and refreshing source meta) // (Ignore soft-deleted records:) if (null == query) { query = new BasicDBObject(); }//w w w .j av a 2 s. co m Object sourceKeyQueryTerm = query.remove(DocumentPojo.sourceKey_); if (null != sourceKeyQueryTerm) { if (query.toString() .contains(new StringBuffer('"').append(DocumentPojo.sourceKey_).append('"').toString())) { throw new RuntimeException( "Can't specify sourceKey as part of complex query term: " + query.toString()); } //TESTED (by hand, "{ \"sourceKey\": \"x\", \"$or\": [ { \"sourceKey\": \"x\" } ] }") if (sourceKeyQueryTerm instanceof String) { query.put(DocumentPojo.sourceKey_, SourcePojo.getDistributedKeyQueryTerm((String) sourceKeyQueryTerm)); } //TESTED (by hand, "{\"sourceKey\": \"feeds.arstechnica.com.arstechnica.index.11.2.\" }") else if (sourceKeyQueryTerm instanceof DBObject) { // find all the _sources_ matching this term, and convert to a big list including distribution BasicDBObject fields = new BasicDBObject(SourcePojo.key_, 1); fields.put(SourcePojo.highestDistributionFactorStored_, 1); DBCursor dbc = sourcesDB.find(new BasicDBObject(SourcePojo.key_, sourceKeyQueryTerm), fields); LinkedList<String> sourceKeys = new LinkedList<String>(); for (DBObject dbo : dbc) { String key = (String) dbo.get(SourcePojo.key_); Integer distributionFactor = (Integer) dbo.get(SourcePojo.highestDistributionFactorStored_); Collection<String> sourceKeysForSource = SourcePojo.getDistributedKeys(key, distributionFactor); sourceKeys.addAll(sourceKeysForSource); } query.put(DocumentPojo.sourceKey_, new BasicDBObject(DbManager.in_, sourceKeys)); } //TESTED (by hand, "{\"sourceKey\": { \"$gt\": \"dev.ikanow\" } }") else { throw new RuntimeException("Can't specify sourceKey as part of complex query term"); } //(actually not possible, just included here for mathematical completeness...) } else { if (query.toString() .contains(new StringBuffer('"').append(DocumentPojo.sourceKey_).append('"').toString())) { throw new RuntimeException("Can't specify sourceKey as part of complex query term"); } //TESTE (by hand, "{ \"$or\": [ { \"sourceKey\": \"x\" } ] }") // Optimize communityId into sourceKeys... if (null != query.get(DocumentPojo.communityId_)) { try { ObjectId commId = query.getObjectId(DocumentPojo.communityId_); BasicDBObject fields = new BasicDBObject(SourcePojo.key_, 1); fields.put(SourcePojo.highestDistributionFactorStored_, 1); DBCursor dbc = sourcesDB.find(new BasicDBObject(SourcePojo.communityIds_, commId), fields); LinkedList<String> sourceKeys = new LinkedList<String>(); int added = 0; for (DBObject dbo : dbc) { String key = (String) dbo.get(SourcePojo.key_); Integer distributionFactor = (Integer) dbo.get(SourcePojo.highestDistributionFactorStored_); Collection<String> sourceKeysForSource = SourcePojo.getDistributedKeys(key, distributionFactor); sourceKeys.addAll(sourceKeysForSource); added += sourceKeysForSource.size(); } query.put(DocumentPojo.sourceKey_, new BasicDBObject(DbManager.in_, sourceKeys)); System.out.println("(Optimized simple community query to " + added + " source key(s))"); } catch (Exception e) { //DEBUG //e.printStackTrace(); System.out.println("(Can't optimize complex community query: " + e.getMessage()); } } //TESTED (by hand - including distributed source version) } // Ignored delete objects Object urlQuery = query.get(DocumentPojo.url_); if (null == urlQuery) { query.put(DocumentPojo.url_, Pattern.compile("^[^?]")); // (ie nothing starting with ?) } //TESTED else if (urlQuery instanceof BasicDBObject) { ((BasicDBObject) urlQuery).append("$regex", "^[^?]"); } //TESTED //DEBUG //System.out.println("COMBINED QUERY= " + query.toString()); // If aggregating, kick off the background aggregation thread if (bAggregate) { EntityBackgroundAggregationManager.startThread(); AssociationBackgroundAggregationManager.startThread(); } //Debug: DBCursor dbc = null; dbc = docsDB.find(query); if (null != chunk) { if (chunk.containsField(DbManager.min_)) { dbc = dbc.addSpecial(DbManager.min_, chunk.get(DbManager.min_)); } if (chunk.containsField(DbManager.max_)) { dbc = dbc.addSpecial(DbManager.max_, chunk.get(DbManager.max_)); } } dbc = dbc.skip(nSkip).limit(nLimit).batchSize(1000); if (null == chunk) { int nCount = dbc.count() - nSkip; if (nCount < 0) nCount = 0; System.out.println( "Found " + nCount + " records to sync, process first " + (0 == nLimit ? nCount : nLimit)); if (0 == nCount) { // Nothing to do... return; } } byte[] storageArray = new byte[200000]; int nSynced = 0; LinkedList<DocumentPojo> docsToTransfer = new LinkedList<DocumentPojo>(); Map<ObjectId, LinkedList<DocumentPojo>> communityList = null; ObjectId currCommunityId = null; while (dbc.hasNext()) { BasicDBObject dbo = (BasicDBObject) dbc.next(); DocumentPojo doc = DocumentPojo.fromDb(dbo, DocumentPojo.class); String sDocIndex = doc.getIndex(); if (null == sDocIndex) { sDocIndex = "document_index"; } if ((null != _deletedIndex) && !_deletedIndex.contains(sDocIndex)) { _deletedIndex.add(sDocIndex); rebuildIndex(sDocIndex); try { // (Just in case the index requires some time to sort itself out) Thread.sleep(1000); } catch (InterruptedException e) { } } //Debug: //System.out.println("Getting content..." + feed.getTitle() + " / " + feed.getUrl()); // Get the content: if ((0 != nMaxContentSize_bytes) && StoreAndIndexManager.docHasExternalContent(doc.getUrl(), doc.getSourceUrl())) { BasicDBObject contentQ = new BasicDBObject(CompressedFullTextPojo.url_, doc.getUrl()); contentQ.put(CompressedFullTextPojo.sourceKey_, new BasicDBObject(MongoDbManager.in_, Arrays.asList(null, doc.getSourceKey()))); BasicDBObject fields = new BasicDBObject(CompressedFullTextPojo.gzip_content_, 1); fields.put(CompressedFullTextPojo.sourceKey_, 1); DBCursor dbcGzip = contentDB.find(contentQ, fields); while (dbcGzip.hasNext()) { BasicDBObject dboContent = (BasicDBObject) dbcGzip.next(); if (!dboContent.containsField(CompressedFullTextPojo.sourceKey_)) { // If this has another version then ignore this one... if (dbc.hasNext()) { continue; } //TESTED (by hand) } byte[] compressedData = ((byte[]) dboContent.get(CompressedFullTextPojo.gzip_content_)); ByteArrayInputStream in = new ByteArrayInputStream(compressedData); GZIPInputStream gzip = new GZIPInputStream(in); int nRead = 0; StringBuffer output = new StringBuffer(); while (nRead >= 0) { nRead = gzip.read(storageArray, 0, 200000); if (nRead > 0) { String s = new String(storageArray, 0, nRead, "UTF-8"); output.append(s); } } doc.setFullText(output.toString()); } } // (else document has full text already) // Get tags, if necessary: // Always overwrite tags - one of the reasons we might choose to migrate // Also may need source in order to support source index filtering SourcePojo src = _sourceCache.get(doc.getSourceKey()); if (null == src) { //TODO (INF-2265): handle search index settings in pipeline mode... (also didn't seem to work?) BasicDBObject srcDbo = (BasicDBObject) sourcesDB .findOne(new BasicDBObject(SourcePojo.key_, doc.getSourceKey())); if (null != srcDbo) { src = SourcePojo.fromDb(srcDbo, SourcePojo.class); if (null != src.getProcessingPipeline()) { try { // Set the index settings HarvestController hc = new HarvestController(); HarvestControllerPipeline hcPipe = new HarvestControllerPipeline(); hcPipe.extractSource_preProcessingPipeline(src, hc); } catch (Exception e) { //DEBUG e.printStackTrace(); } } //TESTED (by hand) _sourceCache.put(doc.getSourceKey(), src); } } doc.setTempSource(src); // (needed for source index filtering) if (null != src) { if (null != src.getTags()) { Set<String> tagsTidied = new TreeSet<String>(); for (String s : src.getTags()) { String ss = s.trim().toLowerCase(); tagsTidied.add(ss); } // May also want to write this back to the DB: //TODO (INF-2223): Handle append tags or not in the pipeline... if ((null == src.getAppendTagsToDocs()) || src.getAppendTagsToDocs()) { if ((null == doc.getTags()) || (doc.getTags().size() < tagsTidied.size())) { BasicDBObject updateQuery = new BasicDBObject(DocumentPojo.sourceKey_, doc.getRawSourceKey()); // (ie including the # if there is one) updateQuery.put(DocumentPojo._id_, doc.getId()); docsDB.update(updateQuery, new BasicDBObject(DbManager.addToSet_, new BasicDBObject(DocumentPojo.tags_, new BasicDBObject(DbManager.each_, tagsTidied)))); } doc.setTags(tagsTidied); // (just copy ptr across) } } } // 2. Update the index with the new document // (Optionally also update entity and assoc features) if (bAggregate) { if (null == currCommunityId) { currCommunityId = doc.getCommunityId(); } else if (!currCommunityId.equals(doc.getCommunityId())) { LinkedList<DocumentPojo> perCommunityDocList = null; if (null == communityList) { // (very first time we see > 1 community) communityList = new TreeMap<ObjectId, LinkedList<DocumentPojo>>(); perCommunityDocList = new LinkedList<DocumentPojo>(); perCommunityDocList.addAll(docsToTransfer); //(NOT including doc, this hasn't been added to docsToTransfer yet) communityList.put(currCommunityId, perCommunityDocList); } currCommunityId = doc.getCommunityId(); perCommunityDocList = communityList.get(currCommunityId); if (null == perCommunityDocList) { perCommunityDocList = new LinkedList<DocumentPojo>(); communityList.put(currCommunityId, perCommunityDocList); } perCommunityDocList.add(doc); } } //TESTED nSynced++; docsToTransfer.add(doc); if (0 == (nSynced % 10000)) { StoreAndIndexManager manager = new StoreAndIndexManager(); if (bAggregate) { // Loop over communities and aggregate each one then store the modified entities/assocs doAggregation(communityList, docsToTransfer); communityList = null; // (in case the next 10,000 docs are all in the same community!) currCommunityId = null; } //TOTEST manager.addToSearch(docsToTransfer); docsToTransfer.clear(); System.out.println("(Synced " + nSynced + " records)"); } } // (End loop over docs) // Sync remaining docs if (!docsToTransfer.isEmpty()) { if (bAggregate) { // Loop over communities and aggregate each one then store the modified entities/assocs doAggregation(communityList, docsToTransfer); } StoreAndIndexManager manager = new StoreAndIndexManager(); manager.addToSearch(docsToTransfer); } if (null != chunk) { System.out.println("Found " + nSynced + " records to sync in chunk"); } if (bAggregate) { System.out.println("Completed. You can hit CTRL+C at any time."); System.out.println( "By default it will keep running for 5 minutes while the background aggregation runs to update the documents' entities."); try { Thread.sleep(300000); } catch (InterruptedException e) { } // Turn off so we can exit EntityBackgroundAggregationManager.stopThreadAndWait(); AssociationBackgroundAggregationManager.stopThreadAndWait(); } }
From source file:com.ikanow.infinit.e.utility.MongoDocumentTxfer.java
License:Apache License
@SuppressWarnings("unused") private void doUnitTest(String sMongoDbHost, String sMongoDbPort, String sElasticHost, String sElasticPort, BasicDBObject query, int nLimit) { ElasticSearchManager elasticManager = null; try {/* w w w . j a v a 2s . co m*/ // Initialize the DB: DBCollection feedsDB = DbManager.getDocument().getMetadata(); DBCollection contentDB = DbManager.getDocument().getContent(); DBCollection sourcesDB = DbManager.getIngest().getSource(); String indexName = "document_index"; // Test/debug recreate the index if (true) { // (delete the index) System.out.println("Deleting index..."); elasticManager = ElasticSearchManager.getIndex(indexName, sElasticHost + ":" + sElasticPort); elasticManager.deleteMe(); //(also deletes the child index - same index, different type) // Create the index if necessary String sMapping = new Gson().toJson(new DocumentPojoIndexMap.Mapping(), DocumentPojoIndexMap.Mapping.class); Builder localSettings = ImmutableSettings.settingsBuilder(); localSettings.put("number_of_shards", 10).put("number_of_replicas", 2); System.out.println("Creating index..." + sMapping); elasticManager = ElasticSearchManager.createIndex(indexName, null, false, sElasticHost + ":" + sElasticPort, sMapping, localSettings); } // Get the index (necessary if already created) if (null == elasticManager) { elasticManager = ElasticSearchManager.getIndex(indexName, sElasticHost + ":" + sElasticPort); } // Get the feeds from the DB: //Debug: // System.out.println("Querying DB..."); DBCursor dbc = feedsDB.find(query).limit(nLimit); byte[] storageArray = new byte[200000]; while (dbc.hasNext()) { BasicDBObject dbo = (BasicDBObject) dbc.next(); DocumentPojo doc = DocumentPojo.fromDb(dbo, DocumentPojo.class); //Debug: System.out.println("Getting content..." + doc.getTitle() + " / " + doc.getUrl()); // Get the content: BasicDBObject contentQ = new BasicDBObject(CompressedFullTextPojo.url_, doc.getUrl()); contentQ.put(CompressedFullTextPojo.sourceKey_, new BasicDBObject(MongoDbManager.in_, Arrays.asList(null, doc.getSourceKey()))); BasicDBObject dboContent = (BasicDBObject) contentDB.findOne(contentQ); if (null != dboContent) { byte[] compressedData = ((byte[]) dboContent.get("gzip_content")); ByteArrayInputStream in = new ByteArrayInputStream(compressedData); GZIPInputStream gzip = new GZIPInputStream(in); int nRead = gzip.read(storageArray, 0, 200000); String s = new String(storageArray, 0, nRead, "UTF-8"); doc.setFullText(s); } // Get tag: SourcePojo src = _sourceCache.get(doc.getSourceKey()); if (null == src) { BasicDBObject srcDbo = (BasicDBObject) sourcesDB .findOne(new BasicDBObject("key", doc.getSourceKey())); if (null != srcDbo) { src = new Gson().fromJson(srcDbo.toString(), SourcePojo.class); _sourceCache.put(doc.getSourceKey(), src); } } if (null != src) { Set<String> tagsTidied = new TreeSet<String>(); for (String s : src.getTags()) { String ss = s.trim().toLowerCase(); tagsTidied.add(ss); } doc.setTags(tagsTidied); } //TEST: set dynamic field // Lots of testing of dynamic dates: // feed.addToMetadata("my_dateISO", Date.parse(feed.getCreated().toGMTString())); // String s1 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").format(feed.getCreated()); // feed.addToMetadata("another_dateISO", s1); // String s1_5 = new SimpleDateFormat().format(feed.getCreated()); // feed.addToMetadata("another_dateTimeJava", s1_5); // String s2 = new SimpleDateFormat("yyyyMMdd").format(feed.getCreated()); // feed.addToMetadata("another_dateYYYYMMDD", s2); // String s3 = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z").format(feed.getCreated()); // feed.addToMetadata("another_dateRFC822", s3); // feed.addToMetadata("another_dateGMT", feed.getCreated().toGMTString()); // // Testing of the string field types // feed.addToMetadata("my_comment", "Testing this ABCDEFG"); // feed.addToMetadata("my_term", "Testing this UVWXYZ"); // feed.addToMetadata("my_text", "Testing this 123456"); // // Test an array of longs: // Long tl[] = new Long[4]; tl[0] = 0L; tl[1] = 1L; tl[2] = 2L; tl[3] = 3L; // feed.addToMetadata("md_long", tl); //TEST: some dummy event timestamp adding code (not seeing much/any in the data) // if (null != feed.getEvents()) { // int i = 0; // for (EventPojo evt: feed.getEvents()) { // //1: Add single date // if (0 == i) { // evt.time_start = "2011-01-01"; // } // //2: Add short span // if (1 == i) { // evt.time_start = "2010-04-06"; // evt.time_end = "2010-08-09"; // } // //3: Add cross-yr span // if (2 == i) { // evt.time_start = "2012-06-05"; // evt.time_end = "2013-09-05"; // } // //4: Add too long span // if (3 == i) { // evt.time_start = "2012-04-06"; // evt.time_end = "2014-04-09"; // } // i++; // } // } // For event adding, see data_model.test.TestCode } } catch (IOException e) { e.printStackTrace(); } finally { //nothing to do } }
From source file:com.ikanow.utility.GridFSRandomAccessFile.java
License:Open Source License
/** Returns a random access file accessor from a database, FS name, and fileId * @param db - com.mongodb.DB, the database name * @param fsName - string, the "collection" name * @param fileId - org.bson.ObjectId, the _id of the file * @throws IOException// w ww . j a v a 2 s . com */ public GridFSRandomAccessFile(DB db, String fsName, ObjectId fileId) throws IOException { DBCollection fileColl = db.getCollection(new StringBuffer(fsName).append(".files").toString()); _chunkCollection = db.getCollection(new StringBuffer(fsName).append(".chunks").toString()); //TEST:System.out.println("GridFSRandomAccessFile1: "+_chunkCollection.getDB().getName()+"."+_chunkCollection.getName()+": "+fileId.toString()); _chunkQuery = new BasicDBObject(_CHUNK_files_id_, fileId); _chunkQuery.put(_CHUNK_n_, 0); _fileObj = (DBObject) fileColl.findOne(new BasicDBObject(_FILE_id_, fileId)); if (null == _fileObj) { throw new IOException("File Not Found"); } _fileId = fileId; _chunkSize = ((Number) _fileObj.get(_FILE_chunkSize_)).intValue(); _fileSize = ((Number) _fileObj.get(_FILE_length_)).longValue(); _lastChunkNum = (int) (_fileSize / _chunkSize); _finalChunkSize = (int) (_fileSize % _chunkSize); _currChunkSize = (_lastChunkNum == 0) ? _finalChunkSize : _chunkSize; //TEST:System.out.println("GridFSRandomAccessFile2: chunkSize="+_chunkSize+" fileSize="+_fileSize+" finalChunkSize="+_finalChunkSize+" currChunkSize="+_currChunkSize); }
From source file:com.images3.data.impl.ImageMetricsServiceImplMongoDB.java
License:Apache License
private DBObject selectMetricsByTempalteId(TemplateIdentity templateId) { DBCollection coll = getDatabase().getCollection("ImageMetrics"); BasicDBObject criteria = new BasicDBObject().append("imagePlantId", templateId.getImagePlantId()) .append("templateName", templateId.getTemplateName()).append("second", 0); DBObject object = coll.findOne(criteria); if (null == object) { throw new NoSuchEntityFoundException("ImageMetrics", templateId.toString(), "No such metrics found."); }/*from ww w . ja va 2s . c o m*/ return object; }
From source file:com.imaginea.mongodb.services.DocumentServiceImpl.java
License:Apache License
/** * Updates a document inside a collection in a database in mongo to which * user is connected to./* w w w. j av a2 s .c o m*/ * * @param dbName * Name of Database * @param collectionName * Name of Collection from which to get all Documents * @param id * Id of Document to be updated * @param newData * new Document value. * @return Update status * @exception EmptyDatabaseNameException * If database name is null * @exception EmptyCollectionNameException * If Collection name is null * @exception EmptyDocumentDataException * If Document data is null * @exception UndefinedDatabaseException * If database is not present * @exception UndefinedCollectionException * If Collection is not present * @exception UpdateDocumentException * Any exception while updating document * @exception DatabaseException * throw super type of UndefinedDatabaseException * @exception ValidationException * throw super type of * EmptyDatabaseNameException,EmptyCollectionNameException * ,EmptyDocumentDataException * @exception CollectionException * throw super type of UndefinedCollectionException * @exception DocumentException * throw super type of UpdateDocumentException * @exception JSONException * */ public String updateDocument(String dbName, String collectionName, ObjectId id, DBObject newData) throws DatabaseException, CollectionException, DocumentException, ValidationException { mongoInstance = mongoInstanceProvider.getMongoInstance(); if (dbName == null) { throw new EmptyDatabaseNameException("Database name is null"); } if (dbName.equals("")) { throw new EmptyDatabaseNameException("Database Name Empty"); } if (collectionName == null) { throw new EmptyCollectionNameException("Collection name is null"); } if (collectionName.equals("")) { throw new EmptyCollectionNameException("Collection Name Empty"); } String result = null; DBObject documentData = null; try { if (!mongoInstance.getDatabaseNames().contains(dbName)) { throw new UndefinedDatabaseException("DB [" + dbName + "] DOES NOT EXIST"); } if (!mongoInstance.getDB(dbName).getCollectionNames().contains(collectionName)) { throw new UndefinedCollectionException( "COLLECTION [ " + collectionName + "] _DOES_NOT_EXIST in Db [ " + dbName + "]"); } if (id == null) { throw new EmptyDocumentDataException("Document is empty"); } String temp = (String) newData.get("_id"); if (temp == null) { throw new DocumentException(ErrorCodes.INVALID_OBJECT_ID, "INVALID_OBJECT_ID"); } if (temp.equals("")) { throw new DocumentException(ErrorCodes.INVALID_OBJECT_ID, "INVALID_OBJECT_ID"); } ObjectId newId = new ObjectId(temp); if (!newId.equals(id)) { throw new DocumentException(ErrorCodes.INVALID_OBJECT_ID, "Cannot Change Object Id of a document"); } else { // Id's equal but putting the id of old document still // as newData as id of string type but we need ObjectId type newData.put("_id", id); } DBObject query = new BasicDBObject("_id", id); DBCollection collection = mongoInstance.getDB(dbName).getCollection(collectionName); DBObject doc = collection.findOne(query); if (doc == null) { throw new UndefinedDocumentException("DOCUMENT_DOES_NOT_EXIST"); } collection.update(doc, newData, true, false); documentData = collection.findOne(query); } catch (IllegalArgumentException e) { // When error converting object Id throw new DocumentException(ErrorCodes.INVALID_OBJECT_ID, "INVALID_OBJECT_ID"); } catch (MongoException e) { throw new UpdateDocumentException("DOCUMENT_UPDATE_EXCEPTION"); } result = "Updated Document: [" + documentData + "]"; return result; }
From source file:com.imaginea.mongodb.services.DocumentServiceImpl.java
License:Apache License
/** * Deletes a document inside a collection in a database in mongo to which * user is connected to.//w ww . ja va2s . co m * * @param dbName * Name of Database * @param collectionName * Name of Collection from which to get all Documents * @param id * Id of Document to be updated * @return Deletion status * @exception EmptyDatabaseNameException * If database name is null * @exception EmptyCollectionNameException * If Collection name is null * @exception EmptyDocumentDataException * If Document data is null * @exception UndefinedDatabaseException * If database is not present * @exception UndefinedCollectionException * If Collection is not present * @exception DeleteDocumentException * Any exception while deleting document * @exception DatabaseException * throw super type of UndefinedDatabaseException * @exception ValidationException * throw super type of * EmptyDatabaseNameException,EmptyCollectionNameException * ,EmptyDocumentDataException * @exception CollectionException * throw super type of UndefinedCollectionException * @exception DocumentException * throw super type of DeleteDocumentException * */ public String deleteDocument(String dbName, String collectionName, ObjectId id) throws DatabaseException, CollectionException, DocumentException, ValidationException { mongoInstance = mongoInstanceProvider.getMongoInstance(); if (dbName == null) { throw new EmptyDatabaseNameException("Database name is null"); } if (dbName.equals("")) { throw new EmptyDatabaseNameException("Database Name Empty"); } if (collectionName == null) { throw new EmptyCollectionNameException("Collection name is null"); } if (collectionName.equals("")) { throw new EmptyCollectionNameException("Collection Name Empty"); } String result = null; DBObject documentData = null; try { if (!mongoInstance.getDatabaseNames().contains(dbName)) { throw new UndefinedDatabaseException("DB [" + dbName + "] DOES NOT EXIST"); } if (!mongoInstance.getDB(dbName).getCollectionNames().contains(collectionName)) { throw new UndefinedCollectionException( "COLLECTION [ " + collectionName + "] _DOES_NOT_EXIST in Db [ " + dbName + "]"); } DBCollection coll = mongoInstance.getDB(dbName).getCollection(collectionName); if (coll.isCapped()) { throw new DocumentException(ErrorCodes.DELETING_FROM_CAPPED_COLLECTION, "Cannot Delete From a Capped Collection"); } if (id == null) { throw new EmptyDocumentDataException("Document is empty"); } DBObject query = new BasicDBObject(); query.put("_id", id); DBCollection collection = this.mongoInstance.getDB(dbName).getCollection(collectionName); documentData = collection.findOne(query); if (documentData == null) { throw new UndefinedDocumentException("DOCUMENT_DOES_NOT_EXIST"); } mongoInstance.getDB(dbName).getCollection(collectionName).remove(documentData); } catch (MongoException e) { throw new DeleteDocumentException("DOCUMENT_DELETION_EXCEPTION"); } result = "Deleted Document with Data : [" + documentData + "]"; return result; }
From source file:com.impetus.client.mongodb.MongoDBClient.java
License:Apache License
/** * Find./*from ww w . jav a 2s . c om*/ * * @param entityClass * the entity class * @param key * the key * @param entityMetadata * the entity metadata * @param metaModel * the meta model * @param managedType * the managed type * @return the object */ private Object find(Class entityClass, Object key, EntityMetadata entityMetadata, MetamodelImpl metaModel, AbstractManagedType managedType) { List<String> relationNames = entityMetadata.getRelationNames(); BasicDBObject query = new BasicDBObject(); if (metaModel.isEmbeddable(entityMetadata.getIdAttribute().getBindableJavaType())) { MongoDBUtils.populateCompoundKey(query, entityMetadata, metaModel, key); } else { query.put("_id", MongoDBUtils.populateValue(key, key.getClass())); } // For secondary tables. List<String> secondaryTables = ((DefaultEntityAnnotationProcessor) managedType.getEntityAnnotation()) .getSecondaryTablesName(); secondaryTables.add(entityMetadata.getTableName()); Object enhancedEntity = null; Map<String, Object> relationValue = null; // Here you need to fetch by sub managed type. EntityType entityType = metaModel.entity(entityMetadata.getEntityClazz()); for (String tableName : secondaryTables) { DBCollection dbCollection = mongoDb.getCollection(tableName); KunderaCoreUtils.printQuery("Find document:" + query, showQuery); DBObject fetchedDocument = dbCollection.findOne(query); if (fetchedDocument != null) { List<AbstractManagedType> subManagedType = ((AbstractManagedType) entityType).getSubManagedType(); EntityMetadata subEntityMetadata = null; if (!subManagedType.isEmpty()) { for (AbstractManagedType subEntity : subManagedType) { String discColumn = subEntity.getDiscriminatorColumn(); String disColValue = subEntity.getDiscriminatorValue(); Object value = fetchedDocument.get(discColumn); if (value != null && value.toString().equals(disColValue)) { subEntityMetadata = KunderaMetadataManager.getEntityMetadata(kunderaMetadata, subEntity.getJavaType()); break; } } enhancedEntity = instantiateEntity(subEntityMetadata.getEntityClazz(), enhancedEntity); relationValue = handler.getEntityFromDocument(subEntityMetadata.getEntityClazz(), enhancedEntity, subEntityMetadata, fetchedDocument, subEntityMetadata.getRelationNames(), relationValue, kunderaMetadata); } else { enhancedEntity = instantiateEntity(entityClass, enhancedEntity); relationValue = handler.getEntityFromDocument(entityMetadata.getEntityClazz(), enhancedEntity, entityMetadata, fetchedDocument, relationNames, relationValue, kunderaMetadata); } } } if (relationValue != null && !relationValue.isEmpty()) { EnhanceEntity entity = new EnhanceEntity(enhancedEntity, PropertyAccessorHelper.getId(enhancedEntity, entityMetadata), relationValue); return entity; } else { return enhancedEntity; } }