List of usage examples for com.mongodb BasicDBObject toString
@SuppressWarnings("deprecation") public String toString()
Returns a JSON serialization of this object
The output will look like: {"a":1, "b":["x","y","z"]} }
From source file:com.ikanow.infinit.e.processing.generic.store_and_index.StoreAndIndexManager.java
License:Open Source License
/** * Remove a doc from the data store, ensures all the fields specified in "fields" are populated (ready for index deletion) * @param col/*from w w w.j ava2 s . co m*/ * @param doc - needs url, sourceKey set * @param fields - fields to retrieve (index, created), set in calling function outside of loop for performance * * CALLED FROM: removeFromDatastore_byURL(col, List<doc>, bDeleteContent) <- ADDS INDEX, CREATED TO FIELDS * removeFromDataStore_byURL(List<doc>, bDeleteContent) [ALSO DELETES FROM INDEX AFTER ADDED FROM HERE] * MongoDocumentTxfer.doDelete(...) <- SETS URL, SOURCE URL, SOURCE KEY, COMMUNITY ID, INDEX, _ID * processDocuments(...) [ always called after harvester: have sourceUrl, sourceKey, * DON'T have _id, BUT do have updateId and index (correct except in many geo cases)] * pruneSource(source, ...) <- SETS URL, SOURCE URL, SOURCE KEY, INDEX * updateHarvestStatus(...) */ private void removeFromDatastore_byURL(DBCollection col, DocumentPojo doc, BasicDBObject fields, boolean bDeleteContent) { // 1] Create the query to soft delete the document BasicDBObject query = new BasicDBObject(); query.put(DocumentPojo.url_, doc.getUrl()); query.put(DocumentPojo.sourceKey_, SourcePojo.getDistributedKeyQueryTerm(doc.getSourceKey())); // 2] Delete the content if needed if (bDeleteContent) { if (docHasExternalContent(doc.getUrl(), doc.getSourceUrl())) { if (!_diagnosticMode) { DbManager.getDocument().getContent().remove(query); } else { System.out.println("StoreAndIndexManager.removeFromDatastore_byUrl(2), delete content: " + doc.getSourceKey() + "/" + doc.getUrl()); } } } //TESTED // 3] Work out which fields we have and which (if any we need to go and fetch): boolean needToFindAndModify = false; if (null == doc.getId()) { // This is called from processDocuments if (null != doc.getUpdateId()) { // update case... doc.setId(doc.getUpdateId()); // (note this is overwritten by addToDatastore later, in update case, so we're good) // (doc.index is populated but may not be correct because of the "many geos" workaround): if (DocumentPojoIndexMap.hasManyGeos(doc)) { doc.setIndex(DocumentPojoIndexMap.manyGeoDocumentIndex_); // (note this check isn't stateless, it actually populates "locs" at the same time // this is handled in addToDatastore (update case), temp removed when adding to DB } //TESTED (2.1.2, diagnostic mode, doc2) } else { // Not an update case, we're going to have to grab the document after all, which is a bit slower needToFindAndModify = true; } } //TESTED (2.1.2, diagnostic mode, doc2) if (!needToFindAndModify) { // set created if we need to, since we're not grabbing it from the datastore if (null != doc.getUpdateId()) { // (this means we have an approx created if we don't need to go fetch the deleted doc) doc.setCreated(new Date(doc.getUpdateId().getTime())); } //TESTED (2.1.2, diagnostic mode, doc2) } // (if we're here and index is not set, then it is intended to be null) // 4] Update the doc_metadata collection BasicDBObject softDelete = getSoftDeleteUpdate(); BasicDBObject deadDoc = null; // (not normally needed) if (needToFindAndModify) { // less pleasant, need to go grab the doc deadDoc = (BasicDBObject) col.findOne(query, fields); } //TESTED (2.1.2) if (!_diagnosticMode) { col.update(query, softDelete, false, true); // (needs to be multi- even though there's a single element for sharding reasons) } //TESTED (2.1.2) // 5] Add fields if necessary if (null != deadDoc) { doc.setCreated((Date) deadDoc.get(DocumentPojo.created_)); // (if getting this doc anyway then might as well get the created) doc.setId((ObjectId) deadDoc.get(DocumentPojo._id_)); doc.setIndex((String) deadDoc.get(DocumentPojo.index_)); if (_diagnosticMode) { System.out .println("StoreAndIndexManager.removeFromDatastore_byUrl(2): found " + deadDoc.toString()); } } //TESTED (2.1.2) else if (_diagnosticMode) { if (!needToFindAndModify) { System.out.println("StoreAndIndexManager.removeFromDatastore_byUrl(2): straight deleted " + doc.toDb().toString()); } else { System.out.println( "StoreAndIndexManager.removeFromDatastore_byUrl(2): didn't find " + query.toString()); } } //TESTED (2.1.2) }
From source file:com.ikanow.infinit.e.utility.MongoDocumentTxfer.java
License:Apache License
private void doTransfer(BasicDBObject query, int nSkip, int nLimit, boolean bAggregate, BasicDBObject chunk) throws IOException { PropertiesManager pm = new PropertiesManager(); int nMaxContentSize_bytes = pm.getMaxContentSize(); // Initialize the DB: DBCollection docsDB = DbManager.getDocument().getMetadata(); DBCollection contentDB = DbManager.getDocument().getContent(); DBCollection sourcesDB = DbManager.getIngest().getSource(); ElasticSearchManager.setDefaultClusterName("infinite-aws"); // 1. Get the documents from the DB (combining data + metadata and refreshing source meta) // (Ignore soft-deleted records:) if (null == query) { query = new BasicDBObject(); }//from w w w . j av a 2 s. com Object sourceKeyQueryTerm = query.remove(DocumentPojo.sourceKey_); if (null != sourceKeyQueryTerm) { if (query.toString() .contains(new StringBuffer('"').append(DocumentPojo.sourceKey_).append('"').toString())) { throw new RuntimeException( "Can't specify sourceKey as part of complex query term: " + query.toString()); } //TESTED (by hand, "{ \"sourceKey\": \"x\", \"$or\": [ { \"sourceKey\": \"x\" } ] }") if (sourceKeyQueryTerm instanceof String) { query.put(DocumentPojo.sourceKey_, SourcePojo.getDistributedKeyQueryTerm((String) sourceKeyQueryTerm)); } //TESTED (by hand, "{\"sourceKey\": \"feeds.arstechnica.com.arstechnica.index.11.2.\" }") else if (sourceKeyQueryTerm instanceof DBObject) { // find all the _sources_ matching this term, and convert to a big list including distribution BasicDBObject fields = new BasicDBObject(SourcePojo.key_, 1); fields.put(SourcePojo.highestDistributionFactorStored_, 1); DBCursor dbc = sourcesDB.find(new BasicDBObject(SourcePojo.key_, sourceKeyQueryTerm), fields); LinkedList<String> sourceKeys = new LinkedList<String>(); for (DBObject dbo : dbc) { String key = (String) dbo.get(SourcePojo.key_); Integer distributionFactor = (Integer) dbo.get(SourcePojo.highestDistributionFactorStored_); Collection<String> sourceKeysForSource = SourcePojo.getDistributedKeys(key, distributionFactor); sourceKeys.addAll(sourceKeysForSource); } query.put(DocumentPojo.sourceKey_, new BasicDBObject(DbManager.in_, sourceKeys)); } //TESTED (by hand, "{\"sourceKey\": { \"$gt\": \"dev.ikanow\" } }") else { throw new RuntimeException("Can't specify sourceKey as part of complex query term"); } //(actually not possible, just included here for mathematical completeness...) } else { if (query.toString() .contains(new StringBuffer('"').append(DocumentPojo.sourceKey_).append('"').toString())) { throw new RuntimeException("Can't specify sourceKey as part of complex query term"); } //TESTE (by hand, "{ \"$or\": [ { \"sourceKey\": \"x\" } ] }") // Optimize communityId into sourceKeys... if (null != query.get(DocumentPojo.communityId_)) { try { ObjectId commId = query.getObjectId(DocumentPojo.communityId_); BasicDBObject fields = new BasicDBObject(SourcePojo.key_, 1); fields.put(SourcePojo.highestDistributionFactorStored_, 1); DBCursor dbc = sourcesDB.find(new BasicDBObject(SourcePojo.communityIds_, commId), fields); LinkedList<String> sourceKeys = new LinkedList<String>(); int added = 0; for (DBObject dbo : dbc) { String key = (String) dbo.get(SourcePojo.key_); Integer distributionFactor = (Integer) dbo.get(SourcePojo.highestDistributionFactorStored_); Collection<String> sourceKeysForSource = SourcePojo.getDistributedKeys(key, distributionFactor); sourceKeys.addAll(sourceKeysForSource); added += sourceKeysForSource.size(); } query.put(DocumentPojo.sourceKey_, new BasicDBObject(DbManager.in_, sourceKeys)); System.out.println("(Optimized simple community query to " + added + " source key(s))"); } catch (Exception e) { //DEBUG //e.printStackTrace(); System.out.println("(Can't optimize complex community query: " + e.getMessage()); } } //TESTED (by hand - including distributed source version) } // Ignored delete objects Object urlQuery = query.get(DocumentPojo.url_); if (null == urlQuery) { query.put(DocumentPojo.url_, Pattern.compile("^[^?]")); // (ie nothing starting with ?) } //TESTED else if (urlQuery instanceof BasicDBObject) { ((BasicDBObject) urlQuery).append("$regex", "^[^?]"); } //TESTED //DEBUG //System.out.println("COMBINED QUERY= " + query.toString()); // If aggregating, kick off the background aggregation thread if (bAggregate) { EntityBackgroundAggregationManager.startThread(); AssociationBackgroundAggregationManager.startThread(); } //Debug: DBCursor dbc = null; dbc = docsDB.find(query); if (null != chunk) { if (chunk.containsField(DbManager.min_)) { dbc = dbc.addSpecial(DbManager.min_, chunk.get(DbManager.min_)); } if (chunk.containsField(DbManager.max_)) { dbc = dbc.addSpecial(DbManager.max_, chunk.get(DbManager.max_)); } } dbc = dbc.skip(nSkip).limit(nLimit).batchSize(1000); if (null == chunk) { int nCount = dbc.count() - nSkip; if (nCount < 0) nCount = 0; System.out.println( "Found " + nCount + " records to sync, process first " + (0 == nLimit ? nCount : nLimit)); if (0 == nCount) { // Nothing to do... return; } } byte[] storageArray = new byte[200000]; int nSynced = 0; LinkedList<DocumentPojo> docsToTransfer = new LinkedList<DocumentPojo>(); Map<ObjectId, LinkedList<DocumentPojo>> communityList = null; ObjectId currCommunityId = null; while (dbc.hasNext()) { BasicDBObject dbo = (BasicDBObject) dbc.next(); DocumentPojo doc = DocumentPojo.fromDb(dbo, DocumentPojo.class); String sDocIndex = doc.getIndex(); if (null == sDocIndex) { sDocIndex = "document_index"; } if ((null != _deletedIndex) && !_deletedIndex.contains(sDocIndex)) { _deletedIndex.add(sDocIndex); rebuildIndex(sDocIndex); try { // (Just in case the index requires some time to sort itself out) Thread.sleep(1000); } catch (InterruptedException e) { } } //Debug: //System.out.println("Getting content..." + feed.getTitle() + " / " + feed.getUrl()); // Get the content: if ((0 != nMaxContentSize_bytes) && StoreAndIndexManager.docHasExternalContent(doc.getUrl(), doc.getSourceUrl())) { BasicDBObject contentQ = new BasicDBObject(CompressedFullTextPojo.url_, doc.getUrl()); contentQ.put(CompressedFullTextPojo.sourceKey_, new BasicDBObject(MongoDbManager.in_, Arrays.asList(null, doc.getSourceKey()))); BasicDBObject fields = new BasicDBObject(CompressedFullTextPojo.gzip_content_, 1); fields.put(CompressedFullTextPojo.sourceKey_, 1); DBCursor dbcGzip = contentDB.find(contentQ, fields); while (dbcGzip.hasNext()) { BasicDBObject dboContent = (BasicDBObject) dbcGzip.next(); if (!dboContent.containsField(CompressedFullTextPojo.sourceKey_)) { // If this has another version then ignore this one... if (dbc.hasNext()) { continue; } //TESTED (by hand) } byte[] compressedData = ((byte[]) dboContent.get(CompressedFullTextPojo.gzip_content_)); ByteArrayInputStream in = new ByteArrayInputStream(compressedData); GZIPInputStream gzip = new GZIPInputStream(in); int nRead = 0; StringBuffer output = new StringBuffer(); while (nRead >= 0) { nRead = gzip.read(storageArray, 0, 200000); if (nRead > 0) { String s = new String(storageArray, 0, nRead, "UTF-8"); output.append(s); } } doc.setFullText(output.toString()); } } // (else document has full text already) // Get tags, if necessary: // Always overwrite tags - one of the reasons we might choose to migrate // Also may need source in order to support source index filtering SourcePojo src = _sourceCache.get(doc.getSourceKey()); if (null == src) { //TODO (INF-2265): handle search index settings in pipeline mode... (also didn't seem to work?) BasicDBObject srcDbo = (BasicDBObject) sourcesDB .findOne(new BasicDBObject(SourcePojo.key_, doc.getSourceKey())); if (null != srcDbo) { src = SourcePojo.fromDb(srcDbo, SourcePojo.class); if (null != src.getProcessingPipeline()) { try { // Set the index settings HarvestController hc = new HarvestController(); HarvestControllerPipeline hcPipe = new HarvestControllerPipeline(); hcPipe.extractSource_preProcessingPipeline(src, hc); } catch (Exception e) { //DEBUG e.printStackTrace(); } } //TESTED (by hand) _sourceCache.put(doc.getSourceKey(), src); } } doc.setTempSource(src); // (needed for source index filtering) if (null != src) { if (null != src.getTags()) { Set<String> tagsTidied = new TreeSet<String>(); for (String s : src.getTags()) { String ss = s.trim().toLowerCase(); tagsTidied.add(ss); } // May also want to write this back to the DB: //TODO (INF-2223): Handle append tags or not in the pipeline... if ((null == src.getAppendTagsToDocs()) || src.getAppendTagsToDocs()) { if ((null == doc.getTags()) || (doc.getTags().size() < tagsTidied.size())) { BasicDBObject updateQuery = new BasicDBObject(DocumentPojo.sourceKey_, doc.getRawSourceKey()); // (ie including the # if there is one) updateQuery.put(DocumentPojo._id_, doc.getId()); docsDB.update(updateQuery, new BasicDBObject(DbManager.addToSet_, new BasicDBObject(DocumentPojo.tags_, new BasicDBObject(DbManager.each_, tagsTidied)))); } doc.setTags(tagsTidied); // (just copy ptr across) } } } // 2. Update the index with the new document // (Optionally also update entity and assoc features) if (bAggregate) { if (null == currCommunityId) { currCommunityId = doc.getCommunityId(); } else if (!currCommunityId.equals(doc.getCommunityId())) { LinkedList<DocumentPojo> perCommunityDocList = null; if (null == communityList) { // (very first time we see > 1 community) communityList = new TreeMap<ObjectId, LinkedList<DocumentPojo>>(); perCommunityDocList = new LinkedList<DocumentPojo>(); perCommunityDocList.addAll(docsToTransfer); //(NOT including doc, this hasn't been added to docsToTransfer yet) communityList.put(currCommunityId, perCommunityDocList); } currCommunityId = doc.getCommunityId(); perCommunityDocList = communityList.get(currCommunityId); if (null == perCommunityDocList) { perCommunityDocList = new LinkedList<DocumentPojo>(); communityList.put(currCommunityId, perCommunityDocList); } perCommunityDocList.add(doc); } } //TESTED nSynced++; docsToTransfer.add(doc); if (0 == (nSynced % 10000)) { StoreAndIndexManager manager = new StoreAndIndexManager(); if (bAggregate) { // Loop over communities and aggregate each one then store the modified entities/assocs doAggregation(communityList, docsToTransfer); communityList = null; // (in case the next 10,000 docs are all in the same community!) currCommunityId = null; } //TOTEST manager.addToSearch(docsToTransfer); docsToTransfer.clear(); System.out.println("(Synced " + nSynced + " records)"); } } // (End loop over docs) // Sync remaining docs if (!docsToTransfer.isEmpty()) { if (bAggregate) { // Loop over communities and aggregate each one then store the modified entities/assocs doAggregation(communityList, docsToTransfer); } StoreAndIndexManager manager = new StoreAndIndexManager(); manager.addToSearch(docsToTransfer); } if (null != chunk) { System.out.println("Found " + nSynced + " records to sync in chunk"); } if (bAggregate) { System.out.println("Completed. You can hit CTRL+C at any time."); System.out.println( "By default it will keep running for 5 minutes while the background aggregation runs to update the documents' entities."); try { Thread.sleep(300000); } catch (InterruptedException e) { } // Turn off so we can exit EntityBackgroundAggregationManager.stopThreadAndWait(); AssociationBackgroundAggregationManager.stopThreadAndWait(); } }
From source file:com.ikanow.infinit.e.utility.MongoDocumentTxfer.java
License:Apache License
@SuppressWarnings("unused") private void doUnitTest(String sMongoDbHost, String sMongoDbPort, String sElasticHost, String sElasticPort, BasicDBObject query, int nLimit) { ElasticSearchManager elasticManager = null; try {//from w w w . j a v a 2 s.co m // Initialize the DB: DBCollection feedsDB = DbManager.getDocument().getMetadata(); DBCollection contentDB = DbManager.getDocument().getContent(); DBCollection sourcesDB = DbManager.getIngest().getSource(); String indexName = "document_index"; // Test/debug recreate the index if (true) { // (delete the index) System.out.println("Deleting index..."); elasticManager = ElasticSearchManager.getIndex(indexName, sElasticHost + ":" + sElasticPort); elasticManager.deleteMe(); //(also deletes the child index - same index, different type) // Create the index if necessary String sMapping = new Gson().toJson(new DocumentPojoIndexMap.Mapping(), DocumentPojoIndexMap.Mapping.class); Builder localSettings = ImmutableSettings.settingsBuilder(); localSettings.put("number_of_shards", 10).put("number_of_replicas", 2); System.out.println("Creating index..." + sMapping); elasticManager = ElasticSearchManager.createIndex(indexName, null, false, sElasticHost + ":" + sElasticPort, sMapping, localSettings); } // Get the index (necessary if already created) if (null == elasticManager) { elasticManager = ElasticSearchManager.getIndex(indexName, sElasticHost + ":" + sElasticPort); } // Get the feeds from the DB: //Debug: // System.out.println("Querying DB..."); DBCursor dbc = feedsDB.find(query).limit(nLimit); byte[] storageArray = new byte[200000]; while (dbc.hasNext()) { BasicDBObject dbo = (BasicDBObject) dbc.next(); DocumentPojo doc = DocumentPojo.fromDb(dbo, DocumentPojo.class); //Debug: System.out.println("Getting content..." + doc.getTitle() + " / " + doc.getUrl()); // Get the content: BasicDBObject contentQ = new BasicDBObject(CompressedFullTextPojo.url_, doc.getUrl()); contentQ.put(CompressedFullTextPojo.sourceKey_, new BasicDBObject(MongoDbManager.in_, Arrays.asList(null, doc.getSourceKey()))); BasicDBObject dboContent = (BasicDBObject) contentDB.findOne(contentQ); if (null != dboContent) { byte[] compressedData = ((byte[]) dboContent.get("gzip_content")); ByteArrayInputStream in = new ByteArrayInputStream(compressedData); GZIPInputStream gzip = new GZIPInputStream(in); int nRead = gzip.read(storageArray, 0, 200000); String s = new String(storageArray, 0, nRead, "UTF-8"); doc.setFullText(s); } // Get tag: SourcePojo src = _sourceCache.get(doc.getSourceKey()); if (null == src) { BasicDBObject srcDbo = (BasicDBObject) sourcesDB .findOne(new BasicDBObject("key", doc.getSourceKey())); if (null != srcDbo) { src = new Gson().fromJson(srcDbo.toString(), SourcePojo.class); _sourceCache.put(doc.getSourceKey(), src); } } if (null != src) { Set<String> tagsTidied = new TreeSet<String>(); for (String s : src.getTags()) { String ss = s.trim().toLowerCase(); tagsTidied.add(ss); } doc.setTags(tagsTidied); } //TEST: set dynamic field // Lots of testing of dynamic dates: // feed.addToMetadata("my_dateISO", Date.parse(feed.getCreated().toGMTString())); // String s1 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").format(feed.getCreated()); // feed.addToMetadata("another_dateISO", s1); // String s1_5 = new SimpleDateFormat().format(feed.getCreated()); // feed.addToMetadata("another_dateTimeJava", s1_5); // String s2 = new SimpleDateFormat("yyyyMMdd").format(feed.getCreated()); // feed.addToMetadata("another_dateYYYYMMDD", s2); // String s3 = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z").format(feed.getCreated()); // feed.addToMetadata("another_dateRFC822", s3); // feed.addToMetadata("another_dateGMT", feed.getCreated().toGMTString()); // // Testing of the string field types // feed.addToMetadata("my_comment", "Testing this ABCDEFG"); // feed.addToMetadata("my_term", "Testing this UVWXYZ"); // feed.addToMetadata("my_text", "Testing this 123456"); // // Test an array of longs: // Long tl[] = new Long[4]; tl[0] = 0L; tl[1] = 1L; tl[2] = 2L; tl[3] = 3L; // feed.addToMetadata("md_long", tl); //TEST: some dummy event timestamp adding code (not seeing much/any in the data) // if (null != feed.getEvents()) { // int i = 0; // for (EventPojo evt: feed.getEvents()) { // //1: Add single date // if (0 == i) { // evt.time_start = "2011-01-01"; // } // //2: Add short span // if (1 == i) { // evt.time_start = "2010-04-06"; // evt.time_end = "2010-08-09"; // } // //3: Add cross-yr span // if (2 == i) { // evt.time_start = "2012-06-05"; // evt.time_end = "2013-09-05"; // } // //4: Add too long span // if (3 == i) { // evt.time_start = "2012-04-06"; // evt.time_end = "2014-04-09"; // } // i++; // } // } // For event adding, see data_model.test.TestCode } } catch (IOException e) { e.printStackTrace(); } finally { //nothing to do } }
From source file:com.ikanow.infinit.e.utility.MongoEntityFeatureTxfer.java
License:Apache License
static void doDelete(BasicDBObject query, int nLimit, boolean automatedRequest) { try {//from w w w .j av a 2 s . c om // Initialize the DB: DBCollection entityFeatureDB = DbManager.getFeature().getEntity(); ElasticSearchManager elasticManager = ElasticSearchManager.getIndex("entity_index"); BasicDBObject fields = new BasicDBObject(); fields.put(EntityFeaturePojo.index_, 1); fields.put(EntityFeaturePojo.communityId_, 1); DBCursor cur = entityFeatureDB.find(query, fields).limit(nLimit); // (this internally works in batches of 1000) if (automatedRequest) { System.out.println("Found " + cur.count() + " records to delete from _id list"); } else { System.out.println("Found " + cur.count() + " records to delete from " + query.toString()); } if (nLimit > 0) { System.out.println("(limited to " + nLimit + " records)"); } int nArraySize = (cur.count() > 1000) ? 1000 : cur.count(); ArrayList<EntityFeaturePojo> batchList = new ArrayList<EntityFeaturePojo>(nArraySize); while (cur.hasNext()) { EntityFeaturePojo gp = EntityFeaturePojo.fromDb(cur.next(), EntityFeaturePojo.class); batchList.add(gp); if (batchList.size() >= nArraySize) { internalDelete(batchList, elasticManager); batchList.clear(); } } if (!batchList.isEmpty()) { internalDelete(batchList, elasticManager); } entityFeatureDB.remove(query); } catch (NumberFormatException e) { e.printStackTrace(); } catch (MongoException e) { e.printStackTrace(); } finally { } }
From source file:com.impetus.client.mongodb.MongoDBClient.java
License:Apache License
/** * Load query data./*from w ww .j a va2 s .co m*/ * * @param <E> * the element type * @param entityMetadata * the entity metadata * @param mongoQuery * the mongo query * @param orderBy * the order by * @param maxResult * the max result * @param firstResult * the first result * @param isCountQuery * the is count query * @param keys * the keys * @param results * the results * @return the list * @throws InstantiationException * the instantiation exception * @throws IllegalAccessException * the illegal access exception */ private <E> List<E> loadQueryData(EntityMetadata entityMetadata, BasicDBObject mongoQuery, BasicDBObject orderBy, int maxResult, int firstResult, boolean isCountQuery, BasicDBObject keys, String... results) throws InstantiationException, IllegalAccessException { String documentName = entityMetadata.getTableName(); List entities = new ArrayList<E>(); Object object = getDBCursorInstance(mongoQuery, orderBy, maxResult, firstResult, keys, documentName, isCountQuery); DBCursor cursor = null; if (object instanceof Long) { List<Long> lst = new ArrayList<Long>(); lst.add((Long) object); return (List<E>) lst; } else { cursor = (DBCursor) object; } if (results != null && results.length > 0) { DBCollection dbCollection = mongoDb.getCollection(documentName); KunderaCoreUtils.printQuery("Find document: " + mongoQuery, showQuery); for (int i = 1; i < results.length; i++) { String result = results[i]; // If User wants search on a column within a particular super // column, // fetch that embedded object collection only // otherwise retrieve whole entity // TODO: improve code if (result != null && result.indexOf(".") >= 0) { // TODO i need to discuss with Amresh before modifying it. entities.addAll(handler.getEmbeddedObjectList(dbCollection, entityMetadata, documentName, mongoQuery, result, orderBy, maxResult, firstResult, keys, kunderaMetadata)); return entities; } } } log.debug("Fetching data from " + documentName + " for Filter " + mongoQuery.toString()); while (cursor.hasNext()) { DBObject fetchedDocument = cursor.next(); populateEntity(entityMetadata, entities, fetchedDocument); } return entities; }
From source file:com.intuit.utils.PopulateTweets.java
public static void main(String[] args) { Date now = new Date(); System.out.println("Current date is: " + now.toString()); MongoClient mongo = new MongoClient("localhost", 27017); DB db = mongo.getDB("tweetsdb"); DBCollection collection = db.getCollection("tweetscollection"); WriteResult result = collection.remove(new BasicDBObject()); String[] users = { "user1", "user2", "user3", "user4", "user5", "user6", "user7", "user8", "user9", "user10" }; // I am not introducing enough randomness in terms of the insertion of // tweets for users at a random time orders, due to lack of time. for (String user : users) { int tweetIndex = 0; for (int i = 1; i <= 10; i++) { BasicDBObject document = new BasicDBObject(); // This is a way to maintain uniqueness of the tweetid value across the system // Ideally, this should be the "_id" value, but due to lack of time, I am skipping // that part. That would help to partition the tweets across multiple shards in a // large scale system. String tweetId = user + "|tweet" + tweetIndex; document.put("tweetId", tweetId); document.put("user", user); document.put("text", "tweet number" + tweetIndex); document.put("tweetedOn", new Date().toString()); System.out.println("tweet number: " + tweetIndex + " " + document.toString()); collection.insert(document); tweetIndex++;/* w ww .ja va 2s . co m*/ try { // Just introducing some delay between tweets to make the testing a bit easy Thread.sleep(3000); } catch (InterruptedException ex) { Logger.getLogger(PopulateTweets.class.getName()).log(Level.SEVERE, null, ex); } } } BasicDBObject indexObj = new BasicDBObject(); indexObj.put("user", 1); indexObj.put("tweetedOn", -1); collection.createIndex(indexObj); BasicDBObject tweetIdObj = new BasicDBObject(); tweetIdObj.put("tweetId", 1); collection.createIndex(tweetIdObj); }
From source file:com.owly.srv.RemoteBasicStatMongoDAOImpl.java
License:Apache License
public void insertRemoteBasicStat(RemoteBasicStat remoteBasicStat) { getLogger().info("MONGODB : Insert Statistic in DataBase"); BasicDBObject obj = new BasicDBObject("StatDate", remoteBasicStat.getDateStatServerDate()) .append("ServerDate", remoteBasicStat.getDateRmtServerDate()) .append("StatType", remoteBasicStat.getTypeOfStat()) .append("NameServer", remoteBasicStat.getNameServer()) .append("Hostname", remoteBasicStat.getHostname()).append("ServerIP", remoteBasicStat.getIpServer()) .append("ServerType", remoteBasicStat.getTypeServer()) .append("Metrics", remoteBasicStat.getMetrics()) .append("MetricType", remoteBasicStat.getMetricType()); getLogger().info("MONGODB : Data to insert : " + obj.toString()); remoteStatCollection.insert(obj);//from w ww . j a v a 2 s . co m getLogger().info("MONGODB :Statistic inserted"); }
From source file:com.owly.srv.RemoteBasicStatMongoDAOImpl.java
License:Apache License
public List<String> getTypeStatFromServer(ArrayList<ShortRemoteServerId> listShortRemoteServerId, Integer minutesToCheck) { getLogger().info("MONGODB : get info of type of stats for an specific server"); GregorianCalendar cal = new GregorianCalendar(); Date nowDate = cal.getTime(); cal.add(Calendar.MINUTE, -minutesToCheck); Date beforeDate = cal.getTime(); SimpleDateFormat dtformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); getLogger().info("MONGODB : Get a metric in a period of time starting : " + dtformat.format(beforeDate) + " and ending : " + dtformat.format(nowDate)); List<String> typeStat = new ArrayList<String>(); //Varaibles to generat a query with or clause BasicDBObject query = new BasicDBObject(); BasicDBList or = new BasicDBList(); //Check all severs that arrive to the method and generate the or clause Iterator<ShortRemoteServerId> iterator = listShortRemoteServerId.iterator(); while (iterator.hasNext()) { ShortRemoteServerId shortRmt = iterator.next(); BasicDBObject clause = new BasicDBObject("ServerIP", shortRmt.getNodeIPAddress()).append("NameServer", shortRmt.getName());//from ww w . j av a2s .co m or.add(clause); } query = new BasicDBObject("$or", or).append("StatDate", new BasicDBObject("$gt", beforeDate)); getLogger().info("MONGODB : Get type of stats in the database for this clause" + query.toString()); typeStat = remoteStatCollection.distinct("StatType", query); getLogger().info("MONGODB : List obtained"); return typeStat; }
From source file:com.owly.srv.RemoteBasicStatMongoDAOImpl.java
License:Apache License
public Set<String> getTypeMetricFromStat(ArrayList<ShortRemoteServerId> listShortRemoteServerId, String typeOfStat, Integer minutesToCheck) { getLogger().info("MONGODB : get info of type of metrcis for an specific server and specific stat"); GregorianCalendar cal = new GregorianCalendar(); Date nowDate = cal.getTime(); cal.add(Calendar.MINUTE, -minutesToCheck); Date beforeDate = cal.getTime(); SimpleDateFormat dtformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); getLogger().info("MONGODB : Get a metric in a period of time starting : " + dtformat.format(beforeDate) + " and ending : " + dtformat.format(nowDate)); //Varaibles to generat a query with or clause BasicDBObject query = new BasicDBObject(); BasicDBList or = new BasicDBList(); //Check all severs that arrive to the method and generate the or clause Iterator<ShortRemoteServerId> iterator = listShortRemoteServerId.iterator(); while (iterator.hasNext()) { ShortRemoteServerId shortRmt = iterator.next(); BasicDBObject clause = new BasicDBObject("ServerIP", shortRmt.getNodeIPAddress()).append("NameServer", shortRmt.getName());// w w w . j av a 2 s. co m or.add(clause); } query = new BasicDBObject("$or", or).append("StatType", typeOfStat).append("StatDate", new BasicDBObject("$gt", beforeDate)); getLogger().info("MONGODB : Get type of stats in the database for this clause" + query.toString()); //logger.debug("MONGODB : query ="+query.toString()); //Create the selector BasicDBObject obj = new BasicDBObject("Metrics", true).append("_id", false); //Create the ordeby BasicDBObject orderBy = new BasicDBObject("StatDate", -1); DBObject res = remoteStatCollection.findOne(query, obj, orderBy); Set<String> typeMetric = (((BSONObject) res.get("Metrics")).toMap()).keySet(); //logger.debug("MONGODB : query result"+typeMetric.toString()); getLogger().info("MONGODB : List obtained"); return typeMetric; }
From source file:com.owly.srv.RemoteBasicStatMongoDAOImpl.java
License:Apache License
public ArrayList<BasicStat> getMetricDetails(ShortRemoteServerId shortRemoteServer, String typeOfStat, String typeOfMetric, Integer minutesToCheck) { getLogger().info("MONGODB : Get a metric in a period of time vbased in minutes"); GregorianCalendar cal = new GregorianCalendar(); Date nowDate = cal.getTime(); cal.add(Calendar.MINUTE, -minutesToCheck); Date beforeDate = cal.getTime(); SimpleDateFormat dtformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); getLogger().info("MONGODB : Get a metric in a period of time starting : " + dtformat.format(beforeDate) + " and ending : " + dtformat.format(nowDate)); //Create a List BasicStat Object ArrayList<BasicStat> ListMetrics = new ArrayList<BasicStat>(); //Create the query BasicDBObject query = new BasicDBObject("NameServer", shortRemoteServer.getName()) .append("ServerIP", shortRemoteServer.getNodeIPAddress()).append("StatType", typeOfStat) .append("StatDate", new BasicDBObject("$gt", beforeDate).append("$lt", nowDate)); getLogger().debug("MONGODB : query =" + query.toString()); //Create the selector BasicDBObject obj = new BasicDBObject("Metrics." + typeOfMetric, true).append("StatDate", true) .append("_id", false); getLogger().debug("MONGODB : selector =" + obj.toString()); //Create the ordeby BasicDBObject orderBy = new BasicDBObject("StatDate", -1); DBCursor cursor = remoteStatCollection.find(query, obj).sort(orderBy); while (cursor.hasNext()) { DBObject cur = cursor.next();//w w w . jav a 2s . co m BasicStat basicStat = new BasicStat(); //logger.debug("MONGODB : query result"+cur.toString()); //logger.debug("MONGODB : query result"+((BSONObject)cur.get("Metrics")).get(typeOfMetric)); basicStat.setValue((String) ((BSONObject) cur.get("Metrics")).get(typeOfMetric).toString()); basicStat.setDate((Date) cur.get("StatDate")); // Add this object into the list. ListMetrics.add(basicStat); } getLogger().debug("MONGODB : List obtained" + ListMetrics.toString()); cursor.close(); getLogger().info("MONGODB : List obtained"); return ListMetrics; }