List of usage examples for com.mongodb.util JSON parse
public static Object parse(final String jsonString)
Parses a JSON string and returns a corresponding Java object.
From source file:fr.eolya.utils.nosql.mongodb.MongoDBHelper.java
License:Apache License
public static BasicDBObject JSON2BasicDBObject(String json) { return (BasicDBObject) JSON.parse(json); }
From source file:fr.gouv.vitam.cases.DbRequest.java
License:Open Source License
private final ResultInterface getRequestDomain(final TypeRequest request, final ResultInterface previous, final boolean useStart) throws InvalidExecOperationException, InstantiationException, IllegalAccessException { // must be MD if (request.isOnlyES) { throw new InvalidExecOperationException("Expression is not valid for Domain"); }//from www .jav a 2 s.c om if (request.requestModel == null) { throw new InvalidExecOperationException( "Expression is not valid for Domain since no Request is available"); } final String srequest = request.requestModel.toString(); final BasicDBObject condition = (BasicDBObject) JSON.parse(srequest); final ResultInterface newResult = CassandraAccess.createOneResult(); newResult.setMinLevel(1); newResult.setMaxLevel(1); if (simulate) { LOGGER.info("ReqDomain: {}\n\t{}", condition, ID_NBCHILD); return createFalseResult(null, 1); } LOGGER.debug("ReqDomain: {}\n\t{}", condition, ID_NBCHILD); if (GlobalDatas.PRINT_REQUEST) { LOGGER.warn("ReqDomain: {}\n\t{}", condition, ID_NBCHILD); } final DBCursor cursor = mdAccess.find(mdAccess.domains, condition, ID_NBCHILD); long tempCount = 0; while (cursor.hasNext()) { final Domain dom = (Domain) cursor.next(); final String mid = dom.getId(); if (useStart) { if (previous.getCurrentDaip().contains(mid)) { newResult.getCurrentDaip().add(mid); tempCount += dom.getLong(Domain.NBCHILD); } } else { newResult.getCurrentDaip().add(mid); tempCount += dom.getLong(Domain.NBCHILD); } } cursor.close(); newResult.setNbSubNodes(tempCount); // filter on Ancestor if (!useStart && !previous.checkAncestor(mdAccess, newResult)) { LOGGER.error("No ancestor"); return null; } // Compute of MinMax if valid since path = 1 length (root) newResult.updateMinMax(); if (GlobalDatas.PRINT_REQUEST) { newResult.putBeforeSave(); LOGGER.warn("Dom: {}", newResult); } return newResult; }
From source file:fr.gouv.vitam.cases.DbRequest.java
License:Open Source License
private final ResultInterface getRequest1LevelMaipFromMD(final TypeRequest request, final ResultInterface previous, final boolean useStart) throws InvalidExecOperationException, InstantiationException, IllegalAccessException { BasicDBObject query = null;/*from ww w . j a va2 s .c o m*/ if (request.requestModel == null) { throw new InvalidExecOperationException( "Expression is not valid for Daip Level 1 with MD only since no MD request is available"); } if (useStart) { query = getInClauseForField(DAip.ID, previous.getCurrentDaip()); } else { if (previous.getMinLevel() == 1) { query = getInClauseForField(CassandraAccess.VitamLinks.Domain2DAip.field2to1, previous.getCurrentDaip()); } else { query = getInClauseForField(CassandraAccess.VitamLinks.DAip2DAip.field2to1, previous.getCurrentDaip()); } } final String srequest = request.requestModel.toString(); final BasicDBObject condition = (BasicDBObject) JSON.parse(srequest); query.putAll((BSONObject) condition); final ResultInterface subresult = CassandraAccess.createOneResult(); if (simulate) { LOGGER.info("Req1LevelMD: {}", query); return createFalseResult(previous, 1); } LOGGER.debug("Req1LevelMD: {}", query); if (GlobalDatas.PRINT_REQUEST) { LOGGER.warn("Req1LevelMD: {}", query); } final DBCursor cursor = mdAccess.find(mdAccess.daips, query, ID_NBCHILD); long tempCount = 0; while (cursor.hasNext()) { final DAip maip = (DAip) cursor.next(); final String mid = maip.getId(); if (useStart) { if (previous.getCurrentDaip().contains(mid)) { subresult.getCurrentDaip().add(mid); tempCount += maip.getLong(Domain.NBCHILD); } } else { subresult.getCurrentDaip().add(mid); tempCount += maip.getLong(Domain.NBCHILD); } } cursor.close(); subresult.setNbSubNodes(tempCount); // filter on Ancestor if (!useStart && !previous.checkAncestor(mdAccess, subresult)) { LOGGER.error("No ancestor for " + query + "\n" + previous.getCurrentDaip() + " not in " + subresult.getCurrentDaip()); return null; } // Not updateMinMax since result is not "valid" path but node UUID and not needed subresult.setMinLevel(previous.getMinLevel() + 1); subresult.setMaxLevel(previous.getMaxLevel() + 1); if (GlobalDatas.PRINT_REQUEST) { subresult.putBeforeSave(); LOGGER.warn("MetaAip2: {}", subresult); } return subresult; }
From source file:fr.gouv.vitam.cases.DbRequest.java
License:Open Source License
private final ResultInterface getRequestNegativeRelativeDepthFromMD(final TypeRequest request, final ResultInterface previous, final boolean useStart) throws InvalidExecOperationException, InstantiationException, IllegalAccessException { BasicDBObject query = null;//from ww w .j a v a 2s .c o m if (request.requestModel == null) { throw new InvalidExecOperationException("Expression is not valid for Daip Level " + request.relativedepth + " with MD only since no MD request is available"); } if (useStart) { throw new InvalidExecOperationException("Cannot make a negative path when starting up"); } if (simulate) { LOGGER.info("Req-xLevelMD"); return createFalseResult(previous, 1); } int distance = -request.relativedepth; Set<String> subset = new HashSet<String>(); for (String prev : previous.getCurrentDaip()) { DAip dprev = DAip.findOne(mdAccess, prev); Map<String, Integer> parents = dprev.getDomDepth(); for (Entry<String, Integer> elt : parents.entrySet()) { if (elt.getValue() == distance) { subset.add(elt.getKey()); } } } // Use ID and not graph dependencies query = getInClauseForField(DAip.ID, subset); final String srequest = request.requestModel.toString(); final BasicDBObject condition = (BasicDBObject) JSON.parse(srequest); query.putAll((BSONObject) condition); final ResultInterface subresult = CassandraAccess.createOneResult(); LOGGER.debug("Req-xLevelMD: {}", query); if (GlobalDatas.PRINT_REQUEST) { LOGGER.warn("Req-xLevelMD: {}", query); } final DBCursor cursor = mdAccess.find(mdAccess.daips, query, ID_NBCHILD); long tempCount = 0; subresult.setMinLevel(previous.getMaxLevel()); subresult.setMaxLevel(0); while (cursor.hasNext()) { final DAip maip = (DAip) cursor.next(); final String mid = maip.getId(); subresult.getCurrentDaip().add(mid); maip.load(mdAccess); tempCount += maip.getLong(DAip.NBCHILD); // Not updateMinMax since result is not "valid" path but node UUID and not needed int max = maip.getMaxDepth(); if (subresult.getMaxLevel() < max) { subresult.setMaxLevel(max); } if (subresult.getMinLevel() > max) { subresult.setMinLevel(max); } } cursor.close(); subresult.setNbSubNodes(tempCount); if (GlobalDatas.PRINT_REQUEST) { subresult.putBeforeSave(); LOGGER.warn("MetaAip2: {}", subresult); } return subresult; }
From source file:fr.gouv.vitam.cbes.MainCouchbaseImport.java
License:Open Source License
/** * @param args 192.168.56.110 VitamLinks file *//*ww w. jav a2 s . c om*/ @SuppressWarnings("unchecked") public static void main(String[] args) { VitamLoggerFactory.setDefaultFactory(new LogbackLoggerFactory(VitamLogLevel.WARN)); LOGGER = VitamLoggerFactory.getInstance(MainIngestFile.class); if (args.length < 3) { LOGGER.error("Need hostname bucketName"); return; } String host = args[0]; String bucketname = args[1]; System.setProperty("com.couchbase.client.queryEnabled", "true"); CouchbaseCluster cluster = new CouchbaseCluster(host); try { Bucket bucket = cluster.openBucket(bucketname).toBlockingObservable().single(); if (bucket == null) { LOGGER.error("no bucket"); return; } String ingest = args[2]; final long date11 = System.currentTimeMillis(); long nb = 0; System.out.println("Starting: "); try { final FileInputStream fstream = new FileInputStream(ingest); final DataInputStream in = new DataInputStream(fstream); final BufferedReader br = new BufferedReader(new InputStreamReader(in)); String strLine; JsonObject content = JsonObject.empty(); Map<String, Object> map = content.toMap(); String id = null; // Read File Line By Line //List<Observable<JsonDocument>> list = new ArrayList<Observable<JsonDocument>>(500); Queue<Observable<JsonDocument>> queue = new LinkedList<Observable<JsonDocument>>(); while ((strLine = br.readLine()) != null) { final BSONObject bson = (BSONObject) JSON.parse(strLine); map.clear(); map.putAll(bson.toMap()); id = (String) bson.get("_id"); JsonDocument doc = JsonDocument.create(id, content); //bucket.upsert(doc); nb++; /* if (nb % 500 == 0) { for (Observable<JsonDocument> observable : list) { observable.toBlockingObservable().single(); } list.clear(); bucket.insert(doc).toBlockingObservable().single(); if (nb % 10000 == 0) { System.out.print("."); } } else { //bucket.insert(doc).toBlockingObservable().single(); list.add(bucket.insert(doc)); } */ if (nb > 500) { Observable<JsonDocument> observable = queue.poll(); if (observable != null) { observable.toBlockingObservable().single(); } queue.add(bucket.insert(doc)); if (nb % 10000 == 0) { System.out.print("."); } } else { queue.add(bucket.insert(doc)); } } /* for (Observable<JsonDocument> observable : list) { observable.toBlockingObservable().single(); } list.clear(); */ Observable<JsonDocument> observable = queue.poll(); while (observable != null) { observable.toBlockingObservable().single(); observable = queue.poll(); } // Close the input stream in.close(); } catch (IOException e) { LOGGER.error("IO", e); } final long date12 = System.currentTimeMillis(); System.out.println("\nEnd"); LOGGER.warn("IngestCouchBase: " + nb + " in " + (date12 - date11) + " = " + ((date12 - date11) / ((double) nb))); // now try requests executeQuery(bucket, "CREATE INDEX daip2daip_up ON VitamLinks(_up)"); final long date13 = System.currentTimeMillis(); executeQuery(bucket, "CREATE INDEX domain2daip_doms ON VitamLinks(_doms)"); final long date14 = System.currentTimeMillis(); executeQuery(bucket, "CREATE INDEX daip2paip_paip ON VitamLinks(_paip)"); final long date15 = System.currentTimeMillis(); executeQuery(bucket, "CREATE INDEX daip2dua_dua ON VitamLinks(_dua)"); final long date16 = System.currentTimeMillis(); executeQuery(bucket, "CREATE INDEX daip_dds ON VitamLinks(_dds)"); final long date17 = System.currentTimeMillis(); LOGGER.warn("SelectCouchBase: " + (date13 - date12) + ":" + (date14 - date13) + ":" + (date15 - date14) + ":" + (date16 - date15) + ":" + (date17 - date16)); } catch (Exception e) { LOGGER.error("All", e); } cluster.disconnect(); LOGGER.warn("Disconnected"); }
From source file:fr.gouv.vitam.mdbes.DbRequest.java
License:Open Source License
private final ResultInterface getRequestDomain(final TypeRequest request, final ResultInterface previous, final boolean useStart) throws InvalidExecOperationException, InstantiationException, IllegalAccessException { // must be MD if (request.isOnlyES) { throw new InvalidExecOperationException("Expression is not valid for Domain"); }//w w w .ja v a 2 s. co m if (request.requestModel == null) { throw new InvalidExecOperationException( "Expression is not valid for Domain since no Request is available"); } final String srequest = request.requestModel.toString(); final BasicDBObject condition = (BasicDBObject) JSON.parse(srequest); final ResultInterface newResult = MongoDbAccess.createOneResult(); newResult.setMinLevel(1); newResult.setMaxLevel(1); if (simulate) { LOGGER.info("ReqDomain: {}\n\t{}", condition, ID_NBCHILD); return createFalseResult(null, 1); } LOGGER.debug("ReqDomain: {}\n\t{}", condition, ID_NBCHILD); if (GlobalDatas.PRINT_REQUEST) { LOGGER.warn("ReqDomain: {}\n\t{}", condition, ID_NBCHILD); } final DBCursor cursor = mdAccess.find(mdAccess.domains, condition, ID_NBCHILD); long tempCount = 0; while (cursor.hasNext()) { final Domain dom = (Domain) cursor.next(); final String mid = dom.getId(); if (useStart) { if (previous.getCurrentDaip().contains(mid)) { newResult.getCurrentDaip().add(mid); tempCount += dom.getLong(Domain.NBCHILD); } } else { newResult.getCurrentDaip().add(mid); tempCount += dom.getLong(Domain.NBCHILD); } } cursor.close(); newResult.setNbSubNodes(tempCount); // filter on Ancestor if (!useStart && !previous.checkAncestor(mdAccess, newResult)) { LOGGER.error("No ancestor"); return null; } // Compute of MinMax if valid since path = 1 length (root) newResult.updateMinMax(); if (GlobalDatas.PRINT_REQUEST) { newResult.putBeforeSave(); LOGGER.warn("Dom: {}", newResult); } return newResult; }
From source file:fr.gouv.vitam.mdbes.DbRequest.java
License:Open Source License
private final ResultInterface getRequest1LevelMaipFromMD(final TypeRequest request, final ResultInterface previous, final boolean useStart) throws InvalidExecOperationException, InstantiationException, IllegalAccessException { BasicDBObject query = null;// w w w . ja va 2s. c om if (request.requestModel == null) { throw new InvalidExecOperationException( "Expression is not valid for Daip Level 1 with MD only since no MD request is available"); } if (useStart) { query = getInClauseForField(DAip.ID, previous.getCurrentDaip()); } else { if (previous.getMinLevel() == 1) { query = getInClauseForField(MongoDbAccess.VitamLinks.Domain2DAip.field2to1, previous.getCurrentDaip()); } else { query = getInClauseForField(MongoDbAccess.VitamLinks.DAip2DAip.field2to1, previous.getCurrentDaip()); } } final String srequest = request.requestModel.toString(); final BasicDBObject condition = (BasicDBObject) JSON.parse(srequest); query.putAll((BSONObject) condition); final ResultInterface subresult = MongoDbAccess.createOneResult(); if (simulate) { LOGGER.info("Req1LevelMD: {}", query); return createFalseResult(previous, 1); } LOGGER.debug("Req1LevelMD: {}", query); if (GlobalDatas.PRINT_REQUEST) { LOGGER.warn("Req1LevelMD: {}", query); } final DBCursor cursor = mdAccess.find(mdAccess.daips, query, ID_NBCHILD); long tempCount = 0; while (cursor.hasNext()) { final DAip maip = (DAip) cursor.next(); final String mid = maip.getId(); if (useStart) { if (previous.getCurrentDaip().contains(mid)) { subresult.getCurrentDaip().add(mid); tempCount += maip.getLong(Domain.NBCHILD); } } else { subresult.getCurrentDaip().add(mid); tempCount += maip.getLong(Domain.NBCHILD); } } cursor.close(); subresult.setNbSubNodes(tempCount); // filter on Ancestor if (!useStart && !previous.checkAncestor(mdAccess, subresult)) { LOGGER.error("No ancestor for " + query + "\n" + previous.getCurrentDaip() + " not in " + subresult.getCurrentDaip()); return null; } // Not updateMinMax since result is not "valid" path but node UUID and not needed subresult.setMinLevel(previous.getMinLevel() + 1); subresult.setMaxLevel(previous.getMaxLevel() + 1); if (GlobalDatas.PRINT_REQUEST) { subresult.putBeforeSave(); LOGGER.warn("MetaAip2: {}", subresult); } return subresult; }
From source file:fr.gouv.vitam.mdbes.DbRequest.java
License:Open Source License
private final ResultInterface getRequestNegativeRelativeDepthFromMD(final TypeRequest request, final ResultInterface previous, final boolean useStart) throws InvalidExecOperationException, InstantiationException, IllegalAccessException { BasicDBObject query = null;/* www .jav a2 s . c o m*/ if (request.requestModel == null) { throw new InvalidExecOperationException("Expression is not valid for Daip Level " + request.relativedepth + " with MD only since no MD request is available"); } if (useStart) { throw new InvalidExecOperationException("Cannot make a negative path when starting up"); } if (simulate) { LOGGER.info("Req-xLevelMD"); return createFalseResult(previous, 1); } int distance = -request.relativedepth; Set<String> subset = new HashSet<String>(); for (String prev : previous.getCurrentDaip()) { DAip dprev = DAip.findOne(mdAccess, prev); Map<String, Integer> parents = dprev.getDomDepth(); for (Entry<String, Integer> elt : parents.entrySet()) { if (elt.getValue() == distance) { subset.add(elt.getKey()); } } } // Use ID and not graph dependencies query = getInClauseForField(DAip.ID, subset); final String srequest = request.requestModel.toString(); final BasicDBObject condition = (BasicDBObject) JSON.parse(srequest); query.putAll((BSONObject) condition); final ResultInterface subresult = MongoDbAccess.createOneResult(); LOGGER.debug("Req-xLevelMD: {}", query); if (GlobalDatas.PRINT_REQUEST) { LOGGER.warn("Req-xLevelMD: {}", query); } final DBCursor cursor = mdAccess.find(mdAccess.daips, query, ID_NBCHILD); long tempCount = 0; subresult.setMinLevel(previous.getMaxLevel()); subresult.setMaxLevel(0); while (cursor.hasNext()) { final DAip maip = (DAip) cursor.next(); final String mid = maip.getId(); subresult.getCurrentDaip().add(mid); maip.load(mdAccess); tempCount += maip.getLong(DAip.NBCHILD); // Not updateMinMax since result is not "valid" path but node UUID and not needed int max = maip.getMaxDepth(); if (subresult.getMaxLevel() < max) { subresult.setMaxLevel(max); } if (subresult.getMinLevel() > max) { subresult.setMinLevel(max); } } cursor.close(); subresult.setNbSubNodes(tempCount); if (GlobalDatas.PRINT_REQUEST) { subresult.putBeforeSave(); LOGGER.warn("MetaAip2: {}", subresult); } return subresult; }
From source file:fr.gouv.vitam.mdbes.MainIngestESFromFile.java
License:Open Source License
private static final void runOnce(final MongoDbAccess dbvitam, final String file, boolean insert) throws InterruptedException, InstantiationException, IllegalAccessException, IOException { System.out.println("Load starting... "); final long date11 = System.currentTimeMillis(); final HashMap<String, String> esIndex = new HashMap<>(); final FileInputStream fstream = new FileInputStream(file); final DataInputStream in = new DataInputStream(fstream); final BufferedReader br = new BufferedReader(new InputStreamReader(in)); String strLine;//from w w w. j a v a 2 s. co m // Read File Line By Line while ((strLine = br.readLine()) != null) { final BSONObject bson = (BSONObject) JSON.parse(strLine); if (insert) { final int nbEs = ElasticSearchAccess.addEsIndex(dbvitam, model, esIndex, bson); MainIngestFile.cptMaip.addAndGet(nbEs); } else { ElasticSearchAccess.addEsIndex(dbvitam, model, bson); MainIngestFile.cptMaip.incrementAndGet(); } } // Close the input stream br.close(); in.close(); fstream.close(); if (!esIndex.isEmpty()) { MainIngestFile.cptMaip.addAndGet(esIndex.size()); System.out.println("Last bulk ES"); dbvitam.addEsEntryIndex(true, esIndex, model); esIndex.clear(); } final long date12 = System.currentTimeMillis(); MainIngestESFromFile.loadt.addAndGet(date12 - date11); System.out.println("Load ended"); /* * System.out.println("All elements\n================================================================"); * DbVitam.printStructure(dbvitam); */ final long nbBigM = dbvitam.getDaipSize(); final long nbBigD = dbvitam.getPaipSize(); System.out.println("\n Big Test (" + MainIngestFile.nbThread + " Threads chacune " + MainIngestFile.nb + " itrations de load, nb MAIP: " + MainIngestFile.cptMaip.get() + ") with MAIP: " + nbBigM + " DATA: " + nbBigD + " => Load:" + (MainIngestESFromFile.loadt.get()) / ((float) MainIngestFile.cptMaip.get())); System.out.println("\nThread;nbLoad;nbTotal;Load"); System.out.println(MainIngestFile.nbThread + ";" + MainIngestFile.cptMaip.get() + ";" + nbBigM + ";" + (MainIngestESFromFile.loadt.get()) / ((float) MainIngestFile.cptMaip.get())); }
From source file:fr.gouv.vitam.mdbes.MainIngestFile.java
License:Open Source License
private static final void runOnce(final MongoDbAccess dbvitam) throws InterruptedException, InstantiationException, IllegalAccessException, IOException { MainIngestFile ingests = null;/*from w w w . j a v a 2 s.co m*/ nb = nb / nbThread; int step = startFrom; final int interval = nb; LOGGER.warn("Load starting... " + nbThread + ":" + nb + ":" + interval); for (int i = 0; i < nbThread; i++) { ingests = new MainIngestFile(); ingests.start = step; ingests.stop = step + interval - 1; step += interval; ingests.run(); } System.out.println("Load ended"); final FileOutputStream outputStream = new FileOutputStream(fileout); final Map<String, String> esIndex = new HashMap<String, String>(); for (DAip daip : ParserIngest.savedDaips.values()) { daip.load(dbvitam); daip.toFile(outputStream); final BSONObject bson = (BSONObject) JSON.parse(daip.toStringDirect()); ElasticSearchAccess.addEsIndex(dbvitam, model, esIndex, bson); } outputStream.close(); if (!esIndex.isEmpty()) { MainIngestFile.cptMaip.addAndGet(esIndex.size()); System.out.println("Last bulk ES"); dbvitam.addEsEntryIndex(true, esIndex, model); esIndex.clear(); } /* * System.out.println("All elements\n================================================================"); * DbVitam.printStructure(dbvitam); */ final long nbBigM = dbvitam.getDaipSize(); final long nbBigD = dbvitam.getPaipSize(); System.out.println("\n Big Test (" + nbThread + " Threads chacune " + nb + " itrations de load, nb MAIP: " + cptMaip.get() + ") with MAIP: " + nbBigM + " DATA: " + nbBigD + " => Load:" + (loadt.get()) / ((float) cptMaip.get())); System.out.println("\nThread;nbLoad;nbTotal;Load"); System.out.println( nbThread + ";" + cptMaip.get() + ";" + nbBigM + ";" + (loadt.get()) / ((float) cptMaip.get())); }