List of usage examples for com.mongodb MapReduceOutput results
@SuppressWarnings("unchecked") public Iterable<DBObject> results()
From source file:com.petpet.c3po.dao.mongo.MongoPersistenceLayer.java
License:Apache License
public DBObject mapReduce(int key, String property, Filter filter, List<Integer> bins) { LOG.debug("Starting mapReduce for the following property: {}", property); long start = System.currentTimeMillis(); Property prop = getCache().getProperty(property); String propType = prop.getType(); String map = ""; String map2 = ""; String reduce = ""; if (propType.equals(PropertyType.STRING.toString()) || propType.equals(PropertyType.BOOL.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " for (mr in this.metadata){\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property)\n" + " {\n" + " if (metadataRecord.status == 'CONFLICT'){\n" + " emit({\n" + " property: property,\n" + " value: 'CONFLICT'\n" + " }, 1);\n" + " } else {\n" + " emit({\n" + " property: property,\n" + " value: metadataRecord.sourcedValues[0].value\n" + " }, 1);\n" + "\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}"; reduce = "function reduce(key, values) {\n" + " var res = 0;\n" + " values.forEach(function(v) {\n" + " res += v;\n" + " });\n" + " return res;\n" + "}"; } else if (propType.equals(PropertyType.INTEGER.toString()) || propType.equals(PropertyType.FLOAT.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " thresholds = " + getBinThresholds(bins) + ";\n" + " for (mr in this.metadata){\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property){\n" + " if (metadataRecord.status == 'CONFLICT'){\n" + " emit({\n" + " property: property,\n" + " value: 'CONFLICT'\n" + " }, 1);\n" + " } else {\n" + " var val=metadataRecord.sourcedValues[0].value;\n" + " var skipped=false;\n" + " if (thresholds.length > 0)\n" + " for (t in thresholds){\n" + " threshold = thresholds[t]; \n" + " if (val>=threshold[0] && val<=threshold[1]){\n" + " emit({\n" + " property: property,\n" + " value: threshold[0]+'-'+threshold[1]\n" + " }, 1);\n" + " skipped=true;\n" + " break;\n" + " }\n" + " }\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}"; reduce = "function reduce(key, values) {\n" + " var res = 0;\n" + " values.forEach(function(v) {\n" + " res += v;\n" + " });\n" + " return res;\n" + "}"; } else if (propType.equals(PropertyType.DATE.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " for (mr in this.metadata){\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property){\n" + " if (metadataRecord.status == 'CONFLICT'){\n" + " emit({\n" + " property: property,\n" + " value: 'CONFLICT'\n" + " }, 1);\n" + " } else {\n" + " var date = new Date(metadataRecord.sourcedValues[0].value);\n" + " var val=date.getFullYear();\n" + " emit({\n" + " property: property,\n" + " value: val\n" + " }, 1);\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}"; reduce = "function reduce(key, values) {\n" + " var res = 0;\n" + " values.forEach(function(v) {\n" + " res += v;\n" + " });\n" + " return res;\n" + "}"; }/* ww w. j a va2 s. co m*/ DBObject query = this.getCachedFilter(filter); LOG.debug("Filter query is:\n{}", query); String queryString = query.toString(); DBCollection elmnts = getCollection(Element.class); MapReduceCommand cmd = new MapReduceCommand(elmnts, map, reduce, null, INLINE, query); MapReduceOutput output = elmnts.mapReduce(cmd); // List<BasicDBObject> results = (List<BasicDBObject>) output.getCommandResult().get( "results" ); Iterator<DBObject> iterator = output.results().iterator(); List<BasicDBObject> results = new ArrayList<BasicDBObject>(); while (iterator.hasNext()) { results.add((BasicDBObject) iterator.next()); } LOG.debug("MapReduce produced {} results", results.size()); DBCollection histCollection = this.db.getCollection(TBL_HISTOGRAMS); BasicDBObject old = new BasicDBObject("_id", key); BasicDBObject res = new BasicDBObject(old.toMap()); res.put("results", results); histCollection.update(old, res, true, false); DBCursor cursor = histCollection.find(new BasicDBObject("_id", key)); if (cursor.count() == 0) { return null; } long end = System.currentTimeMillis(); LOG.debug("MapReduce took {} seconds", (end - start) / 1000); return (DBObject) cursor.next().get("results"); }
From source file:com.petpet.c3po.dao.mongo.MongoPersistenceLayer.java
License:Apache License
public DBObject mapReduceAllValues(int key, String property, Filter filter, List<Integer> bins) { LOG.debug("Starting mapReduce for the following property: {}", property); long start = System.currentTimeMillis(); Property prop = getCache().getProperty(property); String propType = prop.getType(); String map = ""; String map2 = ""; String reduce = ""; if (propType.equals(PropertyType.STRING.toString()) || propType.equals(PropertyType.BOOL.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " for (mr in this.metadata){\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property)\n" + " {\n" + " for (i in metadataRecord.sourcedValues)\n" + " {\n" + " sv=metadataRecord.sourcedValues[i];\n" + " emit({\n" + " property: property,\n" + " value: sv.value\n" + " }, 1);\n" + "\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}"; reduce = "function reduce(key, values) {\n" + " var res = 0;\n" + " values.forEach(function(v) {\n" + " res += v;\n" + " });\n" + " return res;\n" + "}"; } else if (propType.equals(PropertyType.INTEGER.toString()) || propType.equals(PropertyType.FLOAT.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " thresholds = " + getBinThresholds(bins) + ";\n" + " for (mr in this.metadata)" + " {\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property)" + " {\n" + " for (i in metadataRecord.sourcedValues)" + " {\n" + " sv=metadataRecord.sourcedValues[i];\n" + " var val=sv.value;\n" + " if (thresholds.length > 0)\n" + " for (t in thresholds){\n" + " threshold = thresholds[t]; \n" + " if (val>=threshold[0] && val<=threshold[1]){\n" + " emit({\n" + " property: property,\n" + " value: threshold[0]+'-'+threshold[1]\n" + " }, 1);\n" + " }\n" + " }\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}"; reduce = "function reduce(key, values) {\n" + " var res = 0;\n" + " values.forEach(function(v) {\n" + " res += v;\n" + " });\n" + " return res;\n" + "}"; } else if (propType.equals(PropertyType.DATE.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " for (mr in this.metadata){\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property){\n" + " for (i in metadataRecord.sourcedValues){\n" + " sv=metadataRecord.sourcedValues[i];\n" + " var date = new Date(sv.value);\n" + " var val=date.getFullYear();\n" + " emit({\n" + " property: property,\n" + " value: val\n" + " }, 1);\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}"; reduce = "function reduce(key, values) {\n" + " var res = 0;\n" + " values.forEach(function(v) {\n" + " res += v;\n" + " });\n" + " return res;\n" + "}"; }/*from w w w . j ava 2s .c o m*/ DBObject query = this.getCachedFilter(filter); LOG.debug("Filter query is:\n{}", query); String queryString = query.toString(); DBCollection elmnts = getCollection(Element.class); MapReduceCommand cmd = new MapReduceCommand(elmnts, map, reduce, null, INLINE, query); MapReduceOutput output = elmnts.mapReduce(cmd); // List<BasicDBObject> results = (List<BasicDBObject>) output.getCommandResult().get( "results" ); Iterator<DBObject> iterator = output.results().iterator(); List<BasicDBObject> results = new ArrayList<BasicDBObject>(); while (iterator.hasNext()) { results.add((BasicDBObject) iterator.next()); } LOG.debug("MapReduce produced {} results", results.size()); DBCollection histCollection = this.db.getCollection(TBL_HISTOGRAMS); BasicDBObject old = new BasicDBObject("_id", key); BasicDBObject res = new BasicDBObject(old.toMap()); res.put("results", results); histCollection.update(old, res, true, false); DBCursor cursor = histCollection.find(new BasicDBObject("_id", key)); if (cursor.count() == 0) { return null; } long end = System.currentTimeMillis(); LOG.debug("MapReduce took {} seconds", (end - start) / 1000); return (DBObject) cursor.next().get("results"); }
From source file:com.petpet.c3po.dao.mongo.MongoPersistenceLayer.java
License:Apache License
public DBObject mapReduceStats(int key, String property, Filter filter) { LOG.debug("Starting mapReduceStats for the following property: {}", property); long start = System.currentTimeMillis(); Property prop = getCache().getProperty(property); String propType = prop.getType(); String map = ""; String reduce = ""; String finalize = ""; if (propType.equals(PropertyType.INTEGER.toString()) || propType.equals(PropertyType.FLOAT.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " for (mr in this.metadata){\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property){\n" + " {\n" + " emit({\n" + " property: property,\n" + " value: property\n" + " }, \n" + " {\n" + " sum: metadataRecord.sourcedValues[0].value,\n" + " min: metadataRecord.sourcedValues[0].value,\n" + " max: metadataRecord.sourcedValues[0].value,\n" + " count: 1,\n" + " diff: 0\n" + " }\n" + " )\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}\n"; reduce = "function reduce(key, values) {\n" + "var a = values[0];\n" + " for (var i = 1; i < values.length; i++) {\n" + " var b = values[i];\n" + " var delta = a.sum / a.count - b.sum / b.count;\n" + " var weight = (a.count * b.count) / (a.count + b.count);\n" + " a.diff += b.diff + delta * delta * weight;\n" + " a.sum = b.sum*1+ a.sum*1;\n" + " a.count += b.count;\n" + " a.min = Math.min(a.min, b.min);\n" + " a.max = Math.max(a.max, b.max);\n" + " }\n" + "return a;" + "}" ;/*from www . j av a 2 s. com*/ finalize = "function finalize(key, value) {\n" + " value.avg = value.sum / value.count;\n" + " value.variance = value.diff / value.count;\n" + " value.stddev = Math.sqrt(value.variance);\n" + " return value;\n" + "}"; } DBObject query = this.getCachedFilter(filter); LOG.debug("filter query is:\n{}", query); DBCollection elmnts = getCollection(Element.class); MapReduceCommand cmd = new MapReduceCommand(elmnts, map, reduce, null, INLINE, query); cmd.setFinalize(finalize); MapReduceOutput output = elmnts.mapReduce(cmd); //List<BasicDBObject> results = (List<BasicDBObject>) output.getCommandResult().get( "results" ); Iterator<DBObject> iterator = output.results().iterator(); List<BasicDBObject> results = new ArrayList<BasicDBObject>(); while (iterator.hasNext()) { results.add((BasicDBObject) iterator.next()); } LOG.debug("MapReduce produced {} results", results.size()); DBCollection histCollection = this.db.getCollection(TBL_HISTOGRAMS); BasicDBObject old = new BasicDBObject("_id", key); BasicDBObject res = new BasicDBObject(old.toMap()); res.put("results", results); histCollection.update(old, res, true, false); DBCursor cursor = histCollection.find(new BasicDBObject("_id", key)); if (cursor.count() == 0) { return null; } long end = System.currentTimeMillis(); LOG.debug("The map-reduce job took {} seconds", (end - start) / 1000); return (DBObject) cursor.next().get("results"); }
From source file:de.otto.mongodb.profiler.collection.ListSizeDistribution.java
License:Apache License
/** * Calculates the distribution sizes of arrays / lists inside documents of a collection. Executes a map-reduce * operation to aggregate the values. Returns a map where the key determines the length of the array / list * and the value the amount of documents with that length. * * @param collection the collection * @param attribute the name of the attribute that is an array inside documents of the collection * @param resultCollectionName the name of the result collection of the map-reduce * @return the map of array lengths to the amount of documents *//*w w w. ja va2s. com*/ public static Map<Long, Long> calculate(final DBCollection collection, final String attribute, final String resultCollectionName) { checkNotNull(collection); checkNotNull(attribute); final String mapFunction = String.format(MAP_FUNCTION, attribute); final MapReduceCommand.OutputType outputType = resultCollectionName != null ? MapReduceCommand.OutputType.REPLACE : MapReduceCommand.OutputType.INLINE; final MapReduceOutput output = collection.mapReduce(mapFunction, REDUCE_FUNCTION, resultCollectionName, outputType, QUERY_ALL); final Map<Long, Long> result = new TreeMap<>(KEYS_BY_VALUE); for (DBObject dbo : output.results()) { final Number id = ((Number) dbo.get("_id")); final Number count = ((Number) ((DBObject) dbo.get("value")).get("count")); result.put(Long.valueOf(id.longValue()), Long.valueOf(count.longValue())); } return result; }
From source file:de.taimos.dao.mongo.AbstractMongoDAO.java
License:Apache License
protected <T> Iterable<T> mapReduce(String map, String reduce, DBObject query, final IObjectConverter<T> conv) { MapReduceOutput mr = this.collection.getDBCollection().mapReduce(map, reduce, null, OutputType.INLINE, query);//w ww . j a v a 2s . c o m return new ConverterIterable<T>(mr.results().iterator(), conv); }
From source file:de.taimos.dvalin.mongo.AbstractMongoDAO.java
License:Apache License
/** * runs a map-reduce-job on the collection. The functions are read from the classpath in the folder mongodb. The systems reads them from * files called <name>.map.js, <name>.reduce.js and optionally <name>.finalize.js. After this the result is converted * using the given {@link MapReduceResultHandler} * * @param <R> the type of the result class * @param name the name of the map-reduce functions * @param query the query to filter the elements used for the map-reduce * @param sort sort query to sort elements before running map-reduce * @param scope the global scope for the JavaScript run * @param conv the converter to convert the result * @return an {@link Iterable} with the result entries * @throws RuntimeException if resources cannot be read *//* www .j a v a 2 s .c om*/ protected final <R> Iterable<R> mapReduce(String name, DBObject query, DBObject sort, Map<String, Object> scope, final MapReduceResultHandler<R> conv) { String map = this.getMRFunction(name, "map"); String reduce = this.getMRFunction(name, "reduce"); MapReduceCommand mrc = new MapReduceCommand(this.collection.getDBCollection(), map, reduce, null, OutputType.INLINE, query); String finalizeFunction = this.getMRFunction(name, "finalize"); if (finalizeFunction != null) { mrc.setFinalize(finalizeFunction); } if (sort != null) { mrc.setSort(sort); } if (scope != null) { mrc.setScope(scope); } MapReduceOutput mr = this.collection.getDBCollection().mapReduce(mrc); return new ConverterIterable<R>(mr.results().iterator(), conv); }
From source file:edu.wayne.cs.fms.controller.CRUD.java
public static ArrayList MapReduce(String colName, String map, String reduce, String finalize, MongoClient mongoClient) {/*from www. ja v a 2 s .co m*/ //MongoClient mongoClient = Connector.connect("localhost", 27017); DB db = mongoClient.getDB("project"); DBCollection temp = db.getCollection(colName); ArrayList result = new ArrayList(); MapReduceCommand cmd = new MapReduceCommand(temp, map, reduce, null, MapReduceCommand.OutputType.INLINE, null); if (finalize != null) { cmd.setFinalize(finalize); } MapReduceOutput out = temp.mapReduce(cmd); for (DBObject o : out.results()) { result.add(o.toString()); } System.out.println("Done"); //mongoClient.close(); return result; }
From source file:eu.eubrazilcc.lvl.storage.mongodb.MongoDBConnector.java
License:EUPL
/** * Runs a map-reduce aggregation over a collection and saves the result to a temporary collection, which is deleted at the end * of the execution. The results of the operation are returned to the caller in a list of {@code BasicDBObject}. * @param collection - collection whose objects are searched * @param mapFn - map function/* w w w .j a v a2 s. c om*/ * @param reduceFn - reduce function * @param query - (optional) specifies the selection criteria using query operators for determining the documents input to the * map function. Set this parameter to {@code null} to use all documents in the collection * @return a list of {@code BasicDBObject} which contains the results of this map reduce operation. * @see <a href="http://cookbook.mongodb.org/patterns/pivot/">The MongoDB Cookbook - Pivot Data with Map reduce</a> */ public List<BasicDBObject> mapReduce(final String collection, final String mapFn, final String reduceFn, final @Nullable DBObject query) { checkArgument(isNotBlank(collection), "Uninitialized or invalid collection"); checkArgument(isNotBlank(mapFn), "Uninitialized or map function"); checkArgument(isNotBlank(reduceFn), "Uninitialized or reduce function"); final List<BasicDBObject> list = newArrayList(); final DB db = client().getDB(CONFIG_MANAGER.getDbName()); final DBCollection dbcol = db.getCollection(collection); final DBCollection tmpCol = tmpCollection(); try { final MapReduceCommand command = new MapReduceCommand(dbcol, mapFn, reduceFn, tmpCol.getName(), REDUCE, query); final MapReduceOutput output = dbcol.mapReduce(command); final Iterable<DBObject> results = output.results(); for (final DBObject result : results) { list.add((BasicDBObject) result); } } catch (Exception e) { throw new IllegalStateException("Failed to execute map-reduce operation", e); } finally { try { tmpCol.drop(); } catch (Exception mdbe) { LOGGER.warn("Failed to drop temporary collection", mdbe); } } return list; }
From source file:fr.gouv.vitam.mdbes.MainMapReduce.java
License:Open Source License
protected static void oneShot(MongoDbAccess dbvitam) throws InvalidParseOperationException, InvalidExecOperationException, InstantiationException, IllegalAccessException { // Requesting if (map == null) { map = "function() {" + "flattened = serializeDocFiltered(this, MAXDEPTH, FILTER);" + "for (var key in flattened) {" + "var value = flattened[key];" + "var valueType = varietyTypeOf(value);" + "var finalvalue = { types : [ valueType ], occurences : 1};" // + "var finalvalue = { occurences : 1};" + "emit(key, finalvalue); } }"; }/* w w w . j a v a2s . c o m*/ if (reduce == null) { reduce = "function(key,values) {" + "var typeset = new Set();" + "var occur = 0;" + "for (var idx = 0; idx < values.length; idx++) {" + "occur += values[idx].occurences;" + "typeset.add(values[idx].types);" + "} return { types : typeset.asArray(), occurences : occur }; }"; // + "} return { occurences : occur }; }"; } if (output == null) { output = "AttributeUsage"; } MapReduceCommand mapReduceCommand = new MapReduceCommand(dbvitam.daips.collection, map, reduce, output, OutputType.REDUCE, new BasicDBObject()); if (options != null) { String[] optionsArray = options.split(","); Map<String, Object> mapScope = new HashMap<String, Object>(); for (String string : optionsArray) { String[] kv = string.split("="); mapScope.put(kv[0], getParsedString(kv[1])); } mapReduceCommand.setScope(mapScope); } else { Map<String, Object> mapScope = new HashMap<String, Object>(); mapScope.put("MAXDEPTH", 5); mapScope.put("FILTER", "_dds"); mapReduceCommand.setScope(mapScope); } mapReduceCommand.addExtraOption("nonAtomic", true); mapReduceCommand.addExtraOption("jsMode", true); MapReduceOutput output = dbvitam.daips.collection.mapReduce(mapReduceCommand); System.out.println("Duration: " + output.getDuration() + " Saved into: " + output.getCollectionName() + " Input: " + output.getInputCount() + " Emit: " + output.getEmitCount() + " Output: " + output.getOutputCount() + "\nCmd: " + output.getCommand()); Iterable<DBObject> iterable = output.results(); for (DBObject dbObject : iterable) { System.out.println(dbObject.toString()); } }
From source file:govt_import_export.Order.java
private void sort_cars() { MongoClient mongo = null;/*from ww w. j a v a 2s. co m*/ try { mongo = new MongoClient("localhost", 27017); //get database } catch (UnknownHostException | MongoException e) { } String[] st = new String[10]; st[0] = "HYUNDAI"; st[1] = "MARUTISUZUKI"; st[2] = "NISSAN"; st[3] = "BAJAJ"; st[4] = "KTM"; st[5] = "VOLKSVAGEN"; DB db = mongo.getDB("AUTOMOBILEXPO"); DBCollection table; ArrayList<Order> arr = new ArrayList<Order>(); for (int j = 0; j < 6; j++) { table = db.getCollection(st[j]); if (table.findOne() != null) { String map = "function () {" + "emit(this.model,this.Units);" + "}"; String reduce; reduce = "function (key,value) { " + "return Array.sum(value)}"; MapReduceCommand cmd = new MapReduceCommand(table, map, reduce, null, MapReduceCommand.OutputType.INLINE, null); MapReduceOutput out = table.mapReduce(cmd); String str; for (DBObject o : out.results()) { str = o.get("value").toString().trim(); System.out.println(str); int i = 0; // var i=(int)(Convert.ToDouble("1.2")); //int a = int.Parse("1.2".Split('.')[0]); if (!str.equals(null)) { Float f = Float.parseFloat(str); i = (int) Math.ceil(f); arr.add(new Order(o.get("_id").toString(), i) { @Override public int compare(Order o1, Order o2) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } @Override public int compareTo(Order o) { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } }); } } } } Collections.sort(arr, new Order() { }); for (Order a : arr) System.out.println(" " + a.getid() + " " + a.getval()); this.dispose(); Sorted s = new Sorted(arr); s.setVisible(true); }