List of usage examples for com.mongodb MapReduceCommand MapReduceCommand
public MapReduceCommand(final DBCollection inputCollection, final String map, final String reduce, @Nullable final String outputCollection, final OutputType type, final DBObject query)
From source file:com.petpet.c3po.dao.mongo.MongoPersistenceLayer.java
License:Apache License
public DBObject mapReduce(int key, String property, Filter filter, List<Integer> bins) { LOG.debug("Starting mapReduce for the following property: {}", property); long start = System.currentTimeMillis(); Property prop = getCache().getProperty(property); String propType = prop.getType(); String map = ""; String map2 = ""; String reduce = ""; if (propType.equals(PropertyType.STRING.toString()) || propType.equals(PropertyType.BOOL.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " for (mr in this.metadata){\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property)\n" + " {\n" + " if (metadataRecord.status == 'CONFLICT'){\n" + " emit({\n" + " property: property,\n" + " value: 'CONFLICT'\n" + " }, 1);\n" + " } else {\n" + " emit({\n" + " property: property,\n" + " value: metadataRecord.sourcedValues[0].value\n" + " }, 1);\n" + "\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}"; reduce = "function reduce(key, values) {\n" + " var res = 0;\n" + " values.forEach(function(v) {\n" + " res += v;\n" + " });\n" + " return res;\n" + "}"; } else if (propType.equals(PropertyType.INTEGER.toString()) || propType.equals(PropertyType.FLOAT.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " thresholds = " + getBinThresholds(bins) + ";\n" + " for (mr in this.metadata){\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property){\n" + " if (metadataRecord.status == 'CONFLICT'){\n" + " emit({\n" + " property: property,\n" + " value: 'CONFLICT'\n" + " }, 1);\n" + " } else {\n" + " var val=metadataRecord.sourcedValues[0].value;\n" + " var skipped=false;\n" + " if (thresholds.length > 0)\n" + " for (t in thresholds){\n" + " threshold = thresholds[t]; \n" + " if (val>=threshold[0] && val<=threshold[1]){\n" + " emit({\n" + " property: property,\n" + " value: threshold[0]+'-'+threshold[1]\n" + " }, 1);\n" + " skipped=true;\n" + " break;\n" + " }\n" + " }\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}"; reduce = "function reduce(key, values) {\n" + " var res = 0;\n" + " values.forEach(function(v) {\n" + " res += v;\n" + " });\n" + " return res;\n" + "}"; } else if (propType.equals(PropertyType.DATE.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " for (mr in this.metadata){\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property){\n" + " if (metadataRecord.status == 'CONFLICT'){\n" + " emit({\n" + " property: property,\n" + " value: 'CONFLICT'\n" + " }, 1);\n" + " } else {\n" + " var date = new Date(metadataRecord.sourcedValues[0].value);\n" + " var val=date.getFullYear();\n" + " emit({\n" + " property: property,\n" + " value: val\n" + " }, 1);\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}"; reduce = "function reduce(key, values) {\n" + " var res = 0;\n" + " values.forEach(function(v) {\n" + " res += v;\n" + " });\n" + " return res;\n" + "}"; }//from w w w . j a va2 s. c o m DBObject query = this.getCachedFilter(filter); LOG.debug("Filter query is:\n{}", query); String queryString = query.toString(); DBCollection elmnts = getCollection(Element.class); MapReduceCommand cmd = new MapReduceCommand(elmnts, map, reduce, null, INLINE, query); MapReduceOutput output = elmnts.mapReduce(cmd); // List<BasicDBObject> results = (List<BasicDBObject>) output.getCommandResult().get( "results" ); Iterator<DBObject> iterator = output.results().iterator(); List<BasicDBObject> results = new ArrayList<BasicDBObject>(); while (iterator.hasNext()) { results.add((BasicDBObject) iterator.next()); } LOG.debug("MapReduce produced {} results", results.size()); DBCollection histCollection = this.db.getCollection(TBL_HISTOGRAMS); BasicDBObject old = new BasicDBObject("_id", key); BasicDBObject res = new BasicDBObject(old.toMap()); res.put("results", results); histCollection.update(old, res, true, false); DBCursor cursor = histCollection.find(new BasicDBObject("_id", key)); if (cursor.count() == 0) { return null; } long end = System.currentTimeMillis(); LOG.debug("MapReduce took {} seconds", (end - start) / 1000); return (DBObject) cursor.next().get("results"); }
From source file:com.petpet.c3po.dao.mongo.MongoPersistenceLayer.java
License:Apache License
public DBObject mapReduceAllValues(int key, String property, Filter filter, List<Integer> bins) { LOG.debug("Starting mapReduce for the following property: {}", property); long start = System.currentTimeMillis(); Property prop = getCache().getProperty(property); String propType = prop.getType(); String map = ""; String map2 = ""; String reduce = ""; if (propType.equals(PropertyType.STRING.toString()) || propType.equals(PropertyType.BOOL.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " for (mr in this.metadata){\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property)\n" + " {\n" + " for (i in metadataRecord.sourcedValues)\n" + " {\n" + " sv=metadataRecord.sourcedValues[i];\n" + " emit({\n" + " property: property,\n" + " value: sv.value\n" + " }, 1);\n" + "\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}"; reduce = "function reduce(key, values) {\n" + " var res = 0;\n" + " values.forEach(function(v) {\n" + " res += v;\n" + " });\n" + " return res;\n" + "}"; } else if (propType.equals(PropertyType.INTEGER.toString()) || propType.equals(PropertyType.FLOAT.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " thresholds = " + getBinThresholds(bins) + ";\n" + " for (mr in this.metadata)" + " {\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property)" + " {\n" + " for (i in metadataRecord.sourcedValues)" + " {\n" + " sv=metadataRecord.sourcedValues[i];\n" + " var val=sv.value;\n" + " if (thresholds.length > 0)\n" + " for (t in thresholds){\n" + " threshold = thresholds[t]; \n" + " if (val>=threshold[0] && val<=threshold[1]){\n" + " emit({\n" + " property: property,\n" + " value: threshold[0]+'-'+threshold[1]\n" + " }, 1);\n" + " }\n" + " }\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}"; reduce = "function reduce(key, values) {\n" + " var res = 0;\n" + " values.forEach(function(v) {\n" + " res += v;\n" + " });\n" + " return res;\n" + "}"; } else if (propType.equals(PropertyType.DATE.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " for (mr in this.metadata){\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property){\n" + " for (i in metadataRecord.sourcedValues){\n" + " sv=metadataRecord.sourcedValues[i];\n" + " var date = new Date(sv.value);\n" + " var val=date.getFullYear();\n" + " emit({\n" + " property: property,\n" + " value: val\n" + " }, 1);\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}"; reduce = "function reduce(key, values) {\n" + " var res = 0;\n" + " values.forEach(function(v) {\n" + " res += v;\n" + " });\n" + " return res;\n" + "}"; }/* w w w.ja v a2s . c om*/ DBObject query = this.getCachedFilter(filter); LOG.debug("Filter query is:\n{}", query); String queryString = query.toString(); DBCollection elmnts = getCollection(Element.class); MapReduceCommand cmd = new MapReduceCommand(elmnts, map, reduce, null, INLINE, query); MapReduceOutput output = elmnts.mapReduce(cmd); // List<BasicDBObject> results = (List<BasicDBObject>) output.getCommandResult().get( "results" ); Iterator<DBObject> iterator = output.results().iterator(); List<BasicDBObject> results = new ArrayList<BasicDBObject>(); while (iterator.hasNext()) { results.add((BasicDBObject) iterator.next()); } LOG.debug("MapReduce produced {} results", results.size()); DBCollection histCollection = this.db.getCollection(TBL_HISTOGRAMS); BasicDBObject old = new BasicDBObject("_id", key); BasicDBObject res = new BasicDBObject(old.toMap()); res.put("results", results); histCollection.update(old, res, true, false); DBCursor cursor = histCollection.find(new BasicDBObject("_id", key)); if (cursor.count() == 0) { return null; } long end = System.currentTimeMillis(); LOG.debug("MapReduce took {} seconds", (end - start) / 1000); return (DBObject) cursor.next().get("results"); }
From source file:com.petpet.c3po.dao.mongo.MongoPersistenceLayer.java
License:Apache License
public DBObject mapReduceStats(int key, String property, Filter filter) { LOG.debug("Starting mapReduceStats for the following property: {}", property); long start = System.currentTimeMillis(); Property prop = getCache().getProperty(property); String propType = prop.getType(); String map = ""; String reduce = ""; String finalize = ""; if (propType.equals(PropertyType.INTEGER.toString()) || propType.equals(PropertyType.FLOAT.toString())) { map = "function() {\n" + " property = '" + property + "';\n" + " for (mr in this.metadata){\n" + " metadataRecord=this.metadata[mr];\n" + " if(metadataRecord.property == property){\n" + " {\n" + " emit({\n" + " property: property,\n" + " value: property\n" + " }, \n" + " {\n" + " sum: metadataRecord.sourcedValues[0].value,\n" + " min: metadataRecord.sourcedValues[0].value,\n" + " max: metadataRecord.sourcedValues[0].value,\n" + " count: 1,\n" + " diff: 0\n" + " }\n" + " )\n" + " }\n" + " return;\n" + " }\n" + " }\n" + " emit({\n" + " property: property,\n" + " value: 'Unknown'\n" + " }, 1);\n" + "}\n"; reduce = "function reduce(key, values) {\n" + "var a = values[0];\n" + " for (var i = 1; i < values.length; i++) {\n" + " var b = values[i];\n" + " var delta = a.sum / a.count - b.sum / b.count;\n" + " var weight = (a.count * b.count) / (a.count + b.count);\n" + " a.diff += b.diff + delta * delta * weight;\n" + " a.sum = b.sum*1+ a.sum*1;\n" + " a.count += b.count;\n" + " a.min = Math.min(a.min, b.min);\n" + " a.max = Math.max(a.max, b.max);\n" + " }\n" + "return a;" + "}" ;//from www . j ava 2 s . c om finalize = "function finalize(key, value) {\n" + " value.avg = value.sum / value.count;\n" + " value.variance = value.diff / value.count;\n" + " value.stddev = Math.sqrt(value.variance);\n" + " return value;\n" + "}"; } DBObject query = this.getCachedFilter(filter); LOG.debug("filter query is:\n{}", query); DBCollection elmnts = getCollection(Element.class); MapReduceCommand cmd = new MapReduceCommand(elmnts, map, reduce, null, INLINE, query); cmd.setFinalize(finalize); MapReduceOutput output = elmnts.mapReduce(cmd); //List<BasicDBObject> results = (List<BasicDBObject>) output.getCommandResult().get( "results" ); Iterator<DBObject> iterator = output.results().iterator(); List<BasicDBObject> results = new ArrayList<BasicDBObject>(); while (iterator.hasNext()) { results.add((BasicDBObject) iterator.next()); } LOG.debug("MapReduce produced {} results", results.size()); DBCollection histCollection = this.db.getCollection(TBL_HISTOGRAMS); BasicDBObject old = new BasicDBObject("_id", key); BasicDBObject res = new BasicDBObject(old.toMap()); res.put("results", results); histCollection.update(old, res, true, false); DBCursor cursor = histCollection.find(new BasicDBObject("_id", key)); if (cursor.count() == 0) { return null; } long end = System.currentTimeMillis(); LOG.debug("The map-reduce job took {} seconds", (end - start) / 1000); return (DBObject) cursor.next().get("results"); }
From source file:com.stratio.connector.mongodb.core.engine.metadata.DiscoverMetadataUtils.java
License:Apache License
/** * Discover the existing fields stored in the collection. * * @param collection//w ww . j a v a2s. co m * the collection * @return the list of fields including the _id */ public static List<String> discoverField(DBCollection collection) { String map = "function() { for (var field in this) { emit(field, null); }}"; String reduce = "function(field, stuff) { return null; }"; MapReduceCommand mapReduceCommand = new MapReduceCommand(collection, map, reduce, null, OutputType.INLINE, null); DBObject getFieldsCommand = mapReduceCommand.toDBObject(); CommandResult command = collection.getDB().command(getFieldsCommand); BasicDBList results = (BasicDBList) command.get("results"); Set<String> fields = new HashSet<>(); if (results != null) { for (Object object : results) { DBObject bson = (DBObject) object; fields.add((String) bson.get("_id")); } } return new ArrayList<String>(fields); }
From source file:com.stratio.connector.mongodb.core.engine.metadata.DiscoverMetadataUtils.java
License:Apache License
/** * Discover the existing fields stored in the collection and their data types. * * @param collection/*from w ww .j a va 2s. c o m*/ * the collection * @return the list of fields including the _id */ public static HashMap<String, String> discoverFieldsWithType(DBCollection collection, String sample_probability) { String map = "function() { if(Math.random() <= sample_number) {for (var key in this) {var type = typeof(this[key]); if(type == \"object\"){type = \"string\";};emit(key, type);}} } "; String reduce = "function(key, values) { var result = \"\"; for (var i = 0; i < values.length; i++){ var v = values[i];if(v == \"string\"){result = \"string\"; break;} if(v == \"number\"){result = \"number\"} if(v == \"boolean\" && result == \"number\"){result = \"string\"; break;}if(v == \"number\" && result == \"boolean\"){result = \"string\"; break;} if(v==\"boolean\"){result = \"boolean\"}};return result; }"; MapReduceCommand mapReduceCommand = new MapReduceCommand(collection, map, reduce, null, OutputType.INLINE, null); HashMap<String, Object> scope = new HashMap<>(); // connection scope.put("sample_number", sample_probability); mapReduceCommand.setScope(scope); DBObject getFieldsCommand = mapReduceCommand.toDBObject(); CommandResult command = collection.getDB().command(getFieldsCommand); BasicDBList results = (BasicDBList) command.get("results"); HashMap<String, String> fields = new HashMap<>(); if (results != null) { for (Object object : results) { DBObject bson = (DBObject) object; String nameField = (String) bson.get("_id"); String type = (String) bson.get("value"); fields.put(nameField, type); } } return fields; }
From source file:com.sube.daos.mongodb.StatisticDaoMongoImpl.java
License:Apache License
private void executeMapReduce(MapReduce mapReduce, DBObject query, MongoCollection mongoCollection) { MapReduceCommand mapReduceCommand = new MapReduceCommand(getCardUsagesCollection(), mapReduce.getMapExpression(), mapReduce.getReduceExpression(), mongoCollection.name, OutputType.REPLACE, query);/* w ww . j av a 2 s .co m*/ getCardUsagesCollection().mapReduce(mapReduceCommand); }
From source file:de.taimos.dvalin.mongo.AbstractMongoDAO.java
License:Apache License
/** * runs a map-reduce-job on the collection. The functions are read from the classpath in the folder mongodb. The systems reads them from * files called <name>.map.js, <name>.reduce.js and optionally <name>.finalize.js. After this the result is converted * using the given {@link MapReduceResultHandler} * * @param <R> the type of the result class * @param name the name of the map-reduce functions * @param query the query to filter the elements used for the map-reduce * @param sort sort query to sort elements before running map-reduce * @param scope the global scope for the JavaScript run * @param conv the converter to convert the result * @return an {@link Iterable} with the result entries * @throws RuntimeException if resources cannot be read */// w w w. j av a 2 s . co m protected final <R> Iterable<R> mapReduce(String name, DBObject query, DBObject sort, Map<String, Object> scope, final MapReduceResultHandler<R> conv) { String map = this.getMRFunction(name, "map"); String reduce = this.getMRFunction(name, "reduce"); MapReduceCommand mrc = new MapReduceCommand(this.collection.getDBCollection(), map, reduce, null, OutputType.INLINE, query); String finalizeFunction = this.getMRFunction(name, "finalize"); if (finalizeFunction != null) { mrc.setFinalize(finalizeFunction); } if (sort != null) { mrc.setSort(sort); } if (scope != null) { mrc.setScope(scope); } MapReduceOutput mr = this.collection.getDBCollection().mapReduce(mrc); return new ConverterIterable<R>(mr.results().iterator(), conv); }
From source file:edu.wayne.cs.fms.controller.CRUD.java
public static ArrayList MapReduce(String colName, String map, String reduce, String finalize, MongoClient mongoClient) {/* w w w .jav a2 s . c om*/ //MongoClient mongoClient = Connector.connect("localhost", 27017); DB db = mongoClient.getDB("project"); DBCollection temp = db.getCollection(colName); ArrayList result = new ArrayList(); MapReduceCommand cmd = new MapReduceCommand(temp, map, reduce, null, MapReduceCommand.OutputType.INLINE, null); if (finalize != null) { cmd.setFinalize(finalize); } MapReduceOutput out = temp.mapReduce(cmd); for (DBObject o : out.results()) { result.add(o.toString()); } System.out.println("Done"); //mongoClient.close(); return result; }
From source file:eu.eubrazilcc.lvl.storage.mongodb.MongoDBConnector.java
License:EUPL
/** * Runs a map-reduce aggregation over a collection and saves the result to a temporary collection, which is deleted at the end * of the execution. The results of the operation are returned to the caller in a list of {@code BasicDBObject}. * @param collection - collection whose objects are searched * @param mapFn - map function/* w ww. j a va 2 s. c o m*/ * @param reduceFn - reduce function * @param query - (optional) specifies the selection criteria using query operators for determining the documents input to the * map function. Set this parameter to {@code null} to use all documents in the collection * @return a list of {@code BasicDBObject} which contains the results of this map reduce operation. * @see <a href="http://cookbook.mongodb.org/patterns/pivot/">The MongoDB Cookbook - Pivot Data with Map reduce</a> */ public List<BasicDBObject> mapReduce(final String collection, final String mapFn, final String reduceFn, final @Nullable DBObject query) { checkArgument(isNotBlank(collection), "Uninitialized or invalid collection"); checkArgument(isNotBlank(mapFn), "Uninitialized or map function"); checkArgument(isNotBlank(reduceFn), "Uninitialized or reduce function"); final List<BasicDBObject> list = newArrayList(); final DB db = client().getDB(CONFIG_MANAGER.getDbName()); final DBCollection dbcol = db.getCollection(collection); final DBCollection tmpCol = tmpCollection(); try { final MapReduceCommand command = new MapReduceCommand(dbcol, mapFn, reduceFn, tmpCol.getName(), REDUCE, query); final MapReduceOutput output = dbcol.mapReduce(command); final Iterable<DBObject> results = output.results(); for (final DBObject result : results) { list.add((BasicDBObject) result); } } catch (Exception e) { throw new IllegalStateException("Failed to execute map-reduce operation", e); } finally { try { tmpCol.drop(); } catch (Exception mdbe) { LOGGER.warn("Failed to drop temporary collection", mdbe); } } return list; }
From source file:fr.gouv.vitam.mdbes.MainMapReduce.java
License:Open Source License
protected static void oneShot(MongoDbAccess dbvitam) throws InvalidParseOperationException, InvalidExecOperationException, InstantiationException, IllegalAccessException { // Requesting if (map == null) { map = "function() {" + "flattened = serializeDocFiltered(this, MAXDEPTH, FILTER);" + "for (var key in flattened) {" + "var value = flattened[key];" + "var valueType = varietyTypeOf(value);" + "var finalvalue = { types : [ valueType ], occurences : 1};" // + "var finalvalue = { occurences : 1};" + "emit(key, finalvalue); } }"; }/*w w w . ja va 2 s . c om*/ if (reduce == null) { reduce = "function(key,values) {" + "var typeset = new Set();" + "var occur = 0;" + "for (var idx = 0; idx < values.length; idx++) {" + "occur += values[idx].occurences;" + "typeset.add(values[idx].types);" + "} return { types : typeset.asArray(), occurences : occur }; }"; // + "} return { occurences : occur }; }"; } if (output == null) { output = "AttributeUsage"; } MapReduceCommand mapReduceCommand = new MapReduceCommand(dbvitam.daips.collection, map, reduce, output, OutputType.REDUCE, new BasicDBObject()); if (options != null) { String[] optionsArray = options.split(","); Map<String, Object> mapScope = new HashMap<String, Object>(); for (String string : optionsArray) { String[] kv = string.split("="); mapScope.put(kv[0], getParsedString(kv[1])); } mapReduceCommand.setScope(mapScope); } else { Map<String, Object> mapScope = new HashMap<String, Object>(); mapScope.put("MAXDEPTH", 5); mapScope.put("FILTER", "_dds"); mapReduceCommand.setScope(mapScope); } mapReduceCommand.addExtraOption("nonAtomic", true); mapReduceCommand.addExtraOption("jsMode", true); MapReduceOutput output = dbvitam.daips.collection.mapReduce(mapReduceCommand); System.out.println("Duration: " + output.getDuration() + " Saved into: " + output.getCollectionName() + " Input: " + output.getInputCount() + " Emit: " + output.getEmitCount() + " Output: " + output.getOutputCount() + "\nCmd: " + output.getCommand()); Iterable<DBObject> iterable = output.results(); for (DBObject dbObject : iterable) { System.out.println(dbObject.toString()); } }