List of usage examples for com.mongodb MongoClientURI getDatabase
@Nullable
public String getDatabase()
From source file:com.zjy.mongo.splitter.MultiMongoCollectionSplitter.java
License:Apache License
@Override public List<InputSplit> calculateSplits() throws SplitFailedException { List<MongoClientURI> inputURIs = MongoConfigUtil.getMongoURIs(this.getConfiguration(), MongoConfigUtil.INPUT_URI); List<InputSplit> returnVal = new LinkedList<InputSplit>(); List<MongoSplitter> splitters = new LinkedList<MongoSplitter>(); //For each input URI that is specified, get the appropriate //splitter for each implementation. if (inputURIs.size() > 0) { if (LOG.isDebugEnabled()) { LOG.debug("Using global split settings for multiple URIs specified."); }/*ww w . j ava 2 s . c o m*/ //Get options from the hadoop config. //Use these options for all URIs in the list. //Splitter class is ignored here, since it would just be referring //to MultiMongoCollectionSplitter here anyway. If the user needs //to customize the splitter class, they should use the JSON key for //the configuration instead. for (MongoClientURI uri : inputURIs) { MongoCollectionSplitter splitter; Configuration confForThisUri = new Configuration(getConfiguration()); MongoConfigUtil.setInputURI(confForThisUri, uri); confForThisUri.set(MongoConfigUtil.MONGO_SPLITTER_CLASS, ""); splitter = MongoSplitterFactory.getSplitterByStats(uri, confForThisUri); splitters.add(splitter); } } else { //Otherwise the user has set options per-collection. if (LOG.isDebugEnabled()) { LOG.debug("Loading multiple input URIs from JSON stored in " + MULTI_COLLECTION_CONF_KEY); } DBObject multiUriConfig = MongoConfigUtil.getDBObject(this.getConfiguration(), MULTI_COLLECTION_CONF_KEY); if (!(multiUriConfig instanceof List)) { throw new IllegalArgumentException( "Invalid JSON format in multi uri config key: Must be an array where each element " + "is an object describing the URI and config options for each split."); } for (Object obj : (List) multiUriConfig) { Map<String, Object> configMap; MongoClientURI inputURI; Configuration confForThisUri; try { configMap = (Map<String, Object>) obj; if (LOG.isDebugEnabled()) { LOG.debug("building config from " + configMap.toString()); } confForThisUri = MongoConfigUtil.buildConfiguration(configMap); inputURI = MongoConfigUtil.getInputURI(confForThisUri); } catch (ClassCastException e) { throw new IllegalArgumentException( "Invalid JSON format in multi uri config key: each config item must be an " + "object with keys/values describing options for each URI."); } MongoSplitter splitter; Class<? extends MongoSplitter> splitterClass = MongoConfigUtil.getSplitterClass(confForThisUri); if (splitterClass != null) { if (LOG.isDebugEnabled()) { LOG.debug(format("Using custom Splitter class for namespace: %s.%s; hosts: %s", inputURI.getDatabase(), inputURI.getCollection(), inputURI.getHosts())); } //User wants to use a specific custom class for this URI. //Make sure that the custom class isn't this one if (splitterClass == MultiMongoCollectionSplitter.class) { throw new IllegalArgumentException("Can't nest uses of MultiMongoCollectionSplitter"); } //All clear. MongoCollectionSplitter collectionSplitter; collectionSplitter = (MongoCollectionSplitter) ReflectionUtils.newInstance(splitterClass, confForThisUri); //Since we use no-arg constructor, need to inject //configuration and input URI. collectionSplitter.setConfiguration(confForThisUri); splitter = collectionSplitter; } else { if (LOG.isDebugEnabled()) { LOG.debug(format( "Fetching collection stats on namespace: %s.%s; hosts: %s to choose splitter implementation.", inputURI.getDatabase(), inputURI.getCollection(), inputURI.getHosts())); } //No class was specified, so choose one by looking at //collection stats. splitter = MongoSplitterFactory.getSplitterByStats(inputURI, confForThisUri); } splitters.add(splitter); } } //Now we have splitters for all the input collections. //Run through all of em, get all the splits for each, //compile them into one big ol' list. for (MongoSplitter splitter : splitters) { returnVal.addAll(splitter.calculateSplits()); } return returnVal; }
From source file:com.zjy.mongo.splitter.ShardChunkMongoSplitter.java
License:Apache License
@Override public List<InputSplit> calculateSplits() throws SplitFailedException { boolean targetShards = MongoConfigUtil.canReadSplitsFromShards(getConfiguration()); DB configDB = getConfigDB();//w w w . j a v a 2 s . co m DBCollection chunksCollection = configDB.getCollection("chunks"); MongoClientURI inputURI = MongoConfigUtil.getInputURI(getConfiguration()); String inputNS = inputURI.getDatabase() + "." + inputURI.getCollection(); DBCursor cur = chunksCollection.find(new BasicDBObject("ns", inputNS)); int numChunks = 0; Map<String, String> shardsMap = null; if (targetShards) { try { shardsMap = getShardsMap(); } catch (Exception e) { //Something went wrong when trying to //read the shards data from the config server, //so abort the splitting throw new SplitFailedException("Couldn't get shards information from config server", e); } } List<String> mongosHostNames = MongoConfigUtil.getInputMongosHosts(getConfiguration()); if (targetShards && mongosHostNames.size() > 0) { throw new SplitFailedException( "Setting both mongo.input.split.read_from_shards and mongo.input.mongos_hosts" + " does not make sense. "); } if (mongosHostNames.size() > 0) { LOG.info("Using multiple mongos instances (round robin) for reading input."); } Map<String, LinkedList<InputSplit>> shardToSplits = new HashMap<String, LinkedList<InputSplit>>(); try { while (cur.hasNext()) { final BasicDBObject row = (BasicDBObject) cur.next(); BasicDBObject chunkLowerBound = (BasicDBObject) row.get("min"); BasicDBObject chunkUpperBound = (BasicDBObject) row.get("max"); MongoInputSplit chunkSplit = createSplitFromBounds(chunkLowerBound, chunkUpperBound); chunkSplit.setInputURI(inputURI); String shard = (String) row.get("shard"); if (targetShards) { //The job is configured to target shards, so replace the //mongos hostname with the host of the shard's servers String shardHosts = shardsMap.get(shard); if (shardHosts == null) { throw new SplitFailedException("Couldn't find shard ID: " + shard + " in config.shards."); } MongoClientURI newURI = rewriteURI(inputURI, shardHosts); chunkSplit.setInputURI(newURI); } else if (mongosHostNames.size() > 0) { //Multiple mongos hosts are specified, so //choose a host name in round-robin fashion //and rewrite the URI using that hostname. //This evenly distributes the load to avoid //pegging a single mongos instance. String roundRobinHost = mongosHostNames.get(numChunks % mongosHostNames.size()); MongoClientURI newURI = rewriteURI(inputURI, roundRobinHost); chunkSplit.setInputURI(newURI); } LinkedList<InputSplit> shardList = shardToSplits.get(shard); if (shardList == null) { shardList = new LinkedList<InputSplit>(); shardToSplits.put(shard, shardList); } chunkSplit.setKeyField(MongoConfigUtil.getInputKey(getConfiguration())); shardList.add(chunkSplit); numChunks++; } } finally { MongoConfigUtil.close(configDB.getMongo()); } final List<InputSplit> splits = new ArrayList<InputSplit>(numChunks); int splitIndex = 0; while (splitIndex < numChunks) { Set<String> shardSplitsToRemove = new HashSet<String>(); for (Entry<String, LinkedList<InputSplit>> shardSplits : shardToSplits.entrySet()) { LinkedList<InputSplit> shardSplitsList = shardSplits.getValue(); InputSplit split = shardSplitsList.pop(); splits.add(splitIndex, split); splitIndex++; if (shardSplitsList.isEmpty()) { shardSplitsToRemove.add(shardSplits.getKey()); } } for (String shardName : shardSplitsToRemove) { shardToSplits.remove(shardName); } } return splits; }
From source file:com.zjy.mongo.splitter.SingleMongoSplitter.java
License:Apache License
@Override public List<InputSplit> calculateSplits() { MongoClientURI inputURI = MongoConfigUtil.getInputURI(getConfiguration()); if (LOG.isDebugEnabled()) { LOG.debug(format("SingleMongoSplitter calculating splits for namespace: %s.%s; hosts: %s", inputURI.getDatabase(), inputURI.getCollection(), inputURI.getHosts())); }/* w w w. ja va2 s .com*/ final List<InputSplit> splits = new ArrayList<InputSplit>(); MongoInputSplit mongoSplit = new MongoInputSplit(); mongoSplit.setInputURI(MongoConfigUtil.getInputURI(getConfiguration())); mongoSplit.setAuthURI(MongoConfigUtil.getAuthURI(getConfiguration())); mongoSplit.setQuery(MongoConfigUtil.getQuery(getConfiguration())); mongoSplit.setNoTimeout(MongoConfigUtil.isNoTimeout(getConfiguration())); mongoSplit.setFields(MongoConfigUtil.getFields(getConfiguration())); mongoSplit.setSort(MongoConfigUtil.getSort(getConfiguration())); mongoSplit.setKeyField(MongoConfigUtil.getInputKey(getConfiguration())); //Not using any index min/max bounds, so range query is //meaningless here - don't set it //mongoSplit.setUseRangeQuery(...) splits.add(mongoSplit); return splits; }
From source file:com.zjy.mongo.splitter.StandaloneMongoSplitter.java
License:Apache License
@Override public List<InputSplit> calculateSplits() throws SplitFailedException { final DBObject splitKey = MongoConfigUtil.getInputSplitKey(getConfiguration()); final DBObject splitKeyMax = MongoConfigUtil.getMaxSplitKey(getConfiguration()); final DBObject splitKeyMin = MongoConfigUtil.getMinSplitKey(getConfiguration()); final int splitSize = MongoConfigUtil.getSplitSize(getConfiguration()); final MongoClientURI inputURI; DBCollection inputCollection = null; final ArrayList<InputSplit> returnVal; try {//from www . j a va2 s. c o m inputURI = MongoConfigUtil.getInputURI(getConfiguration()); MongoClientURI authURI = MongoConfigUtil.getAuthURI(getConfiguration()); if (authURI != null) { inputCollection = MongoConfigUtil.getCollectionWithAuth(inputURI, authURI); } else { inputCollection = MongoConfigUtil.getCollection(inputURI); } returnVal = new ArrayList<InputSplit>(); final String ns = inputCollection.getFullName(); if (LOG.isDebugEnabled()) { LOG.debug(String.format("Running splitVector on namespace: %s.%s; hosts: %s", inputURI.getDatabase(), inputURI.getCollection(), inputURI.getHosts())); } final DBObject cmd = BasicDBObjectBuilder.start("splitVector", ns).add("keyPattern", splitKey) .add("min", splitKeyMin).add("max", splitKeyMax) // force:True is misbehaving it seems .add("force", false).add("maxChunkSize", splitSize).get(); CommandResult data; boolean ok = true; try { data = inputCollection.getDB().getSisterDB(inputURI.getDatabase()).command(cmd, ReadPreference.primary()); } catch (final MongoException e) { // 2.0 servers throw exceptions rather than info in a CommandResult data = null; LOG.info(e.getMessage(), e); if (e.getMessage().contains("unrecognized command: splitVector")) { ok = false; } else { throw e; } } if (data != null) { if (data.containsField("$err")) { throw new SplitFailedException("Error calculating splits: " + data); } else if (!data.get("ok").equals(1.0)) { ok = false; } } if (!ok) { final CommandResult stats = inputCollection.getStats(); if (stats.containsField("primary")) { final DBCursor shards = inputCollection.getDB().getSisterDB("config").getCollection("shards") .find(new BasicDBObject("_id", stats.getString("primary"))); try { if (shards.hasNext()) { final DBObject shard = shards.next(); final String host = ((String) shard.get("host")).replace(shard.get("_id") + "/", ""); final MongoClientURI shardHost; if (authURI != null) { shardHost = new MongoClientURIBuilder(authURI).host(host).build(); } else { shardHost = new MongoClientURIBuilder(inputURI).host(host).build(); } MongoClient shardClient = null; try { shardClient = new MongoClient(shardHost); data = shardClient.getDB(shardHost.getDatabase()).command(cmd, ReadPreference.primary()); } catch (final Exception e) { LOG.error(e.getMessage(), e); } finally { if (shardClient != null) { shardClient.close(); } } } } finally { shards.close(); } } if (data != null && !data.get("ok").equals(1.0)) { throw new SplitFailedException("Unable to calculate input splits: " + data.get("errmsg")); } } // Comes in a format where "min" and "max" are implicit // and each entry is just a boundary key; not ranged final BasicDBList splitData = (BasicDBList) data.get("splitKeys"); if (splitData.size() == 0) { LOG.warn( "WARNING: No Input Splits were calculated by the split code. Proceeding with a *single* split. Data may be too" + " small, try lowering 'mongo.input.split_size' if this is undesirable."); } BasicDBObject lastKey = null; // Lower boundary of the first min split // If splitKeyMin was given, use it as first boundary. if (!splitKeyMin.toMap().isEmpty()) { lastKey = new BasicDBObject(splitKeyMin.toMap()); } for (final Object aSplitData : splitData) { final BasicDBObject currentKey = (BasicDBObject) aSplitData; returnVal.add(createSplitFromBounds(lastKey, currentKey)); lastKey = currentKey; } BasicDBObject maxKey = null; // If splitKeyMax was given, use it as last boundary. if (!splitKeyMax.toMap().isEmpty()) { maxKey = new BasicDBObject(splitKeyMax.toMap()); } // Last max split final MongoInputSplit lastSplit = createSplitFromBounds(lastKey, maxKey); returnVal.add(lastSplit); } finally { if (inputCollection != null) { MongoConfigUtil.close(inputCollection.getDB().getMongo()); } } return returnVal; }
From source file:com.zjy.mongo.util.MongoConfigUtil.java
License:Apache License
/** * Retrieve a DBCollection from a MongoDB URI. * @param uri the MongoDB URI/* w w w . ja v a 2s . com*/ * @return the DBCollection in the URI */ public static DBCollection getCollection(final MongoClientURI uri) { try { return getMongoClient(uri).getDB(uri.getDatabase()).getCollection(uri.getCollection()); } catch (Exception e) { throw new IllegalArgumentException("Couldn't connect and authenticate to get collection", e); } }
From source file:com.zjy.mongo.util.MongoConfigUtil.java
License:Apache License
/** * Get an authenticated DBCollection from a MongodB URI. * @param authURI the URI with which to authenticate * @param uri the MongoDB URI/* w w w.j a v a 2s. co m*/ * @return the authenticated DBCollection */ public static DBCollection getCollectionWithAuth(final MongoClientURI uri, final MongoClientURI authURI) { //Make sure auth uri is valid and actually has a username/pw to use if (authURI == null || authURI.getUsername() == null || authURI.getPassword() == null) { throw new IllegalArgumentException( "auth URI is empty or does not contain a valid username/password combination."); } DBCollection coll; try { Mongo mongo = getMongoClient(authURI); coll = mongo.getDB(uri.getDatabase()).getCollection(uri.getCollection()); return coll; } catch (Exception e) { throw new IllegalArgumentException("Couldn't connect and authenticate to get collection", e); } }
From source file:DS.Model.java
public void MongoConnect() { MongoClientURI connString = new MongoClientURI(mongoURI); mClient = new MongoClient(connString); db = mClient.getDatabase(connString.getDatabase()); }
From source file:edu.eci.arsw.examples.MongoDBAccessExample.java
public static void main(String ap[]) throws UnknownHostException { MongoClientURI uri = new MongoClientURI("mongodb://test:test@ds031631.mongolab.com:31631/documents"); MongoClient client = new MongoClient(uri); DB db = client.getDB(uri.getDatabase()); DBCollection names = db.getCollection("commontypos"); DBCursor cur = names.find();//from w w w. j av a 2 s . c o m while (cur.hasNext()) { DBObject doc = cur.next(); System.out.println(doc.get("words")); System.out.println(doc.get("pocibilidad")); } }
From source file:io.sip3.tapir.salto.configuration.MongoConfiguration.java
License:Apache License
@Bean public MongoDatabase db(MongoClientURI uri) { MongoClient client = new MongoClient(uri); return client.getDatabase(uri.getDatabase()); }
From source file:localdomain.localhost.MongoDbTroubleshooter.java
License:Open Source License
public void run() throws Exception { try (PrintWriter writer = new PrintWriter(System.out)) { MongoClientURI mongoClientUri = new MongoClientURI(uri); writer.println("" + "host=" + mongoClientUri.getHosts() + ",username=" + mongoClientUri.getUsername() + ",database=" + mongoClientUri.getDatabase() + ",collection=" + mongoClientUri.getCollection() + ""); writer.println();/* w ww.j a v a 2s . c o m*/ writer.println(); writer.println("# MongoClient"); writer.println(); MongoClient mongoClient = new MongoClient(mongoClientUri); writer.println("" + mongoClient + ""); writer.println(); writer.println(); writer.println("# Databases"); writer.println(); try { List<String> databaseNames = mongoClient.getDatabaseNames(); for (String databaseName : databaseNames) { writer.println("* " + databaseName + (databaseName.equals(mongoClientUri.getDatabase()) ? " - default database" : "")); } } catch (Exception e) { writer.println("Could not list the databases of the MongoDB instance: '" + e.getMessage() + "'"); } writer.println(); writer.println(); writer.println("# Database"); writer.println(); DB db = mongoClient.getDB(mongoClientUri.getDatabase()); writer.println("DB: " + db.getName() + ""); writer.println(); writer.println(); writer.println("## Collections"); writer.println(); Set<String> myCollections = db.getCollectionNames(); if (myCollections.isEmpty()) { writer.println("NO COLLECTIONS!"); } else { for (String collection : myCollections) { DBCollection dbCollection = db.getCollection(collection); writer.println("* " + dbCollection.getName() + " - " + dbCollection.getCount() + " entries"); } } } }