List of usage examples for com.mongodb ServerAddress ServerAddress
public ServerAddress(final InetSocketAddress inetSocketAddress)
From source file:com.bosscs.spark.mongodb.extractor.MongoNativeExtractor.java
License:Apache License
@Override public Partition[] getPartitions(S config) { MongoClient mongoClient = null;/*w w w. j a v a2 s. c o m*/ try { mongoDeepJobConfig = initConfig(config, mongoDeepJobConfig); DBCollection collection; ServerAddress address = new ServerAddress(mongoDeepJobConfig.getHost()); List<ServerAddress> addressList = new ArrayList<>(); addressList.add(address); mongoClient = new MongoClient(addressList); //mongoClient.setReadPreference(ReadPreference.nearest()); DB db = mongoClient.getDB(mongoDeepJobConfig.getDatabase()); collection = db.getCollection(mongoDeepJobConfig.getCollection()); return isShardedCollection(collection) ? calculateShardChunks(collection) : calculateSplits(collection); } catch (UnknownHostException e) { throw new GenericException(e); } finally { if (mongoClient != null) { mongoClient.close(); } } }
From source file:com.bosscs.spark.mongodb.extractor.MongoNativeExtractor.java
License:Apache License
/** * Gets server address list./*from w w w .j a v a2 s . c om*/ * * @param addressStringList the address string list * @return the server address list * @throws UnknownHostException the unknown host exception */ private List<ServerAddress> getServerAddressList(List<String> addressStringList) throws UnknownHostException { List<ServerAddress> addressList = new ArrayList<>(); for (String addressString : addressStringList) { addressList.add(new ServerAddress(addressString)); } return addressList; }
From source file:com.bosscs.spark.mongodb.reader.MongoReader.java
License:Apache License
/** * Init void.//from w w w . j a v a 2 s . co m * * @param partition the partition */ public void init(Partition partition) { try { List<ServerAddress> addressList = new ArrayList<>(); for (String s : (List<String>) ((HadoopPartition) partition).splitWrapper().getReplicas()) { addressList.add(new ServerAddress(s)); } //Credentials List<MongoCredential> mongoCredentials = new ArrayList<>(); if (mongoDeepJobConfig.getUsername() != null && mongoDeepJobConfig.getPassword() != null) { MongoCredential credential = MongoCredential.createMongoCRCredential( mongoDeepJobConfig.getUsername(), mongoDeepJobConfig.getDatabase(), mongoDeepJobConfig.getPassword().toCharArray()); mongoCredentials.add(credential); } mongoClient = new MongoClient(addressList, mongoCredentials); //mongoClient.setReadPreference(ReadPreference.valueOf(mongoDeepJobConfig.getReadPreference())); db = mongoClient.getDB(mongoDeepJobConfig.getDatabase()); collection = db.getCollection(mongoDeepJobConfig.getCollection()); dbCursor = collection.find(generateFilterQuery((MongoPartition) partition), mongoDeepJobConfig.getDBFields()); } catch (UnknownHostException e) { throw new ExtractorInitializationException(e); } }
From source file:com.camel.realtimelog.PersistenceMongoAccessor.java
License:Open Source License
private void setMongoClient() { if (mongo == null) { try {/*from w ww .j a v a2 s. c om*/ ServerAddress sa = new ServerAddress(serverAddress); MongoCredential mc = MongoCredential.createMongoCRCredential(username, dbName, password.toCharArray()); List<MongoCredential> mcs = new ArrayList<MongoCredential>(); mcs.add(mc); mongo = new MongoClient(sa, mcs); } catch (UnknownHostException e) { e.printStackTrace(); } db = mongo.getDB(dbName); logs = db.getCollection(collectionName); } }
From source file:com.clavain.muninmxcd.java
License:Apache License
/** * @param args the command line arguments *///from w ww .jav a2 s. c o m public static void main(String[] args) { if (args.length < 1) { System.err.println("Usage: java -jar MuninMXcd.jar <full path to config>"); System.exit(1); } try { initialArgs = args; p = new Properties(); FileInputStream propInFile = null; propInFile = new FileInputStream(args[0]); p.loadFromXML(propInFile); String logfile = p.getProperty("log.file"); socketTimeout = Integer.parseInt(p.getProperty("socket.timeout")); PatternLayout layout = new PatternLayout("%d{ISO8601} %-5p %m%n"); ConsoleAppender consoleAppender = new ConsoleAppender(layout); logger.addAppender(consoleAppender); FileAppender fileAppender = new FileAppender(layout, logfile, false); logger.addAppender(fileAppender); logger.info("MuninMX Collector Daemon - " + version + " starting up..."); logger.info("Loading configuration from <" + args[0] + ">"); String l_strLogLevel = p.getProperty("log.level"); // ALL | DEBUG | INFO | WARN | ERROR | FATAL | OFF: if (l_strLogLevel.equals("ALL")) { logger.setLevel(Level.ALL); } else if (l_strLogLevel.equals("DEBUG")) { logger.setLevel(Level.DEBUG); } else if (l_strLogLevel.equals("INFO")) { logger.setLevel(Level.INFO); } else if (l_strLogLevel.equals("WARN")) { logger.setLevel(Level.WARN); } else if (l_strLogLevel.equals("ERROR")) { logger.setLevel(Level.ERROR); } else if (l_strLogLevel.equals("FATAL")) { logger.setLevel(Level.FATAL); } else { logger.setLevel(Level.OFF); } if (p.getProperty("log.more") != null) { if (p.getProperty("log.more").equals("true")) { logMore = true; } } } catch (Exception ex) { System.err.println("Failed to Init basic logging infastructure: " + ex.getLocalizedMessage()); System.exit(1); } try { // connect to db logger.info("Connecting to TokuMX"); MongoClientOptions.Builder builder = new MongoClientOptions.Builder(); builder.connectionsPerHost(400); builder.autoConnectRetry(true); builder.threadsAllowedToBlockForConnectionMultiplier(10); // speed up inserts, we dont care if we miss some if the shit hits the fan builder.writeConcern(WriteConcern.NONE); m = new MongoClient(new ServerAddress(p.getProperty("mongo.host")), builder.build()); // connect to mysql connectToDatabase(p); // PreFilling Nodes, max 100 in concurrent logger.info("Loading initial MuninNode details. This can take a few minutes..."); v_munin_nodes = new CopyOnWriteArrayList<>(); v_cinterval_plugins = new CopyOnWriteArrayList<>(); v_sockets = new CopyOnWriteArrayList<>(); java.sql.Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery("SELECT * FROM nodes"); while (rs.next()) { MuninNode mn = new MuninNode(); mn.setHostname(rs.getString("hostname")); mn.setNodename(rs.getString("hostname")); mn.setNode_id(rs.getInt("id")); mn.setPort(rs.getInt("port")); mn.setUser_id(rs.getInt("user_id")); mn.setQueryInterval(rs.getInt("query_interval")); mn.setStr_via(rs.getString("via_host")); mn.setAuthpw(rs.getString("authpw")); mn.setGroup(rs.getString("groupname")); v_munin_nodes.add(mn); logger.info("* " + mn.getHostname() + " queued for pluginfetch"); } // launching quartz scheduler logger.info("Launching Scheduler"); SchedulerFactory sf = new StdSchedulerFactory("quartz.properties"); sched = sf.getScheduler(); sched.start(); // launching quartz scheduler for custom interval logger.info("Launching Custom Interval Scheduler"); SchedulerFactory sfc = new StdSchedulerFactory("customquartz.properties"); sched_custom = sfc.getScheduler(); sched_custom.start(); // starting API server new Thread(new JettyLauncher()).start(); int sleepTime = Integer.parseInt(p.getProperty("startup.sleeptime")); int startupIterations = Integer.parseInt(p.getProperty("startup.iterations")); // scheduling jobs int i = 0; for (MuninNode it_mn : v_munin_nodes) { if (i == startupIterations) { Thread.sleep(sleepTime); i = 0; logger.info("Waiting " + sleepTime + "ms for new scheduling slot"); } scheduleJob(it_mn); i++; } // schedule custom interval jobs dbScheduleAllCustomJobs(); // add all alerts dbAddAllAlerts(); // Service Checks logger.info("Launching Service Check Scheduler"); SchedulerFactory sfsc = new StdSchedulerFactory("checksquartz.properties"); sched_checks = sfsc.getScheduler(); sched_checks.start(); // load service checks from database stmt = conn.createStatement(); rs = stmt.executeQuery("SELECT * FROM service_checks"); while (rs.next()) { Gson gson = new Gson(); ServiceCheck tc = gson.fromJson(rs.getString("json"), ServiceCheck.class); tc.setCid(rs.getInt("id")); tc.setUser_id(rs.getInt("user_id")); v_serviceChecks.add(tc); logger.info("* " + tc.getCheckname() + " Service Check added"); } // queue service checks for (ServiceCheck it_sc : v_serviceChecks) { scheduleServiceCheck(it_sc); } // starting MongoExecutor new Thread(new MongoExecutor()).start(); // starting MongoExecutor for Package Tracking and Essential Informations new Thread(new MongoEssentialExecutor()).start(); // starting MongoExecutor for Service Checks new Thread(new MongoCheckExecutor()).start(); // starting newnodewatcher new Thread(new NewNodeWatcher()).start(); // start pushover sending message new Thread(new PushOverLimiter()).start(); // SMS Limiter new Thread(new SMSLimiter()).start(); // TTS Limiter new Thread(new TTSLimiter()).start(); // start DataRetention Worker new Thread(new DataRetentionWorker()).start(); // start Error Notify Inspector new Thread(new ErrorNotifyExecutor()).start(); int curTime; int toTime; int mb = 1024 * 1024; while (true) { Thread.sleep(5000); System.out.println("Mongo Queue Size: " + mongo_queue.size()); System.out.println("Mongo Check Queue Size: " + mongo_check_queue.size()); System.out.println("Mongo Essential Queue Size: " + mongo_essential_queue.size()); Runtime runtime = Runtime.getRuntime(); //Print used memory System.out.println("Used Memory:" + (runtime.totalMemory() - runtime.freeMemory()) / mb); //Print free memory System.out.println("Free Memory:" + runtime.freeMemory() / mb); //Print total available memory System.out.println("Total Memory:" + runtime.totalMemory() / mb); //Print Maximum available memory System.out.println("Max Memory:" + runtime.maxMemory() / mb); System.out.println(" "); if (p.getProperty("kill.sockets").equals("true")) { System.out.println("Sockets: " + v_sockets.size()); // check for sockets that we can kill curTime = getUnixtime(); for (SocketCheck sc : v_sockets) { toTime = curTime - 120; if (sc.getSocketCreated() < toTime) { if (!sc.getSocket().isClosed()) { logger.info("timing out socket... from: " + sc.getHostname()); sc.closeSocket(); v_sockets.remove(sc); } else { v_sockets.remove(sc); } } } } } } catch (Exception ex) { System.err.println("Something went wrong as fuck: " + ex.getLocalizedMessage()); logger.fatal("Something went wrong as fuck. exiting: " + ex.getLocalizedMessage()); ex.printStackTrace(); System.exit(1); } }
From source file:com.crosstreelabs.cognitio.service.mongo.provider.MongoClientProvider.java
License:Apache License
protected List<ServerAddress> getSeeds() { List<ServerAddress> seeds = new ArrayList<>(); for (String seed : config.mongoHosts) { try {//from w ww.j a va 2s . c o m seeds.add(new ServerAddress(seed)); } catch (UnknownHostException ex) { LOGGER.warn("{}", ex); } } return seeds; }
From source file:com.ebay.cloud.cms.mongo.MongoDataSource.java
License:Apache License
static List<ServerAddress> parseServerString(String servers) { CheckConditions.checkArgument(!StringUtils.isNullOrEmpty(servers)); ArrayList<ServerAddress> addrs = new ArrayList<ServerAddress>(); String[] serverList = servers.split(","); for (String server : serverList) { String[] pair = server.split(":"); try {/* w w w .jav a 2 s . c om*/ if (pair.length == 1) { addrs.add(new ServerAddress(pair[0].trim())); } else if (pair.length == 2) { addrs.add(new ServerAddress(pair[0].trim(), Integer.valueOf(pair[1].trim()))); } else { throw new RuntimeException("Bad server string: " + servers); } } catch (UnknownHostException e) { throw new RuntimeException("parse server string error", e); } } return addrs; }
From source file:com.edgytech.umongo.RouterPanel.java
License:Apache License
public void regenConfigDB(ButtonBase button) throws UnknownHostException { MongoClient cmongo = getRouterNode().getMongoClient(); String servers = getStringFieldValue(Item.regenServers); final String db = getStringFieldValue(Item.regenDB); final String col = getStringFieldValue(Item.regenCollection); final String ns = db + "." + col; final DBObject shardKey = ((DocBuilderField) getBoundUnit(Item.regenShardKey)).getDBObject(); final boolean unique = getBooleanFieldValue(Item.regenKeyUnique); final BasicDBObject result = new BasicDBObject(); result.put("ns", ns); result.put("shardKey", shardKey); result.put("unique", unique); // create direct mongo for each replica set String[] serverList = servers.split("\n"); List<ServerAddress> list = new ArrayList<ServerAddress>(); String txt = ""; String primaryShard = null;//from w w w .j a v a 2 s . com final BasicDBObject shardList = new BasicDBObject(); HashMap<MongoClient, String> mongoToShard = new HashMap<MongoClient, String>(); sLoop: for (String server : serverList) { server = server.trim(); String[] tokens = server.split("/"); if (tokens.length != 2) { new InfoDialog(null, "Error", null, "Server format must be like 'hostname:port/shard', one by line") .show(); return; } server = tokens[0]; String shard = tokens[1]; if (primaryShard == null) { primaryShard = shard; } ServerAddress addr = new ServerAddress(server); // filter out if replset already exists for (MongoClient replset : mongoToShard.keySet()) { if (replset.getServerAddressList().contains(addr)) { continue sLoop; } } list.clear(); list.add(addr); MongoClient mongo = new MongoClient(list); // UMongo.instance.addMongoClient(mongo, null); // make request to force server detection mongo.getDatabaseNames(); mongoToShard.put(mongo, shard); String desc = null; if (!mongo.getDatabaseNames().contains(db) || !mongo.getDB(db).getCollectionNames().contains(col)) { desc = "Collection not present!"; } else { // try to see if shard key has index DBObject index = mongo.getDB(db).getCollection("system.indexes") .findOne(new BasicDBObject("key", shardKey)); if (index != null) { desc = "shard key found"; } else { desc = "shard key NOT found!"; } } txt += mongo.toString() + " shard=" + shard + " - " + desc + "\n"; BasicDBObject shardObj = new BasicDBObject("servers", mongo.toString()); shardObj.put("status", desc); if (shardList.containsField(shard)) { new InfoDialog(null, "Error", null, "Duplicate Shard name " + shard).show(); return; } shardList.put(shard, shardObj); } result.put("shards", shardList); FormDialog dia = (FormDialog) getBoundUnit(Item.regenRSList); dia.setStringFieldValue(Item.regenRSListArea, txt); if (!dia.show()) { return; } DB config = cmongo.getDB("config"); // add database record BasicDBObject doc = new BasicDBObject("_id", db); doc.put("partitioned", true); doc.put("primary", primaryShard); config.getCollection("databases").save(doc); // add collection record doc = new BasicDBObject("_id", ns); doc.put("lastmod", new Date()); doc.put("dropped", false); doc.put("key", shardKey); doc.put("unique", unique); config.getCollection("collections").save(doc); final DBCollection chunks = config.getCollection("chunks"); long count = chunks.count(new BasicDBObject("ns", ns)); if (count > 0) { dia = (FormDialog) getBoundUnit(Item.regenDeleteChunks); if (dia.show()) { chunks.remove(new BasicDBObject("ns", ns)); } else { return; } } // add temp collection to sort chunks with shard key final DBCollection tmpchunks = config.getCollection("_tmpchunks_" + col); tmpchunks.drop(); // should be safe environment, and dup keys should be ignored tmpchunks.setWriteConcern(WriteConcern.NORMAL); // can use shardKey as unique _id // tmpchunks.ensureIndex(shardKey, "shardKey", true); // create filter for shard fields final DBObject shardKeyFilter = new BasicDBObject(); // final DBObject shardKeyDescend = new BasicDBObject(); boolean hasId = false; for (String key : shardKey.keySet()) { shardKeyFilter.put(key, 1); if (key.equals("_id")) { hasId = true; } } if (!hasId) { shardKeyFilter.put("_id", 0); } dia = (FormDialog) getBoundUnit(Item.regenConfirm); if (!dia.show()) { return; } // now fetch all records from each shard final AtomicInteger todo = new AtomicInteger(mongoToShard.size()); for (Map.Entry<MongoClient, String> entry : mongoToShard.entrySet()) { final MongoClient mongo = entry.getKey(); final String shard = entry.getValue(); new DbJob() { @Override public Object doRun() throws Exception { BasicDBObject shardObj = (BasicDBObject) shardList.get(shard); long count = mongo.getDB(db).getCollection(col).count(); shardObj.put("count", count); DBCursor cur = mongo.getDB(db).getCollection(col).find(new BasicDBObject(), shardKeyFilter); long i = 0; int inserted = 0; long start = System.currentTimeMillis(); while (cur.hasNext() && !isCancelled()) { BasicDBObject key = (BasicDBObject) cur.next(); setProgress((int) ((++i * 100.0f) / count)); try { BasicDBObject entry = new BasicDBObject("_id", key); entry.put("_shard", shard); tmpchunks.insert(entry); ++inserted; } catch (Exception e) { getLogger().log(Level.WARNING, e.getMessage(), e); } } if (isCancelled()) { shardObj.put("cancelled", true); } shardObj.put("inserted", inserted); shardObj.put("scanTime", System.currentTimeMillis() - start); todo.decrementAndGet(); return null; } @Override public String getNS() { return tmpchunks.getFullName(); } @Override public String getShortName() { return "Scanning " + shard; } @Override public boolean isDeterminate() { return true; } }.addJob(); } new DbJob() { @Override public Object doRun() throws Exception { // wait for all shards to be done long start = System.currentTimeMillis(); while (todo.get() > 0 && !isCancelled()) { Thread.sleep(2000); } if (isCancelled()) { result.put("cancelled", true); return result; } // find highest current timestamp DBCursor cur = chunks.find().sort(new BasicDBObject("lastmod", -1)).batchSize(-1); BasicDBObject chunk = (BasicDBObject) (cur.hasNext() ? cur.next() : null); BSONTimestamp ts = (BSONTimestamp) (chunk != null ? chunk.get("lastmod") : null); // now infer chunk ranges long count = tmpchunks.count(); result.put("uniqueKeys", count); int numChunks = 0; cur = tmpchunks.find().sort(new BasicDBObject("_id", 1)); BasicDBObject prev = (BasicDBObject) cur.next(); BasicDBObject next = null; // snap prev to minkey BasicDBObject theid = (BasicDBObject) prev.get("_id"); for (String key : shardKey.keySet()) { theid.put(key, new MinKey()); } String currentShard = prev.getString("_shard"); int i = 1; while (cur.hasNext()) { next = (BasicDBObject) cur.next(); setProgress((int) ((++i * 100.0f) / count)); String newShard = next.getString("_shard"); if (newShard.equals(currentShard)) continue; // add chunk ts = getNextTimestamp(ts); chunk = getChunk(ns, shardKey, prev, next, ts); chunks.insert(chunk); prev = next; currentShard = prev.getString("_shard"); ++numChunks; } // build max next = new BasicDBObject(); for (String key : shardKey.keySet()) { next.put(key, new MaxKey()); } next = new BasicDBObject("_id", next); ts = getNextTimestamp(ts); chunk = getChunk(ns, shardKey, prev, next, ts); chunks.insert(chunk); ++numChunks; result.put("numChunks", numChunks); result.put("totalTime", System.currentTimeMillis() - start); return result; } @Override public String getNS() { return chunks.getFullName(); } @Override public String getShortName() { return "Creating Chunks"; } @Override public boolean isDeterminate() { return true; } }.addJob(); }
From source file:com.edgytech.umongo.ServerNode.java
License:Apache License
public ServerNode(String host, MongoClientOptions opts, boolean isReplica, boolean isConfig) throws UnknownHostException { setLabel(host);// w w w . j a v a 2 s .c o m this.host = host; this.serverAddress = new ServerAddress(host); serverMongo = new MongoClient(serverAddress, opts); serverMongo.addOption(Bytes.QUERYOPTION_SLAVEOK); this.isReplica = isReplica; this.isConfig = isConfig; try { xmlLoad(Resource.getXmlDir(), Resource.File.serverNode, null); } catch (Exception ex) { getLogger().log(Level.SEVERE, null, ex); } markStructured(); }
From source file:com.facebook.presto.mongodb.MongoClientConfig.java
License:Apache License
private List<ServerAddress> buildSeeds(Iterable<String> hostPorts) { ImmutableList.Builder<ServerAddress> builder = ImmutableList.builder(); for (String hostPort : hostPorts) { List<String> values = PORT_SPLITTER.splitToList(hostPort); checkArgument(values.size() == 1 || values.size() == 2, "Invalid ServerAddress format. Requires host[:port]"); try {//from w ww. ja v a 2 s . c o m if (values.size() == 1) { builder.add(new ServerAddress(values.get(0))); } else { builder.add(new ServerAddress(values.get(0), Integer.parseInt(values.get(1)))); } } catch (NumberFormatException e) { throw Throwables.propagate(e); } } return builder.build(); }