List of usage examples for com.mongodb ServerAddress ServerAddress
public ServerAddress(@Nullable final String host, final int port)
From source file:org.opencb.commons.datastore.mongodb.MongoDataStoreManager.java
License:Apache License
private MongoClient newMongoClient(MongoClientOptions mongoClientOptions, MongoCredential mongoCredential) { MongoClient mc;/* w ww . j a va 2 s .co m*/ if (dataStoreServerAddresses.size() == 1) { if (mongoCredential != null) { mc = new MongoClient( new ServerAddress(dataStoreServerAddresses.get(0).getHost(), dataStoreServerAddresses.get(0).getPort()), Arrays.asList(mongoCredential), mongoClientOptions); } else { mc = new MongoClient(new ServerAddress(dataStoreServerAddresses.get(0).getHost(), dataStoreServerAddresses.get(0).getPort()), mongoClientOptions); } } else { List<ServerAddress> serverAddresses = new ArrayList<>(dataStoreServerAddresses.size()); for (DataStoreServerAddress serverAddress : dataStoreServerAddresses) { serverAddresses.add(new ServerAddress(serverAddress.getHost(), serverAddress.getPort())); } if (mongoCredential != null) { mc = new MongoClient(serverAddresses, Arrays.asList(mongoCredential), mongoClientOptions); } else { mc = new MongoClient(serverAddresses, mongoClientOptions); } } return mc; }
From source file:org.openmama.dbwriter.MamaMongodb.java
/** * @param serverString semi colon delimited list of server address (host:port) */// w ww. java 2 s. c o m private void populateServers(final String serverString) { if (serverString == null || serverString.isEmpty()) { throw new MamaException("No Servers configured for MongoDB"); } for (final String s : serverString.split(",")) { String host = ""; int port = 0; try { final String[] address = s.split(":"); host = address[0]; port = Integer.parseInt(address[1]); servers.add(new ServerAddress(host, port)); } catch (final UnknownHostException ex) { logger.severe(String.format("Server %s:%d was unknown: %s", host, port, ex.getMessage())); System.exit(MamaDatabaseWriter.EXIT_FAIL); } catch (final Exception ex) { logger.severe(String.format("Error parsing servers %s", ex.getMessage())); } } }
From source file:org.openmhealth.reference.data.mongodb.MongoDao.java
License:Apache License
/** * Default constructor, which will create the connection to the MongoDB. * /*from w w w . ja v a2s.c om*/ * @param properties * The user-defined properties to use to setup the connection. * * @throws OmhException * There was a problem setting up the connection to the database. */ public MongoDao(final Properties properties) throws OmhException { super(properties); // Create the singular Mongo instance. try { // Create the empty list of credentials. List<MongoCredential> credentials = new LinkedList<MongoCredential>(); // If a username and password were given, use them. if ((MongoDao.getInstance().getDatabaseUsername() != null) && (MongoDao.getInstance().getDatabasePassword() != null)) { credentials.add(MongoCredential.createMongoCRCredential( MongoDao.getInstance().getDatabaseUsername(), MongoDao.getInstance().getDatabaseName(), MongoDao.getInstance().getDatabaseUsername().toCharArray())); } // Create the MongoClient. mongo = new MongoClient(new ServerAddress(getDatabaseAddress(), getDatabasePort()), credentials); } catch (UnknownHostException e) { throw new OmhException("The database could not setup.", e); } // Instantiate the specific components. new MongoAuthenticationTokenBin(); new MongoAuthorizationCodeBin(); new MongoAuthorizationCodeResponseBin(); new MongoAuthorizationTokenBin(); new MongoDataSet(); new MongoRegistry(); new MongoThirdPartyBin(); new MongoUserBin(); }
From source file:org.opentdc.mongo.AbstractMongodbServiceProvider.java
License:Open Source License
/** * Establish a connection to the MongoDB database. * @throws InternalServerErrorException if the connection could not be established */// ww w . ja v a 2 s . c o m protected static void connect() throws InternalServerErrorException { ServerAddress _serverAddress = new ServerAddress(mongodbHost, mongodbPort); if (mongoClient != null) { logger.warning("re-connecting to an already open mongoDB connection"); } else { try { if (mongodbUser == null || mongodbUser.isEmpty() || mongodbPwd == null || mongodbPwd.isEmpty()) { logger.warning("access to MongoDB is insecure"); mongoClient = new MongoClient(_serverAddress); } else { MongoCredential _credential = MongoCredential.createCredential(mongodbUser, mongodbName, mongodbPwd.toCharArray()); mongoClient = new MongoClient(_serverAddress, Arrays.asList(_credential)); } // default WriteConcern.ACKNOWLEDGED // optionally, choose another WriteConcern eg. _mongoClient.setWriteConcern(WriteConcern.JOURNALED); } catch (MongoException _ex) { logger.info(_ex.getMessage()); logger.info(_ex.getStackTrace().toString()); throw new InternalServerErrorException( "could not establish a connection to the mongodb: MongoException in AbstractMongodbServiceProvider.constructor"); } } }
From source file:org.ossmeter.platform.Configuration.java
License:Open Source License
public Mongo getMongoConnection() throws UnknownHostException { // System.out.println("Mongo hosts: " + properties.getProperty(MONGO_HOSTS, "localhost:27017")); String[] hosts = properties.getProperty(MONGO_HOSTS, "localhost:27017").split(","); if (hosts.length > 1) { List<ServerAddress> mongoHostAddresses = new ArrayList<>(); for (String host : hosts) { String[] s = host.split(":"); mongoHostAddresses.add(new ServerAddress(s[0], Integer.valueOf(s[1]))); }/*from ww w . jav a2 s . c o m*/ // MongoOptions options = new MongoOptions(); // options.connectTimeout = 1000; return new Mongo(mongoHostAddresses);//,options); } else { return new Mongo();//hosts[0]); } }
From source file:org.ossmeter.platform.mining.msr14.Extractor.java
License:Open Source License
public static void main(String[] args) throws Exception { long start = System.currentTimeMillis(); Mongo msrMongo = new Mongo(new ServerAddress("localhost", 1234)); // GitHub challenge data Mongo bioMongo = new Mongo(new ServerAddress("localhost", 12345));// Extracted data // Create indexes Biodiversity bio = new Biodiversity(bioMongo.getDB("biodiversity")); bio.setClearPongoCacheOnSync(true);/*from www .j a va2 s . co m*/ bioMongo.getDB("biodiversity").getCollection("users").ensureIndex(new BasicDBObject("login", 1)); BasicDBObject index = new BasicDBObject(); index.put("name", 1); index.put("ownerName", 1); bioMongo.getDB("biodiversity").getCollection("projects").ensureIndex(index); index = new BasicDBObject(); index.put("projectName", 1); index.put("projectOwner", 1); bioMongo.getDB("biodiversity").getCollection("projectMemberships").ensureIndex(index); index = new BasicDBObject(); index.put("projectName", 1); index.put("userName", 1); bioMongo.getDB("biodiversity").getCollection("projectMemberships").ensureIndex(index); bioMongo.getDB("biodiversity").getCollection("projectMemberships") .ensureIndex(new BasicDBObject("userName", 1)); DB msrDb = msrMongo.getDB("msr14"); // #1 User extraction System.out.println("Extracting users..."); DBCursor cursor = msrDb.getCollection("users").find(); cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); Iterator<DBObject> it = cursor.iterator(); int count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // User user = new User(); // user.setGhId(obj.getString("id")); // user.setLogin(obj.getString("login")); // user.setLocation(obj.getString("location")); // user.setPublicRepos(obj.getInt("public_repos", 0)); // user.setJoinedDate(obj.getString("created_at")); // user.setFollowerCount(obj.getInt("followers", 0)); // user.setFollowingCount(obj.getInt("following", 0)); // user.setPublicGists(obj.getInt("public_gists", 0)); // // bio.getUsers().add(user); // // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // //// #1.2 Project extraction // System.out.println("Extracting projects..."); // cursor = msrDb.getCollection("repos").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // Project project = new Project(); // project.setName(obj.getString("name")); // project.setGhId(obj.getString("id")); // project.setCreatedAt(obj.getString("created_at")); // project.setSize(obj.getInt("size", 0)); // project.setWatchersCount(obj.getInt("watchers",0)); // project.setWatchersCount2(obj.getInt("watchers_count",0)); // project.setLanguage(obj.getString("language")); // project.setForks(obj.getInt("forks", 0)); // project.setForksCount(obj.getInt("forks_count", 0)); // project.setOpenIssues(obj.getInt("open_issues",0)); // project.setOpenIssuesCount(obj.getInt("open_issues_count",0)); // project.setOpenIssues(obj.getInt("open_issues",0)); // project.setNetworkCount(obj.getInt("network_count", 0)); // // BasicDBObject ownerObj = (BasicDBObject) obj.get("owner"); // User owner = null; // if (ownerObj != null) { // owner = bio.getUsers().findOne(User.LOGIN.eq(ownerObj.getString("login"))); // if (owner !=null) { // project.setOwner(owner); // project.setOwnerName(owner.getLogin()); // } // } // bio.getProjects().add(project); // // if (owner != null) { // This comes here as to reference the project, we need to have added to the project list first // ProjectMembership pm = getProjectMembership(bio, owner, project); // pm.setOwner(true); // } // // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // // System.exit(0); //// #2 Follower/following extraction // System.out.println("Extracting followers..."); // cursor = msrDb.getCollection("followers").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String followerLogin = obj.getString("login"); // String followedLogin = obj.getString("follows"); // // User follower = bio.getUsers().findOne(User.LOGIN.eq(followerLogin)); // User followed = bio.getUsers().findOne(User.LOGIN.eq(followedLogin)); // // if (follower != null && followed != null) { // follower.getFollowing().add(followed); // followed.getFollowers().add(follower); // } else{ //// System.err.println("Follower or followed is null. Follower: " +follower + ". followed: " + followed); // } // if (follower != null) follower.setFollowingCount(follower.getFollowingCount()+1); // if (followed != null) followed.setFollowerCount(followed.getFollowerCount()+1); // // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // // System.exit(0); System.out.println("Clearing ProjectMembership commit data"); for (ProjectMembership pm : bio.getProjectMemberships()) { pm.setCommitCount(0); pm.setCommitTotalChanges(0); pm.setCommitAdditions(0); pm.setCommitDeletions(0); pm.setCommitsAsAuthor(0); pm.setCommitsAsCommitter(0); pm.setCommitTotalFiles(0); pm.setAverageFilesPerCommit(0); pm.getCommitTimes().clear(); } bio.sync(); System.out.println("cleared."); // #3 Commits System.out.println("Extracting commits..."); cursor = msrDb.getCollection("commits").find(); cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); it = cursor.iterator(); count = 0; while (it.hasNext()) { BasicDBObject obj = (BasicDBObject) it.next(); // Author and committer BasicDBObject commitAuthor = (BasicDBObject) obj.get("author"); BasicDBObject commitCommitter = (BasicDBObject) obj.get("committer"); String authorLogin = ""; if (commitAuthor != null) authorLogin = commitAuthor.getString("login"); String committerLogin = ""; if (commitCommitter != null) committerLogin = commitCommitter.getString("login"); // Stats BasicDBObject stats = (BasicDBObject) obj.get("stats"); if (stats == null) stats = new BasicDBObject(); // Create a new one so we can get zeroed values int total = stats.getInt("total", 0); int additions = stats.getInt("additions", 0); int deletions = stats.getInt("deletions", 0); String commitDate = ((BasicDBObject) ((BasicDBObject) obj.get("commit")).get("author")) .getString("date"); BasicDBList files = (BasicDBList) obj.get("files"); String[] url = convertUrlIntoProjectNameAndOwner(obj.getString("url")); ProjectMembership authorPm = null; ProjectMembership committerPm = null; if (authorLogin != null) { authorPm = getProjectMembership(bio, authorLogin, url[1], url[0]); authorPm.setCommitCount(authorPm.getCommitCount() + 1); authorPm.setCommitTotalChanges(authorPm.getCommitTotalChanges() + total); authorPm.setCommitAdditions(authorPm.getCommitAdditions() + additions); authorPm.setCommitDeletions(authorPm.getCommitDeletions() + deletions); authorPm.setCommitsAsAuthor(authorPm.getCommitsAsAuthor() + 1); if (files != null) authorPm.setCommitTotalFiles(authorPm.getCommitTotalChanges() + files.size()); authorPm.setAverageFilesPerCommit(authorPm.getCommitTotalFiles() / authorPm.getCommitCount()); authorPm.getCommitTimes().add(commitDate); } if (authorLogin != null && !authorLogin.equals(committerLogin)) { committerPm = getProjectMembership(bio, committerLogin, url[1], url[0]); committerPm.setCommitCount(committerPm.getCommitCount() + 1); // committerPm.setCommitTotalChanges(committerPm.getCommitTotalChanges()+total); // committerPm.setCommitAdditions(committerPm.getCommitAdditions()+additions); // committerPm.setCommitDeletions(committerPm.getCommitDeletions()+deletions); committerPm.setCommitsAsCommitter(committerPm.getCommitsAsCommitter() + 1); committerPm.setCommitTotalFiles(committerPm.getCommitTotalChanges() + files.size()); committerPm.setAverageFilesPerCommit(committerPm.getCommitTotalFiles() / authorPm.getCommitCount()); if (files != null) committerPm.setCommitTotalFiles(committerPm.getCommitTotalChanges() + files.size()); committerPm .setAverageFilesPerCommit(committerPm.getCommitTotalFiles() / committerPm.getCommitCount()); committerPm.getCommitTimes().add(commitDate); } bio.sync(); count++; if (count % 1000 == 0) { System.out.print(count + ", "); bio.sync(); } } cursor.close(); bio.sync(); System.out.println(); System.exit(0); // if (author != null) { //// if (author.getCommits() ==null) author.setCommits(new Commits()); // author.setCommitCount(author.getCommitCount()+1); // author.setCommitTotalChanges(author.getCommitTotalChanges()+total); // author.setCommitAdditions(author.getCommitAdditions()+additions); // author.setCommitDeletions(author.getCommitDeletions()+deletions); // author.setCommitsAsAuthor(author.getCommitsAsAuthor()+1); // author.getCommitTimes().add(commitDate); // } // if (committer != null) { //// if (committer.getCommits() ==null) committer.setCommits(new Commits()); // committer.setCommitCount(committer.getCommitCount()+1); // committer.setCommitTotalChanges(committer.getCommitTotalChanges()+total); // committer.setCommitAdditions(committer.getCommitAdditions()+additions); // committer.setCommitDeletions(committer.getCommitDeletions()+deletions); // committer.setCommitsAsCommitter(committer.getCommitsAsCommitter()+1); // committer.getCommitTimes().add(commitDate); // } // // ProjectMembership authorPm = null; // ProjectMembership committerPm = null; // //// Only a very small number of commit comments actually reference the repo //// Instead we're going to have to strip the string // String[] url = convertUrlIntoProjectNameAndOwner(obj.getString("url")); // Project project = null; // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(url[1]), Project.OWNERNAME.eq(url[0])).iterator(); // if (repoIt.hasNext()) { // project = repoIt.next(); // if (project != null) { // // project.setCommitCount(project.getCommitCount()+1); // project.setCommitTotalChanges(project.getCommitTotalChanges()+total); // project.setCommitAdditions(project.getCommitAdditions()+additions); // project.setCommitDeletions(project.getCommitDeletions()+deletions); // project.getCommitTimes().add(commitDate); // // if (author != null) { // authorPm = getProjectMembership(bio, author, project); // authorPm.setCommitCount(authorPm.getCommitCount()+1); // authorPm.setCommitTotalChanges(authorPm.getCommitTotalChanges()+total); // authorPm.setCommitAdditions(authorPm.getCommitAdditions()+additions); // authorPm.setCommitDeletions(authorPm.getCommitDeletions()+deletions); // authorPm.setCommitsAsAuthor(authorPm.getCommitsAsAuthor()+1); // // // Avoid duplicating information // if (committer != null && author.getLogin().equals(committer.getLogin())) { // authorPm.setCommitsAsCommitter(authorPm.getCommitsAsCommitter()+1); // } // // authorPm.getCommitTimes().add(commitDate); // } // if (committer != null && author != null && !author.getLogin().equals(committer.getLogin())) { // committerPm = getProjectMembership(bio, committer, project); // committerPm.setCommitCount(committerPm.getCommitCount()+1); // committerPm.setCommitTotalChanges(committerPm.getCommitTotalChanges()+total); // committerPm.setCommitAdditions(committerPm.getCommitAdditions()+additions); // committerPm.setCommitDeletions(committerPm.getCommitDeletions()+deletions); // committerPm.setCommitsAsCommitter(committerPm.getCommitsAsCommitter()+1); // // committerPm.getCommitTimes().add(commitDate); // } // } // } // else { // System.err.println("Didn't find project:" + url[0] + ":"+url[1] + ", prestrip: " + obj.getString("url")); // } // bio.getProjectMemberships().sync(); // bio.sync(); // // // // Files // BasicDBList files = (BasicDBList) obj.get("files"); // if (files != null) { // for (Object f : files) { // BasicDBObject file = (BasicDBObject)f; // // String filename = file.getString("filename"); // if (filename.lastIndexOf(".") != -1) { // If it has an extension, we want that. If not, use the entire filename // filename = filename.substring(filename.lastIndexOf(".")); // filename = filename.toLowerCase(); // Ensure consistency // } // // FIXME: Should strip any /'s if there is no '.' - i.e. just the last one // // if (author != null) addArtefact(author, filename); // if (committer != null) addArtefact(committer, filename); //// if (project != null) addArtefact(project, filename); // } // } // // if (author != null && files !=null) { // author.setCommitTotalFiles(author.getCommitTotalFiles()+files.size()); // author.setAverageFilesPerCommit(author.getCommitTotalFiles()/author.getCommitCount()); // } // if (committer != null && files !=null && (author==null || !committer.getLogin().equals(author.getLogin()))) { // committer.setCommitTotalFiles(committer.getCommitTotalFiles()+files.size()); // committer.setAverageFilesPerCommit(committer.getCommitTotalFiles()/committer.getCommitCount()); // } // if (authorPm !=null && files != null) { // authorPm.setCommitTotalFiles(authorPm.getCommitTotalChanges()+files.size()); // authorPm.setAverageFilesPerCommit(authorPm.getCommitTotalFiles()/authorPm.getCommitCount()); // } // if (committerPm != null && files != null) { // committerPm.setCommitTotalFiles(committerPm.getCommitTotalChanges()+files.size()); // committerPm.setAverageFilesPerCommit(committerPm.getCommitTotalFiles()/committerPm.getCommitCount()); // } // // if (project!=null && files != null) { // project.setCommitTotalFiles(project.getCommitTotalChanges()+files.size()); // project.setAverageFilesPerCommit(project.getCommitTotalFiles()/project.getCommitCount()); // } // bio.getProjectMemberships().sync(); // bio.sync(); // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // cursor.close(); // bio.sync(); // System.out.println(); // // System.exit(0); //// #4 Commit comments // System.out.println("Extracting commit comments..."); // cursor = msrDb.getCollection("commit_comments").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "user", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { // System.err.println("Found commit comment with unrecognised user: " + username); // continue; // } // // user.setNumberOfCommitComments(user.getNumberOfCommitComments()+1); // //// if (!user.getDbObject().containsField("commitCommentTimes")) { //// user.getDbObject().put("commitCommentTimes", new BasicDBList()); //// } //// user.getCommitCommentTimes().add(obj.getString("created_at")); // // // Only a very small number of commit comments actually reference the repo // // Instead we're going to have to strip the string // String[] url = convertUrlIntoProjectNameAndOwner(obj.getString("url")); // //// System.out.println("Querying project " + url[1] + " and owner " + url[0]); // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(url[1]), Project.OWNERNAME.eq(url[0])).iterator(); //// if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfCommitComments(project.getNumberOfCommitComments()+1); // // if (!project.getDbObject().containsField("commitCommentTimes")) { // project.getDbObject().put("commitCommentTimes", new BasicDBList()); // } // project.getCommitCommentTimes().add(obj.getString("created_at")); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setNumberOfCommitComments(pm.getNumberOfCommitComments()+1); // // if (!pm.getDbObject().containsField("commitCommentTimes")) { // pm.getDbObject().put("commitCommentTimes", new BasicDBList()); // } // pm.getCommitCommentTimes().add(obj.getString("created_at")); // } //// } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // cursor.close(); // bio.sync(); // System.out.println(); // System.exit(0); //// //FIXME: THIS IS CAUSING THE CPU TO HIS 350% AND THEN KILLS THE LAPTOP?!?!?!?!? // #5 Pull requests System.out.println("Extracting pull requests..."); cursor = msrDb.getCollection("pull_requests").find(); cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); it = cursor.iterator(); System.out.println("Clearing previous data"); for (User u : bio.getUsers()) { if (!u.getDbObject().containsField("pullRequestTimes")) { u.getDbObject().put("pullRequestTimes", new BasicDBList()); } u.getPullRequestTimes().clear(); u.setNumberOfPullRequests(0); } bio.sync(); for (Project u : bio.getProjects()) { if (!u.getDbObject().containsField("pullRequestTimes")) { u.getDbObject().put("pullRequestTimes", new BasicDBList()); } u.getPullRequestTimes().clear(); u.setNumberOfPullRequests(0); } bio.sync(); for (ProjectMembership u : bio.getProjectMemberships()) { if (!u.getDbObject().containsField("pullRequestTimes")) { u.getDbObject().put("pullRequestTimes", new BasicDBList()); } u.getPullRequestTimes().clear(); u.setNumberOfPullRequests(0); } bio.sync(); System.out.println("Cleared!"); count = 0; while (it.hasNext()) { BasicDBObject obj = (BasicDBObject) it.next(); String username = getUserLoginName(bio, "user", "login", obj); User user = bio.getUsers().findOne(User.LOGIN.eq(username)); if (user == null) { // System.err.println("Found pull request with unrecognised user:" + username); continue; } if (!user.getDbObject().containsField("pullRequestTimes")) { user.getDbObject().put("pullRequestTimes", new BasicDBList()); } user.getPullRequestTimes().add(obj.getString("created_at")); user.setNumberOfPullRequests(user.getNumberOfPullRequests() + 1); // Project System.out.println(obj.getString("repo") + " " + obj.getString("owner") + obj.getString("_id")); ProjectMembership pm = getProjectMembership(bio, user.getLogin(), obj.getString("repo"), obj.getString("owner")); pm.setNumberOfPullRequests(pm.getNumberOfPullRequests() + 1); if (!pm.getDbObject().containsField("pullRequestTimes")) { pm.getDbObject().put("pullRequestTimes", new BasicDBList()); } pm.getPullRequestTimes().add(obj.getString("created_at")); // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // FIXME Causes it to run out of heap! // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfPullRequests(project.getNumberOfPullRequests()+1); // // if (!project.getDbObject().containsField("pullRequestTimes")) { // project.getDbObject().put("pullRequestTimes", new BasicDBList()); // } // project.getPullRequestTimes().add(obj.getString("created_at")); // // } // } else { // System.err.println("Didn't find project:" + obj.getString("repo") + ":"+obj.getString("owner")); // } count++; if (count % 1000 == 0) { System.out.print(count + ", "); bio.sync(); System.gc(); } } bio.sync(); System.out.println(); System.exit(0); //// #6 Pull request comments // System.out.println("Extracting pull request comments..."); // cursor = msrDb.getCollection("pull_request_comments").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "user", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { //// System.err.println("Found pull request comment with unrecognised user:" + username); // continue; // } // // if (!user.getDbObject().containsField("pullRequestCommentTimes")) { // user.getDbObject().put("pullRequestCommentTimes", new BasicDBList()); // } // user.getPullRequestCommentTimes().add(obj.getString("created_at")); // user.setNumberOfPullRequestComments(user.getNumberOfPullRequestComments()+1); // // // Project // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); //// if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfPullRequestComments(project.getNumberOfPullRequestComments()+1); // if (!project.getDbObject().containsField("pullRequestCommentTimes")) { // project.getDbObject().put("pullRequestCommentTimes", new BasicDBList()); // } // project.getPullRequestCommentTimes().add(obj.getString("created_at")); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setNumberOfPullRequestComments(pm.getNumberOfPullRequestComments()+1); // // if (!pm.getDbObject().containsField("pullRequestCommentTimes")) { // pm.getDbObject().put("pullRequestCommentTimes", new BasicDBList()); // } // pm.getPullRequestCommentTimes().add(obj.getString("created_at")); // } //// } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // System.exit(0); //// #7 Issues // System.out.println("Extracting issues..."); // cursor = msrDb.getCollection("issues").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "user", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { //// System.err.println("Found issue with unrecognised user:" + username); // continue; // } // // if (!user.getDbObject().containsField("issueTimes")) { // user.getDbObject().put("issueTimes", new BasicDBList()); // } // user.getIssueTimes().add(obj.getString("created_at")); // user.setNumberOfIssues(user.getNumberOfIssues()+1); // // // Project // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfIssues(project.getNumberOfIssues()+1); // // if (!project.getDbObject().containsField("issueTimes")) { // project.getDbObject().put("issueTimes", new BasicDBList()); // } // project.getIssueTimes().add(obj.getString("created_at")); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setNumberOfIssues(pm.getNumberOfIssues()+1); // // if (!pm.getDbObject().containsField("issueTimes")) { // pm.getDbObject().put("issueTimes", new BasicDBList()); // } // pm.getIssueTimes().add(obj.getString("created_at")); // } // } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // System.exit(0); //// #8 Issue comments // System.out.println("Extracting issue comments..."); // cursor = msrDb.getCollection("issue_comments").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "user", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { //// System.err.println("Found issue comment with unrecognised user:" + username); // continue; // } // // if (!user.getDbObject().containsField("issueCommentTimes")) { // user.getDbObject().put("issueCommentTimes", new BasicDBList()); // } // user.getIssueCommentTimes().add(obj.getString("created_at")); // user.setNumberOfIssueComments(user.getNumberOfIssueComments()+1); // // // Project // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfIssueComments(project.getNumberOfIssueComments()+1); // // if (!project.getDbObject().containsField("issueCommentTimes")) { // project.getDbObject().put("issueCommentTimes", new BasicDBList()); // } // project.getIssueCommentTimes().add(obj.getString("created_at")); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setNumberOfIssueComments(pm.getNumberOfIssueComments()+1); // // if (!pm.getDbObject().containsField("issueCommentTimes")) { // pm.getDbObject().put("issueCommentTimes", new BasicDBList()); // } // pm.getIssueCommentTimes().add(obj.getString("created_at")); // } // } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // System.exit(0); //// #9 Issue events // System.out.println("Extracting issue events..."); // cursor = msrDb.getCollection("issue_events").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "actor", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { //// System.err.println("Found issue event with unrecognised user:" + username); // continue; // } // // String eventKind = obj.getString("event"); // IssueEventKind kind = null; //FIXME // // switch (eventKind) { // case "closed": kind = IssueEventKind.CLOSED; break; // case "assigned": kind = IssueEventKind.ASSIGNED; break; // case "mentioned": kind = IssueEventKind.MENTIONED; break; // case "merged": kind = IssueEventKind.MERGED; break; // case "referenced": kind = IssueEventKind.REFERENCED; break; // case "reopened": kind = IssueEventKind.REOPENED; break; // case "subscribed": kind = IssueEventKind.SUBSCRIBED; break; // case "head_ref_deleted" : kind = IssueEventKind.HEAD_REF_DELETED; break; // case "head_ref_restored" : kind = IssueEventKind.HEAD_REF_RESTORED; break; // case "head_ref_cleaned" : kind = IssueEventKind.HEAD_REF_CLEANED; break; // case "unsubscribed" : kind = IssueEventKind.UNSUBSCRIBED; break; // default: // System.err.println("Unrecognised issue event kind: " + eventKind); // } // if (kind == null) continue; // // boolean eventKindFound = false; // // if (!user.getDbObject().containsField("issueEvents")) { // user.getDbObject().put("issueEvents", new BasicDBList()); // } // // for (IssueEvent ie : user.getIssueEvents()) { // if (ie.getEventKind().equals(kind)) { // ie.setCount(ie.getCount()+1); // eventKindFound = true; // break; // } // } // if (!eventKindFound) { // IssueEvent ie = new IssueEvent(); // ie.setEventKind(kind); // ie.setCount(1); // user.getIssueEvents().add(ie); // } // // // Project // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // // if (!project.getDbObject().containsField("issueEvents")) { // project.getDbObject().put("issueEvents", new BasicDBList()); // } // // eventKindFound = false; // for (IssueEvent ie : project.getIssueEvents()) { // if (ie.getEventKind().equals(kind)) { // ie.setCount(ie.getCount()+1); // eventKindFound = true; // break; // } // } // if (!eventKindFound) { // IssueEvent ie = new IssueEvent(); // ie.setEventKind(kind); // ie.setCount(1); // project.getIssueEvents().add(ie); // } // // ProjectMembership pm = getProjectMembership(bio, user, project); // // if (!pm.getDbObject().containsField("issueEvents")) { // pm.getDbObject().put("issueEvents", new BasicDBList()); // } // // eventKindFound = false; // for (IssueEvent ie : pm.getIssueEvents()) { // if (ie.getEventKind().equals(kind)) { // ie.setCount(ie.getCount()+1); // eventKindFound = true; // break; // } // } // if (!eventKindFound) { // IssueEvent ie = new IssueEvent(); // ie.setEventKind(kind); // ie.setCount(1); // pm.getIssueEvents().add(ie); // } // } // // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // System.exit(0); // //// Watchers // System.out.println("Extracting watchers..."); // cursor = msrDb.getCollection("watchers").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // User user = bio.getUsers().findOne(User.LOGIN.eq(obj.getString("login"))); // if (user == null) continue; // // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null && !project.getWatchers().contains(user)) project.getWatchers().add(user); // if (!user.getWatches().contains(project)) user.getWatches().add(project); // } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); //// Org members FIXME: INCOMPLETE: Cannot match the org name against ANYTHING.... // System.out.println("Extracting org members..."); // cursor = msrDb.getCollection("org_members").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String login = obj.getString("login"); // String orgName = obj.getString("org"); // // User user = bio.getUsers().findOne(User.LOGIN.eq(login)); // // User org = bio.getUsers().findOne(User.LOGIN.eq(orgName)); // // if (org!=null){ // System.err.println("Found org! " + orgName); // // } // //// Project project = bio.getProjects().findOne(Project.OWNERNAME.eq("orgName")); //// if (project==null) { //// System.err.println("Didn't find project: " + orgName); //// continue; //// } //// ProjectMembership pm = getProjectMembership(bio, user, project); //// pm.setOrgMember(true); // } // bio.sync(); // System.out.println(); // Repo collaborators // System.out.println("Extracting repo collaborators..."); // cursor = msrDb.getCollection("repo_collaborators").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String login = obj.getString("login"); // String projectName = obj.getString("repo"); // String ownerName = obj.getString("owner"); // // User user = bio.getUsers().findOne(User.LOGIN.eq(login)); // Iterator<Project> repoIt = bio.getProjects().find(Project.OWNERNAME.eq(ownerName), Project.NAME.eq(projectName)).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setCollaborator(true); // } else { // System.err.println("Couldn't find repo. owner: " + ownerName + ", repo: " + projectName); // } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // long end = System.currentTimeMillis(); System.out.println("Finished at " + new Date()); long duration = end - start; System.out.println("Duration: " + duration); }
From source file:org.ossmeter.platform.mining.msr14.stats.KMeansClustering.java
License:Open Source License
public static void main(String[] args) { try {//from w w w . j av a 2s .com Mongo mongo = new Mongo(new ServerAddress("localhost", 12345)); Biodiversity bio = new Biodiversity(mongo.getDB("biodiversity_1")); bio.setClearPongoCacheOnSync(true); List<User> users = new ArrayList<User>(); // Massively inefficient int usss = 0; for (User u : bio.getUsers()) { if (u.getCommitAdditions() == 0 && u.getCommitCount() == 0 && u.getCommitDeletions() == 0 && u.getNumberOfCommitComments() == 0 && u.getNumberOfIssues() == 0 && u.getNumberOfIssueComments() == 0 && u.getNumberOfPullRequests() == 0 && u.getNumberOfPullRequestComments() == 0) { continue; } users.add(u); usss++; if (usss > 5000) break; } KMeansClustering kmeans = new KMeansClustering(); HashMap<Centroid, List<User>> clusters = kmeans.compute(4, users); FileWriter writer = new FileWriter(new File("groups.csv")); writer.write("group,numberOfCommits,numberOfChanges,numberOfAdditions,numberOfDeletions," + "numberOfCommitsAsAuthor,numberOfCommitsAsCommitter,numberOfIssues,numberOfIssueComments," + "numberOfPullRequests,numberOfPullRequestComments,numberOfCommitComments,numberOfForks+\n"); int group = 0; for (Centroid c : clusters.keySet()) { List<User> us = clusters.get(c); for (User u : us) { writer.write(group + ","); writer.write(u.getCommitCount() + ","); writer.write(u.getCommitTotalChanges() + ","); writer.write(u.getCommitAdditions() + ","); writer.write(u.getCommitDeletions() + ","); writer.write(u.getCommitsAsAuthor() + ","); writer.write(u.getCommitsAsCommitter() + ","); writer.write(u.getNumberOfIssues() + ","); writer.write(u.getNumberOfIssueComments() + ","); writer.write(u.getNumberOfPullRequests() + ","); writer.write(u.getNumberOfPullRequestComments() + ","); writer.write(u.getNumberOfCommitComments() + ","); writer.write(u.getNumberOfForks() + "\n"); } group++; writer.flush(); } writer.close(); } catch (UnknownHostException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } }
From source file:org.ow2.play.governance.storage.MongoSubscriptionRegistry.java
License:Open Source License
/** * Returns a List of ServerAddress objects for each host specified in the * hostname property. Returns an empty list if configuration is detected to * be invalid, e.g.:/*from w w w. jav a 2 s .co m*/ * <ul> * <li>Port property doesn't contain either one port or one port per host</li> * <li>After parsing port property to integers, there isn't either one port * or one port per host</li> * </ul> * * @param hostname * Blank space delimited hostnames * @param port * Blank space delimited ports. Must specify one port for all * hosts or a port per host. * @return List of ServerAddresses to connect to */ private List<ServerAddress> getServerAddresses(String hostname, String port) { List<ServerAddress> addresses = new ArrayList<ServerAddress>(); String[] hosts = hostname.split(" "); String[] ports = port.split(" "); if (ports.length != 1 && ports.length != hosts.length) { // errorHandler // .error("MongoDB appender port property must contain one port or a port per host", // null, ErrorCode.ADDRESS_PARSE_FAILURE); } else { List<Integer> portNums = getPortNums(ports); // Validate number of ports again after parsing if (portNums.size() != 1 && portNums.size() != hosts.length) { // error("MongoDB appender port property must contain one port or a valid port per host", // null, ErrorCode.ADDRESS_PARSE_FAILURE); } else { boolean onePort = (portNums.size() == 1); int i = 0; for (String host : hosts) { int portNum = (onePort) ? portNums.get(0) : portNums.get(i); try { addresses.add(new ServerAddress(host.trim(), portNum)); } catch (UnknownHostException e) { // errorHandler // .error("MongoDB appender hostname property contains unknown host", // e, ErrorCode.ADDRESS_PARSE_FAILURE); } i++; } } } return addresses; }
From source file:org.pentaho.mongo.MongoUtils.java
License:Open Source License
public static MongoClient initConnection(String hostsPorts, String singlePort, MongoCredential cred, boolean useAllReplicaSetMembers, String connTimeout, String socketTimeout, String readPreference, String writeConcern, String wTimeout, boolean journaled, List<String> tagSet, VariableSpace vars, LogChannelInterface log) throws KettleException { hostsPorts = vars.environmentSubstitute(hostsPorts); singlePort = vars.environmentSubstitute(singlePort); int singlePortI = -1; try {/*from www. j a v a 2s. c o m*/ singlePortI = Integer.parseInt(singlePort); } catch (NumberFormatException n) { // don't complain } if (Const.isEmpty(hostsPorts)) { throw new KettleException(BaseMessages.getString(PKG, "MongoUtils.Message.Error.EmptyHostsString")); //$NON-NLS-1$ } List<ServerAddress> repSet = new ArrayList<ServerAddress>(); // if (useAllReplicaSetMembers) { // repSet = getReplicaSetMembers(hostsPorts, singlePort, cred, vars, log); // // if (repSet.size() == 0) { // useAllReplicaSetMembers = false; // drop back and just configure using // // what we've been given // } else { // if (log != null) { // StringBuilder builder = new StringBuilder(); // for (ServerAddress s : repSet) { // builder.append(s.toString()).append(" "); // } // log.logBasic(BaseMessages.getString(PKG, // "MongoUtils.Message.UsingTheFollowingReplicaSetMembers") // + " " // + builder.toString()); // } // } // } // if (!useAllReplicaSetMembers) { String[] parts = hostsPorts.trim().split(","); //$NON-NLS-1$ for (String part : parts) { // host:port? int port = singlePortI != -1 ? singlePortI : MONGO_DEFAULT_PORT; String[] hp = part.split(":"); //$NON-NLS-1$ if (hp.length > 2) { throw new KettleException( BaseMessages.getString(PKG, "MongoUtils.Message.Error.MalformedHost", part)); //$NON-NLS-1$ } String host = hp[0]; if (hp.length == 2) { // non-default port try { port = Integer.parseInt(hp[1].trim()); } catch (NumberFormatException n) { throw new KettleException( BaseMessages.getString(PKG, "MongoUtils.Message.Error.UnableToParsePortNumber", hp[1])); //$NON-NLS-1$ } } try { ServerAddress s = new ServerAddress(host, port); repSet.add(s); } catch (UnknownHostException u) { throw new KettleException(u); } } // } MongoClientOptions.Builder mongoOptsBuilder = new MongoClientOptions.Builder(); configureConnectionOptions(mongoOptsBuilder, connTimeout, socketTimeout, readPreference, writeConcern, wTimeout, journaled, tagSet, vars, log); MongoClientOptions opts = mongoOptsBuilder.build(); try { // Mongo's java driver will discover all replica set or shard // members (Mongos) automatically when MongoClient is constructed // using a list of ServerAddresses. The javadocs state that MongoClient // should be constructed using a SingleServer address instance (rather // than a list) when connecting to a stand-alone host - this is why // we differentiate here between a list containing one ServerAddress // and a single ServerAddress instance via the useAllReplicaSetMembers // flag. if (cred == null) { return (repSet.size() > 1 || (useAllReplicaSetMembers && repSet.size() >= 1) ? new MongoClient(repSet, opts) : (repSet.size() == 1 ? new MongoClient(repSet.get(0), opts) : new MongoClient(new ServerAddress("localhost"), opts))); //$NON-NLS-1$ } List<MongoCredential> credList = new ArrayList<MongoCredential>(); credList.add(cred); return (repSet.size() > 1 || (useAllReplicaSetMembers && repSet.size() >= 1) ? new MongoClient(repSet, credList, opts) : (repSet.size() == 1 ? new MongoClient(repSet.get(0), credList, opts) : new MongoClient(new ServerAddress("localhost"), credList, opts))); //$NON-NLS-1$ } catch (UnknownHostException u) { throw new KettleException(u); } }
From source file:org.pentaho.mongo.MongoUtils.java
License:Open Source License
/** * Connect to mongo and retrieve any replica set members defined in the * local.system.replset collection. Note that this method is not actually * needed for configuring a connection to a replica set or sharded mongo * cluster (mongos) as the driver will determine this automatically. * /*w w w. j a va 2 s .c o m*/ * @param hostsPorts the host(s) and port(s) to use for initiating the * connection * @param singlePort default port to use if none specified in the hostsPorts * string * @param username username to use for authenticating * @param password password to use for authenticating * @param vars environment variables * @param log for logging * @return a list of replica set ServerAddresses * @throws KettleException if a problem occurs */ public static List<ServerAddress> getReplicaSetMembers(String hostsPorts, String singlePort, MongoCredential cred, VariableSpace vars, LogChannelInterface log) throws KettleException { List<ServerAddress> replSetMembers = new ArrayList<ServerAddress>(); if (log != null) { log.logBasic( BaseMessages.getString(PKG, "MongoUtils.Message.QueryingForReplicaSetMembers", hostsPorts)); } BasicDBList members = getRepSetMemberRecords(hostsPorts, singlePort, cred, vars, log); try { if (members != null && members.size() > 0) { for (int i = 0; i < members.size(); i++) { Object m = members.get(i); if (m != null) { String hostPort = ((DBObject) m).get("host").toString(); //$NON-NLS-1$ if (!Const.isEmpty(hostPort)) { String[] parts = hostPort.split(":"); //$NON-NLS-1$ if (parts.length == 2) { ServerAddress address = new ServerAddress(parts[0].trim(), Integer.parseInt(parts[1].trim())); replSetMembers.add(address); } else { ServerAddress address = new ServerAddress(parts[0].trim()); replSetMembers.add(address); } } } } } } catch (Exception ex) { throw new KettleException(ex); } return replSetMembers; }