List of usage examples for com.mongodb Mongo getDB
@Deprecated public DB getDB(final String dbName)
From source file:org.ossmeter.platform.admin.ProjectListAnalysis.java
License:Open Source License
@Get("json") public String represent() { Series<Header> responseHeaders = (Series<Header>) getResponse().getAttributes() .get("org.restlet.http.headers"); if (responseHeaders == null) { responseHeaders = new Series(Header.class); getResponse().getAttributes().put("org.restlet.http.headers", responseHeaders); }/*from www.j a va 2 s . c o m*/ responseHeaders.add(new Header("Access-Control-Allow-Origin", "*")); responseHeaders.add(new Header("Access-Control-Allow-Methods", "GET")); try { Mongo mongo = Configuration.getInstance().getMongoConnection(); DB db = mongo.getDB("ossmeter"); DBCollection col = db.getCollection("metricAnalysis"); ObjectMapper mapper = new ObjectMapper(); ArrayNode results = mapper.createArrayNode(); DBCursor cursor = col.find(); try { while (cursor.hasNext()) { results.add(cursor.next().toString()); } } finally { cursor.close(); } mongo.close(); return results.toString(); } catch (UnknownHostException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } return null; }
From source file:org.ossmeter.platform.admin.ProjectMetricAnalysis.java
License:Open Source License
@Get("json") public String represent() { Series<Header> responseHeaders = (Series<Header>) getResponse().getAttributes() .get("org.restlet.http.headers"); if (responseHeaders == null) { responseHeaders = new Series(Header.class); getResponse().getAttributes().put("org.restlet.http.headers", responseHeaders); }//from w ww .ja v a2 s. c o m responseHeaders.add(new Header("Access-Control-Allow-Origin", "*")); responseHeaders.add(new Header("Access-Control-Allow-Methods", "GET")); try { String projectId = (String) getRequest().getAttributes().get("projectId"); String metricId = (String) getRequest().getAttributes().get("metricId"); Mongo mongo = Configuration.getInstance().getMongoConnection(); DB db = mongo.getDB("ossmeter"); DBCollection col = db.getCollection("metricAnalysis"); BasicDBObject query = new BasicDBObject("projectId", projectId); query.append("metricId", metricId); BasicDBObject sort = new BasicDBObject("analysisDate", 1); DBCursor cursor = col.find(query).sort(sort); ObjectMapper mapper = new ObjectMapper(); ArrayNode results = mapper.createArrayNode(); while (cursor.hasNext()) { DBObject obj = cursor.next(); if ("true".equals(getQueryValue("incDate"))) { ObjectNode on = mapper.createObjectNode(); on.put("date", obj.get("analysisDate").toString()); on.put("millis", (Double) obj.get("millisTaken")); results.add(on); } else { results.add((Double) obj.get("millisTaken")); } } mongo.close(); return results.toString(); } catch (UnknownHostException e1) { e1.printStackTrace(); } return null; }
From source file:org.ossmeter.platform.communicationchannel.nntp.local.NNTPDownloader.java
License:Open Source License
public void downloadMessages(NntpNewsGroup newsgroup) throws Exception { final long startTime = System.currentTimeMillis(); long previousTime = startTime; previousTime = printTimeMessage(startTime, previousTime, "Download started"); NNTPClient nntpClient = NntpUtil.connectToNntpServer(newsgroup); NewsgroupInfo newsgroupInfo = NntpUtil.selectNewsgroup(nntpClient, newsgroup); int lastArticleChecked = newsgroupInfo.getFirstArticle(); previousTime = printTimeMessage(startTime, previousTime, "First message in newsgroup:\t" + lastArticleChecked); int lastArticle = newsgroupInfo.getLastArticle(); previousTime = printTimeMessage(startTime, previousTime, "Last message in newsgroup:\t" + lastArticle); previousTime = printTimeMessage(startTime, previousTime, "Articles in newsgroup:\t" + newsgroupInfo.getArticleCount()); System.err.println();/*from w w w . j a va 2s .c o m*/ Mongo mongo = new Mongo(); DB db = mongo.getDB(newsgroup.getName() + "LocalStorage"); Messages dbMessages = new Messages(db); NewsgroupData newsgroupData = dbMessages.getNewsgroup().findOneByName(newsgroup.getName()); if (newsgroupData != null) { int newsgroupLastArticleChecked = Integer.parseInt(newsgroupData.getLastArticleChecked()); if (newsgroupLastArticleChecked > lastArticleChecked) { lastArticleChecked = newsgroupLastArticleChecked; } previousTime = printTimeMessage(startTime, previousTime, "Last article checked set to:\t" + lastArticleChecked); } else { newsgroupData = new NewsgroupData(); newsgroupData.setName(newsgroup.getName()); newsgroupData.setUrl(newsgroup.getUrl()); newsgroupData.setAuthenticationRequired(newsgroup.getAuthenticationRequired()); newsgroupData.setUsername(newsgroup.getUsername()); newsgroupData.setPassword(newsgroup.getPassword()); newsgroupData.setPort(newsgroup.getPort()); newsgroupData.setInterval(newsgroup.getInterval()); newsgroupData.setFirstArticle(lastArticleChecked + ""); dbMessages.getNewsgroup().add(newsgroupData); } int retrievalStep = RETRIEVAL_STEP; while (lastArticleChecked < lastArticle) { if (lastArticleChecked + retrievalStep > lastArticle) { retrievalStep = lastArticle - lastArticleChecked; } Article[] articles = NntpUtil.getArticleInfo(nntpClient, lastArticleChecked + 1, lastArticleChecked + retrievalStep); if (articles.length > 0) { Article lastArticleRetrieved = articles[articles.length - 1]; lastArticleChecked = lastArticleRetrieved.getArticleNumber(); newsgroupData.setLastArticleChecked(lastArticleChecked + ""); } previousTime = printTimeMessage(startTime, previousTime, "downloaded:\t" + articles.length + " nntp articles"); previousTime = printTimeMessage(startTime, previousTime, "downloading contents\t"); for (Article article : articles) { ArticleData articleData = new ArticleData(); articleData.setUrl(newsgroup.getUrl()); articleData.setArticleNumber(article.getArticleNumber()); articleData.setArticleId(article.getArticleId()); articleData.setDate(article.getDate()); articleData.setFrom(article.getFrom()); articleData.setSubject(article.getSubject()); for (String referenceId : article.getReferences()) articleData.getReferences().add(referenceId); articleData.setBody(NntpUtil.getArticleBody(nntpClient, article.getArticleNumber())); dbMessages.getArticles().add(articleData); } dbMessages.sync(); previousTime = printTimeMessage(startTime, previousTime, "stored:\t" + dbMessages.getArticles().size() + " / " + newsgroupInfo.getArticleCount() + " nntp articles sofar"); System.err.println(); } nntpClient.disconnect(); dbMessages.sync(); previousTime = printTimeMessage(startTime, previousTime, "stored:\t" + dbMessages.getArticles().size() + " / " + newsgroupInfo.getArticleCount() + " nntp articles"); }
From source file:org.ossmeter.platform.MetricsRepository.java
License:Open Source License
public MetricsRepository(Project project, Mongo mongo) { super();/*ww w . j a va2 s . c om*/ this.project = project; this.mongo = mongo; this.db = mongo.getDB(project.getShortName()); }
From source file:org.ossmeter.platform.mining.msr14.Extractor.java
License:Open Source License
public static void main(String[] args) throws Exception { long start = System.currentTimeMillis(); Mongo msrMongo = new Mongo(new ServerAddress("localhost", 1234)); // GitHub challenge data Mongo bioMongo = new Mongo(new ServerAddress("localhost", 12345));// Extracted data // Create indexes Biodiversity bio = new Biodiversity(bioMongo.getDB("biodiversity")); bio.setClearPongoCacheOnSync(true);//from w ww.j ava 2 s. co m bioMongo.getDB("biodiversity").getCollection("users").ensureIndex(new BasicDBObject("login", 1)); BasicDBObject index = new BasicDBObject(); index.put("name", 1); index.put("ownerName", 1); bioMongo.getDB("biodiversity").getCollection("projects").ensureIndex(index); index = new BasicDBObject(); index.put("projectName", 1); index.put("projectOwner", 1); bioMongo.getDB("biodiversity").getCollection("projectMemberships").ensureIndex(index); index = new BasicDBObject(); index.put("projectName", 1); index.put("userName", 1); bioMongo.getDB("biodiversity").getCollection("projectMemberships").ensureIndex(index); bioMongo.getDB("biodiversity").getCollection("projectMemberships") .ensureIndex(new BasicDBObject("userName", 1)); DB msrDb = msrMongo.getDB("msr14"); // #1 User extraction System.out.println("Extracting users..."); DBCursor cursor = msrDb.getCollection("users").find(); cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); Iterator<DBObject> it = cursor.iterator(); int count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // User user = new User(); // user.setGhId(obj.getString("id")); // user.setLogin(obj.getString("login")); // user.setLocation(obj.getString("location")); // user.setPublicRepos(obj.getInt("public_repos", 0)); // user.setJoinedDate(obj.getString("created_at")); // user.setFollowerCount(obj.getInt("followers", 0)); // user.setFollowingCount(obj.getInt("following", 0)); // user.setPublicGists(obj.getInt("public_gists", 0)); // // bio.getUsers().add(user); // // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // //// #1.2 Project extraction // System.out.println("Extracting projects..."); // cursor = msrDb.getCollection("repos").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // Project project = new Project(); // project.setName(obj.getString("name")); // project.setGhId(obj.getString("id")); // project.setCreatedAt(obj.getString("created_at")); // project.setSize(obj.getInt("size", 0)); // project.setWatchersCount(obj.getInt("watchers",0)); // project.setWatchersCount2(obj.getInt("watchers_count",0)); // project.setLanguage(obj.getString("language")); // project.setForks(obj.getInt("forks", 0)); // project.setForksCount(obj.getInt("forks_count", 0)); // project.setOpenIssues(obj.getInt("open_issues",0)); // project.setOpenIssuesCount(obj.getInt("open_issues_count",0)); // project.setOpenIssues(obj.getInt("open_issues",0)); // project.setNetworkCount(obj.getInt("network_count", 0)); // // BasicDBObject ownerObj = (BasicDBObject) obj.get("owner"); // User owner = null; // if (ownerObj != null) { // owner = bio.getUsers().findOne(User.LOGIN.eq(ownerObj.getString("login"))); // if (owner !=null) { // project.setOwner(owner); // project.setOwnerName(owner.getLogin()); // } // } // bio.getProjects().add(project); // // if (owner != null) { // This comes here as to reference the project, we need to have added to the project list first // ProjectMembership pm = getProjectMembership(bio, owner, project); // pm.setOwner(true); // } // // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // // System.exit(0); //// #2 Follower/following extraction // System.out.println("Extracting followers..."); // cursor = msrDb.getCollection("followers").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String followerLogin = obj.getString("login"); // String followedLogin = obj.getString("follows"); // // User follower = bio.getUsers().findOne(User.LOGIN.eq(followerLogin)); // User followed = bio.getUsers().findOne(User.LOGIN.eq(followedLogin)); // // if (follower != null && followed != null) { // follower.getFollowing().add(followed); // followed.getFollowers().add(follower); // } else{ //// System.err.println("Follower or followed is null. Follower: " +follower + ". followed: " + followed); // } // if (follower != null) follower.setFollowingCount(follower.getFollowingCount()+1); // if (followed != null) followed.setFollowerCount(followed.getFollowerCount()+1); // // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // // System.exit(0); System.out.println("Clearing ProjectMembership commit data"); for (ProjectMembership pm : bio.getProjectMemberships()) { pm.setCommitCount(0); pm.setCommitTotalChanges(0); pm.setCommitAdditions(0); pm.setCommitDeletions(0); pm.setCommitsAsAuthor(0); pm.setCommitsAsCommitter(0); pm.setCommitTotalFiles(0); pm.setAverageFilesPerCommit(0); pm.getCommitTimes().clear(); } bio.sync(); System.out.println("cleared."); // #3 Commits System.out.println("Extracting commits..."); cursor = msrDb.getCollection("commits").find(); cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); it = cursor.iterator(); count = 0; while (it.hasNext()) { BasicDBObject obj = (BasicDBObject) it.next(); // Author and committer BasicDBObject commitAuthor = (BasicDBObject) obj.get("author"); BasicDBObject commitCommitter = (BasicDBObject) obj.get("committer"); String authorLogin = ""; if (commitAuthor != null) authorLogin = commitAuthor.getString("login"); String committerLogin = ""; if (commitCommitter != null) committerLogin = commitCommitter.getString("login"); // Stats BasicDBObject stats = (BasicDBObject) obj.get("stats"); if (stats == null) stats = new BasicDBObject(); // Create a new one so we can get zeroed values int total = stats.getInt("total", 0); int additions = stats.getInt("additions", 0); int deletions = stats.getInt("deletions", 0); String commitDate = ((BasicDBObject) ((BasicDBObject) obj.get("commit")).get("author")) .getString("date"); BasicDBList files = (BasicDBList) obj.get("files"); String[] url = convertUrlIntoProjectNameAndOwner(obj.getString("url")); ProjectMembership authorPm = null; ProjectMembership committerPm = null; if (authorLogin != null) { authorPm = getProjectMembership(bio, authorLogin, url[1], url[0]); authorPm.setCommitCount(authorPm.getCommitCount() + 1); authorPm.setCommitTotalChanges(authorPm.getCommitTotalChanges() + total); authorPm.setCommitAdditions(authorPm.getCommitAdditions() + additions); authorPm.setCommitDeletions(authorPm.getCommitDeletions() + deletions); authorPm.setCommitsAsAuthor(authorPm.getCommitsAsAuthor() + 1); if (files != null) authorPm.setCommitTotalFiles(authorPm.getCommitTotalChanges() + files.size()); authorPm.setAverageFilesPerCommit(authorPm.getCommitTotalFiles() / authorPm.getCommitCount()); authorPm.getCommitTimes().add(commitDate); } if (authorLogin != null && !authorLogin.equals(committerLogin)) { committerPm = getProjectMembership(bio, committerLogin, url[1], url[0]); committerPm.setCommitCount(committerPm.getCommitCount() + 1); // committerPm.setCommitTotalChanges(committerPm.getCommitTotalChanges()+total); // committerPm.setCommitAdditions(committerPm.getCommitAdditions()+additions); // committerPm.setCommitDeletions(committerPm.getCommitDeletions()+deletions); committerPm.setCommitsAsCommitter(committerPm.getCommitsAsCommitter() + 1); committerPm.setCommitTotalFiles(committerPm.getCommitTotalChanges() + files.size()); committerPm.setAverageFilesPerCommit(committerPm.getCommitTotalFiles() / authorPm.getCommitCount()); if (files != null) committerPm.setCommitTotalFiles(committerPm.getCommitTotalChanges() + files.size()); committerPm .setAverageFilesPerCommit(committerPm.getCommitTotalFiles() / committerPm.getCommitCount()); committerPm.getCommitTimes().add(commitDate); } bio.sync(); count++; if (count % 1000 == 0) { System.out.print(count + ", "); bio.sync(); } } cursor.close(); bio.sync(); System.out.println(); System.exit(0); // if (author != null) { //// if (author.getCommits() ==null) author.setCommits(new Commits()); // author.setCommitCount(author.getCommitCount()+1); // author.setCommitTotalChanges(author.getCommitTotalChanges()+total); // author.setCommitAdditions(author.getCommitAdditions()+additions); // author.setCommitDeletions(author.getCommitDeletions()+deletions); // author.setCommitsAsAuthor(author.getCommitsAsAuthor()+1); // author.getCommitTimes().add(commitDate); // } // if (committer != null) { //// if (committer.getCommits() ==null) committer.setCommits(new Commits()); // committer.setCommitCount(committer.getCommitCount()+1); // committer.setCommitTotalChanges(committer.getCommitTotalChanges()+total); // committer.setCommitAdditions(committer.getCommitAdditions()+additions); // committer.setCommitDeletions(committer.getCommitDeletions()+deletions); // committer.setCommitsAsCommitter(committer.getCommitsAsCommitter()+1); // committer.getCommitTimes().add(commitDate); // } // // ProjectMembership authorPm = null; // ProjectMembership committerPm = null; // //// Only a very small number of commit comments actually reference the repo //// Instead we're going to have to strip the string // String[] url = convertUrlIntoProjectNameAndOwner(obj.getString("url")); // Project project = null; // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(url[1]), Project.OWNERNAME.eq(url[0])).iterator(); // if (repoIt.hasNext()) { // project = repoIt.next(); // if (project != null) { // // project.setCommitCount(project.getCommitCount()+1); // project.setCommitTotalChanges(project.getCommitTotalChanges()+total); // project.setCommitAdditions(project.getCommitAdditions()+additions); // project.setCommitDeletions(project.getCommitDeletions()+deletions); // project.getCommitTimes().add(commitDate); // // if (author != null) { // authorPm = getProjectMembership(bio, author, project); // authorPm.setCommitCount(authorPm.getCommitCount()+1); // authorPm.setCommitTotalChanges(authorPm.getCommitTotalChanges()+total); // authorPm.setCommitAdditions(authorPm.getCommitAdditions()+additions); // authorPm.setCommitDeletions(authorPm.getCommitDeletions()+deletions); // authorPm.setCommitsAsAuthor(authorPm.getCommitsAsAuthor()+1); // // // Avoid duplicating information // if (committer != null && author.getLogin().equals(committer.getLogin())) { // authorPm.setCommitsAsCommitter(authorPm.getCommitsAsCommitter()+1); // } // // authorPm.getCommitTimes().add(commitDate); // } // if (committer != null && author != null && !author.getLogin().equals(committer.getLogin())) { // committerPm = getProjectMembership(bio, committer, project); // committerPm.setCommitCount(committerPm.getCommitCount()+1); // committerPm.setCommitTotalChanges(committerPm.getCommitTotalChanges()+total); // committerPm.setCommitAdditions(committerPm.getCommitAdditions()+additions); // committerPm.setCommitDeletions(committerPm.getCommitDeletions()+deletions); // committerPm.setCommitsAsCommitter(committerPm.getCommitsAsCommitter()+1); // // committerPm.getCommitTimes().add(commitDate); // } // } // } // else { // System.err.println("Didn't find project:" + url[0] + ":"+url[1] + ", prestrip: " + obj.getString("url")); // } // bio.getProjectMemberships().sync(); // bio.sync(); // // // // Files // BasicDBList files = (BasicDBList) obj.get("files"); // if (files != null) { // for (Object f : files) { // BasicDBObject file = (BasicDBObject)f; // // String filename = file.getString("filename"); // if (filename.lastIndexOf(".") != -1) { // If it has an extension, we want that. If not, use the entire filename // filename = filename.substring(filename.lastIndexOf(".")); // filename = filename.toLowerCase(); // Ensure consistency // } // // FIXME: Should strip any /'s if there is no '.' - i.e. just the last one // // if (author != null) addArtefact(author, filename); // if (committer != null) addArtefact(committer, filename); //// if (project != null) addArtefact(project, filename); // } // } // // if (author != null && files !=null) { // author.setCommitTotalFiles(author.getCommitTotalFiles()+files.size()); // author.setAverageFilesPerCommit(author.getCommitTotalFiles()/author.getCommitCount()); // } // if (committer != null && files !=null && (author==null || !committer.getLogin().equals(author.getLogin()))) { // committer.setCommitTotalFiles(committer.getCommitTotalFiles()+files.size()); // committer.setAverageFilesPerCommit(committer.getCommitTotalFiles()/committer.getCommitCount()); // } // if (authorPm !=null && files != null) { // authorPm.setCommitTotalFiles(authorPm.getCommitTotalChanges()+files.size()); // authorPm.setAverageFilesPerCommit(authorPm.getCommitTotalFiles()/authorPm.getCommitCount()); // } // if (committerPm != null && files != null) { // committerPm.setCommitTotalFiles(committerPm.getCommitTotalChanges()+files.size()); // committerPm.setAverageFilesPerCommit(committerPm.getCommitTotalFiles()/committerPm.getCommitCount()); // } // // if (project!=null && files != null) { // project.setCommitTotalFiles(project.getCommitTotalChanges()+files.size()); // project.setAverageFilesPerCommit(project.getCommitTotalFiles()/project.getCommitCount()); // } // bio.getProjectMemberships().sync(); // bio.sync(); // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // cursor.close(); // bio.sync(); // System.out.println(); // // System.exit(0); //// #4 Commit comments // System.out.println("Extracting commit comments..."); // cursor = msrDb.getCollection("commit_comments").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "user", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { // System.err.println("Found commit comment with unrecognised user: " + username); // continue; // } // // user.setNumberOfCommitComments(user.getNumberOfCommitComments()+1); // //// if (!user.getDbObject().containsField("commitCommentTimes")) { //// user.getDbObject().put("commitCommentTimes", new BasicDBList()); //// } //// user.getCommitCommentTimes().add(obj.getString("created_at")); // // // Only a very small number of commit comments actually reference the repo // // Instead we're going to have to strip the string // String[] url = convertUrlIntoProjectNameAndOwner(obj.getString("url")); // //// System.out.println("Querying project " + url[1] + " and owner " + url[0]); // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(url[1]), Project.OWNERNAME.eq(url[0])).iterator(); //// if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfCommitComments(project.getNumberOfCommitComments()+1); // // if (!project.getDbObject().containsField("commitCommentTimes")) { // project.getDbObject().put("commitCommentTimes", new BasicDBList()); // } // project.getCommitCommentTimes().add(obj.getString("created_at")); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setNumberOfCommitComments(pm.getNumberOfCommitComments()+1); // // if (!pm.getDbObject().containsField("commitCommentTimes")) { // pm.getDbObject().put("commitCommentTimes", new BasicDBList()); // } // pm.getCommitCommentTimes().add(obj.getString("created_at")); // } //// } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // cursor.close(); // bio.sync(); // System.out.println(); // System.exit(0); //// //FIXME: THIS IS CAUSING THE CPU TO HIS 350% AND THEN KILLS THE LAPTOP?!?!?!?!? // #5 Pull requests System.out.println("Extracting pull requests..."); cursor = msrDb.getCollection("pull_requests").find(); cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); it = cursor.iterator(); System.out.println("Clearing previous data"); for (User u : bio.getUsers()) { if (!u.getDbObject().containsField("pullRequestTimes")) { u.getDbObject().put("pullRequestTimes", new BasicDBList()); } u.getPullRequestTimes().clear(); u.setNumberOfPullRequests(0); } bio.sync(); for (Project u : bio.getProjects()) { if (!u.getDbObject().containsField("pullRequestTimes")) { u.getDbObject().put("pullRequestTimes", new BasicDBList()); } u.getPullRequestTimes().clear(); u.setNumberOfPullRequests(0); } bio.sync(); for (ProjectMembership u : bio.getProjectMemberships()) { if (!u.getDbObject().containsField("pullRequestTimes")) { u.getDbObject().put("pullRequestTimes", new BasicDBList()); } u.getPullRequestTimes().clear(); u.setNumberOfPullRequests(0); } bio.sync(); System.out.println("Cleared!"); count = 0; while (it.hasNext()) { BasicDBObject obj = (BasicDBObject) it.next(); String username = getUserLoginName(bio, "user", "login", obj); User user = bio.getUsers().findOne(User.LOGIN.eq(username)); if (user == null) { // System.err.println("Found pull request with unrecognised user:" + username); continue; } if (!user.getDbObject().containsField("pullRequestTimes")) { user.getDbObject().put("pullRequestTimes", new BasicDBList()); } user.getPullRequestTimes().add(obj.getString("created_at")); user.setNumberOfPullRequests(user.getNumberOfPullRequests() + 1); // Project System.out.println(obj.getString("repo") + " " + obj.getString("owner") + obj.getString("_id")); ProjectMembership pm = getProjectMembership(bio, user.getLogin(), obj.getString("repo"), obj.getString("owner")); pm.setNumberOfPullRequests(pm.getNumberOfPullRequests() + 1); if (!pm.getDbObject().containsField("pullRequestTimes")) { pm.getDbObject().put("pullRequestTimes", new BasicDBList()); } pm.getPullRequestTimes().add(obj.getString("created_at")); // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // FIXME Causes it to run out of heap! // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfPullRequests(project.getNumberOfPullRequests()+1); // // if (!project.getDbObject().containsField("pullRequestTimes")) { // project.getDbObject().put("pullRequestTimes", new BasicDBList()); // } // project.getPullRequestTimes().add(obj.getString("created_at")); // // } // } else { // System.err.println("Didn't find project:" + obj.getString("repo") + ":"+obj.getString("owner")); // } count++; if (count % 1000 == 0) { System.out.print(count + ", "); bio.sync(); System.gc(); } } bio.sync(); System.out.println(); System.exit(0); //// #6 Pull request comments // System.out.println("Extracting pull request comments..."); // cursor = msrDb.getCollection("pull_request_comments").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "user", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { //// System.err.println("Found pull request comment with unrecognised user:" + username); // continue; // } // // if (!user.getDbObject().containsField("pullRequestCommentTimes")) { // user.getDbObject().put("pullRequestCommentTimes", new BasicDBList()); // } // user.getPullRequestCommentTimes().add(obj.getString("created_at")); // user.setNumberOfPullRequestComments(user.getNumberOfPullRequestComments()+1); // // // Project // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); //// if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfPullRequestComments(project.getNumberOfPullRequestComments()+1); // if (!project.getDbObject().containsField("pullRequestCommentTimes")) { // project.getDbObject().put("pullRequestCommentTimes", new BasicDBList()); // } // project.getPullRequestCommentTimes().add(obj.getString("created_at")); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setNumberOfPullRequestComments(pm.getNumberOfPullRequestComments()+1); // // if (!pm.getDbObject().containsField("pullRequestCommentTimes")) { // pm.getDbObject().put("pullRequestCommentTimes", new BasicDBList()); // } // pm.getPullRequestCommentTimes().add(obj.getString("created_at")); // } //// } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // System.exit(0); //// #7 Issues // System.out.println("Extracting issues..."); // cursor = msrDb.getCollection("issues").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "user", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { //// System.err.println("Found issue with unrecognised user:" + username); // continue; // } // // if (!user.getDbObject().containsField("issueTimes")) { // user.getDbObject().put("issueTimes", new BasicDBList()); // } // user.getIssueTimes().add(obj.getString("created_at")); // user.setNumberOfIssues(user.getNumberOfIssues()+1); // // // Project // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfIssues(project.getNumberOfIssues()+1); // // if (!project.getDbObject().containsField("issueTimes")) { // project.getDbObject().put("issueTimes", new BasicDBList()); // } // project.getIssueTimes().add(obj.getString("created_at")); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setNumberOfIssues(pm.getNumberOfIssues()+1); // // if (!pm.getDbObject().containsField("issueTimes")) { // pm.getDbObject().put("issueTimes", new BasicDBList()); // } // pm.getIssueTimes().add(obj.getString("created_at")); // } // } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // System.exit(0); //// #8 Issue comments // System.out.println("Extracting issue comments..."); // cursor = msrDb.getCollection("issue_comments").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "user", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { //// System.err.println("Found issue comment with unrecognised user:" + username); // continue; // } // // if (!user.getDbObject().containsField("issueCommentTimes")) { // user.getDbObject().put("issueCommentTimes", new BasicDBList()); // } // user.getIssueCommentTimes().add(obj.getString("created_at")); // user.setNumberOfIssueComments(user.getNumberOfIssueComments()+1); // // // Project // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfIssueComments(project.getNumberOfIssueComments()+1); // // if (!project.getDbObject().containsField("issueCommentTimes")) { // project.getDbObject().put("issueCommentTimes", new BasicDBList()); // } // project.getIssueCommentTimes().add(obj.getString("created_at")); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setNumberOfIssueComments(pm.getNumberOfIssueComments()+1); // // if (!pm.getDbObject().containsField("issueCommentTimes")) { // pm.getDbObject().put("issueCommentTimes", new BasicDBList()); // } // pm.getIssueCommentTimes().add(obj.getString("created_at")); // } // } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // System.exit(0); //// #9 Issue events // System.out.println("Extracting issue events..."); // cursor = msrDb.getCollection("issue_events").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "actor", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { //// System.err.println("Found issue event with unrecognised user:" + username); // continue; // } // // String eventKind = obj.getString("event"); // IssueEventKind kind = null; //FIXME // // switch (eventKind) { // case "closed": kind = IssueEventKind.CLOSED; break; // case "assigned": kind = IssueEventKind.ASSIGNED; break; // case "mentioned": kind = IssueEventKind.MENTIONED; break; // case "merged": kind = IssueEventKind.MERGED; break; // case "referenced": kind = IssueEventKind.REFERENCED; break; // case "reopened": kind = IssueEventKind.REOPENED; break; // case "subscribed": kind = IssueEventKind.SUBSCRIBED; break; // case "head_ref_deleted" : kind = IssueEventKind.HEAD_REF_DELETED; break; // case "head_ref_restored" : kind = IssueEventKind.HEAD_REF_RESTORED; break; // case "head_ref_cleaned" : kind = IssueEventKind.HEAD_REF_CLEANED; break; // case "unsubscribed" : kind = IssueEventKind.UNSUBSCRIBED; break; // default: // System.err.println("Unrecognised issue event kind: " + eventKind); // } // if (kind == null) continue; // // boolean eventKindFound = false; // // if (!user.getDbObject().containsField("issueEvents")) { // user.getDbObject().put("issueEvents", new BasicDBList()); // } // // for (IssueEvent ie : user.getIssueEvents()) { // if (ie.getEventKind().equals(kind)) { // ie.setCount(ie.getCount()+1); // eventKindFound = true; // break; // } // } // if (!eventKindFound) { // IssueEvent ie = new IssueEvent(); // ie.setEventKind(kind); // ie.setCount(1); // user.getIssueEvents().add(ie); // } // // // Project // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // // if (!project.getDbObject().containsField("issueEvents")) { // project.getDbObject().put("issueEvents", new BasicDBList()); // } // // eventKindFound = false; // for (IssueEvent ie : project.getIssueEvents()) { // if (ie.getEventKind().equals(kind)) { // ie.setCount(ie.getCount()+1); // eventKindFound = true; // break; // } // } // if (!eventKindFound) { // IssueEvent ie = new IssueEvent(); // ie.setEventKind(kind); // ie.setCount(1); // project.getIssueEvents().add(ie); // } // // ProjectMembership pm = getProjectMembership(bio, user, project); // // if (!pm.getDbObject().containsField("issueEvents")) { // pm.getDbObject().put("issueEvents", new BasicDBList()); // } // // eventKindFound = false; // for (IssueEvent ie : pm.getIssueEvents()) { // if (ie.getEventKind().equals(kind)) { // ie.setCount(ie.getCount()+1); // eventKindFound = true; // break; // } // } // if (!eventKindFound) { // IssueEvent ie = new IssueEvent(); // ie.setEventKind(kind); // ie.setCount(1); // pm.getIssueEvents().add(ie); // } // } // // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // System.exit(0); // //// Watchers // System.out.println("Extracting watchers..."); // cursor = msrDb.getCollection("watchers").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // User user = bio.getUsers().findOne(User.LOGIN.eq(obj.getString("login"))); // if (user == null) continue; // // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null && !project.getWatchers().contains(user)) project.getWatchers().add(user); // if (!user.getWatches().contains(project)) user.getWatches().add(project); // } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); //// Org members FIXME: INCOMPLETE: Cannot match the org name against ANYTHING.... // System.out.println("Extracting org members..."); // cursor = msrDb.getCollection("org_members").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String login = obj.getString("login"); // String orgName = obj.getString("org"); // // User user = bio.getUsers().findOne(User.LOGIN.eq(login)); // // User org = bio.getUsers().findOne(User.LOGIN.eq(orgName)); // // if (org!=null){ // System.err.println("Found org! " + orgName); // // } // //// Project project = bio.getProjects().findOne(Project.OWNERNAME.eq("orgName")); //// if (project==null) { //// System.err.println("Didn't find project: " + orgName); //// continue; //// } //// ProjectMembership pm = getProjectMembership(bio, user, project); //// pm.setOrgMember(true); // } // bio.sync(); // System.out.println(); // Repo collaborators // System.out.println("Extracting repo collaborators..."); // cursor = msrDb.getCollection("repo_collaborators").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String login = obj.getString("login"); // String projectName = obj.getString("repo"); // String ownerName = obj.getString("owner"); // // User user = bio.getUsers().findOne(User.LOGIN.eq(login)); // Iterator<Project> repoIt = bio.getProjects().find(Project.OWNERNAME.eq(ownerName), Project.NAME.eq(projectName)).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setCollaborator(true); // } else { // System.err.println("Couldn't find repo. owner: " + ownerName + ", repo: " + projectName); // } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // long end = System.currentTimeMillis(); System.out.println("Finished at " + new Date()); long duration = end - start; System.out.println("Duration: " + duration); }
From source file:org.ossmeter.platform.mining.msr14.stats.KMeansClustering.java
License:Open Source License
public static void main(String[] args) { try {/* www .ja v a2 s . co m*/ Mongo mongo = new Mongo(new ServerAddress("localhost", 12345)); Biodiversity bio = new Biodiversity(mongo.getDB("biodiversity_1")); bio.setClearPongoCacheOnSync(true); List<User> users = new ArrayList<User>(); // Massively inefficient int usss = 0; for (User u : bio.getUsers()) { if (u.getCommitAdditions() == 0 && u.getCommitCount() == 0 && u.getCommitDeletions() == 0 && u.getNumberOfCommitComments() == 0 && u.getNumberOfIssues() == 0 && u.getNumberOfIssueComments() == 0 && u.getNumberOfPullRequests() == 0 && u.getNumberOfPullRequestComments() == 0) { continue; } users.add(u); usss++; if (usss > 5000) break; } KMeansClustering kmeans = new KMeansClustering(); HashMap<Centroid, List<User>> clusters = kmeans.compute(4, users); FileWriter writer = new FileWriter(new File("groups.csv")); writer.write("group,numberOfCommits,numberOfChanges,numberOfAdditions,numberOfDeletions," + "numberOfCommitsAsAuthor,numberOfCommitsAsCommitter,numberOfIssues,numberOfIssueComments," + "numberOfPullRequests,numberOfPullRequestComments,numberOfCommitComments,numberOfForks+\n"); int group = 0; for (Centroid c : clusters.keySet()) { List<User> us = clusters.get(c); for (User u : us) { writer.write(group + ","); writer.write(u.getCommitCount() + ","); writer.write(u.getCommitTotalChanges() + ","); writer.write(u.getCommitAdditions() + ","); writer.write(u.getCommitDeletions() + ","); writer.write(u.getCommitsAsAuthor() + ","); writer.write(u.getCommitsAsCommitter() + ","); writer.write(u.getNumberOfIssues() + ","); writer.write(u.getNumberOfIssueComments() + ","); writer.write(u.getNumberOfPullRequests() + ","); writer.write(u.getNumberOfPullRequestComments() + ","); writer.write(u.getNumberOfCommitComments() + ","); writer.write(u.getNumberOfForks() + "\n"); } group++; writer.flush(); } writer.close(); } catch (UnknownHostException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } }
From source file:org.ossmeter.platform.visualisation.ChartUtil.java
License:Open Source License
public static DBCollection getCollection(Mongo mongo, String dbName, String collectionName) { DB db = mongo.getDB(dbName); DBCollection collection = db.getCollection(collectionName); return collection; }
From source file:org.sglover.alfrescoextensions.common.MongoDbFactory.java
License:Open Source License
public DB createInstance() throws Exception { DB db = null;//from www .j a v a 2 s .c o m try { if (enabled) { if (embedded) { Mongo mongo = newMongo(); // Mongo mongo = mongoFactory.newMongo(); if (dbName != null) { db = mongo.getDB(dbName); } else { db = mongo.getDB(UUID.randomUUID().toString()); } } else { if (mongoURI == null || dbName == null) { throw new RuntimeException("Must provide mongoURI and dbName or a mongo object"); } MongoClientURI uri = new MongoClientURI(mongoURI); MongoClient mongoClient = new MongoClient(uri); db = mongoClient.getDB(dbName); } if (db == null) { throw new InstantiationException("Could not instantiate a Mongo DB instance"); } if (logger.isDebugEnabled()) { logger.debug("Instatiated DB object for dbName '" + db.getName() + "'"); } CommandResult stats = db.getStats(); if (logger.isTraceEnabled()) { stats = db.getStats(); for (String key : stats.keySet()) { logger.trace("\t" + key + " = " + stats.get(key).toString()); } } } } catch (MongoException.Network e) { throw new MongoDbUnavailableException("Mongo network exception, database down?", e); } catch (UnknownHostException e) { throw new MongoDbUnavailableException("Mongo host not found", e); } return db; }
From source file:org.slc.sli.aggregation.util.IdByNameLookup.java
License:Apache License
/** * getAssessmentId - Look up an assessment ID given the assessments identification code ID. * * @param assmtIDCode - ID code to look up * @return assessment identifier/* w ww . j av a 2 s . c o m*/ * @throws UnknownHostException */ public static String getAssessmentId(String assmtIDCode) throws UnknownHostException { // TODO - parameterize these values. Mongo m = new Mongo("localhost"); DB db = m.getDB("sli"); DBCollection assessments = db.getCollection("assessment"); DBObject assmt = assessments .findOne(new BasicDBObject("body.assessmentIdentificationCode.ID", assmtIDCode)); String assmtId = (String) assmt.get("_id"); return assmtId; }
From source file:org.springframework.data.document.mongodb.MongoDbUtils.java
License:Apache License
public static DB doGetDB(Mongo mongo, String databaseName, String username, char[] password, boolean allowCreate) { Assert.notNull(mongo, "No Mongo instance specified"); DbHolder dbHolder = (DbHolder) TransactionSynchronizationManager.getResource(mongo); if (dbHolder != null && !dbHolder.isEmpty()) { // pre-bound Mongo DB DB db = null;//from ww w. j av a2s . co m if (TransactionSynchronizationManager.isSynchronizationActive() && dbHolder.doesNotHoldNonDefaultDB()) { // Spring transaction management is active -> db = dbHolder.getDB(); if (db != null && !dbHolder.isSynchronizedWithTransaction()) { LOGGER.debug("Registering Spring transaction synchronization for existing Mongo DB"); TransactionSynchronizationManager .registerSynchronization(new MongoSynchronization(dbHolder, mongo)); dbHolder.setSynchronizedWithTransaction(true); } } if (db != null) { return db; } } LOGGER.trace("Getting Mongo Database name=[" + databaseName + "]"); DB db = mongo.getDB(databaseName); boolean credentialsGiven = username != null && password != null; if (credentialsGiven && !db.isAuthenticated()) { //Note, can only authenticate once against the same com.mongodb.DB object. if (!db.authenticate(username, password)) { throw new CannotGetMongoDbConnectionException("Failed to authenticate to database [" + databaseName + "], username = [" + username + "], password = [" + new String(password) + "]", databaseName, username, password); } } // Use same Session for further Mongo actions within the transaction. // Thread object will get removed by synchronization at transaction completion. if (TransactionSynchronizationManager.isSynchronizationActive()) { // We're within a Spring-managed transaction, possibly from JtaTransactionManager. LOGGER.debug("Registering Spring transaction synchronization for new Hibernate Session"); DbHolder holderToUse = dbHolder; if (holderToUse == null) { holderToUse = new DbHolder(db); } else { holderToUse.addDB(db); } TransactionSynchronizationManager.registerSynchronization(new MongoSynchronization(holderToUse, mongo)); holderToUse.setSynchronizedWithTransaction(true); if (holderToUse != dbHolder) { TransactionSynchronizationManager.bindResource(mongo, holderToUse); } } // Check whether we are allowed to return the DB. if (!allowCreate && !isDBTransactional(db, mongo)) { throw new IllegalStateException("No Mongo DB bound to thread, " + "and configuration does not allow creation of non-transactional one here"); } return db; }