List of usage examples for com.mongodb BasicDBObject get
public Object get(final String key)
From source file:org.opencb.cellbase.mongodb.db.variation.VariantAnnotationMongoDBAdaptor.java
License:Apache License
private void solveNonCodingNegativeTranscript(Boolean isInsertion, GenomicVariant variant, HashSet<String> SoNames, BasicDBObject transcriptInfo, Integer transcriptStart, Integer transcriptEnd, BasicDBObject miRnaInfo, Integer variantStart, Integer variantEnd, ConsequenceType consequenceTypeTemplate) { BasicDBList exonInfoList;// w w w.java2 s . c o m BasicDBObject exonInfo; Integer exonStart; Integer exonEnd; String transcriptSequence; Boolean variantAhead; Integer cdnaExonEnd; Integer cdnaVariantStart; Integer cdnaVariantEnd; Boolean splicing; int exonCounter; Integer prevSpliceSite; Boolean[] junctionSolution = { false, false }; exonInfoList = (BasicDBList) transcriptInfo.get("exons"); exonInfo = (BasicDBObject) exonInfoList.get(0); exonStart = (Integer) exonInfo.get("start"); exonEnd = (Integer) exonInfo.get("end"); transcriptSequence = (String) exonInfo.get("sequence"); variantAhead = true; // we need a first iteration within the while to ensure junction is solved in case needed cdnaExonEnd = (exonEnd - exonStart + 1); // cdnaExonEnd poinst to the same base than exonStart cdnaVariantStart = null; // cdnaVariantStart points to the same base than variantEnd cdnaVariantEnd = null; // cdnaVariantEnd points to the same base than variantStart junctionSolution[0] = false; junctionSolution[1] = false; splicing = false; if (variantEnd <= exonEnd) { if (variantEnd >= exonStart) { // Variant end within the exon cdnaVariantStart = cdnaExonEnd - (variantEnd - exonStart); consequenceTypeTemplate.setcDnaPosition(cdnaVariantStart); if (variantStart >= exonStart) { // Both variant start and variant end within the exon ----||||S|||||E||||---- cdnaVariantEnd = cdnaExonEnd - (variantStart - exonStart); } } } else { if (variantStart >= exonStart) { // if(variantEnd >= exonStart) { // Only variant end within the exon ----||||||||||E||||---- // We do not contemplate that variant end can be located before this exon since this is the first exon cdnaVariantEnd = cdnaExonEnd - (variantEnd - exonStart); // } } // Variant includes the whole exon. Variant end is located before the exon, variant start is located after the exon } exonCounter = 1; while (exonCounter < exonInfoList.size() && variantAhead) { // This is not a do-while since we cannot call solveJunction until // while(exonCounter<exonInfoList.size() && !splicing && variantAhead) { // This is not a do-while since we cannot call solveJunction until exonInfo = (BasicDBObject) exonInfoList.get(exonCounter); // next exon has been loaded prevSpliceSite = exonStart - 1; exonStart = (Integer) exonInfo.get("start"); exonEnd = (Integer) exonInfo.get("end"); transcriptSequence = ((String) exonInfo.get("sequence")) + transcriptSequence; solveJunction(isInsertion, exonEnd + 1, prevSpliceSite, variantStart, variantEnd, SoNames, "splice_acceptor_variant", "splice_donor_variant", junctionSolution); splicing = (splicing || junctionSolution[0]); if (variantEnd <= exonEnd) { cdnaExonEnd += (exonEnd - exonStart + 1); if (variantEnd >= exonStart) { // Variant end within the exon cdnaVariantStart = cdnaExonEnd - (variantEnd - exonStart); consequenceTypeTemplate.setcDnaPosition(cdnaVariantStart); if (variantStart >= exonStart) { // Both variant start and variant end within the exon ----||||S|||||E||||---- cdnaVariantEnd = cdnaExonEnd - (variantStart - exonStart); } } } else { if (variantStart >= exonStart) { if (variantStart <= exonEnd) { // Only variant start within the exon ----||||||||||E||||---- cdnaVariantEnd = cdnaExonEnd - (variantStart - exonStart); } else { // Variant does not include this exon, variant is located before this exon variantAhead = false; } } else { // Variant includes the whole exon. Variant start is located before the exon, variant end is located after the exon cdnaExonEnd += (exonEnd - exonStart + 1); } } exonCounter++; } if (miRnaInfo != null) { // miRNA with miRBase data BasicDBList matureMiRnaInfo = (BasicDBList) miRnaInfo.get("matures"); int i = 0; while (i < matureMiRnaInfo.size() && !regionsOverlap((Integer) ((BasicDBObject) matureMiRnaInfo.get(i)).get("cdnaStart"), (Integer) ((BasicDBObject) matureMiRnaInfo.get(i)).get("cdnaEnd"), cdnaVariantStart == null ? 1 : cdnaVariantStart, cdnaVariantEnd == null ? cdnaExonEnd : cdnaVariantEnd)) { i++; } if (i < matureMiRnaInfo.size()) { // Variant overlaps at least one mature miRNA SoNames.add("mature_miRNA_variant"); } else { if (!junctionSolution[1]) { // Exon variant SoNames.add("non_coding_transcript_exon_variant"); } SoNames.add("non_coding_transcript_variant"); } } else { if (!junctionSolution[1]) { // Exon variant SoNames.add("non_coding_transcript_exon_variant"); } SoNames.add("non_coding_transcript_variant"); } }
From source file:org.opencb.cellbase.mongodb.db.variation.VariantAnnotationMongoDBAdaptor.java
License:Apache License
public List<QueryResult> getAnnotationByVariantList(List<GenomicVariant> variantList, QueryOptions queryOptions) {/*w ww .j a v a2 s. c o m*/ // TODO: validation of queryoption 'include' values List<String> includeList = queryOptions.getAsStringList("include"); if (includeList.isEmpty()) { includeList = Arrays.asList("variation", "clinical", "consequenceType", "conservation", "drugInteraction"); } List<QueryResult> annotations = null; List<QueryResult> variationQueryResultList = null; if (includeList.contains("variation")) { variationQueryResultList = variationDBAdaptor.getAllByVariantList(variantList, queryOptions); annotations = variationQueryResultList; } List<QueryResult> clinicalQueryResultList = null; if (includeList.contains("clinical")) { clinicalQueryResultList = clinicalDBAdaptor.getAllByGenomicVariantList(variantList, queryOptions); if (annotations == null) { annotations = clinicalQueryResultList; } } // List<QueryResult> variationConsequenceTypeList = null; // if (includeList.contains("consequenceType")) { // variationConsequenceTypeList = getAllConsequenceTypesByVariantList(variantList, queryOptions); // if (annotations == null) { // annotations = variationConsequenceTypeList; // } // } List<QueryResult> conservedRegionQueryResultList = null; if (includeList.contains("conservation")) { conservedRegionQueryResultList = conservedRegionDBAdaptor .getAllScoresByRegionList(variantListToRegionList(variantList), queryOptions); if (annotations == null) { annotations = conservedRegionQueryResultList; } } for (int i = 0; i < variantList.size(); i++) { // TODO: start & end are both being set to variantList.get(i).getPosition(), modify this for indels VariantAnnotation variantAnnotation = new VariantAnnotation(variantList.get(i).getChromosome(), variantList.get(i).getPosition(), variantList.get(i).getPosition(), variantList.get(i).getReference(), variantList.get(i).getAlternative()); if (clinicalQueryResultList != null) { QueryResult clinicalQueryResult = clinicalQueryResultList.get(i); if (clinicalQueryResult.getResult() != null && clinicalQueryResult.getResult().size() > 0) { variantAnnotation.setClinical((Map<String, Object>) clinicalQueryResult.getResult().get(0)); } } // if (variationConsequenceTypeList != null) { if (includeList.contains("consequenceType")) { variantAnnotation.setConsequenceTypes( (List<ConsequenceType>) getAllConsequenceTypesByVariant(variantList.get(i), new QueryOptions()).getResult()); } if (includeList.contains("drugInteraction")) { Map<String, List<Object>> geneDrugInteractionMap = new HashMap<>(1); geneDrugInteractionMap.put("dgidb", getGeneDrugInteractions(variantList.get(i))); variantAnnotation.setGeneDrugInteraction(geneDrugInteractionMap); } if (conservedRegionQueryResultList != null) { variantAnnotation .setConservationScores((List<Score>) conservedRegionQueryResultList.get(i).getResult()); } List<BasicDBObject> variationDBList = null; if (variationQueryResultList != null) { variationDBList = (List<BasicDBObject>) variationQueryResultList.get(i).getResult(); } if (variationDBList != null && variationDBList.size() > 0) { String id = variationDBList.get(0).get("id").toString(); variantAnnotation.setId(id); BasicDBList freqsDBList = (BasicDBList) variationDBList.get(0).get("populationFrequencies"); if (freqsDBList != null) { BasicDBObject freqDBObject; for (int j = 0; j < freqsDBList.size(); j++) { freqDBObject = ((BasicDBObject) freqsDBList.get(j)); if (freqDBObject != null) { if (freqDBObject.containsKey("study")) { variantAnnotation.addPopulationFrequency(new PopulationFrequency( freqDBObject.get("study").toString(), freqDBObject.get("pop").toString(), freqDBObject.get("superPop").toString(), freqDBObject.get("refAllele").toString(), freqDBObject.get("altAllele").toString(), Float.valueOf(freqDBObject.get("refAlleleFreq").toString()), Float.valueOf(freqDBObject.get("altAlleleFreq").toString()))); } else { variantAnnotation.addPopulationFrequency(new PopulationFrequency("1000G_PHASE_3", freqDBObject.get("pop").toString(), freqDBObject.get("superPop").toString(), freqDBObject.get("refAllele").toString(), freqDBObject.get("altAllele").toString(), Float.valueOf(freqDBObject.get("refAlleleFreq").toString()), Float.valueOf(freqDBObject.get("altAlleleFreq").toString()))); } } } } } List<VariantAnnotation> value = Collections.singletonList(variantAnnotation); annotations.get(i).setResult(value); } return annotations; // return clinicalQueryResultList; }
From source file:org.opencb.opencga.storage.hadoop.variant.VariantHbaseDBAdaptor.java
License:Apache License
public QueryResult getSimpleVariantsByRegion(Region region, String sourceId, QueryOptions options) { Long start, end, dbstart, dbend; start = System.currentTimeMillis(); boolean includeStats; boolean includeEffects; if (!options.containsKey("stats") && !options.containsKey("effects")) { includeStats = true;/*from ww w .j av a 2 s. c om*/ includeEffects = true; } else { includeStats = options.containsKey("stats") && options.getBoolean("stats"); includeEffects = options.containsKey("effects") && options.getBoolean("effects"); } QueryResult<Variant> queryResult = new QueryResult<>( String.format("%s:%d-%d", region.getChromosome(), region.getStart(), region.getEnd())); List<Variant> results = new ArrayList<>(); String startRow = buildRowkey(region.getChromosome(), Long.toString(region.getStart())); String stopRow = buildRowkey(region.getChromosome(), Long.toString(region.getEnd())); BasicDBObject query = new BasicDBObject("position", new BasicDBObject("$gte", startRow).append("$lte", stopRow)).append("sources.sourceId", sourceId); DBCollection collection = db.getCollection("variants"); dbstart = System.currentTimeMillis(); DBCursor variantInStudies = collection.find(query); dbend = System.currentTimeMillis(); queryResult.setDbTime(dbend - dbstart); for (DBObject result : variantInStudies) { String[] rowkeyParts = result.get("position").toString().split("_"); String chromosome = rowkeyParts[0].replaceFirst("^0+(?!$)", ""); int position = Integer.parseInt(rowkeyParts[1]); BasicDBList studies = (BasicDBList) result.get("sources"); BasicDBObject st = (BasicDBObject) studies.get(0); String ref = (String) st.get("ref"); String alt = StringUtils.join((ArrayList<String>) st.get("alt"), ","); // TODO Needs rework Variant variant = new Variant(chromosome, position, position, ref, alt); // Set stats informations if (includeStats) { VariantStats stats = new VariantStats(); BasicDBObject mongoStats = (BasicDBObject) st.get("stats"); stats.setMaf((float) (double) mongoStats.get("maf")); stats.setMafAllele((String) mongoStats.get("alleleMaf")); stats.setMissingGenotypes((int) mongoStats.get("missing")); List<Genotype> genotypeCount = new ArrayList<>(); for (BasicDBObject s : (List<BasicDBObject>) mongoStats.get("genotypeCount")) { for (Map.Entry<String, Object> entry : s.entrySet()) { Genotype genotype = new Genotype(entry.getKey()); genotype.setCount((Integer) entry.getValue()); genotypeCount.add(genotype); } } stats.setGenotypes(genotypeCount); variant.setStats(stats); } // TODO Set consequence type names if (includeEffects) { BasicDBList mongoEffects = (BasicDBList) st.get("effects"); if (mongoEffects != null) { for (Object e : mongoEffects) { String effectObo = e.toString(); VariantEffect effect = new VariantEffect(); effect.setConsequenceTypeObo(effectObo); variant.addEffect(effect); } } } results.add(variant); } queryResult.setResult(results); queryResult.setNumResults(results.size()); end = System.currentTimeMillis(); queryResult.setTime(end - start); return queryResult; }
From source file:org.opencb.opencga.storage.mongodb.variant.DBObjectToSamplesConverter.java
License:Apache License
private VariantSourceEntry getLegacyNoncompressedSamples(BasicDBObject object) { VariantSourceEntry variantSourceEntry = new VariantSourceEntry(object.get(FILEID_FIELD).toString(), object.get(STUDYID_FIELD).toString()); BasicDBObject mongoGenotypes = (BasicDBObject) object.get(GENOTYPES_FIELD); for (Object entry : mongoGenotypes.toMap().entrySet()) { Map.Entry sample = (Map.Entry) entry; variantSourceEntry.addSampleData(sample.getKey().toString(), ((DBObject) sample.getValue()).toMap()); }//from ww w.j av a 2 s . co m // for (String sample : samples) { // Map<String, String> sampleData = ((DBObject) mongoGenotypes.get(sample)).toMap(); // fileWithSamples.addSampleData(sample, sampleData); // System.out.println("Sample processed: " + sample); // } return variantSourceEntry; }
From source file:org.opendaylight.controller.samples.onftappingapp.PortChain.java
License:Apache License
public PortChain(DBObject portChainObj) { this.setObjectId(((ObjectId) portChainObj.get("_id")).toString()); this.setName((String) portChainObj.get("name")); this.setType((String) portChainObj.get("type")); this.setReferenceCount((long) portChainObj.get("refCount")); BasicDBObject outPortObj = (BasicDBObject) portChainObj.get("outPort"); BasicDBObject returnPortObj = (BasicDBObject) portChainObj.get("returnPort"); BasicDBObject outRewriteObj = (BasicDBObject) portChainObj.get("outRewriteParams"); BasicDBObject returnRewriteObj = (BasicDBObject) portChainObj.get("returnRewriteParams"); RewriteParams outRewrite = null;// w w w. j av a 2 s . c o m RewriteParams returnRewrite = null; try { outRewrite = new RewriteParams(outRewriteObj); returnRewrite = new RewriteParams(returnRewriteObj); } catch (UnknownHostException e) { logger.error("Unable to resolve IP address", e); } this.setOutRewrite(outRewrite); this.setReturnRewrite(returnRewrite); SwitchAndPort outPort = new SwitchAndPort(); // outPort.setObjectId((String) outPortObj.get("_id")); outPort.setName((String) outPortObj.get("name")); outPort.setSwitchId((String) outPortObj.get("switchId")); outPort.setSwitchPort((int) outPortObj.get("switchPort")); this.setOutPort(outPort); SwitchAndPort returnPort = new SwitchAndPort(); // returnPort.setObjectId((String) returnPortObj.get("_id")); returnPort.setName((String) returnPortObj.get("name")); returnPort.setSwitchId((String) returnPortObj.get("switchId")); returnPort.setSwitchPort((int) returnPortObj.get("switchPort")); this.setReturnPort(returnPort); }
From source file:org.opendaylight.controller.samples.onftappingapp.RewriteParams.java
License:Apache License
public RewriteParams(BasicDBObject objDoc) throws UnknownHostException { this.setName(objDoc.getString("name")); this.setIPAddress((String) objDoc.get("IPAddress")); this.setMacAddr((String) objDoc.getString("MACAddress")); this.setVlanId((int) objDoc.getInt("VLAN_ID")); }
From source file:org.opengrid.data.impl.PlenarioDataProvider.java
private String getGeoFilter(String options) throws ParseException, UnsupportedEncodingException { String geoType = "Polygon"; // String geoCoordinates = // properties.getStringProperty("plenario.geo.default"); String geoCoordinates = properties.getStringProperty("plenario.geo.default"); if (options != null && options.length() > 0) { BasicDBObject o = (BasicDBObject) JSON.parse(options); if (o.containsField("geoFilter")) { BasicDBObject qo = (BasicDBObject) o.get("geoFilter"); geoType = (String) qo.get("type"); BasicDBList coordinatesList = (BasicDBList) qo.get("coordinates"); geoCoordinates = coordinatesList.toString().replace(" ", "%20"); }//from ww w. ja v a 2 s. co m } return "&location_geom__within={\"type\":\"Feature\",\"properties\":{},\"geometry\":{\"type\":\"" + geoType + "\",\"coordinates\":" + geoCoordinates + "}}"; }
From source file:org.ossmeter.platform.mining.msr14.Extractor.java
License:Open Source License
public static void main(String[] args) throws Exception { long start = System.currentTimeMillis(); Mongo msrMongo = new Mongo(new ServerAddress("localhost", 1234)); // GitHub challenge data Mongo bioMongo = new Mongo(new ServerAddress("localhost", 12345));// Extracted data // Create indexes Biodiversity bio = new Biodiversity(bioMongo.getDB("biodiversity")); bio.setClearPongoCacheOnSync(true);// w w w.ja v a 2 s . c o m bioMongo.getDB("biodiversity").getCollection("users").ensureIndex(new BasicDBObject("login", 1)); BasicDBObject index = new BasicDBObject(); index.put("name", 1); index.put("ownerName", 1); bioMongo.getDB("biodiversity").getCollection("projects").ensureIndex(index); index = new BasicDBObject(); index.put("projectName", 1); index.put("projectOwner", 1); bioMongo.getDB("biodiversity").getCollection("projectMemberships").ensureIndex(index); index = new BasicDBObject(); index.put("projectName", 1); index.put("userName", 1); bioMongo.getDB("biodiversity").getCollection("projectMemberships").ensureIndex(index); bioMongo.getDB("biodiversity").getCollection("projectMemberships") .ensureIndex(new BasicDBObject("userName", 1)); DB msrDb = msrMongo.getDB("msr14"); // #1 User extraction System.out.println("Extracting users..."); DBCursor cursor = msrDb.getCollection("users").find(); cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); Iterator<DBObject> it = cursor.iterator(); int count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // User user = new User(); // user.setGhId(obj.getString("id")); // user.setLogin(obj.getString("login")); // user.setLocation(obj.getString("location")); // user.setPublicRepos(obj.getInt("public_repos", 0)); // user.setJoinedDate(obj.getString("created_at")); // user.setFollowerCount(obj.getInt("followers", 0)); // user.setFollowingCount(obj.getInt("following", 0)); // user.setPublicGists(obj.getInt("public_gists", 0)); // // bio.getUsers().add(user); // // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // //// #1.2 Project extraction // System.out.println("Extracting projects..."); // cursor = msrDb.getCollection("repos").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // Project project = new Project(); // project.setName(obj.getString("name")); // project.setGhId(obj.getString("id")); // project.setCreatedAt(obj.getString("created_at")); // project.setSize(obj.getInt("size", 0)); // project.setWatchersCount(obj.getInt("watchers",0)); // project.setWatchersCount2(obj.getInt("watchers_count",0)); // project.setLanguage(obj.getString("language")); // project.setForks(obj.getInt("forks", 0)); // project.setForksCount(obj.getInt("forks_count", 0)); // project.setOpenIssues(obj.getInt("open_issues",0)); // project.setOpenIssuesCount(obj.getInt("open_issues_count",0)); // project.setOpenIssues(obj.getInt("open_issues",0)); // project.setNetworkCount(obj.getInt("network_count", 0)); // // BasicDBObject ownerObj = (BasicDBObject) obj.get("owner"); // User owner = null; // if (ownerObj != null) { // owner = bio.getUsers().findOne(User.LOGIN.eq(ownerObj.getString("login"))); // if (owner !=null) { // project.setOwner(owner); // project.setOwnerName(owner.getLogin()); // } // } // bio.getProjects().add(project); // // if (owner != null) { // This comes here as to reference the project, we need to have added to the project list first // ProjectMembership pm = getProjectMembership(bio, owner, project); // pm.setOwner(true); // } // // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // // System.exit(0); //// #2 Follower/following extraction // System.out.println("Extracting followers..."); // cursor = msrDb.getCollection("followers").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String followerLogin = obj.getString("login"); // String followedLogin = obj.getString("follows"); // // User follower = bio.getUsers().findOne(User.LOGIN.eq(followerLogin)); // User followed = bio.getUsers().findOne(User.LOGIN.eq(followedLogin)); // // if (follower != null && followed != null) { // follower.getFollowing().add(followed); // followed.getFollowers().add(follower); // } else{ //// System.err.println("Follower or followed is null. Follower: " +follower + ". followed: " + followed); // } // if (follower != null) follower.setFollowingCount(follower.getFollowingCount()+1); // if (followed != null) followed.setFollowerCount(followed.getFollowerCount()+1); // // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // // System.exit(0); System.out.println("Clearing ProjectMembership commit data"); for (ProjectMembership pm : bio.getProjectMemberships()) { pm.setCommitCount(0); pm.setCommitTotalChanges(0); pm.setCommitAdditions(0); pm.setCommitDeletions(0); pm.setCommitsAsAuthor(0); pm.setCommitsAsCommitter(0); pm.setCommitTotalFiles(0); pm.setAverageFilesPerCommit(0); pm.getCommitTimes().clear(); } bio.sync(); System.out.println("cleared."); // #3 Commits System.out.println("Extracting commits..."); cursor = msrDb.getCollection("commits").find(); cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); it = cursor.iterator(); count = 0; while (it.hasNext()) { BasicDBObject obj = (BasicDBObject) it.next(); // Author and committer BasicDBObject commitAuthor = (BasicDBObject) obj.get("author"); BasicDBObject commitCommitter = (BasicDBObject) obj.get("committer"); String authorLogin = ""; if (commitAuthor != null) authorLogin = commitAuthor.getString("login"); String committerLogin = ""; if (commitCommitter != null) committerLogin = commitCommitter.getString("login"); // Stats BasicDBObject stats = (BasicDBObject) obj.get("stats"); if (stats == null) stats = new BasicDBObject(); // Create a new one so we can get zeroed values int total = stats.getInt("total", 0); int additions = stats.getInt("additions", 0); int deletions = stats.getInt("deletions", 0); String commitDate = ((BasicDBObject) ((BasicDBObject) obj.get("commit")).get("author")) .getString("date"); BasicDBList files = (BasicDBList) obj.get("files"); String[] url = convertUrlIntoProjectNameAndOwner(obj.getString("url")); ProjectMembership authorPm = null; ProjectMembership committerPm = null; if (authorLogin != null) { authorPm = getProjectMembership(bio, authorLogin, url[1], url[0]); authorPm.setCommitCount(authorPm.getCommitCount() + 1); authorPm.setCommitTotalChanges(authorPm.getCommitTotalChanges() + total); authorPm.setCommitAdditions(authorPm.getCommitAdditions() + additions); authorPm.setCommitDeletions(authorPm.getCommitDeletions() + deletions); authorPm.setCommitsAsAuthor(authorPm.getCommitsAsAuthor() + 1); if (files != null) authorPm.setCommitTotalFiles(authorPm.getCommitTotalChanges() + files.size()); authorPm.setAverageFilesPerCommit(authorPm.getCommitTotalFiles() / authorPm.getCommitCount()); authorPm.getCommitTimes().add(commitDate); } if (authorLogin != null && !authorLogin.equals(committerLogin)) { committerPm = getProjectMembership(bio, committerLogin, url[1], url[0]); committerPm.setCommitCount(committerPm.getCommitCount() + 1); // committerPm.setCommitTotalChanges(committerPm.getCommitTotalChanges()+total); // committerPm.setCommitAdditions(committerPm.getCommitAdditions()+additions); // committerPm.setCommitDeletions(committerPm.getCommitDeletions()+deletions); committerPm.setCommitsAsCommitter(committerPm.getCommitsAsCommitter() + 1); committerPm.setCommitTotalFiles(committerPm.getCommitTotalChanges() + files.size()); committerPm.setAverageFilesPerCommit(committerPm.getCommitTotalFiles() / authorPm.getCommitCount()); if (files != null) committerPm.setCommitTotalFiles(committerPm.getCommitTotalChanges() + files.size()); committerPm .setAverageFilesPerCommit(committerPm.getCommitTotalFiles() / committerPm.getCommitCount()); committerPm.getCommitTimes().add(commitDate); } bio.sync(); count++; if (count % 1000 == 0) { System.out.print(count + ", "); bio.sync(); } } cursor.close(); bio.sync(); System.out.println(); System.exit(0); // if (author != null) { //// if (author.getCommits() ==null) author.setCommits(new Commits()); // author.setCommitCount(author.getCommitCount()+1); // author.setCommitTotalChanges(author.getCommitTotalChanges()+total); // author.setCommitAdditions(author.getCommitAdditions()+additions); // author.setCommitDeletions(author.getCommitDeletions()+deletions); // author.setCommitsAsAuthor(author.getCommitsAsAuthor()+1); // author.getCommitTimes().add(commitDate); // } // if (committer != null) { //// if (committer.getCommits() ==null) committer.setCommits(new Commits()); // committer.setCommitCount(committer.getCommitCount()+1); // committer.setCommitTotalChanges(committer.getCommitTotalChanges()+total); // committer.setCommitAdditions(committer.getCommitAdditions()+additions); // committer.setCommitDeletions(committer.getCommitDeletions()+deletions); // committer.setCommitsAsCommitter(committer.getCommitsAsCommitter()+1); // committer.getCommitTimes().add(commitDate); // } // // ProjectMembership authorPm = null; // ProjectMembership committerPm = null; // //// Only a very small number of commit comments actually reference the repo //// Instead we're going to have to strip the string // String[] url = convertUrlIntoProjectNameAndOwner(obj.getString("url")); // Project project = null; // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(url[1]), Project.OWNERNAME.eq(url[0])).iterator(); // if (repoIt.hasNext()) { // project = repoIt.next(); // if (project != null) { // // project.setCommitCount(project.getCommitCount()+1); // project.setCommitTotalChanges(project.getCommitTotalChanges()+total); // project.setCommitAdditions(project.getCommitAdditions()+additions); // project.setCommitDeletions(project.getCommitDeletions()+deletions); // project.getCommitTimes().add(commitDate); // // if (author != null) { // authorPm = getProjectMembership(bio, author, project); // authorPm.setCommitCount(authorPm.getCommitCount()+1); // authorPm.setCommitTotalChanges(authorPm.getCommitTotalChanges()+total); // authorPm.setCommitAdditions(authorPm.getCommitAdditions()+additions); // authorPm.setCommitDeletions(authorPm.getCommitDeletions()+deletions); // authorPm.setCommitsAsAuthor(authorPm.getCommitsAsAuthor()+1); // // // Avoid duplicating information // if (committer != null && author.getLogin().equals(committer.getLogin())) { // authorPm.setCommitsAsCommitter(authorPm.getCommitsAsCommitter()+1); // } // // authorPm.getCommitTimes().add(commitDate); // } // if (committer != null && author != null && !author.getLogin().equals(committer.getLogin())) { // committerPm = getProjectMembership(bio, committer, project); // committerPm.setCommitCount(committerPm.getCommitCount()+1); // committerPm.setCommitTotalChanges(committerPm.getCommitTotalChanges()+total); // committerPm.setCommitAdditions(committerPm.getCommitAdditions()+additions); // committerPm.setCommitDeletions(committerPm.getCommitDeletions()+deletions); // committerPm.setCommitsAsCommitter(committerPm.getCommitsAsCommitter()+1); // // committerPm.getCommitTimes().add(commitDate); // } // } // } // else { // System.err.println("Didn't find project:" + url[0] + ":"+url[1] + ", prestrip: " + obj.getString("url")); // } // bio.getProjectMemberships().sync(); // bio.sync(); // // // // Files // BasicDBList files = (BasicDBList) obj.get("files"); // if (files != null) { // for (Object f : files) { // BasicDBObject file = (BasicDBObject)f; // // String filename = file.getString("filename"); // if (filename.lastIndexOf(".") != -1) { // If it has an extension, we want that. If not, use the entire filename // filename = filename.substring(filename.lastIndexOf(".")); // filename = filename.toLowerCase(); // Ensure consistency // } // // FIXME: Should strip any /'s if there is no '.' - i.e. just the last one // // if (author != null) addArtefact(author, filename); // if (committer != null) addArtefact(committer, filename); //// if (project != null) addArtefact(project, filename); // } // } // // if (author != null && files !=null) { // author.setCommitTotalFiles(author.getCommitTotalFiles()+files.size()); // author.setAverageFilesPerCommit(author.getCommitTotalFiles()/author.getCommitCount()); // } // if (committer != null && files !=null && (author==null || !committer.getLogin().equals(author.getLogin()))) { // committer.setCommitTotalFiles(committer.getCommitTotalFiles()+files.size()); // committer.setAverageFilesPerCommit(committer.getCommitTotalFiles()/committer.getCommitCount()); // } // if (authorPm !=null && files != null) { // authorPm.setCommitTotalFiles(authorPm.getCommitTotalChanges()+files.size()); // authorPm.setAverageFilesPerCommit(authorPm.getCommitTotalFiles()/authorPm.getCommitCount()); // } // if (committerPm != null && files != null) { // committerPm.setCommitTotalFiles(committerPm.getCommitTotalChanges()+files.size()); // committerPm.setAverageFilesPerCommit(committerPm.getCommitTotalFiles()/committerPm.getCommitCount()); // } // // if (project!=null && files != null) { // project.setCommitTotalFiles(project.getCommitTotalChanges()+files.size()); // project.setAverageFilesPerCommit(project.getCommitTotalFiles()/project.getCommitCount()); // } // bio.getProjectMemberships().sync(); // bio.sync(); // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // cursor.close(); // bio.sync(); // System.out.println(); // // System.exit(0); //// #4 Commit comments // System.out.println("Extracting commit comments..."); // cursor = msrDb.getCollection("commit_comments").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "user", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { // System.err.println("Found commit comment with unrecognised user: " + username); // continue; // } // // user.setNumberOfCommitComments(user.getNumberOfCommitComments()+1); // //// if (!user.getDbObject().containsField("commitCommentTimes")) { //// user.getDbObject().put("commitCommentTimes", new BasicDBList()); //// } //// user.getCommitCommentTimes().add(obj.getString("created_at")); // // // Only a very small number of commit comments actually reference the repo // // Instead we're going to have to strip the string // String[] url = convertUrlIntoProjectNameAndOwner(obj.getString("url")); // //// System.out.println("Querying project " + url[1] + " and owner " + url[0]); // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(url[1]), Project.OWNERNAME.eq(url[0])).iterator(); //// if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfCommitComments(project.getNumberOfCommitComments()+1); // // if (!project.getDbObject().containsField("commitCommentTimes")) { // project.getDbObject().put("commitCommentTimes", new BasicDBList()); // } // project.getCommitCommentTimes().add(obj.getString("created_at")); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setNumberOfCommitComments(pm.getNumberOfCommitComments()+1); // // if (!pm.getDbObject().containsField("commitCommentTimes")) { // pm.getDbObject().put("commitCommentTimes", new BasicDBList()); // } // pm.getCommitCommentTimes().add(obj.getString("created_at")); // } //// } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // cursor.close(); // bio.sync(); // System.out.println(); // System.exit(0); //// //FIXME: THIS IS CAUSING THE CPU TO HIS 350% AND THEN KILLS THE LAPTOP?!?!?!?!? // #5 Pull requests System.out.println("Extracting pull requests..."); cursor = msrDb.getCollection("pull_requests").find(); cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); it = cursor.iterator(); System.out.println("Clearing previous data"); for (User u : bio.getUsers()) { if (!u.getDbObject().containsField("pullRequestTimes")) { u.getDbObject().put("pullRequestTimes", new BasicDBList()); } u.getPullRequestTimes().clear(); u.setNumberOfPullRequests(0); } bio.sync(); for (Project u : bio.getProjects()) { if (!u.getDbObject().containsField("pullRequestTimes")) { u.getDbObject().put("pullRequestTimes", new BasicDBList()); } u.getPullRequestTimes().clear(); u.setNumberOfPullRequests(0); } bio.sync(); for (ProjectMembership u : bio.getProjectMemberships()) { if (!u.getDbObject().containsField("pullRequestTimes")) { u.getDbObject().put("pullRequestTimes", new BasicDBList()); } u.getPullRequestTimes().clear(); u.setNumberOfPullRequests(0); } bio.sync(); System.out.println("Cleared!"); count = 0; while (it.hasNext()) { BasicDBObject obj = (BasicDBObject) it.next(); String username = getUserLoginName(bio, "user", "login", obj); User user = bio.getUsers().findOne(User.LOGIN.eq(username)); if (user == null) { // System.err.println("Found pull request with unrecognised user:" + username); continue; } if (!user.getDbObject().containsField("pullRequestTimes")) { user.getDbObject().put("pullRequestTimes", new BasicDBList()); } user.getPullRequestTimes().add(obj.getString("created_at")); user.setNumberOfPullRequests(user.getNumberOfPullRequests() + 1); // Project System.out.println(obj.getString("repo") + " " + obj.getString("owner") + obj.getString("_id")); ProjectMembership pm = getProjectMembership(bio, user.getLogin(), obj.getString("repo"), obj.getString("owner")); pm.setNumberOfPullRequests(pm.getNumberOfPullRequests() + 1); if (!pm.getDbObject().containsField("pullRequestTimes")) { pm.getDbObject().put("pullRequestTimes", new BasicDBList()); } pm.getPullRequestTimes().add(obj.getString("created_at")); // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // FIXME Causes it to run out of heap! // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfPullRequests(project.getNumberOfPullRequests()+1); // // if (!project.getDbObject().containsField("pullRequestTimes")) { // project.getDbObject().put("pullRequestTimes", new BasicDBList()); // } // project.getPullRequestTimes().add(obj.getString("created_at")); // // } // } else { // System.err.println("Didn't find project:" + obj.getString("repo") + ":"+obj.getString("owner")); // } count++; if (count % 1000 == 0) { System.out.print(count + ", "); bio.sync(); System.gc(); } } bio.sync(); System.out.println(); System.exit(0); //// #6 Pull request comments // System.out.println("Extracting pull request comments..."); // cursor = msrDb.getCollection("pull_request_comments").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "user", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { //// System.err.println("Found pull request comment with unrecognised user:" + username); // continue; // } // // if (!user.getDbObject().containsField("pullRequestCommentTimes")) { // user.getDbObject().put("pullRequestCommentTimes", new BasicDBList()); // } // user.getPullRequestCommentTimes().add(obj.getString("created_at")); // user.setNumberOfPullRequestComments(user.getNumberOfPullRequestComments()+1); // // // Project // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); //// if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfPullRequestComments(project.getNumberOfPullRequestComments()+1); // if (!project.getDbObject().containsField("pullRequestCommentTimes")) { // project.getDbObject().put("pullRequestCommentTimes", new BasicDBList()); // } // project.getPullRequestCommentTimes().add(obj.getString("created_at")); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setNumberOfPullRequestComments(pm.getNumberOfPullRequestComments()+1); // // if (!pm.getDbObject().containsField("pullRequestCommentTimes")) { // pm.getDbObject().put("pullRequestCommentTimes", new BasicDBList()); // } // pm.getPullRequestCommentTimes().add(obj.getString("created_at")); // } //// } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // System.exit(0); //// #7 Issues // System.out.println("Extracting issues..."); // cursor = msrDb.getCollection("issues").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "user", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { //// System.err.println("Found issue with unrecognised user:" + username); // continue; // } // // if (!user.getDbObject().containsField("issueTimes")) { // user.getDbObject().put("issueTimes", new BasicDBList()); // } // user.getIssueTimes().add(obj.getString("created_at")); // user.setNumberOfIssues(user.getNumberOfIssues()+1); // // // Project // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfIssues(project.getNumberOfIssues()+1); // // if (!project.getDbObject().containsField("issueTimes")) { // project.getDbObject().put("issueTimes", new BasicDBList()); // } // project.getIssueTimes().add(obj.getString("created_at")); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setNumberOfIssues(pm.getNumberOfIssues()+1); // // if (!pm.getDbObject().containsField("issueTimes")) { // pm.getDbObject().put("issueTimes", new BasicDBList()); // } // pm.getIssueTimes().add(obj.getString("created_at")); // } // } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // System.exit(0); //// #8 Issue comments // System.out.println("Extracting issue comments..."); // cursor = msrDb.getCollection("issue_comments").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "user", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { //// System.err.println("Found issue comment with unrecognised user:" + username); // continue; // } // // if (!user.getDbObject().containsField("issueCommentTimes")) { // user.getDbObject().put("issueCommentTimes", new BasicDBList()); // } // user.getIssueCommentTimes().add(obj.getString("created_at")); // user.setNumberOfIssueComments(user.getNumberOfIssueComments()+1); // // // Project // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null) { // project.setNumberOfIssueComments(project.getNumberOfIssueComments()+1); // // if (!project.getDbObject().containsField("issueCommentTimes")) { // project.getDbObject().put("issueCommentTimes", new BasicDBList()); // } // project.getIssueCommentTimes().add(obj.getString("created_at")); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setNumberOfIssueComments(pm.getNumberOfIssueComments()+1); // // if (!pm.getDbObject().containsField("issueCommentTimes")) { // pm.getDbObject().put("issueCommentTimes", new BasicDBList()); // } // pm.getIssueCommentTimes().add(obj.getString("created_at")); // } // } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // System.exit(0); //// #9 Issue events // System.out.println("Extracting issue events..."); // cursor = msrDb.getCollection("issue_events").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String username = getUserLoginName(bio, "actor", "login", obj); // User user = bio.getUsers().findOne(User.LOGIN.eq(username)); // if (user == null) { //// System.err.println("Found issue event with unrecognised user:" + username); // continue; // } // // String eventKind = obj.getString("event"); // IssueEventKind kind = null; //FIXME // // switch (eventKind) { // case "closed": kind = IssueEventKind.CLOSED; break; // case "assigned": kind = IssueEventKind.ASSIGNED; break; // case "mentioned": kind = IssueEventKind.MENTIONED; break; // case "merged": kind = IssueEventKind.MERGED; break; // case "referenced": kind = IssueEventKind.REFERENCED; break; // case "reopened": kind = IssueEventKind.REOPENED; break; // case "subscribed": kind = IssueEventKind.SUBSCRIBED; break; // case "head_ref_deleted" : kind = IssueEventKind.HEAD_REF_DELETED; break; // case "head_ref_restored" : kind = IssueEventKind.HEAD_REF_RESTORED; break; // case "head_ref_cleaned" : kind = IssueEventKind.HEAD_REF_CLEANED; break; // case "unsubscribed" : kind = IssueEventKind.UNSUBSCRIBED; break; // default: // System.err.println("Unrecognised issue event kind: " + eventKind); // } // if (kind == null) continue; // // boolean eventKindFound = false; // // if (!user.getDbObject().containsField("issueEvents")) { // user.getDbObject().put("issueEvents", new BasicDBList()); // } // // for (IssueEvent ie : user.getIssueEvents()) { // if (ie.getEventKind().equals(kind)) { // ie.setCount(ie.getCount()+1); // eventKindFound = true; // break; // } // } // if (!eventKindFound) { // IssueEvent ie = new IssueEvent(); // ie.setEventKind(kind); // ie.setCount(1); // user.getIssueEvents().add(ie); // } // // // Project // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // // if (!project.getDbObject().containsField("issueEvents")) { // project.getDbObject().put("issueEvents", new BasicDBList()); // } // // eventKindFound = false; // for (IssueEvent ie : project.getIssueEvents()) { // if (ie.getEventKind().equals(kind)) { // ie.setCount(ie.getCount()+1); // eventKindFound = true; // break; // } // } // if (!eventKindFound) { // IssueEvent ie = new IssueEvent(); // ie.setEventKind(kind); // ie.setCount(1); // project.getIssueEvents().add(ie); // } // // ProjectMembership pm = getProjectMembership(bio, user, project); // // if (!pm.getDbObject().containsField("issueEvents")) { // pm.getDbObject().put("issueEvents", new BasicDBList()); // } // // eventKindFound = false; // for (IssueEvent ie : pm.getIssueEvents()) { // if (ie.getEventKind().equals(kind)) { // ie.setCount(ie.getCount()+1); // eventKindFound = true; // break; // } // } // if (!eventKindFound) { // IssueEvent ie = new IssueEvent(); // ie.setEventKind(kind); // ie.setCount(1); // pm.getIssueEvents().add(ie); // } // } // // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); // System.exit(0); // //// Watchers // System.out.println("Extracting watchers..."); // cursor = msrDb.getCollection("watchers").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // User user = bio.getUsers().findOne(User.LOGIN.eq(obj.getString("login"))); // if (user == null) continue; // // Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // if (project != null && !project.getWatchers().contains(user)) project.getWatchers().add(user); // if (!user.getWatches().contains(project)) user.getWatches().add(project); // } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // System.out.println(); //// Org members FIXME: INCOMPLETE: Cannot match the org name against ANYTHING.... // System.out.println("Extracting org members..."); // cursor = msrDb.getCollection("org_members").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String login = obj.getString("login"); // String orgName = obj.getString("org"); // // User user = bio.getUsers().findOne(User.LOGIN.eq(login)); // // User org = bio.getUsers().findOne(User.LOGIN.eq(orgName)); // // if (org!=null){ // System.err.println("Found org! " + orgName); // // } // //// Project project = bio.getProjects().findOne(Project.OWNERNAME.eq("orgName")); //// if (project==null) { //// System.err.println("Didn't find project: " + orgName); //// continue; //// } //// ProjectMembership pm = getProjectMembership(bio, user, project); //// pm.setOrgMember(true); // } // bio.sync(); // System.out.println(); // Repo collaborators // System.out.println("Extracting repo collaborators..."); // cursor = msrDb.getCollection("repo_collaborators").find(); // cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); // it = cursor.iterator(); // // count = 0; // while(it.hasNext()){ // BasicDBObject obj = (BasicDBObject) it.next(); // // String login = obj.getString("login"); // String projectName = obj.getString("repo"); // String ownerName = obj.getString("owner"); // // User user = bio.getUsers().findOne(User.LOGIN.eq(login)); // Iterator<Project> repoIt = bio.getProjects().find(Project.OWNERNAME.eq(ownerName), Project.NAME.eq(projectName)).iterator(); // if (repoIt.hasNext()) { // Project project = repoIt.next(); // // ProjectMembership pm = getProjectMembership(bio, user, project); // pm.setCollaborator(true); // } else { // System.err.println("Couldn't find repo. owner: " + ownerName + ", repo: " + projectName); // } // count++; // if (count % 1000 == 0) { // System.out.print(count + ", "); // bio.sync(); // } // } // bio.sync(); // long end = System.currentTimeMillis(); System.out.println("Finished at " + new Date()); long duration = end - start; System.out.println("Duration: " + duration); }
From source file:org.ossmeter.platform.mining.msr14.Extractor.java
License:Open Source License
/** * Some 'user' field in commit_comments are Strings. Some are objects. Seriously. * @param bio// w ww . ja v a2s.com * @param obj * @return */ protected static String getUserLoginName(Biodiversity bio, String userField, String loginField, BasicDBObject obj) { String username = null; if (obj.get(userField) == null) return null; if (obj.get(userField) instanceof String) { username = obj.getString(userField); } else { username = ((BasicDBObject) obj.get(userField)).getString(loginField); } return username; }
From source file:org.ossmeter.platform.visualisation.Chart.java
License:Open Source License
protected ArrayNode createDatatable(JsonNode datatableSpec, DBCollection collection, DBObject query) { String rowName = null;/*from w w w .j a v a 2 s . c o m*/ if (datatableSpec.has("rows")) { // TODO: May need more checking here if rows can be more complex rowName = datatableSpec.path("rows").textValue(); rowName = rowName.replace("$", ""); } ArrayNode colNames = (ArrayNode) datatableSpec.path("cols"); ObjectMapper mapper = new ObjectMapper(); ArrayNode results = mapper.createArrayNode(); // Ensure data is sorted correctly DBObject orderBy = new BasicDBObject("__datetime", 1); if (rowName != null) { Iterator<DBObject> it = collection.find(query).sort(orderBy).iterator(); while (it.hasNext()) { DBObject dbobj = it.next(); BasicDBList rows = (BasicDBList) dbobj.get(rowName); Iterator<Object> rowsIt = rows.iterator(); while (rowsIt.hasNext()) { BasicDBObject row = (BasicDBObject) rowsIt.next(); ObjectNode r = mapper.createObjectNode(); boolean validRow = true; for (int i = 0; i < colNames.size(); i++) { JsonNode col = colNames.get(i); String name = col.get("name").asText(); String field = col.get("field").asText(); field = field.replace("$", ""); Object value = null; if (field.equals("__date")) { value = dbobj.get(field); } else { // U.G.L.Y. FIXME if (field.contains("[")) { String[] _ = field.split("\\["); String[] __ = _[1].split("\\]"); int _index = Integer.valueOf(__[0]); String _field = __[1].replace(".", ""); BasicDBList _row = (BasicDBList) row.get(_[0]); BasicDBObject _entry = (BasicDBObject) _row.get(_index); value = _entry.get(_field); } else { value = row.get(field); } } // Fix invalid data: // - If value is null, then we ignore row. // - If value is NaN or Infinity, then we set to 0. if (value == null) { validRow = false; break; } if (value.toString().equals("NaN")) { value = 0; } else if (value.toString().equals("Infinity")) { value = -1; } r.put(name, mapper.valueToTree(value)); } if (validRow) results.add(r); } } } else { Iterator<DBObject> it = collection.find(query).sort(orderBy).iterator(); while (it.hasNext()) { DBObject dbobj = it.next(); ObjectNode r = mapper.createObjectNode(); boolean validRow = true; for (int i = 0; i < colNames.size(); i++) { JsonNode col = colNames.get(i); String name = col.get("name").asText(); String field = col.get("field").asText(); field = field.replace("$", ""); Object value = null; if (field.equals("__date")) { value = dbobj.get(field); } else { // U.G.L.Y. FIXME if (field.contains("[")) { String[] _ = field.split("\\["); String[] __ = _[1].split("\\]"); int _index = Integer.valueOf(__[0]); String _field = __[1].replace(".", ""); BasicDBList _row = (BasicDBList) dbobj.get(_[0]); BasicDBObject _entry = (BasicDBObject) _row.get(_index); value = _entry.get(_field); } else { value = dbobj.get(field); // if (value.toString().equals("NaN")) { // System.out.println(value); // } } } // Object value = null; // value = dbobj.get(field); // mapper.valueToTree(value); // Fix invalid data: // - If value is null, then we ignore row. // - If value is NaN, then we set to 0. if (value == null) { validRow = false; break; } if (value.toString().equals("NaN")) { value = 0; } else if (value.toString().equals("Infinity")) { value = -1; } r.put(name, mapper.valueToTree(value)); } if (validRow) results.add(r); } } return results; }