Example usage for com.mongodb DBCursor close

List of usage examples for com.mongodb DBCursor close

  1. HOME
  2. Java
  3. com.mongodb
  4. com.mongodb.*
  5. DBCursor
  6. close

Introduction

In this page you can find the example usage for com.mongodb DBCursor close.

Prototype

@Override
    public void close()

Source Link

Usage

From source file:org.opendaylight.controller.samples.onftappingapp.TappingApp.java

License:Apache License

public List<LoggedMessage> loadLoggedMessagesFromDatabase(final Date startDate) {
    DBCollection logTable = database.getCollection(DatabaseNames.getLoggerTableName());

    // Create a query to look for log entries after the start date
    BasicDBObject searchQuery = new BasicDBObject();
    searchQuery.put("timestamp", new BasicDBObject("$gte", startDate));

    DBCursor cursor = logTable.find(searchQuery);
    List<LoggedMessage> loggedMessageList = processLogMessageCursor(cursor);
    cursor.close();

    return loggedMessageList;
}

From source file:org.opendaylight.controller.samples.onftappingapp.TappingApp.java

License:Apache License

public List<TapPolicy> getTapPoliciesForSwitch(SwitchEntry switchEntry) {

    DBCollection table = database.getCollection(DatabaseNames.getTapPolicyTableName());

    List<TapPolicy> tapPoliciesForSwitch = new ArrayList<TapPolicy>();

    logger.info("Searching for Tap Policies for switch with ID " + switchEntry.getObjectId());
    BasicDBObject searchQuery = new BasicDBObject();
    ObjectId switchId = new ObjectId(switchEntry.getObjectId());
    searchQuery.put("switchAndPort.switchEntryId", switchId);
    DBCursor cursor = table.find(searchQuery);

    logger.info("Query result set contains " + cursor.count() + "Tap Policies");

    try {//from  w ww. ja v a2s  .  co m
        while (cursor.hasNext()) {
            TapPolicy tapPolicy;
            try {
                tapPolicy = new TapPolicy(cursor.next());
                tapPoliciesForSwitch.add(tapPolicy);
                logger.info("Tap Policy  " + tapPolicy);
            } catch (NotFoundException e) {
                logger.warn("Cannot find tap policy");
            }
        }
    } finally {
        cursor.close();
    }

    return tapPoliciesForSwitch;
}

From source file:org.ossmeter.platform.admin.LoggingInformation.java

License:Open Source License

@SuppressWarnings({ "rawtypes", "unchecked" })
@Get("json")/*from  w  w w  .  ja v a  2  s  . c  o m*/
public String represent() {
    @SuppressWarnings("unchecked")
    Series<Header> responseHeaders = (Series<Header>) getResponse().getAttributes()
            .get("org.restlet.http.headers");
    if (responseHeaders == null) {
        responseHeaders = new Series(Header.class);
        getResponse().getAttributes().put("org.restlet.http.headers", responseHeaders);
    }
    responseHeaders.add(new Header("Access-Control-Allow-Origin", "*"));
    responseHeaders.add(new Header("Access-Control-Allow-Methods", "GET"));

    try {
        Mongo mongo = new Mongo();

        DB db = mongo.getDB("logging");
        DBCollection col = db.getCollection("log");

        ObjectMapper mapper = new ObjectMapper();
        ArrayNode results = mapper.createArrayNode();

        DBCursor cursor = col.find();
        try {
            while (cursor.hasNext()) {
                DBObject p = cursor.next();
                p.removeField("_id");
                p.removeField("thread");
                p.removeField("host");
                p.removeField("class");
                results.add(mapper.readTree(p.toString()));
            }
        } catch (JsonProcessingException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            cursor.close();
        }

        mongo.close();

        return results.toString();

    } catch (UnknownHostException e1) {
        // TODO Auto-generated catch block
        e1.printStackTrace();
    }

    return null;
}

From source file:org.ossmeter.platform.admin.ProjectListAnalysis.java

License:Open Source License

@Get("json")
public String represent() {
    Series<Header> responseHeaders = (Series<Header>) getResponse().getAttributes()
            .get("org.restlet.http.headers");
    if (responseHeaders == null) {
        responseHeaders = new Series(Header.class);
        getResponse().getAttributes().put("org.restlet.http.headers", responseHeaders);
    }/*from  ww w  . j  a  va 2s  .  co  m*/
    responseHeaders.add(new Header("Access-Control-Allow-Origin", "*"));
    responseHeaders.add(new Header("Access-Control-Allow-Methods", "GET"));

    try {
        Mongo mongo = Configuration.getInstance().getMongoConnection();

        DB db = mongo.getDB("ossmeter");
        DBCollection col = db.getCollection("metricAnalysis");

        ObjectMapper mapper = new ObjectMapper();
        ArrayNode results = mapper.createArrayNode();

        DBCursor cursor = col.find();
        try {
            while (cursor.hasNext()) {
                results.add(cursor.next().toString());
            }
        } finally {
            cursor.close();
        }

        mongo.close();

        return results.toString();

    } catch (UnknownHostException e1) {
        // TODO Auto-generated catch block
        e1.printStackTrace();
    }

    return null;
}

From source file:org.ossmeter.platform.client.api.ProjectListResource.java

License:Open Source License

public Representation doRepresent() {
    // Defaults//  w  ww. jav  a  2s.com
    int pageSize = 10;
    int page = 0;

    // Ready query params
    String _page = getQueryValue("page");
    String _size = getQueryValue("size");
    if (_page != null && !"".equals(_page) && isInteger(_page)) {
        page = Integer.valueOf(_page);
    }
    if (_size != null && !"".equals(_size) && isInteger(_size)) {
        pageSize = Integer.valueOf(_size);
    }

    try {
        platform = new Platform(Configuration.getInstance().getMongoConnection());
    } catch (UnknownHostException e1) {
        e1.printStackTrace();
        ObjectNode m = mapper.createObjectNode();
        m.put("apicall", "list-all-projects");
        return Util.generateErrorMessageRepresentation(m, e1.getMessage());
    }

    ProjectRepository projectRepo = platform.getProjectRepositoryManager().getProjectRepository();

    DBCursor cursor = projectRepo.getProjects().getDbCollection().find().skip(page * pageSize).limit(pageSize);

    ArrayNode projects = mapper.createArrayNode();

    while (cursor.hasNext()) {
        try {
            DBObject p = cursor.next();
            p.removeField("storage");
            p.removeField("metricProviderData");
            p.removeField("_superTypes");
            p.removeField("_id");

            // FIXME: Temporary solution
            p.removeField("licenses");
            p.removeField("persons");

            projects.add(mapper.readTree(p.toString()));
        } catch (Exception e) {
            System.err.println("Error: " + e.getMessage());
            ObjectNode m = mapper.createObjectNode();
            m.put("apicall", "list-all-projects");
            return Util.generateErrorMessageRepresentation(m, e.getMessage());
        }
    }

    cursor.close();

    StringRepresentation resp = new StringRepresentation(projects.toString());
    resp.setMediaType(MediaType.APPLICATION_JSON);
    return resp;

    //      // TODO
    //      boolean paging = getRequest().getAttributes().containsKey("page");
    //      
    //      Platform platform = Platform.getInstance();
    //      ProjectRepository projectRepo = platform.getProjectRepositoryManager().getProjectRepository();
    //      
    //      Iterator<Project> it = projectRepo.getProjects().iterator();
    //   
    //      ObjectMapper mapper = new ObjectMapper();
    //      ArrayNode projects = mapper.createArrayNode();
    //      
    //      while (it.hasNext()) {
    //         try {
    //            Project project  = it.next();
    //            DBObject p = project.getDbObject();
    //            
    //            p.removeField("storage");
    //            p.removeField("metricProviderData");
    //            p.removeField("_superTypes");
    //            p.removeField("_id");
    //            
    //            // FIXME: Temporary solution
    //            p.removeField("licenses");
    //            p.removeField("persons");
    //            
    //            projects.add(mapper.readTree(p.toString())); //TODO: There must be a better way..
    //            
    //         } catch (Exception e) {
    //            System.err.println("Error: " + e.getMessage());
    //            ObjectNode m = mapper.createObjectNode();
    //            m.put("apicall", "list-all-projects");
    //            return Util.generateErrorMessageRepresentation(m, e.getMessage());
    //         }         
    //      }
    //      StringRepresentation resp = new StringRepresentation(projects.toString());
    //      resp.setMediaType(MediaType.APPLICATION_JSON);
    //      return resp;
}

From source file:org.ossmeter.platform.client.api.RawMetricResource.java

License:Open Source License

/**
 * TODO: Incomplete. [12th Sept, 2013]//from w  w w  .j a  va  2  s  .c o  m
 * @return
 */
public Representation doRepresent() {
    String projectId = (String) getRequest().getAttributes().get("projectid");
    String metricId = (String) getRequest().getAttributes().get("metricid");

    // FIXME: This and MetricVisualisationResource.java are EXACTLY the same.
    String agg = getQueryValue("agg");
    String start = getQueryValue("startDate");
    String end = getQueryValue("endDate");

    QueryBuilder builder = QueryBuilder.start();
    if (agg != null && agg != "") {
        //         builder.... // TODO
    }
    try {
        if (start != null && start != "") {
            builder.and("__datetime").greaterThanEquals(new Date(start).toJavaDate());
        }
        if (end != null && end != "") {
            builder.and("__datetime").lessThanEquals(new Date(end).toJavaDate());
        }
    } catch (ParseException e) {
        return Util.generateErrorMessageRepresentation(generateRequestJson(projectId, metricId),
                "Invalid date. Format must be YYYYMMDD.");
    }

    BasicDBObject query = (BasicDBObject) builder.get();

    ProjectRepository projectRepo = platform.getProjectRepositoryManager().getProjectRepository();

    Project project = projectRepo.getProjects().findOneByShortName(projectId);
    if (project == null) {
        getResponse().setStatus(Status.CLIENT_ERROR_BAD_REQUEST);
        return Util.generateErrorMessageRepresentation(generateRequestJson(projectId, metricId),
                "No project was found with the requested name.");
    }

    // Get collection from DB
    DB projectDB = platform.getMetricsRepository(project).getDb();

    MetricVisualisationExtensionPointManager manager = MetricVisualisationExtensionPointManager.getInstance();
    manager.getRegisteredVisualisations();
    MetricVisualisation vis = manager.findVisualisationById(metricId);

    if (vis == null) {
        return Util.generateErrorMessageRepresentation(generateRequestJson(projectId, metricId),
                "No visualiser found with specified ID.");
    }

    // TODO: okay, so we only allow people to get raw HISTORIC metrics? How would we
    //        return multiple collections???
    // TODO: Can we stream it? Page it? Filter and agg?

    DBCursor cursor = projectDB.getCollection(vis.getMetricId()).find(query);
    ArrayNode results = mapper.createArrayNode();

    while (cursor.hasNext()) {
        try {
            results.add(mapper.readTree(cursor.next().toString()));
        } catch (JsonProcessingException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    cursor.close();

    return Util.createJsonRepresentation(results);
}

From source file:org.ossmeter.platform.mining.msr14.Extractor.java

License:Open Source License

public static void main(String[] args) throws Exception {
    long start = System.currentTimeMillis();

    Mongo msrMongo = new Mongo(new ServerAddress("localhost", 1234)); // GitHub challenge data
    Mongo bioMongo = new Mongo(new ServerAddress("localhost", 12345));// Extracted data

    // Create indexes
    Biodiversity bio = new Biodiversity(bioMongo.getDB("biodiversity"));
    bio.setClearPongoCacheOnSync(true);//from  ww w  .  j  a  va2  s  . c o m
    bioMongo.getDB("biodiversity").getCollection("users").ensureIndex(new BasicDBObject("login", 1));

    BasicDBObject index = new BasicDBObject();
    index.put("name", 1);
    index.put("ownerName", 1);
    bioMongo.getDB("biodiversity").getCollection("projects").ensureIndex(index);

    index = new BasicDBObject();
    index.put("projectName", 1);
    index.put("projectOwner", 1);
    bioMongo.getDB("biodiversity").getCollection("projectMemberships").ensureIndex(index);

    index = new BasicDBObject();
    index.put("projectName", 1);
    index.put("userName", 1);
    bioMongo.getDB("biodiversity").getCollection("projectMemberships").ensureIndex(index);

    bioMongo.getDB("biodiversity").getCollection("projectMemberships")
            .ensureIndex(new BasicDBObject("userName", 1));

    DB msrDb = msrMongo.getDB("msr14");

    //      #1 User extraction
    System.out.println("Extracting users...");
    DBCursor cursor = msrDb.getCollection("users").find();
    cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
    Iterator<DBObject> it = cursor.iterator();

    int count = 0;
    //      while(it.hasNext()){
    //         BasicDBObject obj = (BasicDBObject) it.next();
    //         
    //         User user = new User();
    //         user.setGhId(obj.getString("id"));
    //         user.setLogin(obj.getString("login"));
    //         user.setLocation(obj.getString("location"));
    //         user.setPublicRepos(obj.getInt("public_repos", 0));
    //         user.setJoinedDate(obj.getString("created_at"));
    //         user.setFollowerCount(obj.getInt("followers", 0));
    //         user.setFollowingCount(obj.getInt("following", 0));
    //         user.setPublicGists(obj.getInt("public_gists", 0));
    //         
    //         bio.getUsers().add(user);
    //         
    //         count++;
    //         if (count % 1000 == 0) {
    //            System.out.print(count + ", ");
    //            bio.sync();
    //         }
    //      }
    //      bio.sync();
    //      System.out.println();
    //      
    ////      #1.2 Project extraction
    //      System.out.println("Extracting projects...");
    //      cursor = msrDb.getCollection("repos").find();
    //      cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
    //      it = cursor.iterator();
    //
    //      count = 0;
    //      while(it.hasNext()){
    //         BasicDBObject obj = (BasicDBObject) it.next();
    //         
    //         Project project = new Project();
    //         project.setName(obj.getString("name"));
    //         project.setGhId(obj.getString("id"));
    //         project.setCreatedAt(obj.getString("created_at"));
    //         project.setSize(obj.getInt("size", 0));
    //         project.setWatchersCount(obj.getInt("watchers",0));
    //         project.setWatchersCount2(obj.getInt("watchers_count",0));
    //         project.setLanguage(obj.getString("language"));
    //         project.setForks(obj.getInt("forks", 0));
    //         project.setForksCount(obj.getInt("forks_count", 0));
    //         project.setOpenIssues(obj.getInt("open_issues",0));
    //         project.setOpenIssuesCount(obj.getInt("open_issues_count",0));
    //         project.setOpenIssues(obj.getInt("open_issues",0));
    //         project.setNetworkCount(obj.getInt("network_count", 0));
    //         
    //         BasicDBObject ownerObj = (BasicDBObject) obj.get("owner");
    //         User owner = null;
    //         if (ownerObj != null) {
    //            owner = bio.getUsers().findOne(User.LOGIN.eq(ownerObj.getString("login")));
    //            if (owner !=null) {
    //               project.setOwner(owner);
    //               project.setOwnerName(owner.getLogin());
    //            }
    //         }
    //         bio.getProjects().add(project);
    //         
    //         if (owner != null) { // This comes here as to reference the project, we need to have added to the project list first
    //            ProjectMembership pm = getProjectMembership(bio, owner, project);
    //            pm.setOwner(true);
    //         }
    //         
    //         count++;
    //         if (count % 1000 == 0) {
    //            System.out.print(count + ", ");
    //            bio.sync();
    //         }
    //      }
    //      bio.sync();
    //      System.out.println();
    //      
    //      System.exit(0);

    ////      #2 Follower/following extraction
    //      System.out.println("Extracting followers...");
    //      cursor = msrDb.getCollection("followers").find();
    //      cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
    //      it = cursor.iterator();
    //      
    //      count = 0;
    //      while(it.hasNext()){
    //         BasicDBObject obj = (BasicDBObject) it.next();
    //         
    //         String followerLogin = obj.getString("login");
    //         String followedLogin = obj.getString("follows");
    //         
    //         User follower = bio.getUsers().findOne(User.LOGIN.eq(followerLogin));
    //         User followed = bio.getUsers().findOne(User.LOGIN.eq(followedLogin));
    //         
    //         if (follower != null && followed != null) {
    //            follower.getFollowing().add(followed);
    //            followed.getFollowers().add(follower);
    //         } else{
    ////            System.err.println("Follower or followed is null. Follower: " +follower + ". followed: " + followed);
    //         }
    //         if (follower != null) follower.setFollowingCount(follower.getFollowingCount()+1);
    //         if (followed != null) followed.setFollowerCount(followed.getFollowerCount()+1);
    //         
    //         count++;
    //         if (count % 1000 == 0) {
    //            System.out.print(count + ", ");
    //            bio.sync();
    //         }
    //      }
    //      bio.sync();
    //      System.out.println();
    //      
    //      System.exit(0);

    System.out.println("Clearing ProjectMembership commit data");

    for (ProjectMembership pm : bio.getProjectMemberships()) {
        pm.setCommitCount(0);
        pm.setCommitTotalChanges(0);
        pm.setCommitAdditions(0);
        pm.setCommitDeletions(0);
        pm.setCommitsAsAuthor(0);
        pm.setCommitsAsCommitter(0);
        pm.setCommitTotalFiles(0);
        pm.setAverageFilesPerCommit(0);
        pm.getCommitTimes().clear();
    }
    bio.sync();
    System.out.println("cleared.");

    //      #3 Commits
    System.out.println("Extracting commits...");
    cursor = msrDb.getCollection("commits").find();
    cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
    it = cursor.iterator();

    count = 0;
    while (it.hasNext()) {
        BasicDBObject obj = (BasicDBObject) it.next();

        // Author and committer
        BasicDBObject commitAuthor = (BasicDBObject) obj.get("author");
        BasicDBObject commitCommitter = (BasicDBObject) obj.get("committer");

        String authorLogin = "";
        if (commitAuthor != null)
            authorLogin = commitAuthor.getString("login");
        String committerLogin = "";
        if (commitCommitter != null)
            committerLogin = commitCommitter.getString("login");

        // Stats
        BasicDBObject stats = (BasicDBObject) obj.get("stats");
        if (stats == null)
            stats = new BasicDBObject(); // Create a new one so we can get zeroed values
        int total = stats.getInt("total", 0);
        int additions = stats.getInt("additions", 0);
        int deletions = stats.getInt("deletions", 0);

        String commitDate = ((BasicDBObject) ((BasicDBObject) obj.get("commit")).get("author"))
                .getString("date");

        BasicDBList files = (BasicDBList) obj.get("files");
        String[] url = convertUrlIntoProjectNameAndOwner(obj.getString("url"));

        ProjectMembership authorPm = null;
        ProjectMembership committerPm = null;

        if (authorLogin != null) {
            authorPm = getProjectMembership(bio, authorLogin, url[1], url[0]);
            authorPm.setCommitCount(authorPm.getCommitCount() + 1);
            authorPm.setCommitTotalChanges(authorPm.getCommitTotalChanges() + total);
            authorPm.setCommitAdditions(authorPm.getCommitAdditions() + additions);
            authorPm.setCommitDeletions(authorPm.getCommitDeletions() + deletions);
            authorPm.setCommitsAsAuthor(authorPm.getCommitsAsAuthor() + 1);
            if (files != null)
                authorPm.setCommitTotalFiles(authorPm.getCommitTotalChanges() + files.size());
            authorPm.setAverageFilesPerCommit(authorPm.getCommitTotalFiles() / authorPm.getCommitCount());
            authorPm.getCommitTimes().add(commitDate);
        }

        if (authorLogin != null && !authorLogin.equals(committerLogin)) {
            committerPm = getProjectMembership(bio, committerLogin, url[1], url[0]);

            committerPm.setCommitCount(committerPm.getCommitCount() + 1);
            //            committerPm.setCommitTotalChanges(committerPm.getCommitTotalChanges()+total);
            //            committerPm.setCommitAdditions(committerPm.getCommitAdditions()+additions);
            //            committerPm.setCommitDeletions(committerPm.getCommitDeletions()+deletions);
            committerPm.setCommitsAsCommitter(committerPm.getCommitsAsCommitter() + 1);
            committerPm.setCommitTotalFiles(committerPm.getCommitTotalChanges() + files.size());
            committerPm.setAverageFilesPerCommit(committerPm.getCommitTotalFiles() / authorPm.getCommitCount());
            if (files != null)
                committerPm.setCommitTotalFiles(committerPm.getCommitTotalChanges() + files.size());
            committerPm
                    .setAverageFilesPerCommit(committerPm.getCommitTotalFiles() / committerPm.getCommitCount());
            committerPm.getCommitTimes().add(commitDate);
        }

        bio.sync();
        count++;
        if (count % 1000 == 0) {
            System.out.print(count + ", ");
            bio.sync();
        }
    }
    cursor.close();
    bio.sync();
    System.out.println();

    System.exit(0);

    //         if (author != null) {
    ////            if (author.getCommits() ==null) author.setCommits(new Commits());
    //            author.setCommitCount(author.getCommitCount()+1);
    //            author.setCommitTotalChanges(author.getCommitTotalChanges()+total);
    //            author.setCommitAdditions(author.getCommitAdditions()+additions);
    //            author.setCommitDeletions(author.getCommitDeletions()+deletions);
    //            author.setCommitsAsAuthor(author.getCommitsAsAuthor()+1);
    //            author.getCommitTimes().add(commitDate);
    //         }
    //         if (committer != null) {
    ////            if (committer.getCommits() ==null) committer.setCommits(new Commits());
    //            committer.setCommitCount(committer.getCommitCount()+1);
    //            committer.setCommitTotalChanges(committer.getCommitTotalChanges()+total);
    //            committer.setCommitAdditions(committer.getCommitAdditions()+additions);
    //            committer.setCommitDeletions(committer.getCommitDeletions()+deletions);
    //            committer.setCommitsAsCommitter(committer.getCommitsAsCommitter()+1);
    //            committer.getCommitTimes().add(commitDate);
    //         }
    //         
    //         ProjectMembership authorPm = null;
    //         ProjectMembership committerPm = null;
    //         
    ////          Only a very small number of commit comments actually reference the repo
    ////          Instead we're going to have to strip the string 
    //         String[] url = convertUrlIntoProjectNameAndOwner(obj.getString("url"));
    //         Project project = null;
    //         Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(url[1]), Project.OWNERNAME.eq(url[0])).iterator();
    //         if (repoIt.hasNext()) {
    //            project = repoIt.next();
    //            if (project != null) {
    //      
    //               project.setCommitCount(project.getCommitCount()+1);
    //               project.setCommitTotalChanges(project.getCommitTotalChanges()+total);
    //               project.setCommitAdditions(project.getCommitAdditions()+additions);
    //               project.setCommitDeletions(project.getCommitDeletions()+deletions);
    //               project.getCommitTimes().add(commitDate);
    //               
    //               if (author != null) {
    //                  authorPm = getProjectMembership(bio, author, project);
    //                  authorPm.setCommitCount(authorPm.getCommitCount()+1);
    //                  authorPm.setCommitTotalChanges(authorPm.getCommitTotalChanges()+total);
    //                  authorPm.setCommitAdditions(authorPm.getCommitAdditions()+additions);
    //                  authorPm.setCommitDeletions(authorPm.getCommitDeletions()+deletions);
    //                  authorPm.setCommitsAsAuthor(authorPm.getCommitsAsAuthor()+1);
    //                  
    //                  // Avoid duplicating information
    //                  if (committer != null && author.getLogin().equals(committer.getLogin())) {
    //                     authorPm.setCommitsAsCommitter(authorPm.getCommitsAsCommitter()+1);
    //                  }
    //                  
    //                  authorPm.getCommitTimes().add(commitDate);
    //               }
    //               if (committer != null && author != null && !author.getLogin().equals(committer.getLogin())) {
    //                  committerPm = getProjectMembership(bio, committer, project);
    //                  committerPm.setCommitCount(committerPm.getCommitCount()+1);
    //                  committerPm.setCommitTotalChanges(committerPm.getCommitTotalChanges()+total);
    //                  committerPm.setCommitAdditions(committerPm.getCommitAdditions()+additions);
    //                  committerPm.setCommitDeletions(committerPm.getCommitDeletions()+deletions);
    //                  committerPm.setCommitsAsCommitter(committerPm.getCommitsAsCommitter()+1);
    //                  
    //                  committerPm.getCommitTimes().add(commitDate);
    //               }
    //            } 
    //         }
    //         else {
    //            System.err.println("Didn't find project:" + url[0] + ":"+url[1] + ", prestrip: " + obj.getString("url"));
    //         }
    //         bio.getProjectMemberships().sync();
    //         bio.sync();
    //         
    //
    //         // Files
    //         BasicDBList files = (BasicDBList) obj.get("files");
    //         if (files != null) {
    //            for (Object f : files) {
    //               BasicDBObject file = (BasicDBObject)f;
    //               
    //               String filename = file.getString("filename");
    //               if (filename.lastIndexOf(".") != -1) { // If it has an extension, we want that. If not, use the entire filename
    //                  filename = filename.substring(filename.lastIndexOf("."));
    //                  filename = filename.toLowerCase(); // Ensure consistency
    //               }
    //         // FIXME: Should strip any /'s if there is no '.' - i.e. just the last one
    //               
    //               if (author != null) addArtefact(author, filename);
    //               if (committer != null) addArtefact(committer, filename);
    ////               if (project != null) addArtefact(project, filename);
    //            }
    //         }
    //         
    //         if (author != null && files !=null) {
    //            author.setCommitTotalFiles(author.getCommitTotalFiles()+files.size());
    //            author.setAverageFilesPerCommit(author.getCommitTotalFiles()/author.getCommitCount());
    //         }
    //         if (committer != null && files !=null && (author==null || !committer.getLogin().equals(author.getLogin()))) {
    //            committer.setCommitTotalFiles(committer.getCommitTotalFiles()+files.size());
    //            committer.setAverageFilesPerCommit(committer.getCommitTotalFiles()/committer.getCommitCount());
    //         }
    //         if (authorPm !=null && files != null) {
    //            authorPm.setCommitTotalFiles(authorPm.getCommitTotalChanges()+files.size());
    //            authorPm.setAverageFilesPerCommit(authorPm.getCommitTotalFiles()/authorPm.getCommitCount());
    //         }
    //         if (committerPm != null && files != null) {
    //            committerPm.setCommitTotalFiles(committerPm.getCommitTotalChanges()+files.size());
    //            committerPm.setAverageFilesPerCommit(committerPm.getCommitTotalFiles()/committerPm.getCommitCount());
    //         }
    //         
    //         if (project!=null && files != null) {
    //            project.setCommitTotalFiles(project.getCommitTotalChanges()+files.size());
    //            project.setAverageFilesPerCommit(project.getCommitTotalFiles()/project.getCommitCount());
    //         }

    //         bio.getProjectMemberships().sync();
    //         bio.sync();
    //         count++;
    //         if (count % 1000 == 0) {
    //            System.out.print(count + ", ");
    //            bio.sync();
    //         }
    //      }
    //      cursor.close();
    //      bio.sync();
    //      System.out.println();
    //
    //      System.exit(0);

    ////      #4 Commit comments
    //      System.out.println("Extracting commit comments...");
    //      cursor = msrDb.getCollection("commit_comments").find();
    //      cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
    //      it = cursor.iterator();
    //      
    //      count = 0;
    //      while(it.hasNext()){
    //         BasicDBObject obj = (BasicDBObject) it.next();
    //      
    //         String username = getUserLoginName(bio, "user", "login", obj);
    //         User user = bio.getUsers().findOne(User.LOGIN.eq(username));
    //         if (user == null) {
    //            System.err.println("Found commit comment with unrecognised user: " + username);
    //            continue;
    //         }
    //         
    //         user.setNumberOfCommitComments(user.getNumberOfCommitComments()+1);
    //         
    ////         if (!user.getDbObject().containsField("commitCommentTimes")) {
    ////            user.getDbObject().put("commitCommentTimes", new BasicDBList());
    ////         }
    ////         user.getCommitCommentTimes().add(obj.getString("created_at"));
    //         
    //         // Only a very small number of commit comments actually reference the repo
    //         // Instead we're going to have to strip the string 
    //         String[] url = convertUrlIntoProjectNameAndOwner(obj.getString("url"));
    //         
    ////         System.out.println("Querying project " + url[1] + " and owner " + url[0]);
    //         Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(url[1]), Project.OWNERNAME.eq(url[0])).iterator();
    ////         if (repoIt.hasNext()) {
    //            Project project = repoIt.next();
    //            if (project != null) {
    //               project.setNumberOfCommitComments(project.getNumberOfCommitComments()+1);
    //               
    //               if (!project.getDbObject().containsField("commitCommentTimes")) {
    //                  project.getDbObject().put("commitCommentTimes", new BasicDBList());
    //               }
    //               project.getCommitCommentTimes().add(obj.getString("created_at"));
    //               
    //               ProjectMembership pm = getProjectMembership(bio, user, project);
    //               pm.setNumberOfCommitComments(pm.getNumberOfCommitComments()+1);
    //               
    //               if (!pm.getDbObject().containsField("commitCommentTimes")) {
    //                  pm.getDbObject().put("commitCommentTimes", new BasicDBList());
    //               }
    //               pm.getCommitCommentTimes().add(obj.getString("created_at"));
    //            }
    ////         }
    //         count++;
    //         if (count % 1000 == 0) {
    //            System.out.print(count + ", ");
    //            bio.sync();
    //         }
    //      }
    //      cursor.close();
    //      bio.sync();
    //      System.out.println();
    //      System.exit(0);

    ////      //FIXME: THIS IS CAUSING THE CPU TO HIS 350% AND THEN KILLS THE LAPTOP?!?!?!?!? 
    //      #5 Pull requests
    System.out.println("Extracting pull requests...");
    cursor = msrDb.getCollection("pull_requests").find();
    cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
    it = cursor.iterator();

    System.out.println("Clearing previous data");
    for (User u : bio.getUsers()) {
        if (!u.getDbObject().containsField("pullRequestTimes")) {
            u.getDbObject().put("pullRequestTimes", new BasicDBList());
        }
        u.getPullRequestTimes().clear();
        u.setNumberOfPullRequests(0);
    }
    bio.sync();
    for (Project u : bio.getProjects()) {
        if (!u.getDbObject().containsField("pullRequestTimes")) {
            u.getDbObject().put("pullRequestTimes", new BasicDBList());
        }
        u.getPullRequestTimes().clear();
        u.setNumberOfPullRequests(0);
    }
    bio.sync();

    for (ProjectMembership u : bio.getProjectMemberships()) {
        if (!u.getDbObject().containsField("pullRequestTimes")) {
            u.getDbObject().put("pullRequestTimes", new BasicDBList());
        }
        u.getPullRequestTimes().clear();
        u.setNumberOfPullRequests(0);
    }
    bio.sync();

    System.out.println("Cleared!");

    count = 0;
    while (it.hasNext()) {

        BasicDBObject obj = (BasicDBObject) it.next();

        String username = getUserLoginName(bio, "user", "login", obj);
        User user = bio.getUsers().findOne(User.LOGIN.eq(username));
        if (user == null) {
            //            System.err.println("Found pull request with unrecognised user:" + username);
            continue;
        }

        if (!user.getDbObject().containsField("pullRequestTimes")) {
            user.getDbObject().put("pullRequestTimes", new BasicDBList());
        }
        user.getPullRequestTimes().add(obj.getString("created_at"));

        user.setNumberOfPullRequests(user.getNumberOfPullRequests() + 1);

        // Project
        System.out.println(obj.getString("repo") + " " + obj.getString("owner") + obj.getString("_id"));

        ProjectMembership pm = getProjectMembership(bio, user.getLogin(), obj.getString("repo"),
                obj.getString("owner"));
        pm.setNumberOfPullRequests(pm.getNumberOfPullRequests() + 1);

        if (!pm.getDbObject().containsField("pullRequestTimes")) {
            pm.getDbObject().put("pullRequestTimes", new BasicDBList());
        }
        pm.getPullRequestTimes().add(obj.getString("created_at"));

        //         Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator();
        //         if (repoIt.hasNext()) { // FIXME Causes it to run out of heap!
        //            Project project = repoIt.next();
        //            if (project != null) {
        //               project.setNumberOfPullRequests(project.getNumberOfPullRequests()+1);
        //               
        //               if (!project.getDbObject().containsField("pullRequestTimes")) {
        //                  project.getDbObject().put("pullRequestTimes", new BasicDBList());
        //               }
        //               project.getPullRequestTimes().add(obj.getString("created_at"));
        //               
        //            }
        //         } else {
        //            System.err.println("Didn't find project:" + obj.getString("repo") + ":"+obj.getString("owner"));
        //         }

        count++;
        if (count % 1000 == 0) {
            System.out.print(count + ", ");
            bio.sync();
            System.gc();
        }
    }
    bio.sync();
    System.out.println();
    System.exit(0);

    ////      #6 Pull request comments
    //      System.out.println("Extracting pull request comments...");
    //      cursor = msrDb.getCollection("pull_request_comments").find();
    //      cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
    //      it = cursor.iterator();
    //      
    //      count = 0;
    //      while(it.hasNext()){
    //         BasicDBObject obj = (BasicDBObject) it.next();
    //      
    //         String username = getUserLoginName(bio, "user", "login", obj);
    //         User user = bio.getUsers().findOne(User.LOGIN.eq(username));
    //         if (user == null) {
    ////            System.err.println("Found pull request comment with unrecognised user:" + username);
    //            continue;
    //         }
    //         
    //         if (!user.getDbObject().containsField("pullRequestCommentTimes")) {
    //            user.getDbObject().put("pullRequestCommentTimes", new BasicDBList());
    //         }
    //         user.getPullRequestCommentTimes().add(obj.getString("created_at"));
    //         user.setNumberOfPullRequestComments(user.getNumberOfPullRequestComments()+1);
    //         
    //         // Project
    //         Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator();
    ////         if (repoIt.hasNext()) {
    //            Project project = repoIt.next();
    //            if (project != null) {
    //               project.setNumberOfPullRequestComments(project.getNumberOfPullRequestComments()+1);
    //               if (!project.getDbObject().containsField("pullRequestCommentTimes")) {
    //                  project.getDbObject().put("pullRequestCommentTimes", new BasicDBList());
    //               }
    //               project.getPullRequestCommentTimes().add(obj.getString("created_at"));
    //               
    //               ProjectMembership pm = getProjectMembership(bio, user, project);
    //               pm.setNumberOfPullRequestComments(pm.getNumberOfPullRequestComments()+1);
    //               
    //               if (!pm.getDbObject().containsField("pullRequestCommentTimes")) {
    //                  pm.getDbObject().put("pullRequestCommentTimes", new BasicDBList());
    //               }
    //               pm.getPullRequestCommentTimes().add(obj.getString("created_at"));
    //            }
    ////         }
    //         count++;
    //         if (count % 1000 == 0) {
    //            System.out.print(count + ", ");
    //            bio.sync();
    //         }
    //      }
    //      bio.sync();
    //      System.out.println();
    //      System.exit(0);

    ////      #7 Issues
    //      System.out.println("Extracting issues...");
    //      cursor = msrDb.getCollection("issues").find();
    //      cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
    //      it = cursor.iterator();
    //      
    //      count = 0;
    //      while(it.hasNext()){
    //         BasicDBObject obj = (BasicDBObject) it.next();
    //      
    //         String username = getUserLoginName(bio, "user", "login", obj);
    //         User user = bio.getUsers().findOne(User.LOGIN.eq(username));
    //         if (user == null) {
    ////            System.err.println("Found issue with unrecognised user:" + username);
    //            continue;
    //         }
    //         
    //         if (!user.getDbObject().containsField("issueTimes")) {
    //            user.getDbObject().put("issueTimes", new BasicDBList());
    //         }
    //         user.getIssueTimes().add(obj.getString("created_at"));
    //         user.setNumberOfIssues(user.getNumberOfIssues()+1);
    //         
    //         // Project
    //         Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator();
    //         if (repoIt.hasNext()) {
    //            Project project = repoIt.next();
    //            if (project != null) {
    //               project.setNumberOfIssues(project.getNumberOfIssues()+1);
    //               
    //               if (!project.getDbObject().containsField("issueTimes")) {
    //                  project.getDbObject().put("issueTimes", new BasicDBList());
    //               }
    //               project.getIssueTimes().add(obj.getString("created_at"));
    //               
    //               ProjectMembership pm = getProjectMembership(bio, user, project);
    //               pm.setNumberOfIssues(pm.getNumberOfIssues()+1);
    //               
    //               if (!pm.getDbObject().containsField("issueTimes")) {
    //                  pm.getDbObject().put("issueTimes", new BasicDBList());
    //               }
    //               pm.getIssueTimes().add(obj.getString("created_at"));
    //            }
    //         }
    //         count++;
    //         if (count % 1000 == 0) {
    //            System.out.print(count + ", ");
    //            bio.sync();
    //         }
    //      }
    //      bio.sync();
    //      System.out.println();
    //      System.exit(0);

    ////      #8 Issue comments
    //      System.out.println("Extracting issue comments...");
    //      cursor = msrDb.getCollection("issue_comments").find();
    //      cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
    //      it = cursor.iterator();
    //      
    //      count = 0;
    //      while(it.hasNext()){
    //         BasicDBObject obj = (BasicDBObject) it.next();
    //      
    //         String username = getUserLoginName(bio, "user", "login", obj);
    //         User user = bio.getUsers().findOne(User.LOGIN.eq(username));
    //         if (user == null) {
    ////            System.err.println("Found issue comment with unrecognised user:" + username);
    //            continue;
    //         }
    //         
    //         if (!user.getDbObject().containsField("issueCommentTimes")) {
    //            user.getDbObject().put("issueCommentTimes", new BasicDBList());
    //         }
    //         user.getIssueCommentTimes().add(obj.getString("created_at"));
    //         user.setNumberOfIssueComments(user.getNumberOfIssueComments()+1);
    //         
    //         // Project
    //         Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator();
    //         if (repoIt.hasNext()) {
    //            Project project = repoIt.next();
    //            if (project != null) {
    //               project.setNumberOfIssueComments(project.getNumberOfIssueComments()+1);
    //               
    //               if (!project.getDbObject().containsField("issueCommentTimes")) {
    //                  project.getDbObject().put("issueCommentTimes", new BasicDBList());
    //               }
    //               project.getIssueCommentTimes().add(obj.getString("created_at"));
    //               
    //               ProjectMembership pm = getProjectMembership(bio, user, project);
    //               pm.setNumberOfIssueComments(pm.getNumberOfIssueComments()+1);
    //               
    //               if (!pm.getDbObject().containsField("issueCommentTimes")) {
    //                  pm.getDbObject().put("issueCommentTimes", new BasicDBList());
    //               }
    //               pm.getIssueCommentTimes().add(obj.getString("created_at"));
    //            }
    //         }
    //         count++;
    //         if (count % 1000 == 0) {
    //            System.out.print(count + ", ");
    //            bio.sync();
    //         }
    //      }
    //      bio.sync();
    //      System.out.println();
    //      System.exit(0);

    ////      #9 Issue events
    //      System.out.println("Extracting issue events...");
    //      cursor = msrDb.getCollection("issue_events").find();
    //      cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
    //      it = cursor.iterator();
    //      
    //      count = 0;
    //      while(it.hasNext()){
    //         BasicDBObject obj = (BasicDBObject) it.next();
    //      
    //         String username = getUserLoginName(bio, "actor", "login", obj);
    //         User user = bio.getUsers().findOne(User.LOGIN.eq(username));
    //         if (user == null) {
    ////            System.err.println("Found issue event with unrecognised user:" + username);
    //            continue;
    //         }
    //         
    //         String eventKind = obj.getString("event");
    //         IssueEventKind kind = null; //FIXME
    //         
    //         switch (eventKind) {
    //            case "closed": kind = IssueEventKind.CLOSED; break;
    //            case "assigned": kind = IssueEventKind.ASSIGNED; break;
    //            case "mentioned": kind = IssueEventKind.MENTIONED; break;
    //            case "merged": kind = IssueEventKind.MERGED; break;
    //            case "referenced": kind = IssueEventKind.REFERENCED; break;
    //            case "reopened": kind = IssueEventKind.REOPENED; break;
    //            case "subscribed": kind = IssueEventKind.SUBSCRIBED; break;
    //            case "head_ref_deleted" : kind = IssueEventKind.HEAD_REF_DELETED; break;
    //            case "head_ref_restored" : kind = IssueEventKind.HEAD_REF_RESTORED; break;
    //            case "head_ref_cleaned" : kind = IssueEventKind.HEAD_REF_CLEANED; break;
    //            case "unsubscribed" : kind = IssueEventKind.UNSUBSCRIBED; break;
    //            default:
    //               System.err.println("Unrecognised issue event kind: " + eventKind);
    //         }
    //         if (kind == null) continue;
    //
    //         boolean eventKindFound = false;
    //         
    //         if (!user.getDbObject().containsField("issueEvents")) {
    //            user.getDbObject().put("issueEvents", new BasicDBList());
    //         }
    //         
    //         for (IssueEvent ie : user.getIssueEvents()) {
    //            if (ie.getEventKind().equals(kind)) {
    //               ie.setCount(ie.getCount()+1);
    //               eventKindFound = true;
    //               break;
    //            }
    //         }
    //         if (!eventKindFound) {
    //            IssueEvent ie = new IssueEvent();
    //            ie.setEventKind(kind);
    //            ie.setCount(1);
    //            user.getIssueEvents().add(ie);
    //         }
    //         
    //         // Project
    //         Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator();
    //         if (repoIt.hasNext()) {
    //            Project project = repoIt.next();
    //            
    //            if (!project.getDbObject().containsField("issueEvents")) {
    //               project.getDbObject().put("issueEvents", new BasicDBList());
    //            }
    //         
    //            eventKindFound = false;
    //            for (IssueEvent ie : project.getIssueEvents()) {
    //               if (ie.getEventKind().equals(kind)) {
    //                  ie.setCount(ie.getCount()+1);
    //                  eventKindFound = true;
    //                  break;
    //               }
    //            }
    //            if (!eventKindFound) {
    //               IssueEvent ie = new IssueEvent();
    //               ie.setEventKind(kind);
    //               ie.setCount(1);
    //               project.getIssueEvents().add(ie);
    //            }
    //            
    //            ProjectMembership pm = getProjectMembership(bio, user, project);
    //            
    //            if (!pm.getDbObject().containsField("issueEvents")) {
    //               pm.getDbObject().put("issueEvents", new BasicDBList());
    //            }
    //            
    //            eventKindFound = false;
    //            for (IssueEvent ie : pm.getIssueEvents()) {
    //               if (ie.getEventKind().equals(kind)) {
    //                  ie.setCount(ie.getCount()+1);
    //                  eventKindFound = true;
    //                  break;
    //               }
    //            }
    //            if (!eventKindFound) {
    //               IssueEvent ie = new IssueEvent();
    //               ie.setEventKind(kind);
    //               ie.setCount(1);
    //               pm.getIssueEvents().add(ie);
    //            }
    //         }
    //         
    //         count++;
    //         if (count % 1000 == 0) {
    //            System.out.print(count + ", ");
    //            bio.sync();
    //         }
    //      }
    //      bio.sync();
    //      System.out.println();
    //      System.exit(0);
    //      
    ////      Watchers
    //      System.out.println("Extracting watchers...");
    //      cursor = msrDb.getCollection("watchers").find();
    //      cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
    //      it = cursor.iterator();
    //      
    //      count = 0;
    //      while(it.hasNext()){
    //         BasicDBObject obj = (BasicDBObject) it.next();
    //         
    //         User user = bio.getUsers().findOne(User.LOGIN.eq(obj.getString("login")));
    //         if (user == null) continue;
    //         
    //         Iterator<Project> repoIt = bio.getProjects().find(Project.NAME.eq(obj.getString("repo")), Project.OWNERNAME.eq(obj.getString("owner"))).iterator();
    //         if (repoIt.hasNext()) {
    //            Project project = repoIt.next();
    //            if (project != null && !project.getWatchers().contains(user)) project.getWatchers().add(user);
    //            if (!user.getWatches().contains(project)) user.getWatches().add(project);
    //         }
    //         count++;
    //         if (count % 1000 == 0) {
    //            System.out.print(count + ", ");
    //            bio.sync();
    //         }
    //      }
    //      bio.sync();
    //      System.out.println();

    ////      Org members FIXME: INCOMPLETE: Cannot match the org name against ANYTHING....
    //      System.out.println("Extracting org members...");
    //      cursor = msrDb.getCollection("org_members").find();
    //      cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
    //      it = cursor.iterator();
    //      
    //      count = 0;
    //      while(it.hasNext()){
    //         BasicDBObject obj = (BasicDBObject) it.next();
    //         
    //         String login = obj.getString("login");
    //         String orgName = obj.getString("org");
    //         
    //         User user = bio.getUsers().findOne(User.LOGIN.eq(login));
    //         
    //         User org = bio.getUsers().findOne(User.LOGIN.eq(orgName));
    //         
    //         if (org!=null){
    //            System.err.println("Found org! " + orgName);
    //            
    //         }
    //         
    ////         Project project = bio.getProjects().findOne(Project.OWNERNAME.eq("orgName"));
    ////         if (project==null) {
    ////            System.err.println("Didn't find project: " + orgName);
    ////            continue;
    ////         }
    ////         ProjectMembership pm = getProjectMembership(bio, user, project);
    ////         pm.setOrgMember(true);
    //      }
    //      bio.sync();
    //      System.out.println();

    //      Repo collaborators
    //      System.out.println("Extracting repo collaborators...");
    //      cursor = msrDb.getCollection("repo_collaborators").find();
    //      cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
    //      it = cursor.iterator();
    //      
    //      count = 0;
    //      while(it.hasNext()){
    //         BasicDBObject obj = (BasicDBObject) it.next();
    //         
    //         String login = obj.getString("login");
    //         String projectName = obj.getString("repo");
    //         String ownerName = obj.getString("owner");
    //         
    //         User user = bio.getUsers().findOne(User.LOGIN.eq(login));
    //         Iterator<Project> repoIt = bio.getProjects().find(Project.OWNERNAME.eq(ownerName), Project.NAME.eq(projectName)).iterator();
    //         if (repoIt.hasNext()) {
    //            Project project = repoIt.next();
    //   
    //            ProjectMembership pm = getProjectMembership(bio, user, project);
    //            pm.setCollaborator(true);
    //         } else {
    //            System.err.println("Couldn't find repo. owner: " + ownerName + ", repo: " + projectName);
    //         }
    //         count++;
    //         if (count % 1000 == 0) {
    //            System.out.print(count + ", ");
    //            bio.sync();
    //         }
    //      }
    //      bio.sync();
    //      
    long end = System.currentTimeMillis();
    System.out.println("Finished at " + new Date());

    long duration = end - start;
    System.out.println("Duration: " + duration);

}

From source file:org.pentaho.mongo.wrapper.field.MongodbInputDiscoverFieldsImpl.java

License:Open Source License

public List<MongoField> discoverFields(final MongoProperties.Builder properties, final String db,
        final String collection, final String query, final String fields, final boolean isPipeline,
        final int docsToSample, MongoDbInputMeta step) throws KettleException {
    MongoClientWrapper clientWrapper = null;
    try {/*from ww  w.j a v a  2s. c o m*/
        clientWrapper = MongoWrapperUtil.createMongoClientWrapper(properties, null);
    } catch (MongoDbException e) {
        throw new KettleException(e);
    }
    try {
        return clientWrapper.perform(db, new MongoDBAction<List<MongoField>>() {
            @Override
            public List<MongoField> perform(DB db) throws MongoDbException {
                DBCursor cursor = null;
                int numDocsToSample = docsToSample;
                if (numDocsToSample < 1) {
                    numDocsToSample = 100; // default
                }

                List<MongoField> discoveredFields = new ArrayList<MongoField>();
                Map<String, MongoField> fieldLookup = new HashMap<String, MongoField>();
                try {
                    if (Const.isEmpty(collection)) {
                        throw new KettleException(BaseMessages.getString(PKG,
                                "MongoNoAuthWrapper.ErrorMessage.NoCollectionSpecified")); //$NON-NLS-1$
                    }
                    DBCollection dbcollection = db.getCollection(collection);

                    Iterator<DBObject> pipeSample = null;

                    if (isPipeline) {
                        pipeSample = setUpPipelineSample(query, numDocsToSample, dbcollection);
                    } else {
                        if (Const.isEmpty(query) && Const.isEmpty(fields)) {
                            cursor = dbcollection.find().limit(numDocsToSample);
                        } else {
                            DBObject dbObject = (DBObject) JSON.parse(Const.isEmpty(query) ? "{}" : query); //$NON-NLS-1$
                            DBObject dbObject2 = (DBObject) JSON.parse(fields);
                            cursor = dbcollection.find(dbObject, dbObject2).limit(numDocsToSample);
                        }
                    }

                    int actualCount = 0;
                    while (cursor != null ? cursor.hasNext() : pipeSample.hasNext()) {
                        actualCount++;
                        DBObject nextDoc = (cursor != null ? cursor.next() : pipeSample.next());
                        docToFields(nextDoc, fieldLookup);
                    }

                    postProcessPaths(fieldLookup, discoveredFields, actualCount);

                    return discoveredFields;
                } catch (Exception e) {
                    throw new MongoDbException(e);
                } finally {
                    if (cursor != null) {
                        cursor.close();
                    }
                }
            }
        });
    } catch (Exception ex) {
        if (ex instanceof KettleException) {
            throw (KettleException) ex;
        } else {
            throw new KettleException(
                    BaseMessages.getString(PKG, "MongoNoAuthWrapper.ErrorMessage.UnableToDiscoverFields"), ex); //$NON-NLS-1$
        }
    } finally {
        try {
            clientWrapper.dispose();
        } catch (MongoDbException e) {
            //Ignore
        }
    }
}

From source file:org.sead.va.dataone.util.DataOneLogService.java

License:Apache License

public List<LogEvent> queryLog(BasicDBObject query, String countStr, int start) {

    int count = 80;
    if (countStr != null && !countStr.equals(Constants.INFINITE))
        count = Integer.parseInt(countStr);
    start = start < 0 ? 0 : start;//ww  w  .java 2s  .  c om

    DBCursor cursor;
    if (countStr != null && countStr.equals(Constants.INFINITE)) {
        cursor = eventCollection.find(query).skip(start).sort(new BasicDBObject("id", 1));
    } else {
        cursor = eventCollection.find(query).limit(count).skip(start).sort(new BasicDBObject("id", 1));
    }
    List<LogEvent> logEvents = new ArrayList<LogEvent>();
    try {
        while (cursor.hasNext()) {
            DBObject dbobj = cursor.next();
            //Converting BasicDBObject to a custom Class(LogEvent)
            LogEvent logEvent = (new Gson()).fromJson(dbobj.toString(), LogEvent.class);
            logEvents.add(logEvent);
        }
    } finally {
        cursor.close();
    }
    return logEvents;
}

From source file:org.seasr.meandre.components.tools.db.mongodb.MongoDBQuery.java

License:Open Source License

@Override
public void executeCallBack(ComponentContext cc) throws Exception {

    if (cc.isInputAvailable(IN_MONGO_CLIENT)) {
        if (_client == null) {
            _client = (MongoClient) cc.getDataComponentFromInput(IN_MONGO_CLIENT);
            _db = _client.getDB(_dbName);
            _collection = _db.getCollection(_collectionName);
        }//  w  w  w .  j a v a2 s .c  o m
    }

    DBObject query = (DBObject) JSON
            .parse(StringUtils.join(DataTypeParser.parseAsString(cc.getDataComponentFromInput(IN_QUERY)), " "));
    DBObject projection = (DBObject) JSON.parse(
            StringUtils.join(DataTypeParser.parseAsString(cc.getDataComponentFromInput(IN_PROJECTION)), " "));

    if (_wrapStream)
        cc.pushDataComponentToOutput(OUT_BSON, new StreamInitiator(streamId));

    DBCursor cursor = _collection.find(query, projection);
    try {
        while (cursor.hasNext()) {
            console.finest("Pushing next item");
            cc.pushDataComponentToOutput(OUT_BSON, cursor.next());
        }
    } finally {
        cursor.close();
    }

    if (_wrapStream)
        cc.pushDataComponentToOutput(OUT_BSON, new StreamTerminator(streamId));
}