Example usage for com.mongodb MongoClient MongoClient

List of usage examples for com.mongodb MongoClient MongoClient

Introduction

In this page you can find the example usage for com.mongodb MongoClient MongoClient.

Prototype

public MongoClient(final MongoClientURI uri, final MongoDriverInformation mongoDriverInformation) 

Source Link

Document

Creates a Mongo described by a URI.

Usage

From source file:com.maoyan.pf.webcollector.spider.AttendrateCrawler.java

License:Open Source License

public static void main(String[] args) throws Exception {
    Executor executor = new Executor() {
        @Override/*from   ww  w . j a va2s.c om*/
        public void execute(CrawlDatum datum, CrawlDatums next) throws Exception {
            MongoClient mongoClient = new MongoClient("localhost", 27017);
            // ?
            // DBCollection dbCollection = mongoClient.getDB("maoyan_crawler").getCollection("rankings_am"); 
            DB db = mongoClient.getDB("maoyan_crawler");
            // ?????
            Set<String> colls = db.getCollectionNames();
            for (String s : colls) {
                // Collection(?"")
                if (s.equals("attend_rate")) {
                    db.getCollection(s).drop();
                }
            }
            DBCollection dbCollection = db.getCollection("attend_rate");
            HtmlUnitDriver driver = new HtmlUnitDriver();
            driver.setJavascriptEnabled(false);
            driver.get(datum.getUrl());
            //                System.out.println(driver.getPageSource());
            WebElement element = driver.findElementByCssSelector("div#seat_table");
            List<WebElement> movie_name = element.findElements(By.className("c1 lineDot"));
            List<WebElement> boxoffice_rate = element.findElements(By.className("c2 red"));
            List<WebElement> visit_pershow = element.findElements(By.className("c3 gray"));
            WebElement cityarea = driver.findElementByCssSelector("span[class='today']");
            System.out.println(cityarea.getText());
            for (int i = 0; i < movie_name.size(); i++) {
                System.out.println(movie_name.get(i).getText());
                System.out.println(boxoffice_rate.get(i).getText());
                System.out.println(visit_pershow.get(i).getText());
                BasicDBObject dbObject = new BasicDBObject();
                dbObject.append("title", cityarea.getText()).append("movie_name", movie_name.get(i).getText())
                        .append("boxoffice_rate", boxoffice_rate.get(i).getText())
                        .append("visit_pershow", visit_pershow.get(i).getText());
                dbCollection.insert(dbObject);
            }
            mongoClient.close();
        }
    };

    //DBDBManager
    DBManager manager = new BerkeleyDBManager("maoyan");
    //Crawler?DBManagerExecutor
    Crawler crawler = new Crawler(manager, executor);
    crawler.addSeed("http://pf.maoyan.com/attend/rate");
    crawler.start(1);
}

From source file:com.maoyan.pf.webcollector.spider.MoviestoreCrawler.java

License:Open Source License

public static void main(String[] args) throws Exception {
    Executor executor = new Executor() {
        @Override//w  w  w . j  a va2s .co  m
        public void execute(CrawlDatum datum, CrawlDatums next) throws Exception {
            MongoClient mongoClient = new MongoClient("127.0.0.1", 27017);
            Logger mongoLogger = Logger.getLogger("org.mongodb.driver");
            mongoLogger.setLevel(Level.ERROR);
            // ?
            // DBCollection dbCollection =
            // mongoClient.getDB("maoyan_crawler").getCollection("rankings_am");
            DB db = mongoClient.getDB("maoyan_crawler");
            // ?????
            DBCollection dbCollection = db.getCollection("movie_store");
            HtmlUnitDriver driver = new HtmlUnitDriver();
            driver.setJavascriptEnabled(false);
            driver.get(datum.getUrl());
            // System.out.println(driver.getPageSource());
            System.out.println(driver.findElement(By.xpath("//html/body/header/h1")).getText());
            // ProfilesIni pi = new ProfilesIni();
            // FirefoxProfile profile = pi.getProfile("default");
            // WebClient webClient = new
            // WebClient(BrowserVersion.FIREFOX_38);
            // driver.setJavascriptEnabled(false);
            // webClient.getOptions().setCssEnabled(true);
            // HtmlPage page = webClient.getPage(datum.getUrl());
            // System.out.println(page.asXml());
            // List<?> title =
            // page.getByXPath("//html/body/header/h1/text()");
            // List<?> score =
            // page.getByXPath("//html/body/section[1]/article[1]/aside/hgroup/article[1]/i/text()");
            // List<?> score_num =
            // page.getByXPath("//html/body/section[1]/article[1]/aside/hgroup/article[1]/span/text()");
            // List<?> want_num =
            // page.getByXPath("//html/body/section[1]/article[1]/aside/hgroup/article[2]/i/text()");
            // List<?> type =
            // page.getByXPath("//html/body/section[1]/article[1]/aside/p[1]/text()");
            // List<?> timing =
            // page.getByXPath("//html/body/section[1]/article[1]/aside/p[2]/text()");
            // List<?> system =
            // page.getByXPath("//html/body/section[1]/article[1]/aside/p[3]/text()");
            // List<?> onscreendate =
            // page.getByXPath("//html/body/section[1]/article[1]/aside/p[4]/text()");
            // List<?> total_rev =
            // page.getByXPath("//html/body/section[1]/article[2]/span[1]/text()");
            // List<?> firstweek_rev =
            // page.getByXPath("//html/body/section[1]/article[2]/span[2]/text()");
            // List<?> ahead_rev =
            // page.getByXPath("//html/body/section[1]/article[2]/span[3]/text()");
            // List<?> director =
            // page.getByXPath("//*[@id='infoContent']/article[1]/div/div[1]/div/div[2]/div[1]/div[2]/text()");
            // List<?> actor =
            // page.getByXPath("//*[@id='infoContent']/article[1]/div/div[1]/div/div[2]/div[2]/div[2]/text()");
            // List<?> production =
            // page.getByXPath("//*[@id='infoContent']/article[2]/div/div[1]/div/div[2]/text()");
            // List<?> distribution =
            // page.getByXPath("//*[@id='infoContent']/article[4]/div/div[1]/div/div[2]/text()");
            // List<?> technique =
            // page.getByXPath("//*[@id='infoContent']/article[5]/div/div[1]/div/div[2]/text()");
            // List<?> desc =
            // page.getByXPath("//*[@id='infoContent']/article[6]/div/div[2]/text()");
            // List<?> want_by_city =
            // page.getByXPath("//*[@id='wantCity']/ul/li[2]/b/text()");
            // List<?> want_val=
            // page.getByXPath("//*[@id='wantCity']/ul/li[3]/div/@style");
            // System.out.println(title);
            //// System.out.println(page.getByXPath("//span[@class='today']/em/text()"));
            //// System.out.println(page.getByXPath("//span[@class='today']/text()"));
            //// List<?> movie_name =
            // page.getByXPath("//div[@id='seat_table']//ul//li[@class='c1
            // lineDot']/text()");
            //// List<?> boxoffice_rate =
            // page.getByXPath("//div[@id='seat_table']//ul//li[@class='c2
            // red']/text()");
            //// List<?> visit_pershow =
            // page.getByXPath("//div[@id='seat_table']//ul//li[@class='c3
            // gray']/text()");
            //// for(int i = 0;i<movie_name.size();i++){
            //// System.out.println(movie_name.get(i));
            //// System.out.println(boxoffice_rate.get(i));
            //// System.out.println(visit_pershow.get(i));
            //// }
            // List<?> hbList = page.getByXPath("//div");
            // HtmlDivision hb = (HtmlDivision)hbList.get(0);
            // System.out.println(hb.toString());
            // System.out.println(ReturnString(driver,By.xpath("//*[@id='infoContent']/article[4]/div/div[1]/div/div[2]")));
            if (ReturnString(driver, (By.xpath("//html/body/header/h1"))).length() != 0) {
                BasicDBObject dbObject = new BasicDBObject();
                dbObject.append("title", ReturnString(driver, (By.xpath("//html/body/header/h1"))))
                        .append("url", driver.getCurrentUrl())
                        .append("score",
                                ReturnString(driver,
                                        By.xpath(
                                                "//html/body/section[1]/article[1]/aside/hgroup/article[1]/i")))
                        .append("score_num",
                                ReturnString(driver, By.xpath(
                                        "//html/body/section[1]/article[1]/aside/hgroup/article[1]/span")))
                        .append("want_num",
                                ReturnString(driver,
                                        By.xpath(
                                                "//html/body/section[1]/article[1]/aside/hgroup/article[2]/i")))
                        .append("type",
                                ReturnString(driver, By.xpath("//html/body/section[1]/article[1]/aside/p[1]")))
                        .append("timing",
                                ReturnString(driver, By.xpath("//html/body/section[1]/article[1]/aside/p[2]")))
                        .append("system",
                                ReturnString(driver, By.xpath("//html/body/section[1]/article[1]/aside/p[3]")))
                        .append("onscreendate",
                                ReturnString(driver, By.xpath("//html/body/section[1]/article[1]/aside/p[4]")))
                        .append("total_rev",
                                ReturnString(driver, By.xpath("//html/body/section[1]/article[2]/span[1]")))
                        .append("firstweek_rev",
                                ReturnString(driver, By.xpath("//html/body/section[1]/article[2]/span[2]")))
                        .append("ahead_rev",
                                ReturnString(driver, By.xpath("//html/body/section[1]/article[2]/span[3]")))
                        .append("director", ReturnString(driver, By.xpath(
                                "//*[@id='infoContent']/article[@class='m-info-crews m-info-section']/div/div[1]/div/div[2]/div[1]/div[2]")))
                        .append("actor", ReturnString(driver, By.xpath(
                                "//*[@id='infoContent']/article[@class='m-info-crews m-info-section']/div/div[1]/div/div[2]/div[2]/div[2]")))
                        .append("production", ReturnString(driver, By.xpath(
                                "//*[@id='infoContent']/article[@class='production-companies m-info-section']/div/div[1]/div/div[2]")))
                        .append("jointproduction", ReturnString(driver, By.xpath(
                                "//*[@id='infoContent']/article[@class='joint-production-companies m-info-section']/div/div[1]/div/div[2]")))
                        .append("distribution", ReturnString(driver, By.xpath(
                                "//*[@id='infoContent']/article[@class='distribution-firm m-info-section']/div/div[1]/div/div[2]")))
                        .append("techpara", ReturnString(driver, By.xpath(
                                "//*[@id='infoContent']/article[tech-params m-info-section]/div/div[1]/div/div[2]")))
                        .append("infodrama", ReturnString(driver, By.xpath(
                                "//*[@id='infoContent']/article[m-info-drama m-info-section]/div/div[2]")))
                        .append("want_by_city", ReturnString(driver, By.xpath("//*[@id='wantCity']")));
                dbCollection.insert(dbObject);
                driver.close();
                driver.quit();
                mongoClient.close();
            }
        }
    };
    MongoClient mongoClient = new MongoClient("127.0.0.1", 27017);
    // ?
    // DBCollection dbCollection =
    // mongoClient.getDB("maoyan_crawler").getCollection("rankings_am");
    DB db = mongoClient.getDB("maoyan_crawler");
    // ?????
    Set<String> colls = db.getCollectionNames();
    for (String s : colls) {
        // Collection(?"")
        if (s.equals("movie_store")) {
            db.getCollection(s).drop();
        }
    }

    // DBDBManager
    DBManager manager = new BerkeleyDBManager("maoyan");
    // Crawler?DBManagerExecutor
    for (int round = 200; round < 300; round++) {
        System.out.println("Round " + round + " crawling.../n");
        Crawler crawler = new Crawler(manager, executor);
        for (int i = 1 + 100 * round; i <= 100 + 100 * round; i++) {
            crawler.addSeed("http://pf.maoyan.com/movie/" + i + "?_v_=yes");
        }
        /* ??visit */
        crawler.setVisitInterval(50);
        /* ?http? */
        crawler.setRetryInterval(100);
        crawler.setThreads(10);
        crawler.start(1);
    }
}

From source file:com.maoyan.pf.webcollector.spider.ShowrateCrawler.java

License:Open Source License

public static void main(String[] args) throws Exception {
    Executor executor = new Executor() {
        @Override/* w  w w  .java2  s . c  o  m*/
        public void execute(CrawlDatum datum, CrawlDatums next) throws Exception {
            MongoClient mongoClient = new MongoClient("localhost", 27017);
            // ?
            // DBCollection dbCollection = mongoClient.getDB("maoyan_crawler").getCollection("rankings_am"); 
            DB db = mongoClient.getDB("maoyan_crawler");
            // ?????
            Set<String> colls = db.getCollectionNames();
            for (String s : colls) {
                // Collection(?"")
                if (s.equals("show_rate")) {
                    db.getCollection(s).drop();
                }
            }
            DBCollection dbCollection = db.getCollection("show_rate");
            //                ProfilesIni pi = new ProfilesIni();
            //                FirefoxProfile profile = pi.getProfile("default");
            WebDriver driver = new FirefoxDriver();
            driver.manage().window().maximize();
            driver.manage().timeouts().pageLoadTimeout(3, TimeUnit.SECONDS);
            //                driver.setJavascriptEnabled(false);
            System.out.println("??\n");
            driver.get(datum.getUrl());
            //                System.out.println(driver.getPageSource());
            List<WebElement> movie_name = driver
                    .findElements(By.xpath("//div[@id='playPlan_table']/ul/li[@class='c1 lineDot']"));
            List<WebElement> boxoffice_rate = driver
                    .findElements(By.xpath("//div[@id='playPlan_table']/ul/li[@class='c2 red']"));
            List<WebElement> visit_pershow = driver
                    .findElements(By.xpath("//div[@id='playPlan_table']/ul/li[@class='c3 gray']"));
            WebElement title = driver.findElement(By.xpath("//p[@id='pieTip']"));
            for (int i = 0; i < movie_name.size(); i++) {
                String movie_name_val = movie_name.get(i).getText();
                String boxofficerate_val = boxoffice_rate.get(i).getText();
                String visit_pershow_val = visit_pershow.get(i).getText();
                BasicDBObject dbObject = new BasicDBObject();
                dbObject.append("title", title.getText()).append("is_gold", "?")
                        .append("show_type", "?").append("movie_name", movie_name_val)
                        .append("boxoffice_rate", boxofficerate_val).append("visit_pershow", visit_pershow_val);
                dbCollection.insert(dbObject);
            }
            System.out.println("?\n");
            WebElement click_gold = driver.findElement(By.id("playPlan_time"));
            click_gold.click();
            String gold_seat = driver.getWindowHandle();
            driver.switchTo().window(gold_seat);
            List<WebElement> movie_name_gold = driver
                    .findElements(By.xpath("//div[@id='playPlan_table']/ul/li[@class='c1 lineDot']"));
            List<WebElement> boxoffice_rate_gold = driver
                    .findElements(By.xpath("//div[@id='playPlan_table']/ul/li[@class='c2 red']"));
            List<WebElement> visit_pershow_gold = driver
                    .findElements(By.xpath("//div[@id='playPlan_table']/ul/li[@class='c3 gray']"));
            WebElement title_gold = driver.findElement(By.xpath("//p[@id='pieTip']"));
            for (int i = 0; i < movie_name_gold.size(); i++) {
                String movie_name_val = movie_name_gold.get(i).getText();
                String boxofficerate_val = boxoffice_rate_gold.get(i).getText();
                String visit_pershow_val = visit_pershow_gold.get(i).getText();
                BasicDBObject dbObject = new BasicDBObject();
                dbObject.append("title", title_gold.getText()).append("is_gold", "")
                        .append("show_type", "?").append("movie_name", movie_name_val)
                        .append("boxoffice_rate", boxofficerate_val).append("visit_pershow", visit_pershow_val);
                dbCollection.insert(dbObject);
            }
            System.out.println("?\n");
            WebElement click_vist = driver.findElement(By.xpath("//*[@id='show--type']"));
            click_vist.click();
            String gold_vist = driver.getWindowHandle();
            driver.switchTo().window(gold_vist);
            List<WebElement> movie_name_gold_visit = driver
                    .findElements(By.xpath("//div[@id='playPlan_table']/ul/li[@class='c1 lineDot']"));
            List<WebElement> boxoffice_rate_gold_visit = driver
                    .findElements(By.xpath("//div[@id='playPlan_table']/ul/li[@class='c2 red']"));
            List<WebElement> visit_pershow_gold_visit = driver
                    .findElements(By.xpath("//div[@id='playPlan_table']/ul/li[@class='c3 gray']"));
            WebElement title_gold_visit = driver.findElement(By.xpath("//p[@id='pieTip']"));
            for (int i = 0; i < movie_name_gold_visit.size(); i++) {
                String movie_name_val = movie_name_gold_visit.get(i).getText();
                String boxofficerate_val = boxoffice_rate_gold_visit.get(i).getText();
                String visit_pershow_val = visit_pershow_gold_visit.get(i).getText();
                BasicDBObject dbObject = new BasicDBObject();
                dbObject.append("title", title_gold_visit.getText()).append("is_gold", "")
                        .append("show_type", "").append("movie_name", movie_name_val)
                        .append("boxoffice_rate", boxofficerate_val).append("visit_pershow", visit_pershow_val);
                dbCollection.insert(dbObject);
            }
            System.out.println("?\n");
            click_gold.click();
            String normal_seat = driver.getWindowHandle();
            driver.switchTo().window(normal_seat);
            List<WebElement> movie_name_normal_seat = driver
                    .findElements(By.xpath("//div[@id='playPlan_table']/ul/li[@class='c1 lineDot']"));
            List<WebElement> boxoffice_rate_normal_seat = driver
                    .findElements(By.xpath("//div[@id='playPlan_table']/ul/li[@class='c2 red']"));
            List<WebElement> visit_pershow_normal_seat = driver
                    .findElements(By.xpath("//div[@id='playPlan_table']/ul/li[@class='c3 gray']"));
            WebElement title_normal_seat = driver.findElement(By.xpath("//p[@id='pieTip']"));
            for (int i = 0; i < movie_name_normal_seat.size(); i++) {
                String movie_name_val = movie_name_normal_seat.get(i).getText();
                String boxofficerate_val = boxoffice_rate_normal_seat.get(i).getText();
                String visit_pershow_val = visit_pershow_normal_seat.get(i).getText();
                BasicDBObject dbObject = new BasicDBObject();
                dbObject.append("title", title_normal_seat.getText()).append("is_gold", "?")
                        .append("show_type", "").append("movie_name", movie_name_val)
                        .append("boxoffice_rate", boxofficerate_val).append("visit_pershow", visit_pershow_val);
                dbCollection.insert(dbObject);
            }

            driver.close();
            driver.quit();
            mongoClient.close();
        }
    };

    //DBDBManager
    DBManager manager = new BerkeleyDBManager("maoyan");
    //Crawler?DBManagerExecutor
    Crawler crawler = new Crawler(manager, executor);
    crawler.addSeed("http://pf.maoyan.com/show/rate");
    crawler.start(1);
}

From source file:com.maschel.lca.lcacloud.analytics.Analytics.java

License:Open Source License

protected Analytics() {
    mongoClient = new MongoClient(ConfigurationHelper.getAnalyticsDBHost(),
            ConfigurationHelper.getAnalyticsDBPort());
    database = mongoClient.getDatabase(ANALYTICS_DATABASE);
}

From source file:com.mballem.simplemongodb.dao.MongoConnection.java

public DB getDB() {
    if (mongo == null) {
        try {/*from w  w w.  ja va  2 s. c  o  m*/
            mongo = new MongoClient(HOST, PORT);
            db = mongo.getDB(DB_NAME);
            System.out.println("Mongo instance equals :> " + mongoInstance++);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    return db;
}

From source file:com.meltmedia.dropwizard.mongo.junit.MongoRule.java

License:Apache License

/**
 * Wraps the provided statement with code to create and then clean up a
 * MongoClient object./*from w w  w.j  av  a 2 s . com*/
 */
@Override
public Statement apply(final Statement statement, final Description description) {
    return new Statement() {
        @Override
        public void evaluate() throws Throwable {
            try {
                mongoClient = new MongoClient(host, port);
                statement.evaluate();
            } finally {
                if (mongoClient != null) {
                    try {
                        mongoClient.close();
                    } catch (Exception e) {
                    }
                    mongoClient = null;
                }
            }
        }
    };
}

From source file:com.mingo.executor.QueryExecutorFactory.java

License:Apache License

/**
 * Creates query executor./* w w w  .j ava2s  . com*/
 *
 * @param context mingo context {@link Context}
 * @return query executor {@link QueryExecutor}
 */
public static QueryExecutor create(Context context) {
    QueryExecutor queryExecutor = null;
    Validate.notNull(context, "context cannot be null");
    try {
        queryExecutor = create(new MongoClient(context.getDatabaseHost(), context.getDatabasePort()), context);
    } catch (UnknownHostException e) {
        LOGGER.error(ExceptionUtils.getMessage(e));
    }
    return queryExecutor;

}

From source file:com.miya.twit.mongodb.DBConnectSentiment.java

public static DBCollection dbConnection() {

    try {//w  ww  . jav a  2s. co  m
        // Connect to mongodb
        MongoClient mongo = new MongoClient("localhost", 27017);
        DB db = mongo.getDB("SentimentDB");
        DBCollection collection = db.getCollection("gokhan");
        return collection;
    } catch (UnknownHostException e) {
        System.out.println("dbConnection hatas : " + e.getCause());
        e.printStackTrace();
        return null;
    } catch (MongoException e) {
        System.out.println("dbConnection hatas : " + e.getCause() + "  --  " + e.getMessage());
        e.printStackTrace();
        return null;
    }
}

From source file:com.miya.twit.mongodb.DBConnectSentiment.java

public static DBCollection dbConnectionForExpression() {

    try {/*w  w  w .j  a v  a 2  s  .  c om*/
        // Connect to mongodb
        MongoClient mongo = new MongoClient("localhost", 27017);
        DB db = mongo.getDB("SentimentExpressionDB");
        DBCollection collection = db.getCollection("gokhan");
        return collection;
    } catch (UnknownHostException e) {
        System.out.println("dbConnectionForExpression hatas : " + e.getCause());
        e.printStackTrace();
        return null;
    } catch (MongoException e) {
        System.out.println("dbConnectionForExpression hatas : " + e.getCause() + "  --  " + e.getMessage());
        e.printStackTrace();
        return null;
    }
}

From source file:com.mongodash.appender.MongoDBAppender.java

License:Apache License

@Override
public void start() {
    try {/*from  ww w .  j a va  2 s  . c o  m*/
        _mongo = new MongoClient(_dbHost, _dbPort);
        DB db = _mongo.getDB(_dbName);
        _collection = db.getCollection(_dbCollection);
        DBObject index = new BasicDBObject();
        index.put("ts", 1);
        index.put("expireAfterSeconds", _dbRetainDataInSeconds);
        _collection.ensureIndex(index);
    } catch (Exception e) {
        addStatus(new ErrorStatus("Failed to initialize MondoDB", this, e));
        return;
    }
    super.start();
}