Example usage for org.jsoup.nodes Document select

List of usage examples for org.jsoup.nodes Document select

  1. HOME
  2. Java
  3. org.jsoup
  4. org.jsoup.nodes.*
  5. Document
  6. select

Introduction

In this page you can find the example usage for org.jsoup.nodes Document select.

Prototype

public Elements select(String cssQuery)

Source Link

Document

Find elements that match the Selector CSS query, with this element as the starting context.

Usage

From source file:com.sastix.cms.server.services.content.impl.ZipFileHandlerServiceImpl.java

@Override
public String findParentResource(Map<String, byte[]> bytesMap) {
    byte[] dataXml = bytesMap.get(METADATA_XML_FILE);
    if (dataXml == null) {
        return null;
    }// w  w w .j  a  v a2 s .  c o  m
    String xml = "";
    String ret = "";
    try {
        xml = new String(dataXml, "UTF-8");
    } catch (UnsupportedEncodingException e) {
        LOG.error("Error in finding parent resource name: {}", e.getLocalizedMessage());
    }
    Document doc = Jsoup.parse(xml, "", Parser.xmlParser());
    for (Element e : doc.select("resources")) {
        ret = e.select("resource").get(0).attr("href");
    }
    return ret;
}

From source file:com.anhao.spring.service.impl.PhotosServiceImpl.java

@Override
public void process(Document doc) {
    Elements links = doc.select("section ul li");
    for (Element li : links) {
        Element figure = li.child(0);
        String wallpaperId = figure.attr("data-wallpaper-id");
        /**//from  w w  w .  ja v a2  s  .co  m
         * ???
         */
        String tempUUID = jobPhotosDAO.findByWallpaperId(wallpaperId);

        if (StringUtils.isNotEmpty(tempUUID)) {
            logger.info("wallpapers id {} thumbnail  exist.", wallpaperId);
            //??wallpaperID? ???
            continue;
        }

        String thumbnail = "http://alpha.wallhaven.cc/wallpapers/thumb/small/th-" + wallpaperId + ".jpg";
        String full = "http://wallpapers.wallhaven.cc/wallpapers/full/wallhaven-" + wallpaperId + ".jpg";
        /**
         * fastdfs?fastdfs?/tmp/
         */
        boolean smallStatus = download(thumbnail, "/tmp/small" + wallpaperId + ".jpg");
        boolean fullStatus = download(full, "/tmp/full" + wallpaperId + ".jpg");

        if (smallStatus && fullStatus) {
            File thumbnailFile = new File("/tmp/small" + wallpaperId + ".jpg");
            String thumbnailPath = storageService.upload(thumbnailFile);

            thumbnailFile.delete();

            File sourceFile = new File("/tmp/full" + wallpaperId + ".jpg");
            String sourceFilePath = storageService.upload(sourceFile);
            EasyImage easyImage = new EasyImage(sourceFile);

            //???
            //sourceFile.delete();
            String uuid = UUID.randomUUID().toString();
            Photos photos = new Photos();
            photos.setId(uuid);

            photos.setTitle(wallpaperId);
            photos.setWidth(easyImage.getWidth());
            photos.setHeight(easyImage.getHeight());
            photos.setSize(sourceFile.length());

            photos.setCreate_date(new Date());
            photos.setModify_date(new Date());

            photos.setLarge(sourceFilePath);
            photos.setMedium(thumbnailPath);
            photos.setOrders(1);
            photos.setSource(sourceFilePath);
            photos.setThumbnail(thumbnailPath);
            photos.setAlbum_id("ff8081814f7e13d8014f7e18a95a0000");
            photos.setMember_id("1");

            photos.setWallhaven(wallpaperId);
            photos.setStorage_host("http://123.57.240.11");
            jobPhotosDAO.add(photos);

            //2015-10-18 ? 
            getWallpaperTags(wallpaperId);
            //2015-10-18 ? ?
            //?
            photosColorsService.generateColors(sourceFile, uuid);

        } else {
            logger.info("wallpapers id {} thumbnail or fullImage not exist.", wallpaperId);
        }
    }
}

From source file:virgil.meanback.HistoryInfo.java

/**
 *
 * @param url/*from   w ww .j  av a  2  s  .  c o m*/
 * @return
 * @throws Exception
 */
@SuppressWarnings("")
public Stock parse(String url) throws Exception {
    java.util.logging.Logger.getLogger("com.gargoylesoftware.htmlunit").setLevel(Level.OFF);
    Stock stock = new Stock();
    List<DayInfo> list = new ArrayList<>();
    /**
     * HtmlUnitweb?
     */
    WebClient wc = new WebClient(BrowserVersion.CHROME);
    wc.getOptions().setUseInsecureSSL(true);
    wc.getOptions().setJavaScriptEnabled(true); // ?JStrue  
    wc.getOptions().setCssEnabled(false); // ?css?  
    wc.getOptions().setThrowExceptionOnScriptError(false); // js??  
    wc.getOptions().setTimeout(50000); //  10S0?  
    wc.getOptions().setDoNotTrackEnabled(false);
    HtmlPage page = wc.getPage(url);
    HtmlElement documentElement = page.getDocumentElement();
    Document doc = Jsoup.parse(documentElement.asXml());
    String name = doc.select("#BIZ_IS_Name").text();
    String code = doc.select(".BIZ_IS_price_id span").text();
    code = code.substring(code.indexOf("(") + 2, code.length() - 1);
    Elements els = doc.select("#BIZ_hq_historySearch tbody tr");
    stock.setCode(code);
    stock.setName(name);
    int count = 0;
    for (Element el : els) {
        if (!el.html().contains("sum")) {
            DayInfo dayInfo = new DayInfo();
            String dateString = el.select("td.e1").text();
            SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
            Date date = format.parse(dateString);
            String open = el.select("td").eq(1).text();
            String close = el.select("td").eq(2).text();
            double cd = Double.parseDouble(close);
            String low = el.select("td").eq(5).text();
            String high = el.select("td").eq(6).text();
            String volume = el.select("td").eq(7).text();
            dayInfo.setClose(close);
            dayInfo.setDateString(dateString);
            dayInfo.setHigh(high);
            dayInfo.setLow(low);
            dayInfo.setOpen(open);
            dayInfo.setVolume(volume);
            dayInfo.setDate(date);
            list.add(dayInfo);
            count++;
            if (list.size() > 79) {
                break;
            }
        }
    }
    stock.setList(list);
    return stock;
}

From source file:com.aestasit.markdown.slidery.converters.DeckJSConverter.java

protected void transformDocument(final Document slidesDocument, final Configuration config) {
    super.transformDocument(slidesDocument, config);
    if (config.listsIncremented()) {
        for (final Element list : slidesDocument.select("div li")) {
            list.addClass("slide");
        }/*from   ww  w  .j  a va 2  s .c  o m*/
    }
}

From source file:com.liato.bankdroid.banking.banks.AbsIkanoPartner.java

@Override
public void update() throws BankException, LoginException, BankChoiceException {
    super.update();
    if (username == null || password == null || username.length() == 0 || password.length() == 0) {
        throw new LoginException(res.getText(R.string.invalid_username_password).toString());
    }/*from www . j a  v a  2s .c  om*/

    urlopen = login();
    Document d = Jsoup.parse(response);
    Element element = d.select("#primary-nav > li:eq(1) > a").first();
    if (element != null && element.attr("href") != null) {
        String myAccountUrl = element.attr("href");
        try {
            response = urlopen.open("https://partner.ikanobank.se/" + myAccountUrl);
            d = Jsoup.parse(response);
            Elements es = d.select("#CustomerAccountInformationSpan > span > span");
            int accId = 0;
            for (Element el : es) {
                Element name = el.select("> span > span:eq(0)").first();
                Element balance = el.select("> span:eq(1)").first();
                Element currency = el.select("> span:eq(2)").first();
                if (name != null && balance != null && currency != null) {
                    Account account = new Account(name.text().trim(), Helpers.parseBalance(balance.text()),
                            Integer.toString(accId));
                    account.setCurrency(Helpers.parseCurrency(currency.text(), "SEK"));
                    if (accId > 0) {
                        account.setAliasfor("0");
                    }
                    accounts.add(account);
                    accId++;
                }
            }
            if (accounts.isEmpty()) {
                throw new BankException(res.getText(R.string.no_accounts_found).toString());
            }
            // Use the amount from "Kvar att handla fr" which should be the
            // last account in the list.
            this.balance = accounts.get(accounts.size() - 1).getBalance();
            ArrayList<Transaction> transactions = new ArrayList<Transaction>();
            es = d.select("#ShowCustomerTransactionPurchasesInformationDiv table tr:has(td)");
            for (Element el : es) {
                if (el.childNodeSize() == 6) {
                    Transaction transaction = new Transaction(el.child(0).text().trim(),
                            el.child(1).text().trim(), Helpers.parseBalance(el.child(2).text()));
                    transaction.setCurrency(Helpers.parseCurrency(el.child(3).text().trim(), "SEK"));
                    transactions.add(transaction);
                }
            }
            accounts.get(0).setTransactions(transactions);
        }

        catch (ClientProtocolException e) {
            throw new BankException(e.getMessage());
        } catch (IOException e) {
            throw new BankException(e.getMessage());
        }
    }
    if (accounts.isEmpty()) {
        throw new BankException(res.getText(R.string.no_accounts_found).toString());
    }
    super.updateComplete();
}

From source file:com.sastix.cms.server.services.content.impl.ZipFileHandlerServiceImpl.java

@Override
public boolean isScormType(Map<String, byte[]> bytesMap) {
    byte[] dataXml = bytesMap.get(METADATA_XML_FILE);
    if (dataXml == null) {
        return false;
    } else {//ww  w.  ja  va  2 s .c om
        String xml = "";
        String ret = "";
        try {
            xml = new String(dataXml, "UTF-8");
        } catch (UnsupportedEncodingException e) {
            LOG.error("Error in determining if it is a scorm type: {}", e.getLocalizedMessage());
        }
        Document doc = Jsoup.parse(xml, "", Parser.xmlParser());
        for (Element e : doc.select("resources")) {
            ret = e.select("resource").get(0).attr("adlcp:scormType");
        }
        return ret.equals("sco");
    }
}

From source file:mobi.jenkinsci.ci.client.sso.GoogleSsoHandler.java

private HttpPost getOtpFormPost(final HttpContext httpContext, final HttpResponse response, final String otp)
        throws IOException, MalformedURLException, UnsupportedEncodingException {
    final String requestUri = JenkinsFormAuthHttpClient.getLatestRedirectedUrl(httpContext);
    final String requestBaseUrl = requestUri.substring(0, requestUri.lastIndexOf('/'));
    log.debug("Looking for HTML input form retrieved from " + requestUri);
    final Document doc = Jsoup.parse(response.getEntity().getContent(), "UTF-8", requestBaseUrl);
    final org.jsoup.nodes.Element form = doc.select("form[id=verify-form]").first();

    if (otp == null) {
        final Element otpLabel = doc.select("div[id=verifyText]").first();
        throw new TwoPhaseAuthenticationRequiredException("Google 2-step Authenticator: \n"
                + "1. Tap on AuthApp to authenticate.\n" + "2. " + getDivText(otpLabel),
                GOOGLE_ANDROID_APPS_AUTHENTICATOR2_APP_ID);
    }// ww  w.j  a  va2s .c  o  m

    final HashMap<String, String> formMapping = new HashMap<String, String>();
    formMapping.put("smsUserPin", otp);
    return JenkinsFormAuthHttpClient.getPostForm(requestBaseUrl, form, formMapping);
}

From source file:com.stratio.qa.utils.GosecSSOUtils.java

/**
 * This method provide dcos and sso token to be used to generate client cookie
 * @return cookieToken list of token generated
 * @throws Exception exception/*  w  ww. j  av a 2 s  . co  m*/
 */
public HashMap<String, String> ssoTokenGenerator() throws Exception {
    String protocol = "https://";
    HashMap<String, String> cookieToken = new HashMap<>();

    SSLContext sslContext = SSLContext.getInstance("SSL");
    // set up a TrustManager that trusts everything
    sslContext.init(null, ALL_TRUSTING_TRUST_MANAGER, new SecureRandom());
    HttpClientContext context = HttpClientContext.create();
    HttpGet httpGet = new HttpGet(protocol + ssoHost + "/login");
    HttpClient client = HttpClientBuilder.create().setSslcontext(sslContext)
            .setRedirectStrategy(new LaxRedirectStrategy())
            .setDefaultRequestConfig(RequestConfig.custom().setCircularRedirectsAllowed(true).build()).build();
    try {
        HttpResponse firstResponse = client.execute(httpGet, context);

        logger.debug(firstResponse.getStatusLine().toString());
        Document doc = Jsoup.parse(getStringFromIS(firstResponse.getEntity().getContent()));
        Elements code = doc.select("[name=lt]");
        String loginCode = code.attr("value");
        String executionCode = doc.select("[name=execution]").attr("value");
        for (Header oneHeader : firstResponse.getAllHeaders()) {
            logger.debug(oneHeader.getName() + ":" + oneHeader.getValue());
        }

        URI redirect = context.getRedirectLocations().get(context.getRedirectLocations().size() - 1);

        List<NameValuePair> params = new ArrayList<>();
        params.add(new BasicNameValuePair("_eventId", "submit"));
        params.add(new BasicNameValuePair("submit", "LOGIN"));
        params.add(new BasicNameValuePair("username", userName));
        params.add(new BasicNameValuePair("password", passWord));
        params.add(new BasicNameValuePair("lt", loginCode));
        params.add(new BasicNameValuePair("execution", executionCode));
        HttpPost httpPost = new HttpPost(redirect);
        httpPost.setEntity(new UrlEncodedFormEntity(params));
        HttpResponse secondResponse = client.execute(httpPost, context);

        for (Header oneHeader : secondResponse.getAllHeaders()) {
            logger.debug(oneHeader.getName() + ":" + oneHeader.getValue());
        }

        HttpGet managementGet = new HttpGet(protocol + ssoHost + managementHost);
        client.execute(managementGet, context);

        for (Cookie oneCookie : context.getCookieStore().getCookies()) {
            logger.debug(oneCookie.getName() + ":" + oneCookie.getValue());
            cookieToken.put(oneCookie.getName(), oneCookie.getValue());
        }

    } catch (Exception e) {
        e.getStackTrace();
    }
    return cookieToken;
}

From source file:com.qubole.rubix.hadoop1.Hadoop1ClusterManager.java

private List<String> extractNodes(String dfsnodelist) {
    Document doc = Jsoup.parse(dfsnodelist);

    String title = doc.title();/*  w  w w.  ja v  a 2  s . c  o m*/
    List<String> workers = new ArrayList<String>();

    Elements links = doc.select(".name");
    for (int i = 0; i < links.size(); i++) {
        Elements nodes = links.get(i).select("a[href]");
        if (nodes != null && nodes.size() > 0) {
            String node = nodes.get(0).ownText();
            if (node != null && !node.isEmpty()) {
                workers.add(node);
            }
        }
    }

    Collections.sort(workers);
    return workers;
}

From source file:net.pixomania.crawler.W3C.parser.rules.editors.EditorsRule7.java

@Override
public ArrayList<Person> run(String url, Document doc) {
    ArrayList<Person> editorList = new ArrayList<>();

    Elements editors = doc.select("dt:contains(Authors/Editors) ~ dd, dt:contains(Author/Editor) ~ dd");
    if (editors.size() == 0)
        return null;

    boolean skip = false;
    for (Element editor : editors) {
        Element prev = editor.previousElementSibling();
        if (prev.tagName().equals("dt")) {
            if (!prev.text().trim().toLowerCase().startsWith("authors/editors")
                    && !prev.text().trim().toLowerCase().startsWith("author/editor")) {
                skip = true;/*from   w  w w.j a v  a  2s  .co  m*/
            }
        }

        if (skip) {
            Element next = editor.nextElementSibling();
            if (next != null) {
                if (next.text().trim().toLowerCase().startsWith("authors/editors")
                        || next.text().trim().toLowerCase().startsWith("author/editor")) {
                    skip = false;
                    continue;
                }
            }
            continue;
        }

        if (StringUtils.countMatches(editor.text(), " - ") > 2) {
            Log.log("warning", url + ": This editor may be a list of editors separated by  - ");
            EditorsRule5 ed5 = new EditorsRule5();

            return ed5.run(url, doc);
        }

        String[] splitted = editor.html().split("<br />|<br clear=\"none\" />");

        if (splitted.length < 2) {
            if (editor.text().equals("WHATWG:") || editor.text().equals("W3C:"))
                continue;
            Person result = NameParser.parse(editor.text());
            if (result == null)
                continue;

            for (int i = 0; i < editor.select("a").size(); i++) {
                if (!editor.select("a").get(i).attr("href").isEmpty()) {
                    if (editor.select("a").get(i).attr("href").contains("@")) {
                        result.setEmail(editor.select("a").get(i).attr("href").replace("mailto:", ""));
                    } else {
                        result.addWebsite(editor.select("a").get(i).attr("href"));
                    }
                }
            }

            editorList.add(result);
        } else {
            for (String split : splitted) {
                if (!split.isEmpty()) {
                    if (split.equals("WHATWG:") || split.equals("W3C:"))
                        continue;
                    Document newdoc = Jsoup.parse(split.replaceAll("\n", ""));
                    Person result = NameParser.parse(newdoc.text());
                    if (result == null)
                        continue;

                    for (int i = 0; i < newdoc.select("a").size(); i++) {
                        if (!newdoc.select("a").get(i).attr("href").isEmpty()) {
                            if (newdoc.select("a").get(i).attr("href").contains("@")) {
                                result.setEmail(newdoc.select("a").get(i).attr("href").replace("mailto:", ""));
                            } else {
                                result.addWebsite(newdoc.select("a").get(i).attr("href"));
                            }
                        }
                    }

                    editorList.add(result);
                }
            }
        }

        Element next = editor.nextElementSibling();
        if (next != null)
            if (next.tag().getName().equals("dt"))
                break;
    }

    if (editorList.size() == 0)
        return null;

    return editorList;
}