Example usage for org.jsoup.nodes Element attributes

List of usage examples for org.jsoup.nodes Element attributes

  1. HOME
  2. Java
  3. org.jsoup
  4. org.jsoup.nodes.*
  5. Element
  6. attributes

Introduction

In this page you can find the example usage for org.jsoup.nodes Element attributes.

Prototype

Attributes attributes

To view the source code for org.jsoup.nodes Element attributes.

Click Source Link

Usage

From source file:downloadwolkflow.getWorkFlowList.java

public static void main(String args[]) {
    CloseableHttpClient httpclient = HttpClients.createDefault();
    String[] pageList = getPageList();
    System.out.println(pageList.length);
    for (int i = 1; i < pageList.length; i++) {
        System.out.println(pageList[i]);
        System.out.println("---------------------------------------------------------------------------");
        HttpGet httpget = new HttpGet(pageList[i]);
        try {//ww  w .j a v  a 2  s  .  c  o m
            HttpResponse response = httpclient.execute(httpget);
            String page = EntityUtils.toString(response.getEntity());
            Document mainDoc = Jsoup.parse(page);
            Elements resultList = mainDoc.select("div.resource_list_item");
            for (int j = 0; j < resultList.size(); j++) {
                Element workflowResult = resultList.get(j);
                Element detailInfo = workflowResult.select("div.main_panel").first().select("p.title.inline")
                        .first().select("a").first();
                String detailUrl = "http://www.myexperiment.org" + detailInfo.attributes().get("href")
                        + ".html";
                System.out.println(detailUrl);
                downloadWorkFlow(detailUrl, httpclient);
                Thread.sleep(1000);
            }
        } catch (IOException ex) {
            Logger.getLogger(getWorkFlowList.class.getName()).log(Level.SEVERE, null, ex);
        } catch (InterruptedException ex) {
            Logger.getLogger(getWorkFlowList.class.getName()).log(Level.SEVERE, null, ex);
        }

    }

    try {
        httpclient.close();
    } catch (IOException ex) {
        Logger.getLogger(getWorkFlowList.class.getName()).log(Level.SEVERE, null, ex);
    }

}

From source file:downloadwolkflow.getWorkFlowList.java

private static void downloadWorkFlow(String detailUrl, CloseableHttpClient httpclient) {
    try {/*from w w  w. ja  va 2  s  .  com*/
        HttpGet httpget = new HttpGet(detailUrl);
        HttpResponse response = httpclient.execute(httpget);
        String page = EntityUtils.toString(response.getEntity());
        Document mainDoc = Jsoup.parse(page);
        Element downloadEle = mainDoc.select("div#myexp_content ul li a").first();
        if (downloadEle == null) {
            downloadEle = mainDoc.select("div#myexp_content ul li:nth-child(1) span a").first();
        }

        String downloadUrl = downloadEle.attributes().get("href");
        Thread.sleep(500);
        if (downloadUrl.contains("download")) {
            downloadFiles(downloadUrl, httpclient);
        } else {
            System.out.println(detailUrl + " do not contain valuable resource");
        }
    } catch (IOException ex) {
        Logger.getLogger(getWorkFlowList.class.getName()).log(Level.SEVERE, null, ex);
    } catch (InterruptedException ex) {
        Logger.getLogger(getWorkFlowList.class.getName()).log(Level.SEVERE, null, ex);
    }
}

From source file:com.shareplaylearn.OauthPasswordFlow.java

public static LoginInfo googleLogin(String username, String password, String clientId, String callbackUri)
        throws URISyntaxException, IOException, AuthorizationException, UnauthorizedException {

    CloseableHttpClient httpClient = HttpClients.custom().build();
    String oAuthQuery = "client_id=" + clientId + "&";
    oAuthQuery += "response_type=code&";
    oAuthQuery += "scope=openid email&";
    oAuthQuery += "redirect_uri=" + callbackUri;
    URI oAuthUrl = new URI("https", null, "accounts.google.com", 443, "/o/oauth2/auth", oAuthQuery, null);
    Connection oauthGetCoonnection = Jsoup.connect(oAuthUrl.toString());
    Connection.Response oauthResponse = oauthGetCoonnection.method(Connection.Method.GET).execute();
    if (oauthResponse.statusCode() != 200) {
        String errorMessage = "Error contacting Google's oauth endpoint: " + oauthResponse.statusCode() + " / "
                + oauthResponse.statusMessage();
        if (oauthResponse.body() != null) {
            errorMessage += oauthResponse.body();
        }// w  ww.j a v a2s  .  c o m
        throw new AuthorizationException(errorMessage);
    }
    Map<String, String> oauthCookies = oauthResponse.cookies();
    Document oauthPage = oauthResponse.parse();
    Element oauthForm = oauthPage.getElementById("gaia_loginform");
    System.out.println(oauthForm.toString());
    Connection oauthPostConnection = Jsoup.connect("https://accounts.google.com/ServiceLoginAuth");
    HashMap<String, String> formParams = new HashMap<>();

    for (Element child : oauthForm.children()) {
        System.out.println("Tag name: " + child.tagName());
        System.out.println("attrs: " + Arrays.toString(child.attributes().asList().toArray()));
        if (child.tagName().equals("input") && child.hasAttr("name")) {

            String keyName = child.attr("name");
            String keyValue = null;

            if (child.hasAttr("value")) {
                keyValue = child.attr("value");
            }

            if (keyName != null && keyName.trim().length() != 0 && keyValue != null
                    && keyValue.trim().length() != 0) {
                oauthPostConnection.data(keyName, keyValue);
                formParams.put(keyName, keyValue);
            }
        }
    }
    oauthPostConnection.cookies(oauthCookies);
    formParams.put("Email", username);
    formParams.put("Passwd-hidden", password);
    //oauthPostConnection.followRedirects(false);
    System.out.println("form post params were: ");
    for (Map.Entry<String, String> kvp : formParams.entrySet()) {
        //DO NOT let passwords end up in the logs ;)
        if (kvp.getKey().equals("Passwd")) {
            continue;
        }
        System.out.println(kvp.getKey() + "," + kvp.getValue());
    }
    System.out.println("form cookies were: ");
    for (Map.Entry<String, String> cookie : oauthCookies.entrySet()) {
        System.out.println(cookie.getKey() + "," + cookie.getValue());
    }
    //System.exit(0);
    Connection.Response postResponse = null;
    try {
        postResponse = oauthPostConnection.method(Connection.Method.POST).timeout(5000).execute();
    } catch (Throwable t) {
        System.out.println("Failed to post login information to googles endpoint :/ " + t.getMessage());
        System.out.println("This usually means the connection is bad, shareplaylearn.com is down, or "
                + " google is being a punk - login manually and check.");
        assertTrue(false);
    }
    if (postResponse.statusCode() != 200) {
        String errorMessage = "Failed to validate credentials: " + oauthResponse.statusCode() + " / "
                + oauthResponse.statusMessage();
        if (oauthResponse.body() != null) {
            errorMessage += oauthResponse.body();
        }
        throw new UnauthorizedException(errorMessage);
    }
    System.out.println("Response headers (after post to google form & following redirect):");
    for (Map.Entry<String, String> header : postResponse.headers().entrySet()) {
        System.out.println(header.getKey() + "," + header.getValue());
    }
    System.out.println("Final response url was: " + postResponse.url().toString());
    String[] args = postResponse.url().toString().split("&");
    LoginInfo loginInfo = new LoginInfo();
    for (String arg : args) {
        if (arg.startsWith("access_token")) {
            loginInfo.accessToken = arg.split("=")[1].trim();
        } else if (arg.startsWith("id_token")) {
            loginInfo.idToken = arg.split("=")[1].trim();
        } else if (arg.startsWith("expires_in")) {
            loginInfo.expiry = arg.split("=")[1].trim();
        }
    }

    //Google doesn't actually throw a 401 or anything - it just doesn't redirect
    //and sends you back to it's login page to try again.
    //So this is what happens with an invalid password.
    if (loginInfo.accessToken == null || loginInfo.idToken == null) {
        //Document oauthPostResponse = postResponse.parse();
        //System.out.println("*** Oauth response from google *** ");
        //System.out.println(oauthPostResponse.toString());
        throw new UnauthorizedException(
                "Error retrieving authorization: did you use the correct username/password?");
    }
    String[] idTokenFields = loginInfo.idToken.split("\\.");
    if (idTokenFields.length < 3) {
        throw new AuthorizationException("Error parsing id token " + loginInfo.idToken + "\n" + "it only had "
                + idTokenFields.length + " field!");
    }
    String jwtBody = new String(Base64.decodeBase64(idTokenFields[1]), StandardCharsets.UTF_8);
    loginInfo.idTokenBody = new Gson().fromJson(jwtBody, OauthJwt.class);
    loginInfo.id = loginInfo.idTokenBody.sub;
    return loginInfo;
}

From source file:com.astamuse.asta4d.render.RenderUtil.java

public final static void applyMessages(Element target) {
    Context context = Context.getCurrentThreadContext();
    List<Element> msgElems = target.select(ExtNodeConstants.MSG_NODE_TAG_SELECTOR);
    for (final Element msgElem : msgElems) {
        Attributes attributes = msgElem.attributes();
        String key = attributes.get(ExtNodeConstants.MSG_NODE_ATTR_KEY);
        // List<String> externalizeParamKeys = getExternalizeParamKeys(attributes);
        Object defaultMsg = new Object() {
            @Override//from w  ww  .java2 s . c  o  m
            public String toString() {
                return ExtNodeConstants.MSG_NODE_ATTRVALUE_HTML_PREFIX + msgElem.html();
            }
        };
        Locale locale = LocalizeUtil.getLocale(attributes.get(ExtNodeConstants.MSG_NODE_ATTR_LOCALE));
        String currentTemplatePath = attributes.get(ExtNodeConstants.ATTR_TEMPLATE_PATH);
        if (StringUtils.isEmpty(currentTemplatePath)) {
            logger.warn("There is a msg tag which does not hold corresponding template file path:{}",
                    msgElem.outerHtml());
        } else {
            context.setData(TRACE_VAR_TEMPLATE_PATH, currentTemplatePath);
        }

        final Map<String, Object> paramMap = getMessageParams(attributes, locale, key);
        String text;
        switch (I18nMessageHelperTypeAssistant.configuredHelperType()) {
        case Mapped:
            text = I18nMessageHelperTypeAssistant.getConfiguredMappedHelper().getMessageWithDefault(locale, key,
                    defaultMsg, paramMap);
            break;
        case Ordered:
        default:
            // convert map to array
            List<Object> numberedParamNameList = new ArrayList<>();
            for (int index = 0; paramMap
                    .containsKey(ExtNodeConstants.MSG_NODE_ATTR_PARAM_PREFIX + index); index++) {
                numberedParamNameList.add(paramMap.get(ExtNodeConstants.MSG_NODE_ATTR_PARAM_PREFIX + index));
            }
            text = I18nMessageHelperTypeAssistant.getConfiguredOrderedHelper().getMessageWithDefault(locale,
                    key, defaultMsg, numberedParamNameList.toArray());
        }

        Node node;
        if (text.startsWith(ExtNodeConstants.MSG_NODE_ATTRVALUE_TEXT_PREFIX)) {
            node = ElementUtil.text(text.substring(ExtNodeConstants.MSG_NODE_ATTRVALUE_TEXT_PREFIX.length()));
        } else if (text.startsWith(ExtNodeConstants.MSG_NODE_ATTRVALUE_HTML_PREFIX)) {
            node = ElementUtil
                    .parseAsSingle(text.substring(ExtNodeConstants.MSG_NODE_ATTRVALUE_HTML_PREFIX.length()));
        } else {
            node = ElementUtil.text(text);
        }
        msgElem.replaceWith(node);

        context.setData(TRACE_VAR_TEMPLATE_PATH, null);
    }
}

From source file:org.abondar.experimental.eventsearch.EventFinder.java

public void getCategorizedEvents(String type) {
    try {/*  w  ww  .ja  v  a  2 s  .  c om*/

        doc = Jsoup.connect("https://afisha.yandex.ru/msk/events/?category=" + type + "&limit=1000").get();

        Elements els = doc.select("a[href]");

        for (Element e : els) {

            for (Attribute attr : e.attributes().asList()) {

                if (attr.getValue().contains("clck.yandex.ru")) {

                    if (attr.getValue().charAt(97) != '/') {
                        getEvent(attr.getValue().substring(90, 96), type);

                    } else {
                        getEvent(attr.getValue().substring(90, 97), type);

                    }
                }
            }

        }
    } catch (IOException ex) {
        Logger.getLogger(EventFinder.class.getName()).log(Level.SEVERE, null, ex);
    }
}

From source file:org.abondar.experimental.eventsearch.EventFinder.java

public void getEvent(String eventId, String evType) {
    try {//from www. j a v  a2 s. c o m

        Document dc = Jsoup.connect("https://afisha.yandex.ru/msk/events/" + eventId + "/").get();

        Event eb = new Event();
        eb.setEventID(eventId);
        eb.setCategory(eventTypes.get(evType));
        Elements elems = dc.select("meta");

        for (Element e : elems) {
            if (e.attributes().get("property").contains("og:description")) {
                eb.setDescription(e.attributes().get("content"));

            }

        }

        elems = dc.select("title");

        for (Element e : elems) {

            eb.setName(e.html().substring(0, e.html().indexOf("")));
        }

        elems = dc.select("a[href]");

        for (Element e : elems) {

            for (Attribute attr : e.attributes().asList()) {

                if (attr.getValue().contains("/msk/places/")) {

                    eb.setPlace(getEventPlaces(attr.getValue()));

                }
            }

        }

        elems = dc.select("tr[id]");

        for (Element e : elems) {
            for (Attribute attr : e.attributes().asList()) {

                if (attr.getValue().contains("f")) {

                    eb.setDate(e.children().first().html());

                    try {
                        Element e1 = e.child(1).children().first();
                        Element e2 = e1.children().first();
                        Element e3 = e2.children().first();
                        Element e4 = e3.children().first();

                        eb.setTime(e4.html());

                    } catch (NullPointerException ex) {

                        Element e1 = e.child(2).children().first();
                        Element e2 = e1.children().first();
                        Element e3 = e2.children().first();
                        Element e4 = e3.children().first();
                        eb.setTime(e4.html());
                    }
                }
            }

        }

        geoCode(eb);
        formJson(eb);

    } catch (IOException ex) {
        Logger.getLogger(EventFinder.class.getName()).log(Level.SEVERE, null, ex);
    }

}

From source file:com.romeikat.datamessie.core.base.service.download.ContentDownloader.java

public DownloadResult downloadContent(String url) {
    LOG.debug("Downloading content from {}", url);
    // In case of a new redirection for that source, use redirected URL
    URLConnection urlConnection = null;
    String originalUrl = null;/* w  w w  .j  a  va2 s.  c o  m*/
    org.jsoup.nodes.Document jsoupDocument = null;
    Integer statusCode = null;
    final LocalDateTime downloaded = LocalDateTime.now();
    try {
        urlConnection = getConnection(url);
        // Server-side redirection
        final String responseUrl = getResponseUrl(urlConnection);
        if (responseUrl != null) {
            final String redirectedUrl = getRedirectedUrl(url, responseUrl);
            if (isValidRedirection(url, redirectedUrl)) {
                originalUrl = url;
                url = redirectedUrl;
                closeUrlConnection(urlConnection);
                urlConnection = getConnection(url);
                LOG.debug("Redirection (server): {} -> {}", originalUrl, url);
            }
        }
        // Download content for further redirects
        final InputStream urlInputStream = asInputStream(urlConnection, true, false);
        final Charset charset = getCharset(urlConnection);
        jsoupDocument = Jsoup.parse(urlInputStream, charset.name(), url);
        final Elements metaTagsHtmlHeadLink;
        Elements metaTagsHtmlHeadMeta = null;
        // Meta redirection (<link rel="canonical" .../>)
        if (originalUrl == null) {
            metaTagsHtmlHeadLink = jsoupDocument.select("html head link");
            for (final Element metaTag : metaTagsHtmlHeadLink) {
                final Attributes metaTagAttributes = metaTag.attributes();
                if (metaTagAttributes.hasKey("rel")
                        && metaTagAttributes.get("rel").equalsIgnoreCase("canonical")
                        && metaTagAttributes.hasKey("href")) {
                    final String redirectedUrl = metaTagAttributes.get("href").trim();
                    if (isValidRedirection(url, redirectedUrl)) {
                        originalUrl = url;
                        url = redirectedUrl;
                        jsoupDocument = null;
                        LOG.debug("Redirection (<link rel=\"canonical\" .../>): {} -> {}", originalUrl, url);
                        break;
                    }
                }
            }
        }
        // Meta redirection (<meta http-equiv="refresh" .../>)
        if (originalUrl == null) {
            metaTagsHtmlHeadMeta = jsoupDocument.select("html head meta");
            for (final Element metaTag : metaTagsHtmlHeadMeta) {
                final Attributes metaTagAttributes = metaTag.attributes();
                if (metaTagAttributes.hasKey("http-equiv")
                        && metaTagAttributes.get("http-equiv").equalsIgnoreCase("refresh")
                        && metaTagAttributes.hasKey("content")) {
                    final String[] parts = metaTagAttributes.get("content").replace(" ", "").split("=", 2);
                    if (parts.length > 1) {
                        final String redirectedUrl = parts[1];
                        if (isValidRedirection(url, redirectedUrl)) {
                            originalUrl = url;
                            url = redirectedUrl;
                            jsoupDocument = null;
                            LOG.debug("Redirection (<meta http-equiv=\"refresh\" .../>): {} -> {}", originalUrl,
                                    url);
                            break;
                        }
                    }
                }
            }
        }
        // Meta redirection (<meta property="og:url" .../>)
        if (originalUrl == null) {
            for (final Element metaTag : metaTagsHtmlHeadMeta) {
                final Attributes metaTagAttributes = metaTag.attributes();
                if (metaTagAttributes.hasKey("property")
                        && metaTagAttributes.get("property").equalsIgnoreCase("og:url")
                        && metaTagAttributes.hasKey("content")) {
                    final String redirectedUrl = metaTagAttributes.get("content").trim();
                    if (isValidRedirection(url, redirectedUrl)) {
                        originalUrl = url;
                        url = redirectedUrl;
                        jsoupDocument = null;
                        LOG.debug("Redirection (<meta property=\"og:url\" .../>): {} -> {}", originalUrl, url);
                        break;
                    }
                }
            }
        }
    } catch (final Exception e) {
        if (e instanceof HttpStatusException) {
            statusCode = ((HttpStatusException) e).getStatusCode();
        }
        LOG.warn("Could not determine redirected URL for " + url, e);
    } finally {
        closeUrlConnection(urlConnection);
    }
    // Download content (if not yet done)
    String content = null;
    try {
        if (jsoupDocument == null) {
            LOG.debug("Downloading content from {}", url);
            urlConnection = getConnection(url);
            final InputStream urlInputStream = asInputStream(urlConnection, true, false);
            final Charset charset = getCharset(urlConnection);
            jsoupDocument = Jsoup.parse(urlInputStream, charset.name(), url);
        }
    } catch (final Exception e) {
        if (e instanceof HttpStatusException) {
            statusCode = ((HttpStatusException) e).getStatusCode();
        }
        // If the redirected URL does not exist, use the original URL instead
        if (originalUrl == null) {
            LOG.warn("Could not download content from " + url, e);
        }
        // If the redirected URL does not exist and a original URL is available, use the
        // original URL instead
        else {
            try {
                LOG.debug(
                        "Could not download content from redirected URL {}, downloading content from original URL {} instead",
                        url, originalUrl);
                urlConnection = getConnection(originalUrl);
                final InputStream urlInputStream = asInputStream(urlConnection, true, false);
                final Charset charset = getCharset(urlConnection);
                jsoupDocument = Jsoup.parse(urlInputStream, charset.name(), url);
                url = originalUrl;
                originalUrl = null;
                statusCode = null;
            } catch (final Exception e2) {
                LOG.warn("Could not download content from original URL " + url, e);
            }
        }
    } finally {
        closeUrlConnection(urlConnection);
    }
    if (jsoupDocument != null) {
        content = jsoupDocument.html();
    }
    // Strip non-valid characters as specified by the XML 1.0 standard
    final String validContent = xmlUtil.stripNonValidXMLCharacters(content);
    // Unescape HTML characters
    final String unescapedContent = StringEscapeUtils.unescapeHtml4(validContent);
    // Done
    final DownloadResult downloadResult = new DownloadResult(originalUrl, url, unescapedContent, downloaded,
            statusCode);
    return downloadResult;
}

From source file:cn.wanghaomiao.xpath.core.XpathEvaluator.java

/**
 * ?xpath/*from w ww  .j a  v  a2  s  .  c o  m*/
 *
 * @param xpath
 * @param root
 * @return
 */
public List<JXNode> evaluate(String xpath, Elements root) throws NoSuchAxisException, NoSuchFunctionException {
    List<JXNode> res = new LinkedList<JXNode>();
    Elements context = root;
    List<Node> xpathNodes = getXpathNodeTree(xpath);
    for (int i = 0; i < xpathNodes.size(); i++) {
        Node n = xpathNodes.get(i);
        LinkedList<Element> contextTmp = new LinkedList<Element>();
        if (n.getScopeEm() == ScopeEm.RECURSIVE || n.getScopeEm() == ScopeEm.CURREC) {
            if (n.getTagName().startsWith("@")) {
                for (Element e : context) {
                    //?
                    String key = n.getTagName().substring(1);
                    if (key.equals("*")) {
                        res.add(JXNode.t(e.attributes().toString()));
                    } else {
                        String value = e.attr(key);
                        if (StringUtils.isNotBlank(value)) {
                            res.add(JXNode.t(value));
                        }
                    }
                    //??
                    for (Element dep : e.getAllElements()) {
                        if (key.equals("*")) {
                            res.add(JXNode.t(dep.attributes().toString()));
                        } else {
                            String value = dep.attr(key);
                            if (StringUtils.isNotBlank(value)) {
                                res.add(JXNode.t(value));
                            }
                        }
                    }
                }
            } else if (n.getTagName().endsWith("()")) {
                //??text()
                res.add(JXNode.t(context.text()));
            } else {
                Elements searchRes = context.select(n.getTagName());
                for (Element e : searchRes) {
                    Element filterR = filter(e, n);
                    if (filterR != null) {
                        contextTmp.add(filterR);
                    }
                }
                context = new Elements(contextTmp);
                if (i == xpathNodes.size() - 1) {
                    for (Element e : contextTmp) {
                        res.add(JXNode.e(e));
                    }
                }
            }

        } else {
            if (n.getTagName().startsWith("@")) {
                for (Element e : context) {
                    String key = n.getTagName().substring(1);
                    if (key.equals("*")) {
                        res.add(JXNode.t(e.attributes().toString()));
                    } else {
                        String value = e.attr(key);
                        if (StringUtils.isNotBlank(value)) {
                            res.add(JXNode.t(value));
                        }
                    }
                }
            } else if (n.getTagName().endsWith("()")) {
                res = (List<JXNode>) callFunc(n.getTagName().substring(0, n.getTagName().length() - 2),
                        context);
            } else {
                for (Element e : context) {
                    Elements filterScope = e.children();
                    if (StringUtils.isNotBlank(n.getAxis())) {
                        filterScope = getAxisScopeEls(n.getAxis(), e);
                    }
                    for (Element chi : filterScope) {
                        Element fchi = filter(chi, n);
                        if (fchi != null) {
                            contextTmp.add(fchi);
                        }
                    }
                }
                context = new Elements(contextTmp);
                if (i == xpathNodes.size() - 1) {
                    for (Element e : contextTmp) {
                        res.add(JXNode.e(e));
                    }
                }
            }
        }
    }
    return res;
}

From source file:net.vexelon.mobileops.GLBClient.java

private byte[] findInvoiceExportParams() throws HttpClientException, InvoiceException {

    BufferedReader reader = null;
    long bytesCount = 0;
    StringBuilder xmlUrl = new StringBuilder(100);
    String invoiceDate = Long.toString(new Date().getTime()); // today

    byte[] resultData = null;
    try {//from  w  ww  .  j  a v  a2 s . c om
        // Get invoice check page
        StringBuilder fullUrl = new StringBuilder(100);
        fullUrl.append(HTTP_MYTELENOR).append(GLBRequestType.PAGE_INVOICE.getPath());

        HttpGet httpGet = new HttpGet(fullUrl.toString());
        HttpResponse resp = httpClient.execute(httpGet, httpContext);
        StatusLine status = resp.getStatusLine();

        // Construct invoice id url
        fullUrl.setLength(0);
        fullUrl.append(HTTP_MYTELENOR);

        if (status.getStatusCode() == HttpStatus.SC_OK) {
            // bytes downloaded
            bytesCount += resp.getEntity().getContentLength() > 0 ? resp.getEntity().getContentLength() : 0;
            // Find invoice id
            Document doc = Jsoup.parse(resp.getEntity().getContent(), RESPONSE_ENCODING, "");
            Elements links = doc.select("a");
            for (Element el : links) {
                String href = el.attributes().get("href");
                if (href != null && href.contains("invId")) {
                    fullUrl.append("/").append(href);
                    break;
                }
            }
        } else {
            throw new HttpClientException(status.getReasonPhrase(), status.getStatusCode());
        }

        // Fetch invoice download parameters
        httpGet = new HttpGet(fullUrl.toString());
        resp = httpClient.execute(httpGet, httpContext);
        status = resp.getStatusLine();
        if (status.getStatusCode() == HttpStatus.SC_OK) {
            String line = null;
            reader = new BufferedReader(new InputStreamReader(resp.getEntity().getContent()));
            while ((line = reader.readLine()) != null) {
                // bytes downloaded
                bytesCount += line.length();

                if (line.contains("ei2_open_file") && line.contains("xml") && line.contains("summary")) {
                    if (Defs.LOG_ENABLED)
                        Log.d(Defs.LOG_TAG, line);

                    /*
                     * This is a g'damn hack. We don't need fancy stuff ;)
                     */
                    xmlUrl.append(HTTP_MYTELENOR).append(GLBRequestType.GET_INVOICE.getPath())
                            .append("?file_name=summary").append("&file_type=xml").append("&lower_bound=0")
                            .append("&upper_bound=0");

                    //                  ?file_name=summary
                    //                  &file_type=xml
                    //                  &lower_bound=0
                    //                  &upper_bound=0
                    //                  &invoiceNumber=1234567890
                    //                  &bill_acc_id=1231231
                    //                  &custnum=001234567
                    //                  &servicetype=1
                    //                  &period=1414792800000
                    //                  &cust_acc_id=1011111
                    //                  &custCode10=1.111111
                    //                  &prgCode=1

                    // extract keyword
                    String keys[] = { "file_type", "file_name", "lower_bound", "upper_bound", "invoiceNumber",
                            "bill_acc_id", "custnum", "servicetype", "period", "cust_acc_id", "custCode10",
                            "prgCode" };
                    String parts[] = line.split(",");
                    if (parts.length > 5) {
                        for (int i = 5; i < parts.length - 1; i++) {
                            String value = parts[i].replace("'", "").trim();

                            xmlUrl.append("&").append(keys[i]).append("=").append(value); // strip single quotes

                            // we need the invoice date
                            if (keys[i].equals("period")) {
                                invoiceDate = value;
                            }
                        }
                        // the last param is a bit tricky, because we need to remove the <a> data
                        int lastidx = parts.length - 1;
                        parts = parts[lastidx].split("\\)");
                        xmlUrl.append("&").append(keys[lastidx]).append("=").append(parts[0].replace("'", ""));
                    } else {
                        Log.e(Defs.LOG_TAG, "Got line: " + line);
                        throw new IOException("Invalid invoice fingerprint!");
                    }
                    break;
                }
            }
        } else {
            throw new HttpClientException(status.getReasonPhrase(), status.getStatusCode());
        }

        // close current stream reader
        if (reader != null)
            try {
                reader.close();
            } catch (IOException e) {
            }
        ;

        // Fetch Invoice XML
        if (Defs.LOG_ENABLED)
            Log.v(Defs.LOG_TAG, "Fetching invoice XML from: " + xmlUrl.toString());

        if (xmlUrl.length() == 0) {
            throw new InvoiceException("Invoice HTTP url not available!");
        }

        httpGet = new HttpGet(xmlUrl.toString());

        resp = httpClient.execute(httpGet, httpContext);
        status = resp.getStatusLine();
        if (status.getStatusCode() == HttpStatus.SC_OK) {
            resultData = Utils.read(resp.getEntity().getContent());
            // add loaded bytes
            bytesCount += resultData.length;
        } else {
            throw new HttpClientException(status.getReasonPhrase(), status.getStatusCode());
        }

    } catch (ClientProtocolException e) {
        throw new HttpClientException("Client protocol error!" + e.getMessage(), e);
    } catch (IOException e) {
        throw new HttpClientException("Client error!" + e.getMessage(), e);
    } finally {
        if (reader != null)
            try {
                reader.close();
            } catch (IOException e) {
            }
        ;

        addDownloadedBytesCount(bytesCount);
    }
    // parse invoice datetime
    try {
        invoiceDateTime = Long.parseLong(invoiceDate); //invoiceDate.substring(0, invoiceDate.length() - 3));
    } catch (NumberFormatException e) {
        // default 
        invoiceDateTime = new Date().getTime();
    }

    return resultData;
}

From source file:net.vexelon.mobileops.GLBClient.java

private List<NameValuePair> findLoginParams() throws HttpClientException {

    List<NameValuePair> result = new ArrayList<NameValuePair>();
    BufferedReader reader = null;
    long bytesCount = 0;

    try {/*from w  w w .ja va  2  s . c o  m*/
        // Get invoice check page
        StringBuilder fullUrl = new StringBuilder(100);
        fullUrl.append(GLBRequestType.LOGIN.getPath()).append("?").append(GLBRequestType.LOGIN.getParams());

        HttpGet httpGet = new HttpGet(fullUrl.toString());
        HttpResponse resp = httpClient.execute(httpGet, httpContext);
        StatusLine status = resp.getStatusLine();
        if (status.getStatusCode() == HttpStatus.SC_OK) {

            Document doc = Jsoup.parse(resp.getEntity().getContent(), RESPONSE_ENCODING, "");
            Elements inputs = doc.select("input");
            for (Element el : inputs) {
                //               if (Defs.LOG_ENABLED) {
                //                  Log.v(Defs.LOG_TAG, "ELEMENT: " + el.tagName());
                //               }
                Attributes attrs = el.attributes();
                //               for (Attribute attr : attrs) {
                //                  if (Defs.LOG_ENABLED) {
                //                     Log.v(Defs.LOG_TAG, " " + attr.getKey() + "=" + attr.getValue());
                //                  }
                //               }

                String elName = attrs.get("name");
                if (elName.equalsIgnoreCase("lt") || elName.equalsIgnoreCase("execution")) {
                    result.add(new BasicNameValuePair(elName, attrs.get("value")));
                }
            }
        } else {
            throw new HttpClientException(status.getReasonPhrase(), status.getStatusCode());
        }

        // close current stream reader
        //         if (reader != null) try { reader.close(); } catch (IOException e) {};

    } catch (ClientProtocolException e) {
        throw new HttpClientException("Client protocol error!" + e.getMessage(), e);
    } catch (IOException e) {
        throw new HttpClientException("Client error!" + e.getMessage(), e);
    } finally {
        if (reader != null)
            try {
                reader.close();
            } catch (IOException e) {
            }
        ;

        addDownloadedBytesCount(bytesCount);
    }

    return result;
}