List of usage examples for org.jsoup.nodes Element attr
public String attr(String attributeKey)
From source file:com.shareplaylearn.OauthPasswordFlow.java
public static LoginInfo googleLogin(String username, String password, String clientId, String callbackUri) throws URISyntaxException, IOException, AuthorizationException, UnauthorizedException { CloseableHttpClient httpClient = HttpClients.custom().build(); String oAuthQuery = "client_id=" + clientId + "&"; oAuthQuery += "response_type=code&"; oAuthQuery += "scope=openid email&"; oAuthQuery += "redirect_uri=" + callbackUri; URI oAuthUrl = new URI("https", null, "accounts.google.com", 443, "/o/oauth2/auth", oAuthQuery, null); Connection oauthGetCoonnection = Jsoup.connect(oAuthUrl.toString()); Connection.Response oauthResponse = oauthGetCoonnection.method(Connection.Method.GET).execute(); if (oauthResponse.statusCode() != 200) { String errorMessage = "Error contacting Google's oauth endpoint: " + oauthResponse.statusCode() + " / " + oauthResponse.statusMessage(); if (oauthResponse.body() != null) { errorMessage += oauthResponse.body(); }//from w w w . ja v a 2 s . c o m throw new AuthorizationException(errorMessage); } Map<String, String> oauthCookies = oauthResponse.cookies(); Document oauthPage = oauthResponse.parse(); Element oauthForm = oauthPage.getElementById("gaia_loginform"); System.out.println(oauthForm.toString()); Connection oauthPostConnection = Jsoup.connect("https://accounts.google.com/ServiceLoginAuth"); HashMap<String, String> formParams = new HashMap<>(); for (Element child : oauthForm.children()) { System.out.println("Tag name: " + child.tagName()); System.out.println("attrs: " + Arrays.toString(child.attributes().asList().toArray())); if (child.tagName().equals("input") && child.hasAttr("name")) { String keyName = child.attr("name"); String keyValue = null; if (child.hasAttr("value")) { keyValue = child.attr("value"); } if (keyName != null && keyName.trim().length() != 0 && keyValue != null && keyValue.trim().length() != 0) { oauthPostConnection.data(keyName, keyValue); formParams.put(keyName, keyValue); } } } oauthPostConnection.cookies(oauthCookies); formParams.put("Email", username); formParams.put("Passwd-hidden", password); //oauthPostConnection.followRedirects(false); System.out.println("form post params were: "); for (Map.Entry<String, String> kvp : formParams.entrySet()) { //DO NOT let passwords end up in the logs ;) if (kvp.getKey().equals("Passwd")) { continue; } System.out.println(kvp.getKey() + "," + kvp.getValue()); } System.out.println("form cookies were: "); for (Map.Entry<String, String> cookie : oauthCookies.entrySet()) { System.out.println(cookie.getKey() + "," + cookie.getValue()); } //System.exit(0); Connection.Response postResponse = null; try { postResponse = oauthPostConnection.method(Connection.Method.POST).timeout(5000).execute(); } catch (Throwable t) { System.out.println("Failed to post login information to googles endpoint :/ " + t.getMessage()); System.out.println("This usually means the connection is bad, shareplaylearn.com is down, or " + " google is being a punk - login manually and check."); assertTrue(false); } if (postResponse.statusCode() != 200) { String errorMessage = "Failed to validate credentials: " + oauthResponse.statusCode() + " / " + oauthResponse.statusMessage(); if (oauthResponse.body() != null) { errorMessage += oauthResponse.body(); } throw new UnauthorizedException(errorMessage); } System.out.println("Response headers (after post to google form & following redirect):"); for (Map.Entry<String, String> header : postResponse.headers().entrySet()) { System.out.println(header.getKey() + "," + header.getValue()); } System.out.println("Final response url was: " + postResponse.url().toString()); String[] args = postResponse.url().toString().split("&"); LoginInfo loginInfo = new LoginInfo(); for (String arg : args) { if (arg.startsWith("access_token")) { loginInfo.accessToken = arg.split("=")[1].trim(); } else if (arg.startsWith("id_token")) { loginInfo.idToken = arg.split("=")[1].trim(); } else if (arg.startsWith("expires_in")) { loginInfo.expiry = arg.split("=")[1].trim(); } } //Google doesn't actually throw a 401 or anything - it just doesn't redirect //and sends you back to it's login page to try again. //So this is what happens with an invalid password. if (loginInfo.accessToken == null || loginInfo.idToken == null) { //Document oauthPostResponse = postResponse.parse(); //System.out.println("*** Oauth response from google *** "); //System.out.println(oauthPostResponse.toString()); throw new UnauthorizedException( "Error retrieving authorization: did you use the correct username/password?"); } String[] idTokenFields = loginInfo.idToken.split("\\."); if (idTokenFields.length < 3) { throw new AuthorizationException("Error parsing id token " + loginInfo.idToken + "\n" + "it only had " + idTokenFields.length + " field!"); } String jwtBody = new String(Base64.decodeBase64(idTokenFields[1]), StandardCharsets.UTF_8); loginInfo.idTokenBody = new Gson().fromJson(jwtBody, OauthJwt.class); loginInfo.id = loginInfo.idTokenBody.sub; return loginInfo; }
From source file:io.jari.geenstijl.API.API.java
/** * Get article and comments (note that getArticles doesn't get the comments) * * @param url The direct url to the geenstijl article * @return Artikel The fetched article/* w w w. ja va 2s .co m*/ * @throws IOException * @throws ParseException */ public static Artikel getArticle(String url, Context context) throws IOException, ParseException { ensureCookies(); domain = context.getSharedPreferences("geenstijl", 0).getString("gsdomain", "www.geenstijl.nl"); Artikel artikel; Log.i(TAG, "GETARTICLE STEP 1/2: Getting/parsing article page & images... " + url); Document document = Jsoup.connect(url).get(); Element artikel_el = document.select("#content>article").first(); artikel = parseArtikel(artikel_el, context); Log.i(TAG, "GETARTICLE STEP 2/2: Parsing comments..."); ArrayList<Comment> comments = new ArrayList<Comment>(); int i = 0; Elements comments_el = document.select("#comments article"); for (Element comment_el : comments_el) { i++; Comment comment = new Comment(); comment.id = Integer.parseInt(comment_el.attr("id").substring(1)); Element footer = comment_el.select("footer").first(); StringTokenizer footer_items = new StringTokenizer(footer.text(), "|"); comment.auteur = footer_items.nextToken().trim(); try { SimpleDateFormat simpleDateFormat = new SimpleDateFormat("dd-MM-yyHH:mm", Locale.US); comment.datum = simpleDateFormat .parse(footer_items.nextToken().trim() + footer_items.nextToken().trim()); } catch (ParseException parseEx) { //fuck gebruikers met pipe chars in hun naam, pech, gehad. continue; } comment.inhoud = comment_el.select("p").first().html(); Log.d(TAG + ".perf", "CommentParser: Parsed " + comment.id + ": " + i + "/" + comments_el.size()); comments.add(comment); } Comment[] comm = new Comment[comments.size()]; comments.toArray(comm); artikel.comments = comm; Log.i(TAG, "GETARTICLE: DONE"); return artikel; }
From source file:app.sunstreak.yourpisd.net.Parser.java
/** Parses average of each term from GradeSummary.aspx. * NOTICE: Does not work for second semester classes in which the second semester schedule * is different from the first semester schedule. * /*from w w w.j av a 2 s. c o m*/ * @param doc the Jsoup element of GradeSummary.aspx * @param classList classList as returned by Init.aspx * @throws org.json.JSONException * @return [ * [classId, avg0, avg1, ...], * [classId, avg0, avg1, ...], * ] */ public static int[][] gradeSummary(Element doc, JSONArray classList) { List<int[]> gradeSummary = new ArrayList<int[]>(); Element reportTable = doc.getElementsByClass("reportTable").get(0).getElementsByTag("tbody").get(0); Elements rows = reportTable.getElementsByTag("tr"); int rowIndex = 0; while (rowIndex < rows.size()) { int[] classAverages = new int[11]; Arrays.fill(classAverages, -3); Element row = rows.get(rowIndex); Elements columns = row.getElementsByTag("td"); classAverages[0] = getClassId(row); for (int col = 0; col < 10; col++) { Element column = columns.get(col); String text = column.text(); // -2 for disabled class if (column.attr("class").equals("disabledCell")) text = "-2"; classAverages[col + 1] = text.equals("") ? -1 : Integer.parseInt(text); } gradeSummary.add(classAverages); rowIndex++; } /* * [ * [classId, avg0, avg1, ...], * [classId, avg0, avg1, ...], * ] */ int[][] result = new int[gradeSummary.size()][]; for (int i = 0; i < result.length; i++) { result[i] = new int[gradeSummary.get(i).length]; for (int j = 0; j < result[i].length; j++) result[i][j] = gradeSummary.get(i)[j]; } return result; }
From source file:com.astamuse.asta4d.render.RenderUtil.java
private final static boolean isBlockedByParentSnippet(Document doc, Element elem) { boolean isBlocked; String blockingId = elem.attr(ExtNodeConstants.SNIPPET_NODE_ATTR_BLOCK); if (blockingId.isEmpty()) { // empty block id means there is no parent snippet that need to be // aware. if the original block is from a embed template, it means // that all of the parent snippets have been finished or this // element would not be imported now. isBlocked = false;// w w w .j a va 2 s.c o m } else { String parentSelector = SelectorUtil.attr(ExtNodeConstants.SNIPPET_NODE_TAG_SELECTOR, ExtNodeConstants.ATTR_SNIPPET_REF, blockingId); Elements parentSnippetSearch = elem.parents().select(parentSelector); if (parentSnippetSearch.isEmpty()) { isBlocked = false; } else { Element parentSnippet = parentSnippetSearch.first(); if (parentSnippet.attr(ExtNodeConstants.SNIPPET_NODE_ATTR_STATUS) .equals(ExtNodeConstants.SNIPPET_NODE_ATTR_STATUS_FINISHED)) { isBlocked = false; } else { isBlocked = true; } } } return isBlocked; }
From source file:io.jari.geenstijl.API.API.java
private static Artikel parseArtikel(Element artikel_el, Context context) throws ParseException { Artikel artikel = new Artikel(); //id//from w ww . j ava 2 s.co m artikel.id = Integer.parseInt(artikel_el.attr("id").substring(1)); //summary artikel.summary = artikel_el.select("a.more").first() != null; //titel artikel.titel = artikel_el.select("h1").text(); //plaatje if (PreferenceManager.getDefaultSharedPreferences(context).getBoolean("show_images", true)) { Element plaatje = artikel_el.select("img").first(); if (plaatje != null) { try { String url = plaatje.attr("src"); Log.d(TAG, "Downloading " + url); // artikel.plaatje = Drawable.createFromStream(((java.io.InputStream)new URL(plaatje.attr("src")).getContent()), null); artikel.plaatje = readBytes((InputStream) new URL(plaatje.attr("src")).getContent()); artikel.groot_plaatje = plaatje.hasClass("groot"); if (plaatje.hasAttr("width") && plaatje.hasAttr("height")) if (!plaatje.attr("width").equals("100") || !plaatje.attr("height").equals("100")) artikel.groot_plaatje = true; if (artikel.groot_plaatje) Log.i(TAG, " Done. Big image."); else Log.i(TAG, " Done."); } catch (Exception ex) { Log.w(TAG, "Unable to download image, Falling back... Reason: " + ex.getMessage()); artikel.plaatje = null; } } } //embed if (artikel_el.select("div.embed").first() != null) { //atm alleen support voor iframes Element frame = artikel_el.select("div.embed>iframe").first(); if (frame != null) artikel.embed = frame.attr("src"); } //embed (geenstijl.tv) if (!domain.equals("www.geenstijl.nl")) { //extract url from script Element scriptEl = artikel_el.select("script").first(); if (scriptEl != null) { String script = scriptEl.html(); Pattern pattern = Pattern.compile("'(.*)', fall"); Matcher matcher = pattern.matcher(script); if (matcher.find() && matcher.groupCount() == 1) { artikel.embed = matcher.group(1); } } } //footer shit Element footer = artikel_el.select("footer").first(); SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm", Locale.US); artikel.datum = simpleDateFormat.parse(footer.select("time").first().attr("datetime")); StringTokenizer footer_items = new StringTokenizer(footer.text(), "|"); artikel.auteur = footer_items.nextToken().trim(); artikel.reacties = Integer.parseInt(footer.select("a.comments").text().replace(" reacties", "")); artikel.link = footer.select("a").first().attr("href"); //clean up artikel_el.select("h1").remove(); artikel_el.select(".embed").remove(); artikel_el.select("img").remove(); artikel_el.select("footer").remove(); artikel_el.select("a.more").remove(); artikel_el.select("script").remove(); //inhoud artikel.inhoud = artikel_el.html(); return artikel; }
From source file:net.sf.texprinter.utils.StringUtils.java
/** * Escapes HTML entities and tags to a TeX format. This method tries to * replace HTML code by the TeX equivalent macros. * * @param text The input text./* w w w .ja v a 2 s . co m*/ * @return A new text formatted from HTML to TeX. */ public static String escapeHTMLtoTeX(String text) { // replace bold tags String newText = text.replaceAll("<b>", "\\\\textbf{"); newText = newText.replaceAll("</b>", "}"); // replace bold tags newText = newText.replaceAll("<strong>", "\\\\textbf{"); newText = newText.replaceAll("</strong>", "}"); // replace italic tags newText = newText.replaceAll("<i>", "\\\\textit{"); newText = newText.replaceAll("</i>", "}"); // replace emphasized tags newText = newText.replaceAll("<em>", "\\\\emph{"); newText = newText.replaceAll("</em>", "}"); // replace paragraphs tags newText = newText.replaceAll("<p>", ""); newText = newText.replaceAll("</p>", "\n\n"); // replace ordered lists tags newText = newText.replaceAll("<ol>", "\\\\begin{enumerate}\n"); newText = newText.replaceAll("</ol>", "\\\\end{enumerate}\n"); // replace unordered lists tags newText = newText.replaceAll("<ul>", "\\\\begin{itemize}\n"); newText = newText.replaceAll("</ul>", "\\\\end{itemize}\n"); // replace item tags newText = newText.replaceAll("<li>", "\\\\item "); newText = newText.replaceAll("</li>", "\n"); // replace blockquote tags newText = newText.replaceAll("<blockquote>", "\\\\begin{quotation}\n"); newText = newText.replaceAll("</blockquote>", "\\\\end{quotation}\n"); // replace code tags newText = newText.replaceAll("<pre><code>", "\\\\begin{TeXPrinterListing}\n"); newText = newText.replaceAll("<pre class=.*\"><code>", "\\\\begin{TeXPrinterListing}\n"); newText = newText.replaceAll("</code></pre>", "\\\\end{TeXPrinterListing}\n\n"); // replace inline code tags newText = newText.replaceAll("<code>", "\\\\lstinline|"); newText = newText.replaceAll("</code>", "|"); // replace links tags newText = newText.replaceAll("alt=\".*\" ", ""); // parse the text Document docLinks = Jsoup.parse(newText); // get all the links Elements links = docLinks.getElementsByTag("a"); // if there are links if (links.size() > 0) { // for every link for (Element link : links) { // get the outer HTML String temp = link.outerHtml(); // replace it newText = newText.replaceFirst(Pattern.quote(temp), "\\\\href{" + link.attr("href") + "}{" + link.text() + "}"); } } // create a list of images ArrayList<ImageGroup> images = new ArrayList<ImageGroup>(); // parse the current text Document doc = Jsoup.parse(text); // fetch all the media found Elements media = doc.select("[src]"); // for all media found for (Element m : media) { // if it's an image tag if (m.tagName().equals("img")) { // create a new image group with the image link ImageGroup image = new ImageGroup(m.attr("abs:src")); // add to the list of images images.add(image); // set the current image to null image = null; } } // create a new loop saver LoopSaver lps = null; // for every image in the list of images for (ImageGroup img : images) { // create a new object lps = new LoopSaver(); // while there are references for that image in the text while (newText.indexOf(img.getURL()) != -1) { // tick loop lps.tick(); // replace the occurrence of that image newText = newText.replaceFirst("<img src=\"" + img.getURL() + "\" />", "\\\\begin{figure}[h!]\n\\\\centering\n\\\\includegraphics[scale=0.5]{" + img.getName() + "}\n\\\\end{figure}"); } // lets try try { // finally, download the image to the current directory Downloader.download(img.getURL(), img.getName()); } catch (Exception exception) { // log message log.log(Level.WARNING, "An error occurred while getting the current image. Trying to set the replacement image instead. MESSAGE: {0}", StringUtils.printStackTrace(exception)); // image could not be downloaded for any reason try { // open a file stream FileOutputStream f = new FileOutputStream(img.getName()); // write a replacement image f.write(Base64.decode( "iVBORw0KGgoAAAANSUhEUgAAALAAAABKCAIAAACU3El2AAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAAcjSURBVHhe7VzrmeMgDExdKSj1pJptZovZi3lqhAQ4n7HhrPt1STAaRoOELa0ff/bPGCAMPIwNY4AyYIIwPQADJggThAnCNKAzYBHC1GERwjRgEcI00MmApYxOou4yzARxF093rtME0UnUXYaZIO7i6c51miA6ibrLMBPEXTzduc4vBPHzejwez/cvt+C+f7x+GpY7h/2+nz2zdS5z6LCFoLZ5qAli8wj/9xHCzIK4xDeXGG27No7YB68dIfx8JCJoguiHOG7kvsUfhOMSo/3Y98H7XhBvlyFIWAfDPjFIOWT/MLgC9IlW/E/xH0tqeWiR7tx1z7dfkE95dKo00TbF8/2TrMTsiIyLhsh8eJVHG00EJGAizUeScT+8KieSqL4VRGCOHggILySI/L5feNzYP0wThGBF2Q2frwPn0oBImuhgZ8T/ElzjZqK6JHPKhmhM/Xlt15e7wpkISIiFIBaF54AJ4LkP6vhmYPlWEFHT4JR0DPR4hINnWjSB3RxWFQRaaYdHIeGxpMjmyIqAS1EFwvk3jy5NaiYASSkaIpQULhR4sl+acvj7GyOItJuEnKGkDH57ohGupAwI9cKdDgnkhVQl97DDdNYvRHevRm1FwZAgUkSTU2uPILRrfVSob9QLBeFNs71HvgOfNYZpEQKP0dL2CSPoBGqEwD0nha12hBANCavTzuXdgmjCmytC5Exapb9zGGREksfFy0WqHc+eQymZ8S1MEzDdU/QMISdp2RALahsQxUSPILRr1QiBP7SCxKCUEU5HrbuMzmEkA4VTvveudHn6DjZRjrOvl7tXoI/V2jGdnNqer1e8l5EOoYqhjBTuWUhiIodK6fyLQmdpg8ILKxOimXqqA420BdGS1KDfBS8NstQ9rRbouydYYeC0gpiQ/QkhHS+xKQURA2yzLnI8H7UZTRDn8m3WZmBgyggxAzF3xWCCuKvnlXWbIEwQa9x2mp+uYaAeIbRndiOx7nsAsW/0cbilp+2HzX7Ior5EuIogeElH7kU4zCXNib6kuzmvGzCvIPrwHztKZqOsHfj65iHcfbGAmwqC0B3qNq1mHrKTc8GAbW94Vo8tQ6qLIXkRbzBBkOpG0fXHLJGqQ+oLVi5PgknXhIqGWJigdRahGk1KwNt07Ras2JgDvVUfSHWqOcJe0ddTBhdEKAtF3txyiaty/bFUEusbAEe6KYSWD7KIHkEoc4qooDzse7oqkDwQcg0tfArtSbwpKhBGCq6EOr9yuXwqfR/r/EINTEPYq4bPuJ2CaBfigu0MzW8DV110vEiRHhSB8qDzQSsb3YjNOUVUWPVksaZEIRQQs1tTrMjRK0+4/c9VWTecIdSmWny9pQUfl4uJCqnG/kyla60ikIMFgckh96yw/0EU5N24REEZuJx1YFvzc2euvQuoyp4u/XKPAp3B/c7yI673M7XPDLEVIowGb0PMis2IXAFlCAjs5ZgUkXx5yjlSEHSPZeQ0L0sdXn3hDFIGuYTYxM2Uxsio4s+ZNuVypkmBbmkTk95tL4XPF5up0Nsd0mNbEKy5Ja1FXpQWw/oo9qMOFwTJk879JEJSXJqD5bY7TKV0noKZ4k/HeIiOqIpdqkMqQ0R5hpCSaVj80+nBr+H5+ZAgdggCFIFJqOwBo0EBEO5QxJGCoGGYNCaxWIyHx9wzhE8Wcgj2i+mIEHlYmhT607eD65bI6eHDjcxVdg1qJDT9Do1b+GccoEh0S/gkd2+KKSPnqrAmgT3oAdMQdktieC1DCGOTtTl0c3WLgaMFgWf3VlS+BeVzL3K0IFK05/cSc9NyX3QnCOK+5K64chPEil4biNkEMZDcFac2QazotYGYTRADyV1x6l2CaD7dXZEBwwwMdD+pTM8B+TPEOQlltcs5Qc6IygQxo1cuxFQTRPHKppAyirdLffDTmqYUQ8jv8ck1LRxAETG/7ikUpppvf2J/CA4F1qIlQLLrC0/C+6M6lnah9waY3h8h6m+XgrceJbz08OFfskQfYpMiXXRlEA37qDY1lfNrKUOxGxs06i9ochf/55WY/YIoO3wY+SVt5WFU6iEoezz4G2g0Q8JhVxGEZld720ZzaQP26LVTHiEIVjRmJWWpM1ptBGIOkPxRvv1Jcr4sCNWuJojW0q513gjrhwmicvPB3RALXqwPMTUc5qgsCaI0JMyvtedLEaJ8oVgedb8b7cZzCCQEPpEPrao2eIycIcouo3qE6Ho1k59fe7ESXYLch4Zy1ZbWWvKIzXvKnK0HU+nAnk6CQpdw5LBsf0pryAd/7EpkjUANQeiGKvOzkAK3IM3mJc3ibQVxiirNyDwMtCLEPEgNySkMmCBOoXkdIyaIdXx1ClITxCk0r2PEBLGOr05BaoI4heZ1jJgg1vHVKUhNEKfQvI4RE8Q6vjoFqQniFJrXMWKCWMdXpyA1QZxC8zpGTBDr+OoUpP8Arv92hCPEu+kAAAAASUVORK5CYII=")); // close the file f.close(); } catch (IOException ioexception) { // log message log.log(Level.SEVERE, "An IO exception occured while trying to create the image replacement. MESSAGE: {0}", StringUtils.printStackTrace(ioexception)); } catch (Exception except) { // log message log.log(Level.SEVERE, "An error occured while trying to create the image replacement. MESSAGE: {0}", StringUtils.printStackTrace(except)); } } } // unescape all HTML entities newText = StringEscapeUtils.unescapeHtml(newText); // return new text return newText; }
From source file:com.entertailion.android.slideshow.utils.Utils.java
/** * Determine if there is a high resolution icon available for the web site. * /* w w w. j ava 2 s. co m*/ * @param context * @param url * @return */ public static final String getWebSiteIcon(Context context, String url) { String icon = null; if (url != null) { String data = Utils.getCachedData(context, url, true); if (data != null) { Document doc = Jsoup.parse(data); if (doc != null) { String href = null; Elements metas = doc.select("meta[itemprop=image]"); if (metas.size() > 0) { Element meta = metas.first(); href = meta.attr("abs:content"); // weird jsoup bug: abs doesn't always work if (href == null || href.trim().length() == 0) { href = url + meta.attr("content"); } } if (href == null || href.trim().length() == 0) { // Find the Microsoft tile icon metas = doc.select("meta[name=msapplication-TileImage]"); if (metas.size() > 0) { Element meta = metas.first(); href = meta.attr("abs:content"); // weird jsoup bug: abs doesn't always work if (href == null || href.trim().length() == 0) { href = url + meta.attr("content"); } } } if (href == null || href.trim().length() == 0) { // Find the Apple touch icon Elements links = doc.select("link[rel=apple-touch-icon]"); if (links.size() > 0) { Element link = links.first(); href = link.attr("abs:href"); // weird jsoup bug: abs doesn't always work if (href == null || href.trim().length() == 0) { href = url + link.attr("href"); } } } if (href == null || href.trim().length() == 0) { // Find the Facebook open graph icon metas = doc.select("meta[property=og:image]"); if (metas.size() > 0) { Element link = metas.first(); href = link.attr("abs:content"); // weird jsoup bug: abs doesn't always work if (href == null || href.trim().length() == 0) { href = url + link.attr("content"); } } } if (href != null && href.trim().length() > 0) { try { Bitmap bitmap = Utils.getBitmapFromURL(href); if (bitmap != null) { icon = "web_site_icon_" + Utils.clean(href) + ".png"; Utils.saveToFile(context, bitmap, bitmap.getWidth(), bitmap.getHeight(), icon); bitmap.recycle(); } } catch (Exception e) { Log.d(LOG_TAG, "getWebSiteIcon", e); } } } } } return icon; }
From source file:com.entertailion.android.slideshow.utils.Utils.java
/** * Get the RSS feed for a web site.//from w ww . ja v a 2 s.c om * * @param url * @param context * @param refresh * @return */ public static String getRssFeed(String url, Context context, boolean refresh) { String rss = Utils.getCachedData(context, url, refresh); if (rss != null) { rss = rss.trim(); if (rss.startsWith(XML_PREFIX)) { return rss; } else { try { Document doc = Jsoup.parse(rss); Element link = doc.select("link[type=application/rss+xml]").first(); if (link != null && link.attr("rel").equalsIgnoreCase("alternate")) { String href = link.attr("href"); if (href != null) { rss = Utils.getCachedData(context, href, refresh); return rss; } } } catch (Exception e) { Log.e(LOG_TAG, "Jsoup exception", e); } } } return rss; }
From source file:com.nineash.hutsync.client.NetworkUtilities.java
/** * Connects to the SampleSync test server, authenticates the provided * username and password./*from w w w . ja v a2 s . c om*/ * * @param username The server account username * @param password The server account password * @return String The authentication token returned by the server (or null) */ public static String authenticate(String username, String password, Context context) { try { final HttpResponse resp; final HttpResponse init_resp; final ArrayList<NameValuePair> params = new ArrayList<NameValuePair>(); DefaultHttpClient hClient = getHttpClient(context); final HttpPost init_post = new HttpPost(BASE_URL); final int first_cookies; final ArrayList<SerializableCookie> saveCooks = new ArrayList<SerializableCookie>(); BasicCookieStore bcs = new BasicCookieStore(); params.add(new BasicNameValuePair(PARAM_USERNAME, username)); params.add(new BasicNameValuePair(PARAM_PASSWORD, password)); params.add(new BasicNameValuePair(PARAM_REMEMBER, "yes")); init_resp = hClient.execute(init_post); String respString = EntityUtils.toString(init_resp.getEntity()); List<Cookie> cookies = hClient.getCookieStore().getCookies(); if (cookies.isEmpty()) { Log.e(TAG, "No cookies gathered first time round"); } first_cookies = cookies.size(); if (first_cookies != 2) { Log.e(TAG, "Should be two cookie to start off with"); } Document doc = Jsoup.parse(respString); Elements hiddens = doc.select("div.homepage input[type=hidden]"); for (Element hidden : hiddens) { params.add(new BasicNameValuePair(hidden.attr("name"), hidden.attr("value"))); } final HttpEntity entity; try { entity = new UrlEncodedFormEntity(params); } catch (final UnsupportedEncodingException e) { // this should never happen. throw new IllegalStateException(e); } Log.i(TAG, "Authenticating to: " + AUTH_URI); final HttpPost post = new HttpPost(AUTH_URI); post.addHeader(entity.getContentType()); post.setEntity(entity); resp = hClient.execute(post); String authToken = null; if (resp.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { //set authtoken here cookies = hClient.getCookieStore().getCookies(); if (cookies.isEmpty()) { Log.e(TAG, "No cookies gathered"); } else { if (cookies.size() == first_cookies + 2) { //we get two new cookies when we log in for (int i = 0; i < cookies.size(); i++) { Cookie cur_cookie = cookies.get(i); if (cur_cookie.isPersistent()) { saveCooks.add(new SerializableCookie(cur_cookie)); } } authToken = toString(saveCooks); } } } if ((authToken != null) && (authToken.length() > 0)) { Log.v(TAG, "Successful authentication"); return authToken; } else { Log.e(TAG, "Error authenticating" + resp.getStatusLine()); return null; } } catch (final IOException e) { Log.e(TAG, "IOException when getting authtoken", e); return null; } finally { Log.v(TAG, "getAuthtoken completing"); } }
From source file:dsll.pinterest.crawler.Reduce.java
private static Text getPinContent(String url, DBCollection pinsCollection) throws JSONException { Document html = null;//from w ww . j a v a2 s .c om JSONObject pin = new JSONObject(); try { html = Jsoup.connect(url).get(); } catch (Exception e) { return new Text("HTTP connection failed..."); } // Gather major pins data Element doc = html.select("body").first(); // Pin ID String id = (url.split("pin/")[1].split("/")[0]); pin.append("ID", id); // Pin image String imageURL = ""; Element tmp = doc.select("div[class=pinImageSourceWrapper]").first(); try { tmp = tmp.select("div[class=imageContainer]").select("img").first(); imageURL = tmp.attr("src"); } catch (Exception e) { } // try{ // ByteArrayOutputStream pimg=new ByteArrayOutputStream(), cimg = new ByteArrayOutputStream(); // for(int i=0; i<3; i++){ // BufferedImage img=dummyImage; // try{ // img = ImageIO.read(new URL(imageURL)); // // }catch(Exception e){} // ImageIO.write(img, "jpg", cimg); // if(pimg.size()<cimg.size()){ // pimg = cimg; // } // } // // save to hdfs // Configuration conf = new Configuration(); // FileSystem fs = FileSystem.get(conf); // Path outFile = new Path("/home/hadoop/"+id+".png"); // FSDataOutputStream out = fs.create(outFile); // out.write(pimg.toByteArray()); // // }catch(Exception e){ // e.printStackTrace(); // } pin.append("image", imageURL); //Pin name tmp = doc.select("h2[itemprop=name]").first(); String name = ""; if (tmp != null) { name = tmp.text().trim(); } pin.append("name", name); // Pin source Element sourceCont = doc.select("div[class=sourceFlagWrapper]").first(); JSONObject source = new JSONObject(); if (sourceCont != null) { String title = sourceCont.text().trim(); String src = sourceCont.select("a").first().attr("href"); source.append("title", title); source.append("src", src); } pin.append("source", source); //pin credit JSONObject pinCredit = new JSONObject(); Element credit = doc.select("div[class=pinCredits]").first(); String creditName = "", creditTitle = "", creditSource = ""; try { creditName = credit.select("div[class=creditName]").text().trim(); } catch (Exception e) { } try { creditTitle = credit.select("div[class=creditTitle]").text().trim(); } catch (Exception e) { } try { creditSource = credit.select("a").attr("href"); } catch (Exception e) { } pinCredit.append("name", creditName); pinCredit.append("title", creditTitle); pinCredit.append("src", creditSource); pin.append("credit", pinCredit); //comments JSONArray comments = new JSONArray(); Elements commentsConts = doc.select("div[class=commenterNameCommentText]"); for (Element commentCont : commentsConts) { JSONObject comment = new JSONObject(); Element creatorEle = commentCont.select("div[class=commenterWrapper] a").first(); String creatorName = creatorEle.text().trim(); String creatorSrc = creatorEle.attr("href"); String content = "", raw = ""; Element commentContent = commentCont.select(".commentDescriptionContent").first(); try { content = commentContent.text().trim(); raw = commentContent.html(); comment.append("creator", creatorName); comment.append("creator_url", creatorSrc); comment.append("content", content); comment.append("content_raw", raw); comments.put(comment); } catch (Exception e) { } } pin.append("comments", comments); //pin board link and related pins Element bottomDoc = doc.select("div[class=Module CloseupSidebar]").first(); //pin board JSONArray board = new JSONArray(); if (bottomDoc != null) { Element boardEle = bottomDoc.select("div[class=boardHeader]").first(); JSONObject b = new JSONObject(); String boardName = ""; try { boardName = boardEle.select("h3[class=title]").text().trim(); } catch (Exception ee) { } String boardSrc = ""; try { boardSrc = "https://www.pinterest.com" + boardEle.select("a").attr("href").trim(); } catch (Exception ee) { } b.append("name", boardName); b.append("src", boardSrc); board.put(b); } pin.append("board", board); //CAUTION: what if a pin shows up in different boards? //related pins bottomDoc = doc .select("div[class=closeupBottom] div[class=Module CloseupBottom] div[class=relatedPinsWrapper]") .first(); JSONArray relatedPins = new JSONArray(); if (bottomDoc != null) { Elements relatedPinsConts = bottomDoc.select("div[class=pinWrapper]"); for (Element relatedPinsCont : relatedPinsConts) { JSONObject relatedPin = new JSONObject(); try { relatedPin.append("src", "https://www.pinterest.com" + relatedPinsCont.select("div[class=pinHolder] > a").attr("href")); } catch (Exception e) { } relatedPins.put(relatedPin); } } pin.append("related_pins", relatedPins); // Optional: push data to database BasicDBObject dbObject = (BasicDBObject) JSON.parse(pin.toString()); pinsCollection.insert(dbObject); return new Text(pin.toString()); }