List of usage examples for org.jsoup.nodes Element attr
public String attr(String attributeKey)
From source file:net.slkdev.swagger.confluence.service.impl.XHtmlToConfluenceServiceImpl.java
private void addLinksByType(final Map<String, ConfluenceLink> confluenceLinkMap, final Elements elements, final PageType pageType, final Integer numericPrefix) { final SwaggerConfluenceConfig swaggerConfluenceConfig = SWAGGER_CONFLUENCE_CONFIG.get(); int linkCount = 1; for (final Element element : elements) { final String confluenceLinkMarkup; final String originalTarget = element.attr("href"); final String text = element.text(); final String confluencePageTitle; if (pageType == INDIVIDUAL) { confluencePageTitle = buildConfluenceTitle(text, numericPrefix, linkCount); } else {//from w ww .j av a 2s .c om confluencePageTitle = buildConfluenceTitle(text, linkCount, null); } switch (swaggerConfluenceConfig.getPaginationMode()) { case SINGLE_PAGE: confluenceLinkMarkup = formatSinglePageLink(text); break; case CATEGORY_PAGES: confluenceLinkMarkup = formatCategoryPageLink(text, confluencePageTitle, pageType); break; case INDIVIDUAL_PAGES: confluenceLinkMarkup = formatIndividualPageLink(text, confluencePageTitle); break; default: throw new SwaggerConfluenceConfigurationException("Unhandled Pagination Mode!"); } final ConfluenceLink confluenceLink = ConfluenceLinkBuilder.aConfluenceLink().withPageType(pageType) .withOriginalHref(originalTarget).withText(text).withConfluenceLinkMarkup(confluenceLinkMarkup) .build(); LOG.debug("LINK MAP: {} -> {}", originalTarget, confluenceLinkMarkup); confluenceLinkMap.put(originalTarget, confluenceLink); linkCount++; } }
From source file:gov.medicaid.screening.dao.impl.BBHTLicenseDAOBean.java
/** * Performs a search for all possible results. * * @param criteria The search criteria.// w w w. jav a 2s . c om * @param byName flag indicating it is a name search * @return the search result for licenses * * @throws URISyntaxException if an error occurs while building the URL. * @throws ClientProtocolException if client does not support protocol used. * @throws IOException if an error occurs while parsing response. * @throws ParseException if an error occurs while parsing response. * @throws ServiceException for any other problems encountered */ private SearchResult<License> getAllResults(BBHTLicenseSearchCriteria criteria, boolean byName) throws URISyntaxException, ClientProtocolException, IOException, ParseException, ServiceException { DefaultHttpClient client = new DefaultHttpClient(getLaxSSLConnectionManager()); client.setRedirectStrategy(new LaxRedirectStrategy()); HttpGet getSearch = new HttpGet(new URIBuilder(getSearchURL()).build()); HttpResponse response = client.execute(getSearch); verifyAndAuditCall(getSearchURL(), response); Document page = Jsoup.parse(EntityUtils.toString(response.getEntity())); HttpPost search = new HttpPost(new URIBuilder(getSearchURL()).build()); List<License> allLicenses = new ArrayList<License>(); // switch to search by name screen if (byName) { HttpEntity entity = postForm(getSearchURL(), client, search, new String[][] { { "__EVENTTARGET", "_ctl7_rbtnSearch_1" }, { "__EVENTARGUMENT", "" }, { "_ctl7:ddlbLicenseType", "CD" }, { "_ctl7:rbtnSearch", "2" }, { "_ctl7:txtLicenseNumber", "" }, { "__VIEWSTATE", page.select("input[name=__VIEWSTATE]").first().val() } }, true); page = Jsoup.parse(EntityUtils.toString(entity)); entity = getResultPage(criteria, client, page, search, "_ctl7:cmdSearch", getSearchURL()); page = Jsoup.parse(EntityUtils.toString(entity)); // get the data grid entries if (page.select("table#_ctl7_grdSearchResults").size() < 1) { throw new ParsingException(ErrorCode.MITA50002.getDesc()); } Elements rows = page.select(GRID_ROW_SELECTOR); while (rows.size() > 0) { for (Element row : rows) { String url = row.select("a").first().attr("href"); String licenseNo = row.select("td:eq(5)").text(); HttpGet getDetail = new HttpGet(Util.replaceLastURLPart(getSearchURL(), url)); response = client.execute(getDetail); verifyAndAuditCall(getSearchURL(), response); Document licenseDetails = Jsoup.parse(EntityUtils.toString(response.getEntity())); allLicenses.add(parseLicense(licenseDetails, licenseNo)); } rows.clear(); // check for next page Element currentPage = page.select("#_ctl7_grdSearchResults tr.TablePager span").first(); if (getLog() != null) { getLog().log(Level.DEBUG, "Current page is: " + currentPage.text()); } Element pageLink = currentPage.nextElementSibling(); if (pageLink != null && pageLink.hasAttr("href")) { if (getLog() != null) { getLog().log(Level.DEBUG, "There are more results, getting the next page."); } String target = parseEventTarget(pageLink.attr("href")); entity = getResultPage(criteria, client, page, search, target, getSearchURL()); page = Jsoup.parse(EntityUtils.toString(entity)); rows = page.select(GRID_ROW_SELECTOR); } } } else { // search by license number (site supports only exact match) HttpEntity entity = postForm(getSearchURL(), client, search, new String[][] { { "__EVENTTARGET", "_ctl7:cmdSearch" }, { "__EVENTARGUMENT", "" }, { "_ctl7:ddlbLicenseType", Util.defaultString(criteria.getLicenseType().getName()) }, { "_ctl7:rbtnSearch", "1" }, { "_ctl7:txtLicenseNumber", Util.defaultString(criteria.getIdentifier()) }, { "__VIEWSTATE", page.select("input[name=__VIEWSTATE]").first().val() } }, true); page = Jsoup.parse(EntityUtils.toString(entity)); if (page.select("span#lblFormTitle").text().equals("License Details")) { String prefLicenseNo = criteria.getIdentifier(); allLicenses.add(parseLicense(page, prefLicenseNo)); } } SearchResult<License> searchResult = new SearchResult<License>(); searchResult.setItems(allLicenses); return searchResult; }
From source file:com.digitalpebble.stormcrawler.bolt.JSoupParserBolt.java
@Override public void execute(Tuple tuple) { byte[] content = tuple.getBinaryByField("content"); String url = tuple.getStringByField("url"); Metadata metadata = (Metadata) tuple.getValueByField("metadata"); LOG.info("Parsing : starting {}", url); // check that its content type is HTML // look at value found in HTTP headers boolean CT_OK = false; String mimeType = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE); if (detectMimeType) { mimeType = guessMimeType(url, mimeType, content); // store identified type in md metadata.setValue("parse.Content-Type", mimeType); }//from w ww.j a v a2 s.co m if (StringUtils.isNotBlank(mimeType)) { if (mimeType.toLowerCase().contains("html")) { CT_OK = true; } } // go ahead even if no mimetype is available else { CT_OK = true; } if (!CT_OK) { if (this.treat_non_html_as_error) { String errorMessage = "Exception content-type " + mimeType + " for " + url; RuntimeException e = new RuntimeException(errorMessage); handleException(url, e, metadata, tuple, "content-type checking", errorMessage); } else { LOG.info("Incorrect mimetype - passing on : {}", url); collector.emit(tuple, new Values(url, content, metadata, "")); collector.ack(tuple); } return; } long start = System.currentTimeMillis(); String charset = getContentCharset(content, metadata); // get the robots tags from the fetch metadata RobotsTags robotsTags = new RobotsTags(metadata); Map<String, List<String>> slinks; String text = ""; DocumentFragment fragment; try (ByteArrayInputStream bais = new ByteArrayInputStream(content)) { org.jsoup.nodes.Document jsoupDoc = Jsoup.parse(bais, charset, url); fragment = JSoupDOMBuilder.jsoup2HTML(jsoupDoc); // extracts the robots directives from the meta tags robotsTags.extractMetaTags(fragment); // store a normalised representation in metadata // so that the indexer is aware of it robotsTags.normaliseToMetadata(metadata); // do not extract the links if no follow has been set // and we are in strict mode if (robotsTags.isNoFollow() && robots_noFollow_strict) { slinks = new HashMap<>(0); } else { Elements links = jsoupDoc.select("a[href]"); slinks = new HashMap<>(links.size()); for (Element link : links) { // abs:href tells jsoup to return fully qualified domains // for // relative urls. // e.g.: /foo will resolve to http://shopstyle.com/foo String targetURL = link.attr("abs:href"); // nofollow boolean noFollow = "nofollow".equalsIgnoreCase(link.attr("rel")); // remove altogether if (noFollow && robots_noFollow_strict) { continue; } // link not specifically marked as no follow // but whole page is if (!noFollow && robotsTags.isNoFollow()) { noFollow = true; } String anchor = link.text(); if (StringUtils.isNotBlank(targetURL)) { // any existing anchors for the same target? List<String> anchors = slinks.get(targetURL); if (anchors == null) { anchors = new LinkedList<>(); slinks.put(targetURL, anchors); } // track the anchors only if no follow is false if (!noFollow && StringUtils.isNotBlank(anchor)) { anchors.add(anchor); } } } } Element body = jsoupDoc.body(); if (body != null) { text = body.text(); } } catch (Throwable e) { String errorMessage = "Exception while parsing " + url + ": " + e; handleException(url, e, metadata, tuple, "content parsing", errorMessage); return; } // store identified charset in md metadata.setValue("parse.Content-Encoding", charset); long duration = System.currentTimeMillis() - start; LOG.info("Parsed {} in {} msec", url, duration); List<Outlink> outlinks = toOutlinks(url, metadata, slinks); ParseResult parse = new ParseResult(); parse.setOutlinks(outlinks); // parse data of the parent URL ParseData parseData = parse.get(url); parseData.setMetadata(metadata); parseData.setText(text); parseData.setContent(content); // apply the parse filters if any try { parseFilters.filter(url, content, fragment, parse); } catch (RuntimeException e) { String errorMessage = "Exception while running parse filters on " + url + ": " + e; handleException(url, e, metadata, tuple, "content filtering", errorMessage); return; } if (emitOutlinks) { for (Outlink outlink : parse.getOutlinks()) { collector.emit(StatusStreamName, tuple, new Values(outlink.getTargetURL(), outlink.getMetadata(), Status.DISCOVERED)); } } // emit each document/subdocument in the ParseResult object // there should be at least one ParseData item for the "parent" URL for (Map.Entry<String, ParseData> doc : parse) { ParseData parseDoc = doc.getValue(); collector.emit(tuple, new Values(doc.getKey(), parseDoc.getContent(), parseDoc.getMetadata(), parseDoc.getText())); } collector.ack(tuple); eventCounter.scope("tuple_success").incr(); }
From source file:org.keycloak.testsuite.util.saml.ModifySamlResponseStepBuilder.java
private HttpUriRequest handlePostBinding(CloseableHttpResponse currentResponse) throws Exception { assertThat(currentResponse, statusCodeIsHC(Status.OK)); final String htmlBody = EntityUtils.toString(currentResponse.getEntity()); assertThat(htmlBody, Matchers.containsString("SAML")); org.jsoup.nodes.Document theResponsePage = Jsoup.parse(htmlBody); Elements samlResponses = theResponsePage.select("input[name=SAMLResponse]"); Elements samlRequests = theResponsePage.select("input[name=SAMLRequest]"); Elements forms = theResponsePage.select("form"); Elements relayStates = theResponsePage.select("input[name=RelayState]"); int size = samlResponses.size() + samlRequests.size(); assertThat("Checking uniqueness of SAMLResponse/SAMLRequest input field in the page", size, is(1)); assertThat("Checking uniqueness of forms in the page", forms, hasSize(1)); Element respElement = samlResponses.isEmpty() ? samlRequests.first() : samlResponses.first(); Element form = forms.first(); String base64EncodedSamlDoc = respElement.val(); InputStream decoded = PostBindingUtil.base64DecodeAsStream(base64EncodedSamlDoc); String samlDoc = IOUtils.toString(decoded, GeneralConstants.SAML_CHARSET); IOUtils.closeQuietly(decoded);/*from w w w . j av a 2s . c om*/ String transformed = getTransformer().transform(samlDoc); if (transformed == null) { return null; } final String attributeName = this.targetAttribute != null ? this.targetAttribute : respElement.attr("name"); List<NameValuePair> parameters = new LinkedList<>(); if (!relayStates.isEmpty()) { parameters.add(new BasicNameValuePair(GeneralConstants.RELAY_STATE, relayStates.first().val())); } URI locationUri = this.targetUri != null ? this.targetUri : URI.create(form.attr("action")); return createRequest(locationUri, attributeName, transformed, parameters); }
From source file:biz.shadowservices.DegreesToolbox.DataFetcher.java
public FetchResult updateData(Context context, boolean force) { //Open database DBOpenHelper dbhelper = new DBOpenHelper(context); SQLiteDatabase db = dbhelper.getWritableDatabase(); // check for internet connectivity try {/* w ww. j a va 2 s.co m*/ if (!isOnline(context)) { Log.d(TAG, "We do not seem to be online. Skipping Update."); return FetchResult.NOTONLINE; } } catch (Exception e) { exceptionReporter.reportException(Thread.currentThread(), e, "Exception during isOnline()"); } SharedPreferences sp = PreferenceManager.getDefaultSharedPreferences(context); if (!force) { try { if (sp.getBoolean("loginFailed", false) == true) { Log.d(TAG, "Previous login failed. Skipping Update."); DBLog.insertMessage(context, "i", TAG, "Previous login failed. Skipping Update."); return FetchResult.LOGINFAILED; } if (sp.getBoolean("autoupdates", true) == false) { Log.d(TAG, "Automatic updates not enabled. Skipping Update."); DBLog.insertMessage(context, "i", TAG, "Automatic updates not enabled. Skipping Update."); return FetchResult.NOTALLOWED; } if (!isBackgroundDataEnabled(context) && sp.getBoolean("obeyBackgroundData", true)) { Log.d(TAG, "Background data not enabled. Skipping Update."); DBLog.insertMessage(context, "i", TAG, "Background data not enabled. Skipping Update."); return FetchResult.NOTALLOWED; } if (!isAutoSyncEnabled() && sp.getBoolean("obeyAutoSync", true) && sp.getBoolean("obeyBackgroundData", true)) { Log.d(TAG, "Auto sync not enabled. Skipping Update."); DBLog.insertMessage(context, "i", TAG, "Auto sync not enabled. Skipping Update."); return FetchResult.NOTALLOWED; } if (isWifi(context) && !sp.getBoolean("wifiUpdates", true)) { Log.d(TAG, "On wifi, and wifi auto updates not allowed. Skipping Update"); DBLog.insertMessage(context, "i", TAG, "On wifi, and wifi auto updates not allowed. Skipping Update"); return FetchResult.NOTALLOWED; } else if (!isWifi(context)) { Log.d(TAG, "We are not on wifi."); if (!isRoaming(context) && !sp.getBoolean("2DData", true)) { Log.d(TAG, "Automatic updates on 2Degrees data not enabled. Skipping Update."); DBLog.insertMessage(context, "i", TAG, "Automatic updates on 2Degrees data not enabled. Skipping Update."); return FetchResult.NOTALLOWED; } else if (isRoaming(context) && !sp.getBoolean("roamingData", false)) { Log.d(TAG, "Automatic updates on roaming mobile data not enabled. Skipping Update."); DBLog.insertMessage(context, "i", TAG, "Automatic updates on roaming mobile data not enabled. Skipping Update."); return FetchResult.NOTALLOWED; } } } catch (Exception e) { exceptionReporter.reportException(Thread.currentThread(), e, "Exception while finding if to update."); } } else { Log.d(TAG, "Update Forced"); } try { String username = sp.getString("username", null); String password = sp.getString("password", null); if (username == null || password == null) { DBLog.insertMessage(context, "i", TAG, "Username or password not set."); return FetchResult.USERNAMEPASSWORDNOTSET; } // Find the URL of the page to send login data to. Log.d(TAG, "Finding Action. "); HttpGetter loginPageGet = new HttpGetter("https://secure.2degreesmobile.co.nz/web/ip/login"); String loginPageString = loginPageGet.execute(); if (loginPageString != null) { Document loginPage = Jsoup.parse(loginPageString, "https://secure.2degreesmobile.co.nz/web/ip/login"); Element loginForm = loginPage.getElementsByAttributeValue("name", "loginFrm").first(); String loginAction = loginForm.attr("action"); // Send login form List<NameValuePair> loginValues = new ArrayList<NameValuePair>(); loginValues.add(new BasicNameValuePair("externalURLRedirect", "")); loginValues.add(new BasicNameValuePair("hdnAction", "login_userlogin")); loginValues.add(new BasicNameValuePair("hdnAuthenticationType", "M")); loginValues.add(new BasicNameValuePair("hdnlocale", "")); loginValues.add(new BasicNameValuePair("userid", username)); loginValues.add(new BasicNameValuePair("password", password)); Log.d(TAG, "Sending Login "); HttpPoster sendLoginPoster = new HttpPoster(loginAction, loginValues); // Parse result String loginResponse = sendLoginPoster.execute(); Document loginResponseParsed = Jsoup.parse(loginResponse); // Determine if this is a pre-pay or post-paid account. boolean postPaid; if (loginResponseParsed .getElementById("p_CustomerPortalPostPaidHomePage_WAR_customerportalhomepage") == null) { Log.d(TAG, "Pre-pay account or no account."); postPaid = false; } else { Log.d(TAG, "Post-paid account."); postPaid = true; } String homepageUrl = "https://secure.2degreesmobile.co.nz/group/ip/home"; if (postPaid) { homepageUrl = "https://secure.2degreesmobile.co.nz/group/ip/postpaid"; } HttpGetter homepageGetter = new HttpGetter(homepageUrl); String homepageHTML = homepageGetter.execute(); Document homePage = Jsoup.parse(homepageHTML); Element accountSummary = homePage.getElementById("accountSummary"); if (accountSummary == null) { Log.d(TAG, "Login failed."); return FetchResult.LOGINFAILED; } db.delete("cache", "", null); /* This code fetched some extra details for postpaid users, but on reflection they aren't that useful. * Might reconsider this. * if (postPaid) { Element accountBalanceSummaryTable = accountSummary.getElementsByClass("tableBillSummary").first(); Elements rows = accountBalanceSummaryTable.getElementsByTag("tr"); int rowno = 0; for (Element row : rows) { if (rowno > 1) { break; } //Log.d(TAG, "Starting row"); //Log.d(TAG, row.html()); Double value; try { Element amount = row.getElementsByClass("tableBillamount").first(); String amountHTML = amount.html(); Log.d(TAG, amountHTML.substring(1)); value = Double.parseDouble(amountHTML.substring(1)); } catch (Exception e) { Log.d(TAG, "Failed to parse amount from row."); value = null; } String expiresDetails = ""; String expiresDate = null; String name = null; try { Element details = row.getElementsByClass("tableBilldetail").first(); name = details.ownText(); Element expires = details.getElementsByTag("em").first(); if (expires != null) { expiresDetails = expires.text(); } Log.d(TAG, expiresDetails); Pattern pattern; pattern = Pattern.compile("\\(payment is due (.*)\\)"); Matcher matcher = pattern.matcher(expiresDetails); if (matcher.find()) { /*Log.d(TAG, "matched expires"); Log.d(TAG, "group 0:" + matcher.group(0)); Log.d(TAG, "group 1:" + matcher.group(1)); Log.d(TAG, "group 2:" + matcher.group(2)); * String expiresDateString = matcher.group(1); Date expiresDateObj; if (expiresDateString != null) { if (expiresDateString.length() > 0) { try { expiresDateObj = DateFormatters.EXPIRESDATE.parse(expiresDateString); expiresDate = DateFormatters.ISO8601DATEONLYFORMAT.format(expiresDateObj); } catch (java.text.ParseException e) { Log.d(TAG, "Could not parse date: " + expiresDateString); } } } } } catch (Exception e) { Log.d(TAG, "Failed to parse details from row."); } String expirev = null; ContentValues values = new ContentValues(); values.put("name", name); values.put("value", value); values.put("units", "$NZ"); values.put("expires_value", expirev ); values.put("expires_date", expiresDate); db.insert("cache", "value", values ); rowno++; } } */ Element accountSummaryTable = accountSummary.getElementsByClass("tableAccountSummary").first(); Elements rows = accountSummaryTable.getElementsByTag("tr"); for (Element row : rows) { // We are now looking at each of the rows in the data table. //Log.d(TAG, "Starting row"); //Log.d(TAG, row.html()); Double value; String units; try { Element amount = row.getElementsByClass("tableBillamount").first(); String amountHTML = amount.html(); //Log.d(TAG, amountHTML); String[] amountParts = amountHTML.split(" ", 2); //Log.d(TAG, amountParts[0]); //Log.d(TAG, amountParts[1]); if (amountParts[0].contains("Included") || amountParts[0].equals("All You Need") || amountParts[0].equals("Unlimited Text*")) { value = Values.INCLUDED; } else { try { value = Double.parseDouble(amountParts[0]); } catch (NumberFormatException e) { exceptionReporter.reportException(Thread.currentThread(), e, "Decoding value."); value = 0.0; } } units = amountParts[1]; } catch (NullPointerException e) { //Log.d(TAG, "Failed to parse amount from row."); value = null; units = null; } Element details = row.getElementsByClass("tableBilldetail").first(); String name = details.getElementsByTag("strong").first().text(); Element expires = details.getElementsByTag("em").first(); String expiresDetails = ""; if (expires != null) { expiresDetails = expires.text(); } Log.d(TAG, expiresDetails); Pattern pattern; if (postPaid == false) { pattern = Pattern.compile("\\(([\\d\\.]*) ?\\w*? ?expiring on (.*)\\)"); } else { pattern = Pattern.compile("\\(([\\d\\.]*) ?\\w*? ?will expire on (.*)\\)"); } Matcher matcher = pattern.matcher(expiresDetails); Double expiresValue = null; String expiresDate = null; if (matcher.find()) { /*Log.d(TAG, "matched expires"); Log.d(TAG, "group 0:" + matcher.group(0)); Log.d(TAG, "group 1:" + matcher.group(1)); Log.d(TAG, "group 2:" + matcher.group(2)); */ try { expiresValue = Double.parseDouble(matcher.group(1)); } catch (NumberFormatException e) { expiresValue = null; } String expiresDateString = matcher.group(2); Date expiresDateObj; if (expiresDateString != null) { if (expiresDateString.length() > 0) { try { expiresDateObj = DateFormatters.EXPIRESDATE.parse(expiresDateString); expiresDate = DateFormatters.ISO8601DATEONLYFORMAT.format(expiresDateObj); } catch (java.text.ParseException e) { Log.d(TAG, "Could not parse date: " + expiresDateString); } } } } ContentValues values = new ContentValues(); values.put("name", name); values.put("value", value); values.put("units", units); values.put("expires_value", expiresValue); values.put("expires_date", expiresDate); db.insert("cache", "value", values); } if (postPaid == false) { Log.d(TAG, "Getting Value packs..."); // Find value packs HttpGetter valuePacksPageGet = new HttpGetter( "https://secure.2degreesmobile.co.nz/group/ip/prevaluepack"); String valuePacksPageString = valuePacksPageGet.execute(); //DBLog.insertMessage(context, "d", "", valuePacksPageString); if (valuePacksPageString != null) { Document valuePacksPage = Jsoup.parse(valuePacksPageString); Elements enabledPacks = valuePacksPage.getElementsByClass("yellow"); for (Element enabledPack : enabledPacks) { Element offerNameElemt = enabledPack .getElementsByAttributeValueStarting("name", "offername").first(); if (offerNameElemt != null) { String offerName = offerNameElemt.val(); DBLog.insertMessage(context, "d", "", "Got element: " + offerName); ValuePack[] packs = Values.valuePacks.get(offerName); if (packs == null) { DBLog.insertMessage(context, "d", "", "Offer name: " + offerName + " not matched."); } else { for (ValuePack pack : packs) { ContentValues values = new ContentValues(); values.put("plan_startamount", pack.value); values.put("plan_name", offerName); DBLog.insertMessage(context, "d", "", "Pack " + pack.type.id + " start value set to " + pack.value); db.update("cache", values, "name = '" + pack.type.id + "'", null); } } } } } } SharedPreferences.Editor prefedit = sp.edit(); Date now = new Date(); prefedit.putString("updateDate", DateFormatters.ISO8601FORMAT.format(now)); prefedit.putBoolean("loginFailed", false); prefedit.putBoolean("networkError", false); prefedit.commit(); DBLog.insertMessage(context, "i", TAG, "Update Successful"); return FetchResult.SUCCESS; } } catch (ClientProtocolException e) { DBLog.insertMessage(context, "w", TAG, "Network error: " + e.getMessage()); return FetchResult.NETWORKERROR; } catch (IOException e) { DBLog.insertMessage(context, "w", TAG, "Network error: " + e.getMessage()); return FetchResult.NETWORKERROR; } finally { db.close(); } return null; }
From source file:com.aquest.emailmarketing.web.controllers.BroadcastController.java
/** * Adds the tracking./*from w ww .j a va 2 s . c om*/ * * @param model the model * @param urls the urls * @param principal the principal * @param id the id * @param trackingFlg the tracking flg * @param openGAflg the open g aflg * @param openPixelFlg the open pixel flg * @param trackingType the tracking type * @return the string */ @RequestMapping(value = "/generateUrls", method = RequestMethod.POST) public String addTracking(Model model, Urls urls, Principal principal, @RequestParam(value = "id") int id, @RequestParam(value = "trackingFlg", required = false) boolean trackingFlg, @RequestParam(value = "openGAflg", required = false) boolean openGAflg, @RequestParam(value = "openPixelFlg", required = false) boolean openPixelFlg, @RequestParam(value = "trackingType", required = false) String trackingType) { TrackingConfig trackingConfig = new TrackingConfig(); Broadcast broadcast = broadcastService.getBroadcastById(id); String workingHtml = broadcast.getHtmlbody(); if (trackingFlg == true) { if (openGAflg == true) { workingHtml = emailTracking.addGaOpenEmailTracking(workingHtml, urls); System.out.println("GA Open: " + workingHtml); } if (openPixelFlg == true) { workingHtml = emailTracking.addPixelOpenEmailTracking(workingHtml); System.out.println("Pixel Open: " + workingHtml); } if (trackingType.equals("ga")) { workingHtml = emailTracking.addGaTrackingToUrl(workingHtml, urls); System.out.println("GA Click added: " + workingHtml); } else if (trackingType.equals("intTrack")) { workingHtml = emailTracking.addIntTrackingToUrl(workingHtml, urls); System.out.println("Internal Tracking: " + workingHtml); } else { workingHtml = emailTracking.addBothTrackingToUrl(workingHtml, urls); } } broadcast.setHtmlbody_tracking(workingHtml); System.out.println(broadcast.getHtmlbody_tracking()); String confirm = broadcastService.SaveOrUpdate(broadcast); System.out.println(confirm); System.out.println(trackingFlg); System.out.println(openGAflg); System.out.println(openPixelFlg); System.out.println(trackingType); if (confirm == broadcast.getBroadcast_id()) { trackingConfig.setBroadcast_id(broadcast.getBroadcast_id()); // taking care of tracking flg int tracking_flg = 0; if (trackingFlg == true) { tracking_flg = 1; } trackingConfig.setTracking_flg(tracking_flg); // taking care of openGAflg int open_ga_flg = 0; if (openGAflg == true) { open_ga_flg = 1; } trackingConfig.setOpen_ga_flg(open_ga_flg); // taking care of openPixelFlg int open_pixel_flg = 0; if (openPixelFlg == true) { open_pixel_flg = 1; } trackingConfig.setOpen_pixel_flg(open_pixel_flg); // set tracking type trackingConfig.setTracking_type(trackingType); // seting utm's trackingConfig.setUtm_campaign(urls.getUtmCampaign()); trackingConfig.setUtm_content(urls.getUtmContent()); trackingConfig.setUtm_medium(urls.getUtmMedium()); trackingConfig.setUtm_source(urls.getUtmSource()); trackingConfigService.SaveOrUpdate(trackingConfig); } // find images in html to be able to embed images in email as in-line attachments EmbeddedImage embeddedImage = new EmbeddedImage(); //HashSet to avoid duplicates Set<String> imgList = new HashSet<String>(); String html = broadcast.getHtmlbody(); Document doc = Jsoup.parse(html); Elements media = doc.select("[src]"); for (Element src : media) { if (src.tagName().equals("img")) { imgList.add(src.attr("abs:src")); } } model.addAttribute("imgList", imgList); model.addAttribute("embeddedImage", embeddedImage); model.addAttribute("broadcast", broadcast); return "embeddedimage"; }
From source file:gov.medicaid.screening.dao.impl.NursingLicenseDAOBean.java
/** * Performs a search for all possible results. * * @param criteria The search criteria./* www . java2 s.co m*/ * @param byName flag indicating it is a name search * @return the search result for licenses * * @throws URISyntaxException if an error occurs while building the URL. * @throws ClientProtocolException if client does not support protocol used. * @throws IOException if an error occurs while parsing response. * @throws ParseException if an error occurs while parsing response. * @throws ServiceException for any other problems encountered */ private SearchResult<License> getAllResults(NursingLicenseSearchCriteria criteria, boolean byName) throws URISyntaxException, ClientProtocolException, IOException, ParseException, ServiceException { DefaultHttpClient client = new DefaultHttpClient(getLaxSSLConnectionManager()); client.setRedirectStrategy(new LaxRedirectStrategy()); client.setCookieStore(loginAsPublicUser()); HttpGet getSearch = new HttpGet(new URIBuilder(getSearchURL()).build()); HttpResponse response = client.execute(getSearch); verifyAndAuditCall(getSearchURL(), response); Document page = Jsoup.parse(EntityUtils.toString(response.getEntity())); HttpPost search = new HttpPost(new URIBuilder(getSearchURL()).build()); List<License> allLicenses = new ArrayList<License>(); // switch to search by name screen if (byName) { HttpEntity entity = postForm(getSearchURL(), client, search, new String[][] { { "__EVENTTARGET", "_ctl7_rbtnSearch_1" }, { "__EVENTARGUMENT", "" }, { "_ctl7:ddlbLicenseType", "R" }, { "_ctl7:rbtnSearch", "2" }, { "_ctl7:txtCheckDigit", "" }, { "_ctl7:txtLicenseNumber", "" }, { "__VIEWSTATE", page.select("input[name=__VIEWSTATE]").first().val() } }, true); page = Jsoup.parse(EntityUtils.toString(entity)); entity = getResultPage(criteria, client, page, search, "_ctl7:cmdSearch", getSearchURL()); page = Jsoup.parse(EntityUtils.toString(entity)); // get the data grid entries if (page.select("table#_ctl7_grdSearchResults").size() < 1) { throw new ParsingException(ErrorCode.MITA50002.getDesc()); } Elements rows = page.select(GRID_ROW_SELECTOR); while (rows.size() > 0) { for (Element row : rows) { String url = row.select("a").first().attr("href"); String licenseNo = row.select("td:eq(4)").text(); HttpGet getDetail = new HttpGet(Util.replaceLastURLPart(getSearchURL(), url)); response = client.execute(getDetail); verifyAndAuditCall(getSearchURL(), response); Document licenseDetails = Jsoup.parse(EntityUtils.toString(response.getEntity())); allLicenses.add(parseLicense(licenseDetails, licenseNo.substring(0, 1))); } rows.clear(); // check for next page Element currentPage = page.select("#_ctl7_grdSearchResults tr.TablePager span").first(); if (getLog() != null) { getLog().log(Level.DEBUG, "Current page is: " + currentPage.text()); } Element pageLink = currentPage.nextElementSibling(); if (pageLink != null && pageLink.hasAttr("href")) { if (getLog() != null) { getLog().log(Level.DEBUG, "There are more results, getting the next page."); } String target = parseEventTarget(pageLink.attr("href")); entity = getResultPage(criteria, client, page, search, target, getSearchURL()); page = Jsoup.parse(EntityUtils.toString(entity)); rows = page.select(GRID_ROW_SELECTOR); } } } else { // search by license number (site supports only exact match) HttpEntity entity = postForm(getSearchURL(), client, search, new String[][] { { "__EVENTTARGET", "_ctl7:cmdSearch" }, { "__EVENTARGUMENT", "" }, { "_ctl7:ddlbLicenseType", Util.defaultString(criteria.getLicenseType().getName()) }, { "_ctl7:rbtnSearch", "1" }, { "_ctl7:txtCheckDigit", Util.defaultString(criteria.getCheckDigit()) }, { "_ctl7:txtLicenseNumber", Util.defaultString(criteria.getIdentifier()) }, { "__VIEWSTATE", page.select("input[name=__VIEWSTATE]").first().val() } }, true); page = Jsoup.parse(EntityUtils.toString(entity)); if (page.select("span#lblFormTitle").text().equals("License Details")) { String prefLicenseType = criteria.getLicenseType().getName(); allLicenses.add(parseLicense(page, prefLicenseType)); } } SearchResult<License> searchResult = new SearchResult<License>(); searchResult.setItems(allLicenses); return searchResult; }
From source file:com.lumata.lib.lupa.extractor.internal.HtmlBiggestImageExtractor.java
@Override public Image extractBestImage(URL sourceUrl, Elements htmlSection, ImageExtractionRequirements requirements) { Map<String, Image> imagesToExplore = new HashMap<String, Image>(); Set<ImageDownloadTask> imagesToDownload = new HashSet<ImageDownloadTask>(); Iterator<org.jsoup.nodes.Element> it = htmlSection.iterator(); // collect valid images while (it.hasNext() && imagesToExplore.size() < requirements.getMaxImagesToExplore()) { Element imageElement = it.next(); String imageUrl = imageElement.absUrl("src"); // Do not process empty img tags, duplicated images or tracking // pixels and other assorted ads if (imageUrl == null || imagesToExplore.containsKey(imageUrl) || isTrackingPixelOrAd(imageUrl)) { continue; }/*from ww w . j a v a 2 s .c o m*/ // remember this image Image imageContent = new Image(imageUrl); if (imageElement.hasAttr(WIDTH_ATTRIBUTE)) { // TODO: We need to convert other picture size units supported by html (there must be a lib for this) imageContent.setWidth(Integer.parseInt(imageElement.attr(WIDTH_ATTRIBUTE).replace("px", ""))); } if (imageElement.hasAttr(HEIGHT_ATTRIBUTE)) { imageContent.setHeight(Integer.parseInt(imageElement.attr(HEIGHT_ATTRIBUTE).replace("px", ""))); } if (imageContent.getWidth() == null || imageContent.getHeight() == null) {// mark image to download imagesToDownload.add(new ImageDownloadTask(imageContent)); } imagesToExplore.put(imageUrl, imageContent); } // if dimensions are empty -> download image if (CollectionUtils.isNotEmpty(imagesToDownload)) { try { ExecutorService pool = Executors.newFixedThreadPool(imagesToDownload.size(), getThreadFactory(sourceUrl)); pool.invokeAll(imagesToDownload); pool.shutdown(); } catch (InterruptedException e) { LOG.error("InterruptedException while downloading images", e); } } // select biggest image Image biggestImage = null; try { biggestImage = Collections.max(imagesToExplore.values(), new Comparator<Image>() { @Override public int compare(Image o1, Image o2) { return getSquarePixels(o1) - getSquarePixels(o2); } }); } catch (NoSuchElementException e) { return null; } // if image is too small, discard return (biggestImage.getWidth() < requirements.getMinImageSize() || biggestImage.getHeight() < requirements.getMinImageSize()) ? null : biggestImage; }
From source file:de.geeksfactory.opacclient.apis.Zones22.java
private Document login(Account acc) throws IOException, OpacErrorException { String html = httpGet(/*from ww w.j a v a 2 s.c om*/ opac_url + "/APS_ZONES?fn=MyZone&Style=Portal3&SubStyle=&Lang=GER&ResponseEncoding=utf-8", getDefaultEncoding()); Document doc = Jsoup.parse(html); doc.setBaseUri(opac_url + "/APS_ZONES"); if (doc.select(".AccountSummaryCounterLink").size() > 0) { return doc; } if (doc.select("#LoginForm").size() == 0) { throw new NotReachableException(); } List<NameValuePair> params = new ArrayList<NameValuePair>(); for (Element input : doc.select("#LoginForm input")) { if (!input.attr("name").equals("BRWR") && !input.attr("name").equals("PIN")) params.add(new BasicNameValuePair(input.attr("name"), input.attr("value"))); } params.add(new BasicNameValuePair("BRWR", acc.getName())); params.add(new BasicNameValuePair("PIN", acc.getPassword())); String loginHtml; try { loginHtml = httpPost(doc.select("#LoginForm").get(0).absUrl("action"), new UrlEncodedFormEntity(params), getDefaultEncoding()); } catch (UnsupportedEncodingException e) { e.printStackTrace(); return null; } catch (IOException e) { e.printStackTrace(); return null; } if (!loginHtml.contains("Kontostand")) { throw new OpacErrorException(stringProvider.getString(StringProvider.LOGIN_FAILED)); } Document doc2 = Jsoup.parse(loginHtml); Pattern objid_pat = Pattern.compile("Obj_([0-9]+)\\?.*"); for (Element a : doc2.select("a")) { Matcher objid_matcher = objid_pat.matcher(a.attr("href")); if (objid_matcher.matches()) { accountobj = objid_matcher.group(1); } } return doc2; }
From source file:com.liato.bankdroid.banking.banks.AbsIkanoPartner.java
@Override protected LoginPackage preLogin() throws BankException, ClientProtocolException, IOException { urlopen = new Urllib(context, CertificateReader.getCertificates(context, R.raw.cert_ikanopartner)); response = urlopen.open("https://partner.ikanobank.se/web/engines/page.aspx?structid=" + structId); Document d = Jsoup.parse(response); Element viewstate = d.getElementById("__VIEWSTATE"); if (viewstate == null || TextUtils.isEmpty(viewstate.val())) { throw new BankException(res.getText(R.string.unable_to_find).toString() + " ViewState."); }/* ww w .ja va 2 s .c om*/ Element eventvalidation = d.getElementById("__EVENTVALIDATION"); if (eventvalidation == null || TextUtils.isEmpty(eventvalidation.val())) { throw new BankException(res.getText(R.string.unable_to_find).toString() + " EventValidation."); } Element userField = d.select("#LoginSpan input[type=text]").first(); Element passField = d.select("#LoginSpan input[type=password]").first(); Element submitField = d.select("#LoginCustomerDiv input[type=submit]").first(); if (userField == null || passField == null || submitField == null) { throw new BankException(res.getText(R.string.unable_to_find).toString() + " login fields."); } List<NameValuePair> postData = new ArrayList<NameValuePair>(); postData.add(new BasicNameValuePair("__VIEWSTATE", viewstate.val())); postData.add(new BasicNameValuePair("__EVENTVALIDATION", eventvalidation.val())); postData.add(new BasicNameValuePair(userField.attr("name"), username)); postData.add(new BasicNameValuePair(passField.attr("name"), password)); postData.add(new BasicNameValuePair(submitField.attr("name"), submitField.val())); return new LoginPackage(urlopen, postData, response, "https://partner.ikanobank.se/web/engines/page.aspx?structid=" + structId); }