List of usage examples for org.jsoup.nodes Document select
public Elements select(String cssQuery)
From source file:org.bungeni.ext.integration.bungeniportal.BungeniServiceAccess.java
private List<BasicNameValuePair> getActionsViewButtonInfo(Document doc) { List<BasicNameValuePair> nvp = new ArrayList<BasicNameValuePair>(0); Elements inputList = doc.select("div#actionsView input"); for (int i = 0; i < inputList.size(); i++) { Element inputItem = inputList.get(i); nvp.add(new BasicNameValuePair(inputItem.attr("name"), inputItem.attr("value"))); }//from ww w. j a v a 2 s. c o m return nvp; }
From source file:com.app.rest.ExperianIntegrationService.java
@POST @Path("/landingPageSubmit") @Produces({ MediaType.APPLICATION_JSON }) @Consumes(MediaType.APPLICATION_JSON)/* w ww .j a v a 2 s. com*/ public ResponseModel getlandingPageDetails(String inputJsonObj) { //BasicConfigurator.configure(); String logMarker = null; ResponseModel responseMap = new ResponseModel(); try { //String requestParams = (String) inputJsonObj.get("input"); Map map = parseJson(inputJsonObj); logger.info("getlandingPageDetails ~ " + map.get("LOG_MARKER") == null ? "NOT_GIVEN" : map.get("LOG_MARKER") + " ~Entry"); logMarker = map.get("LOG_MARKER").toString(); //String voucherCode = "CMD1UjUz9"; if (map.get("clientName").toString() == null) { responseMap.setErrorMessage("Client Name is blank"); return responseMap; } ArrayList<NameValuePair> params = new ArrayList<NameValuePair>(); params.add(new BasicNameValuePair("clientName", map.get("clientName").toString())); params.add(new BasicNameValuePair("allowInput", map.get("allowInput").toString())); params.add(new BasicNameValuePair("allowEdit", map.get("allowEdit").toString())); params.add(new BasicNameValuePair("allowCaptcha", map.get("allowCaptcha").toString())); params.add(new BasicNameValuePair("allowConsent", map.get("allowConsent").toString())); params.add(new BasicNameValuePair("allowConsent_additional", map.get("allowConsent_additional").toString())); params.add(new BasicNameValuePair("allowEmailVerify", map.get("allowEmailVerify").toString())); params.add(new BasicNameValuePair("allowVoucher", map.get("allowVoucher").toString())); params.add(new BasicNameValuePair("voucherCode", map.get("voucherCode").toString())); params.add(new BasicNameValuePair("firstName", map.get("firstName").toString())); params.add(new BasicNameValuePair("surname", map.get("surName").toString())); params.add(new BasicNameValuePair("dateOfBirth", map.get("dateOfBirth").toString())); params.add(new BasicNameValuePair("gender", map.get("gender").toString())); params.add(new BasicNameValuePair("mobileNo", map.get("mobileNo").toString())); params.add(new BasicNameValuePair("email", map.get("email").toString())); params.add(new BasicNameValuePair("flatno", map.get("flatno").toString())); params.add(new BasicNameValuePair("city", map.get("city").toString())); params.add(new BasicNameValuePair("state", map.get("state").toString())); params.add(new BasicNameValuePair("pincode", map.get("pincode").toString())); params.add(new BasicNameValuePair("pan", map.get("pan").toString())); params.add(new BasicNameValuePair("reason", map.get("reason").toString())); params.add(new BasicNameValuePair("middleName", map.get("middleName").toString())); params.add(new BasicNameValuePair("telephoneNo", map.get("telephoneNo").toString())); params.add(new BasicNameValuePair("telephoneType", map.get("telephoneType").toString())); params.add(new BasicNameValuePair("passport", map.get("passport").toString())); params.add(new BasicNameValuePair("voterid", map.get("voterid").toString())); params.add(new BasicNameValuePair("aadhaar", map.get("aadhaar").toString())); params.add(new BasicNameValuePair("driverlicense", map.get("driverlicense").toString())); String request = getQuery(params); String jsessionId = HttpConnection.landingPageSubmit(request); String stageOneRequestId = HttpConnection.openCustomerDetailsFormAction(jsessionId, ""); responseMap.setuniqueId(stageOneRequestId.toString()); logger.info("getlandingPageDetails ~ " + (map.get("LOG_MARKER") == null ? "NOT_GIVEN" : map.get("LOG_MARKER")) + " ~ request : " + (request == null ? "null" : request) + " jsessionId: " + (jsessionId == null ? "null" : jsessionId) + " stageOneRequestId: " + (stageOneRequestId == null ? "null" : stageOneRequestId) + "~ Log Marker 1"); params.clear(); params = new ArrayList<NameValuePair>(); params.add(new BasicNameValuePair("clientName", map.get("clientName").toString())); params.add(new BasicNameValuePair("allowInput", map.get("allowInput").toString())); params.add(new BasicNameValuePair("allowEdit", map.get("allowEdit").toString())); params.add(new BasicNameValuePair("allowCaptcha", map.get("allowCaptcha").toString())); params.add(new BasicNameValuePair("allowConsent", map.get("allowConsent").toString())); params.add(new BasicNameValuePair("allowConsent_additional", map.get("allowConsent_additional").toString())); params.add(new BasicNameValuePair("allowEmailVerify", map.get("allowEmailVerify").toString())); params.add(new BasicNameValuePair("allowVoucher", map.get("allowVoucher").toString())); params.add(new BasicNameValuePair("voucherCode", map.get("voucherCode").toString())); params.add(new BasicNameValuePair("firstName", map.get("firstName").toString())); params.add(new BasicNameValuePair("surname", map.get("surName").toString())); params.add(new BasicNameValuePair("dob", map.get("dateOfBirth").toString())); params.add(new BasicNameValuePair("gender", map.get("gender").toString())); params.add(new BasicNameValuePair("mobileNo", map.get("mobileNo").toString())); params.add(new BasicNameValuePair("email", map.get("email").toString())); params.add(new BasicNameValuePair("flatPlotHouseNo", map.get("flatno").toString())); params.add(new BasicNameValuePair("city", map.get("city").toString())); params.add(new BasicNameValuePair("state", map.get("stateid").toString())); params.add(new BasicNameValuePair("pincode", map.get("pincode").toString())); params.add(new BasicNameValuePair("panNo", map.get("pan").toString())); params.add(new BasicNameValuePair("reason", map.get("reason").toString())); params.add(new BasicNameValuePair("requestReason", map.get("reason").toString())); params.add(new BasicNameValuePair("middleName", map.get("middleName").toString())); params.add(new BasicNameValuePair("telephoneNo", map.get("telephoneNo").toString())); params.add(new BasicNameValuePair("telephoneType", map.get("telephoneType").toString())); params.add(new BasicNameValuePair("passportNo", map.get("passport").toString())); params.add(new BasicNameValuePair("voterIdNo", map.get("voterid").toString())); params.add(new BasicNameValuePair("universalIdNo", map.get("aadhaar").toString())); params.add(new BasicNameValuePair("driverLicenseNo", map.get("driverlicense").toString())); params.add(new BasicNameValuePair("hitId", stageOneRequestId)); request = getQuery(params); logger.info("getlandingPageDetails ~ " + (map.get("LOG_MARKER") == null ? "NOT_GIVEN" : map.get("LOG_MARKER")) + " ~ request : " + (request == null ? "null" : request) + " jsessionId: " + (jsessionId == null ? "null" : jsessionId) + " stageOneRequestId: " + (stageOneRequestId == null ? "null" : stageOneRequestId) + "~ Log Marker 2"); HttpConnection.fetchScreenMetaDataAction(jsessionId, request); String resp = HttpConnection.submitRequest(jsessionId, request); logger.info("getlandingPageDetails ~ " + (map.get("LOG_MARKER") == null ? "NOT_GIVEN" : map.get("LOG_MARKER")) + " ~ request : " + (request == null ? "null" : request) + " jsessionId: " + (jsessionId == null ? "null" : jsessionId) + " stageOneRequestId: " + (stageOneRequestId == null ? "null" : stageOneRequestId) + " resp: " + (resp == null ? "null" : resp) + "~ Log Marker 4"); if (resp == null) { responseMap.setErrorMessage("RESPONSE_NULL"); return responseMap; } if (resp.equals("")) { responseMap.setErrorMessage("RESPONSE_BLANK"); return responseMap; } if (resp.startsWith("error")) { responseMap.setErrorMessage(resp.replace("error ", "")); return responseMap; } params.clear(); String stageTwoRequestId = HttpConnection.directCRQRequest(resp, jsessionId, params); logger.info("getlandingPageDetails ~ " + (map.get("LOG_MARKER") == null ? "NOT_GIVEN" : map.get("LOG_MARKER")) + " ~ request : " + (request == null ? "null" : request) + " jsessionId: " + (jsessionId == null ? "null" : jsessionId) + " stageOneRequestId: " + (stageOneRequestId == null ? "null" : stageOneRequestId) + " stageTwoRequestId: " + (stageTwoRequestId == null ? "null" : stageTwoRequestId) + "~ Log Marker 5"); params.add(new BasicNameValuePair("captchCode", "-999")); params.add(new BasicNameValuePair("payFlag", "true")); params.add(new BasicNameValuePair("voucherCode", map.get("voucherCode").toString())); params.add(new BasicNameValuePair("stgOneHitId", stageOneRequestId)); params.add(new BasicNameValuePair("stgTwoHitId", stageTwoRequestId)); request = getQuery(params); String jsessionIdResp = HttpConnection.paymentSubmitRequest(request); logger.info("getlandingPageDetails ~ " + (map.get("LOG_MARKER") == null ? "NOT_GIVEN" : map.get("LOG_MARKER")) + " ~ request : " + (request == null ? "null" : request) + " jsessionId: " + (jsessionId == null ? "null" : jsessionId) + " stageOneRequestId: " + (stageOneRequestId == null ? "null" : stageOneRequestId) + " stageTwoRequestId: " + (stageTwoRequestId == null ? "null" : stageTwoRequestId) + " jsessionIdResp: " + (jsessionIdResp == null ? "null" : jsessionIdResp) + "~ Log Marker 6"); if (jsessionIdResp.equalsIgnoreCase("customError")) { responseMap.setErrorMessage(jsessionIdResp); logger.info("getlandingPageDetails ~ " + (map.get("LOG_MARKER") == null ? "NOT_GIVEN" : map.get("LOG_MARKER")) + "~ customError ~ Log Marker 6"); return responseMap; } if (jsessionIdResp.equalsIgnoreCase("Invalid Voucher Code")) { responseMap.setErrorMessage("voucherExpired"); logger.info("getlandingPageDetails ~ " + (map.get("LOG_MARKER") == null ? "NOT_GIVEN" : map.get("LOG_MARKER")) + "~ voucherExpired ~ Log Marker 6"); return responseMap; } String responseJson = null; String message = ""; String answer = ""; String qId = ""; while (true) { logger.info("getlandingPageDetails ~ " + (map.get("LOG_MARKER") == null ? "NOT_GIVEN" : map.get("LOG_MARKER")) + "~ Log Marker 6"); params.clear(); params = new ArrayList<NameValuePair>(); params.add(new BasicNameValuePair("stgOneHitId", stageOneRequestId)); params.add(new BasicNameValuePair("stgTwoHitId", stageTwoRequestId)); request = getQuery(params); Map questionMap = HttpConnection.generateQuestionForConsumer(jsessionIdResp, request); responseJson = (String) questionMap.get("responseJson"); logger.info("getlandingPageDetails ~ " + (map.get("LOG_MARKER") == null ? "NOT_GIVEN" : map.get("LOG_MARKER")) + " ~ request : " + (request == null ? "null" : request) + " jsessionId: " + (jsessionId == null ? "null" : jsessionId) + " stageOneRequestId: " + (stageOneRequestId == null ? "null" : stageOneRequestId) + " stageTwoRequestId: " + (stageTwoRequestId == null ? "null" : stageTwoRequestId) + " responseJson: " + (responseJson == null ? "null" : responseJson) + "~ Log Marker 7"); if (responseJson.equalsIgnoreCase("passedReport")) { String pdfData = (String) questionMap.get("showHtmlReportForCreditReport"); Document doc = Jsoup.parse(pdfData); Element input = doc.select("input[name=xmlResponse]").first(); String response = input.attr("value"); responseMap.setXmlResponse(response); } if (responseJson.equalsIgnoreCase("next")) { questionMap.put("jsessionId2", jsessionIdResp); responseMap.setResponseMap(questionMap); } if (responseJson.equalsIgnoreCase("systemError")) { responseMap.setErrorMessage("systemError"); } if (responseJson.equalsIgnoreCase("inCorrectAnswersGiven")) { responseMap.setErrorMessage("inCorrectAnswersGiven"); } if (responseJson.equalsIgnoreCase("insufficientQuestion")) { responseMap.setErrorMessage("insufficientQuestion"); } if (responseJson.equalsIgnoreCase("creditReportEmpty")) { responseMap.setErrorMessage("creditReportEmpty"); } if (responseJson.equalsIgnoreCase("error")) { responseMap.setErrorMessage("error"); } return responseMap; } } catch (Exception e) { logger.info("getlandingPageDetails ~ " + (logMarker == null ? "NOT_GIVEN" : logMarker) + e.toString() + "~ Log Marker 8 "); responseMap.setErrorMessage("Error occured"); responseMap.setExceptionString(e.toString()); return responseMap; } }
From source file:com.weavers.duqhan.business.impl.ProductServiceImpl.java
@Override public List<StatusBean> getTempProductLinks(String link) { boolean status = true; //success String startDate = new Date().toString(); Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, "(==I==)DATE: " + startDate + "Product link collection start.....\n For the link ( " + link + " )"); Elements productUrlList = null; List<StatusBean> statusBeans = new ArrayList<>(); // Elements nexturl = null; boolean contd = true; String productList = link /*"https://www.aliexpress.com/wholesale?minPrice=&maxPrice=&isBigSale=n&isFreeShip=y&isFavorite=all&isMobileExclusive=n&isLocalReturn=n&shipFromCountry=&shipCompanies=&SearchText=jwelry+for+women&CatId=1509&g=y&initiative_id=SB_20170330225112&needQuery=n&isrefine=y"*/; Temtproductlinklist temtproductlinklist; Temtproductlinklist savedTemtproductlinklist; String nexturl = null;/* w w w . jav a 2 s . c o m*/ String firstPart = null; String secondPart = null; int[] pageNumber = new int[199]; Random randomObj1 = new Random(); for (int i = 0; i < 198; i++) { pageNumber[i] = (randomObj1.ints(2, 200).findFirst().getAsInt()); } try { Document doc = Jsoup.connect(productList).get(); productUrlList = doc.select("div.ui-pagination-navi a"); if (!productUrlList.isEmpty()) { nexturl = productUrlList.get(0).attr("abs:href"); firstPart = nexturl.split(".html")[0]; firstPart = firstPart.substring(0, firstPart.length() - 1); secondPart = nexturl.split(".html")[1]; secondPart = ".html" + secondPart; for (int i = 0; i < 198; i++) { nexturl = firstPart + pageNumber[i] + secondPart; doc = Jsoup.connect(nexturl).get(); productUrlList = doc.select(".son-list .list-item .pic a[href]"); //=================== Random sleep START ===================// Random randomObj = new Random(); TimeUnit.SECONDS.sleep(randomObj.ints(30, 60).findFirst().getAsInt()); //=================== Random sleep END =====================// if (!productUrlList.isEmpty()) { for (Element element : productUrlList) { temtproductlinklist = temtproductlinklistDao .getTemtproductlinklistByLink(element.attr("abs:href")); if (temtproductlinklist == null) { StatusBean statusBean = new StatusBean(); temtproductlinklist = new Temtproductlinklist(); temtproductlinklist.setLink(element.attr("abs:href")); temtproductlinklist.setStatus(0); //System.out.println("element.toString()" + element.attr("abs:href")); savedTemtproductlinklist = temtproductlinklistDao.save(temtproductlinklist); statusBean.setStatus(String.valueOf(savedTemtproductlinklist.getStatus())); statusBean.setStatusCode(savedTemtproductlinklist.getLink()); statusBean.setId(savedTemtproductlinklist.getId()); statusBeans.add(statusBean); } } } } } } catch (Exception ex) { status = false; //failure System.out.println("(=============================================)DATE: " + new Date().toString() + "Product link collection get exception.....\n Which started on: " + startDate + "\n" + ex.getLocalizedMessage()); Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, "(==E==)DATE: " + new Date().toString() + "Product link collection get exception.....\n Which started on: " + startDate + "\n", ex); String body = "DATE: " + new Date().toString() + "Product link collection get exception.....\nNext link not found.\n Which started on: " + startDate; // MailSender.sendEmail("krisanu.nandi@pkweb.in", "Error", body, "subhendu.sett@pkweb.in"); } if (status) { System.out.println("=============================================DATE: " + new Date().toString() + "Product link collection end.....\n Which started on: " + startDate); Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, "(==I==)DATE: " + new Date().toString() + "Product link collection end.....\n Which started on: " + startDate); String body = "DATE: " + new Date().toString() + "Product link collection end.....\n Which started on: " + startDate; // MailSender.sendEmail("krisanu.nandi@pkweb.in", "Success", body, "subhendu.sett@pkweb.in"); } return statusBeans; }
From source file:com.thesmartweb.swebrank.WebParser.java
/** * Method to get all the elements with a specific html feature (not used in SWebRank's current version) * @param link_html the url to check//from w w w . j a v a 2s . co m * @param dir the directory to save the file * @return a list with the text of all the elements */ public List<String> getbold(String link_html, String dir) { List<String> SEwords = new ArrayList<String>(); try { //link_html="http://www.themismavridis.com/"; Document doc = Jsoup.connect(link_html).get(); //---------to select the rest of the terms Elements bold = doc.select("em"); //bold=bold.select("b"); for (Element btext : bold) { String stringtosplit = btext.text().toString().toString(); if (!(stringtosplit == null) && (!(stringtosplit.equalsIgnoreCase("")))) { stringtosplit = stringtosplit.replaceAll("[\\W&&[^\\s]]", ""); if (!(stringtosplit == null) && (!(stringtosplit.equalsIgnoreCase("")))) { String[] tokenizedTerms = stringtosplit.split("\\W+"); for (int j = 0; j < tokenizedTerms.length; j++) { if (!(tokenizedTerms[j] == null) && (!(tokenizedTerms[j].equalsIgnoreCase("")))) { SEwords.add(tokenizedTerms[j]); } } } } } File file_thelist = new File(dir + "Javawords.txt"); FileUtils.writeLines(file_thelist, SEwords); return SEwords; } catch (IOException ex) { Logger.getLogger(WebParser.class.getName()).log(Level.SEVERE, null, ex); System.out.print("can not create the content file for SEwords"); return SEwords; } }
From source file:gov.medicaid.screening.dao.impl.OIGDAOBean.java
/** * Performs a search for all exclusion types. * * @return the search result for provider profiles * * @throws URISyntaxException if an error occurs while building the URL. * @throws ClientProtocolException if client does not support protocol used. * @throws IOException if an error occurs while parsing response. * @throws ServiceException for any other problems encountered * * @deprecated not updated in new site layout. *//*from w ww. j a va 2 s.c o m*/ @Deprecated private List<ExclusionType> getAllExclusions() throws URISyntaxException, ClientProtocolException, IOException, ServiceException { DefaultHttpClient client = new DefaultHttpClient(); HttpGet getSearch = new HttpGet(new URIBuilder(getExclusionURL()).build()); HttpResponse response = client.execute(getSearch); verifyAndAuditCall(getExclusionURL(), response); Document page = Jsoup.parse(EntityUtils.toString(response.getEntity())); List<ExclusionType> allExclusions = new ArrayList<ExclusionType>(); Elements links = page.select("table#CountList tbody tr td a"); for (Element link : links) { ExclusionType exclusionType = new ExclusionType(); exclusionType.setName(link.text()); allExclusions.add(exclusionType); } return allExclusions; }
From source file:company.gonapps.loghut.dao.PostDao.java
public PostDto get(PostDto postObject) throws IOException, InvalidTagNameException { PostDto post = getPostObject(postObject.getYear(), postObject.getMonth(), postObject.getDay(), postObject.getNumber(), postObject.getSecretEnabled()); rrwl.readLock().lock();//from w w w . j a v a 2 s . co m Document document; try { document = Jsoup.parse(new File(getPostPathString(post)), "UTF-8"); } finally { rrwl.readLock().unlock(); } post.setTitle(document.select("#loghut-post-title").first().text()); post.setText(document.select("#loghut-post-text").first().html()); List<TagDto> tags = new LinkedList<>(); for (Element tagElement : document.select(".loghut-post-tag")) { tags.add(new TagDto().setName(tagElement.text())); } post.setTags(tags); return post; }
From source file:me.vertretungsplan.parser.DaVinciParser.java
@Override public SubstitutionSchedule getSubstitutionSchedule() throws IOException, JSONException, CredentialInvalidException { new LoginHandler(scheduleData, credential, cookieProvider).handleLogin(executor, cookieStore); SubstitutionSchedule schedule = SubstitutionSchedule.fromData(scheduleData); List<String> urls = new ArrayList<>(); if (scheduleData.getData().has(PARAM_URLS)) { JSONArray urlsArray = scheduleData.getData().getJSONArray(PARAM_URLS); for (int i = 0; i < urlsArray.length(); i++) { urls.add(urlsArray.getString(i)); }// w w w . j a v a2s.c om } else { urls.add(scheduleData.getData().getString(PARAM_URL)); } for (String url : urls) { Document doc = Jsoup.parse(httpGet(url, ENCODING)); List<String> dayUrls = getDayUrls(url, doc); if (scheduleData.getData().has(PARAM_EMBEDDED_CONTENT_SELECTOR)) { for (Element el : doc.select(scheduleData.getData().getString(PARAM_EMBEDDED_CONTENT_SELECTOR))) { parsePage(el, schedule); } } else { for (String dayUrl : dayUrls) { Document dayDoc; if (dayUrl.equals(url)) { dayDoc = doc; } else { dayDoc = Jsoup.parse(httpGet(dayUrl, ENCODING)); } parsePage(dayDoc, schedule); } } } if (scheduleData.getData().has(PARAM_WEBSITE)) { schedule.setWebsite(scheduleData.getData().getString(PARAM_WEBSITE)); } else { schedule.setWebsite(urls.get(0)); } schedule.setClasses(getAllClasses()); schedule.setTeachers(getAllTeachers()); return schedule; }
From source file:gov.medicaid.screening.dao.impl.ChiropracticLicenseDAOBean.java
/** * Retrieves all the practioners from the disciplined list. * // w w w .ja v a2 s. c o m * @return the list of practitioners in the disciplined list * * @throws URISyntaxException * if an error occurs while building the URL. * @throws ClientProtocolException * if client does not support protocol used. * @throws IOException * if an error occurs while parsing response. * @throws ParseException * if an error occurs while parsing response. * @throws ServiceException * for any other problems encountered */ private SearchResult<License> getAllDisciplinedResults() throws URISyntaxException, ClientProtocolException, IOException, ParseException, ServiceException { DefaultHttpClient client = new DefaultHttpClient(); client.setRedirectStrategy(new LaxRedirectStrategy()); HttpGet search = new HttpGet(new URIBuilder(getDisciplineURL()).build()); HttpResponse response = client.execute(search); verifyAndAuditCall(getDisciplineURL(), response); Document page = Jsoup.parse(EntityUtils.toString(response.getEntity())); List<License> allLicenses = new ArrayList<License>(); Elements rows = page.select("blockquote table tr:gt(0)"); for (Element row : rows) { String fullname = row.select("td:eq(0)").text(); License license = new License(); ProviderProfile profile = new ProviderProfile(); license.setProfile(profile); profile.setUser(parseDeniedListFullname(fullname)); LicenseStatus status = new LicenseStatus(); String action = row.select("td:eq(1)").text(); status.setName(action); String date = row.select("td:eq(2)").text(); status.setDate(parseDate(date, DATE_FORMAT)); license.setStatus(status); allLicenses.add(license); } SearchResult<License> searchResult = new SearchResult<License>(); searchResult.setItems(allLicenses); return searchResult; }
From source file:gov.medicaid.screening.dao.impl.ChiropracticLicenseDAOBean.java
/** * Performs a search for all possible results. * //from w w w. j ava2s . c o m * @param criteria * The search criteria. * @param searchType * the type of search to execute. * @return the search result for licenses * * @throws URISyntaxException * if an error occurs while building the URL. * @throws ClientProtocolException * if client does not support protocol used. * @throws IOException * if an error occurs while parsing response. * @throws ParseException * if an error occurs while parsing response. * @throws ServiceException * for any other problems encountered */ private SearchResult<License> getAllResults(ChiropracticLicenseSearchCriteria criteria, String searchType) throws URISyntaxException, ClientProtocolException, IOException, ParseException, ServiceException { DefaultHttpClient client = new DefaultHttpClient(); client.setRedirectStrategy(new LaxRedirectStrategy()); URIBuilder uriBuilder = new URIBuilder(getSearchURL()); uriBuilder.addParameter("strName", Util.defaultString(criteria.getLastName())); uriBuilder.addParameter("strSoundex", Util.defaultString(criteria.getLastName())); uriBuilder.addParameter("strCity", Util.defaultString(criteria.getCity())); uriBuilder.addParameter("strZIP", Util.defaultString(criteria.getZipCode())); uriBuilder.addParameter(searchType, TYPES.get(searchType)); HttpGet search = new HttpGet(uriBuilder.build()); HttpResponse response = client.execute(search); verifyAndAuditCall(getSearchURL(), response); Document page = Jsoup.parse(EntityUtils.toString(response.getEntity())); List<License> allLicenses = new ArrayList<License>(); Elements rows = page.select("tr:gt(0)"); for (Element row : rows) { String href = row.select("a").first().attr("href"); // detail link String licenseType = row.select("td:eq(2)").text(); HttpGet getDetails = new HttpGet(Util.replaceLastURLPart(uriBuilder.build().toString(), href)); response = client.execute(getDetails); verifyAndAuditCall(getSearchURL(), response); page = Jsoup.parse(EntityUtils.toString(response.getEntity())); allLicenses.add(parseLicense(page, licenseType)); } SearchResult<License> searchResult = new SearchResult<License>(); searchResult.setItems(allLicenses); return searchResult; }
From source file:neembuu.release1.externalImpl.linkhandler.YoutubeLinkHandlerProvider.java
private BasicLinkHandler.Builder linkYoutubeExtraction(TrialLinkHandler tlh, int retryCount) throws Exception { String url = tlh.getReferenceLinkString(); BasicLinkHandler.Builder linkHandlerBuilder = BasicLinkHandler.Builder.create(); try {//from ww w. j a va2 s .c o m DefaultHttpClient httpClient = NHttpClient.getNewInstance(); String requestUrl = "http://www.linkyoutube.com/watch/index.php?video=" + URLEncoder.encode(url, "UTF-8"); final String responseString = NHttpClientUtils.getData(requestUrl, httpClient); //Set the group name as the name of the video String nameOfVideo = getVideoName(url); String fileName = "text"; linkHandlerBuilder.setGroupName(nameOfVideo); long c_duration = -1; Document doc = Jsoup.parse(responseString); Elements elements = doc.select("#download_links a"); for (Element element : elements) { String singleUrl = element.attr("href"); fileName = element.text(); if (!singleUrl.equals("#")) { long length = NHttpClientUtils.calculateLength(singleUrl, httpClient); singleUrl = Utils.normalize(singleUrl); LOGGER.log(Level.INFO, "Normalized URL: " + singleUrl); if (length == 0) { length = NHttpClientUtils.calculateLength(singleUrl, httpClient); } //LOGGER.log(Level.INFO,"Length: " + length); if (length <= 0) { continue; /*skip this url*/ } BasicOnlineFile.Builder fileBuilder = linkHandlerBuilder.createFile(); try { // finding video/audio length String dur = StringUtils.stringBetweenTwoStrings(singleUrl, "dur=", "&"); long duration = (int) (Double.parseDouble(dur) * 1000); if (c_duration < 0) { c_duration = duration; } fileBuilder.putLongPropertyValue( PropertyProvider.LongProperty.MEDIA_DURATION_IN_MILLISECONDS, duration); LOGGER.log(Level.INFO, "dur=" + dur); } catch (NumberFormatException a) { // ignore } try { // finding the quality short name String type = fileName.substring(fileName.indexOf("(") + 1); type = type.substring(0, type.indexOf(")")); fileBuilder.putStringPropertyValue(PropertyProvider.StringProperty.VARIANT_DESCRIPTION, type); LOGGER.log(Level.INFO, "type=" + type); } catch (Exception a) { a.printStackTrace(); } fileName = nameOfVideo + " " + fileName; fileBuilder.setName(fileName).setUrl(singleUrl).setSize(length).next(); } } for (OnlineFile of : linkHandlerBuilder.getFiles()) { long dur = of.getPropertyProvider() .getLongPropertyValue(PropertyProvider.LongProperty.MEDIA_DURATION_IN_MILLISECONDS); if (dur < 0 && c_duration > 0 && of.getPropertyProvider() instanceof BasicPropertyProvider) { ((BasicPropertyProvider) of.getPropertyProvider()).putLongPropertyValue( PropertyProvider.LongProperty.MEDIA_DURATION_IN_MILLISECONDS, c_duration); } } } catch (Exception ex) { int retryLimit = ((YT_TLH) tlh).retryLimit; ex.printStackTrace(); LOGGER.log(Level.INFO, "retry no. = " + retryCount); if (retryCount > retryLimit) throw ex; return linkYoutubeExtraction(tlh, retryCount + 1); } return linkHandlerBuilder; }