List of usage examples for org.jsoup.nodes Element attr
public String attr(String attributeKey)
From source file:autoInsurance.BeiJPiccImpl.java
public String queryBaseData(String in, Map<String, String> map) { // TODO Auto-generated method stub JSONObject jsonObject = JSONObject.fromObject(in); String chepNu = jsonObject.getString("chepNu"); String chejh = jsonObject.getString("chejh"); String fadjh = jsonObject.getString("fadjh"); String enrollDate1, enrollDate2; Map<String, Object> outMap = new HashMap<String, Object>(); outMap.put("uuid", map.get("uuid")); if (chepNu != null && !chepNu.equals("")) chepNu = chepNu.toUpperCase();/*from w ww . j ava 2 s . co m*/ else return queryBaseData2(chejh, fadjh, map); map2map(templateData, map); map.put("prpCitemCar.licenseNo", chepNu); outMap.put("licenseNo", chepNu); String url = "http://10.134.136.48:8000/prpall/carInf/getDataFromCiCarInfo.do"; String respStr = httpClientUtil.doPost(url, map, "gbk"); System.out.println(respStr); Map carMap = JackJson.fromJsonToObject(respStr, Map.class); if (((List) carMap.get("data")).size() > 0) { Map data = (Map) ((List) carMap.get("data")).get(0); map.put("prpCitemCar.frameNo", (String) data.get("rackNo")); outMap.put("frameNo", data.get("rackNo")); map.put("prpCitemCar.vinNo", (String) data.get("rackNo")); outMap.put("vinNo", data.get("rackNo")); map.put("prpCitemCar.engineNo", (String) data.get("engineNo")); outMap.put("engineNo", data.get("engineNo")); map.put("prpCitemCar.enrollDate", timeStamp2Date("" + (Long) ((Map) data.get("enrollDate")).get("time"), "yyyy-M-d")); outMap.put("enrollDate", map.get("prpCitemCar.enrollDate")); // enrollDate1 = map.get("prpCitemCar.enrollDate"); int eny = 0; try { eny = new SimpleDateFormat("yyyy-M-d").parse(map.get("prpCitemCar.enrollDate")).getYear(); } catch (ParseException e) { // TODO Auto-generated catch block e.printStackTrace(); } map.put("prpCitemCar.useYears", "" + (new Date().getYear() - eny)); outMap.put("useYears", map.get("prpCitemCar.useYears")); map.put("prpCcarShipTax.prePayTaxYear", "" + (Calendar.getInstance().get(Calendar.YEAR) - 1)); map.put("prpCitemCar.carKindCode", "A01"); //map.put("prpCitemCar.carKindCode", (String)data.get("carKindCode")); map.put("CarKindCodeDes", carTypeMap.get((String) data.get("carKindCode"))); if (StringUtils.startsWith(((String) data.get("carKindCode")), "K")) { map.put("prpCitemCar.licenseType", "80"); } else if (StringUtils.startsWith(((String) data.get("carKindCode")), "M")) { map.put("prpCitemCar.licenseType", "81"); } else { map.put("prpCitemCar.licenseType", (String) ((Map) data.get("id")).get("licenseType")); } outMap.put("licenseType", map.get("prpCitemCar.licenseType")); String carOwner = (String) data.get("carOwner"); if (null != carOwner) { map.put("insuredCarOwner", carOwner); outMap.put("insuredCarOwner", map.get("insuredCarOwner")); map.put("prpCinsureds[0].insuredName", carOwner); outMap.put("insuredName", map.get("prpCinsureds[0].insuredName")); map.put("owner", carOwner); outMap.put("owner", map.get("owner")); map.put("prpCcarShipTax.taxPayerName", carOwner); } String tonCount = data.get("tonCount") == null ? "0" : data.get("tonCount") + ""; map.put("prpCitemCar.tonCount", tonCount); // outMap.put("tonCount", map.get("prpCitemCar.tonCount")); String seatCount = "" + (Integer) data.get("seatCount"); if (StringUtils.isNotBlank(seatCount)) { map.put("prpCitemCar.seatCount", seatCount); outMap.put("seatCount", map.get("prpCitemCar.seatCount")); } } else return "{\"success\": flase, \"msg\": \"" + chepNu + "\"}"; ; url = "http://10.134.136.48:8000/prpall/carInf/getCarModelInfo.do"; respStr = httpClientUtil.doPost(url, map, "gbk"); System.out.println(respStr); Map car2Map = JackJson.fromJsonToObject(respStr, Map.class); List<Map> dataList = (List<Map>) car2Map.get("data"); if (dataList.size() > 0) { Map itemMap = dataList.get(0); if (itemMap.get("refCode2") != null && !itemMap.get("refCode2").equals("")) return "{\"success\": flase, \"msg\": \"" + itemMap.get("refCode2") + "\"}"; map.put("prpCitemCar.brandName", (String) itemMap.get("modelName")); outMap.put("brandName", map.get("prpCitemCar.brandName")); map.put("prpCitemCar.countryNature", (String) itemMap.get("vehicleType")); map.put("prpCitemCar.modelCode", (String) itemMap.get("modelCode")); outMap.put("modelCode", map.get("prpCitemCar.modelCode")); map.put("CarActualValueTrue", "" + itemMap.get("replaceMentValue")); map.put("prpCitemCar.purchasePrice", "" + itemMap.get("replaceMentValue")); map.put("purchasePriceOld", "" + itemMap.get("replaceMentValue")); if (itemMap.get("disPlaceMent") != null) { map.put("prpCitemCar.exhaustScale", "" + Integer.parseInt(itemMap.get("disPlaceMent") + "") / 1000.00); } else { map.put("prpCitemCar.exhaustScale", ""); } outMap.put("exhaustScale", map.get("prpCitemCar.exhaustScale")); if (!map.get("comCode").startsWith("11")) { System.out.println("comCode 11"); return null; } else { String seatCount = map.get("prpCitemCar.seatCount"); String l = "" + itemMap.get("rateDPassengercapacity"); String w = map.get("riskCode"); if (seatCount.equals("0") || seatCount.equals("") && l != null) { map.put("prpCitemCar.seatCount", l); } if ("DAV".equals(w) && Integer.parseInt(seatCount) >= 9) { map.put("prpCitemCar.brandName", ""); map.put("prpCitemCar.modelCode", ""); } String F = itemMap.get("tonnage") == null ? "0" : itemMap.get("tonnage") + ""; if (F != null && (map.get("prpCitemCar.tonCount").equals("0") || map.get("prpCitemCar.tonCount").equals(""))) { map.put("prpCitemCar.tonCount", F); } map.put("prpCitemCar.modelDemandNo", (String) itemMap.get("modelCode")); map.put("prpCitemCar.modelDemandNo", (String) ((Map) itemMap.get("id")).get("pmQueryNo")); map.put("isQueryCarModelFlag", "1"); } map.put("_insuredName", (String) itemMap.get("owner")); url = "http://10.134.136.48:8000/prpall/business/calActualValue.do"; respStr = httpClientUtil.doPost(url, map, "gbk"); System.out.println(respStr); map.put("prpCitemCar.actualValue", respStr); outMap.put("actualValue", respStr); map.put("premiumChangeFlag", "1"); } else { System.out.println("getCarModelInfo "); return null; } // url = "http://10.134.136.48:8000/prpall/business/selectRenewal.do"; Map<String, String> map4xub = null; try { map4xub = parse2Map("prpCrenewalVo.licenseNo=" + chepNu + "&prpCrenewalVo.licenseType=02"); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } respStr = httpClientUtil.doPost(url, map4xub, "GBK"); String lastPolicyNo = ""; JSONObject jObj = JSONObject.fromObject(respStr); JSONArray jdatas = jObj.getJSONArray("data"); Iterator<Object> it = jdatas.iterator(); while (it.hasNext()) { JSONObject obj = (JSONObject) it.next(); lastPolicyNo = obj.getString("policyNo"); } //outMap.put("lastPolicyNo", lastPolicyNo); System.out.println("lastPolicyNo: " + lastPolicyNo); Map<String, Object> xubCopyMap = new HashMap<String, Object>(); List<Map<String, String>> list = new ArrayList<Map<String, String>>(); if (!lastPolicyNo.equals("")) { url = "http://10.134.136.48:8000/prpall/business/quickProposalEditRenewalCopy.do?bizNo=" + lastPolicyNo; System.out.println(": " + url); respStr = httpClientUtil.doPost(url, new HashMap<String, String>(), "GBK"); // PrintWriter out; // try { // out = new PrintWriter("d:\\1.html"); // out.write(respStr); // respStr2 = readFile2Strng("d:\\1.html"); // // } catch (Exception e) { // // TODO Auto-generated catch block // e.printStackTrace(); // } // write2Html(respStr); Document doc = Jsoup.parse(respStr); //if(doc.getElementById("prpCitemCar.licenseNo") != null) { //return "{\"success\": flase, \"msg\": \"\"}"; if (doc.getElementById("prpCmainHeadInput") != null) { String lastYearBaoQi = ""; Elements elements = doc.select("#prpCmainHeadInput strong"); for (Element element : elements) { if (element.toString().contains("")) { lastYearBaoQi = element.text(); break; } } System.out.println(": " + lastYearBaoQi); xubCopyMap.put("lastYearBaoQi", lastYearBaoQi); } if (doc.getElementById("prpCitemCar.licenseNo") != null) { String licenseNo = doc.getElementById("prpCitemCar.licenseNo").attr("value"); System.out.println(": " + licenseNo); xubCopyMap.put("licenseNo", licenseNo); } if (doc.getElementById("prpCitemCar.modelCodeAlias") != null) { String modelCodeAlias = doc.getElementById("prpCitemCar.modelCodeAlias").attr("value"); System.out.println(": " + modelCodeAlias); xubCopyMap.put("modelCodeAlias", modelCodeAlias); } String new_engineNo = ""; if (doc.getElementById("prpCitemCar.engineNo") != null) { String engineNo = doc.getElementById("prpCitemCar.engineNo").attr("value"); System.out.println(": " + engineNo); xubCopyMap.put("engineNo", engineNo); new_engineNo = engineNo; } String new_frameNo = ""; if (doc.getElementById("prpCitemCar.frameNo") != null) { String frameNo = doc.getElementById("prpCitemCar.frameNo").attr("value"); System.out.println(": " + frameNo); xubCopyMap.put("frameNo", frameNo); new_frameNo = frameNo; } if (doc.getElementById("prpCitemCar.useNatureCode") != null) { String useNatureCode = doc.getElementById("prpCitemCar.useNatureCode").attr("title"); System.out.println(": " + useNatureCode); xubCopyMap.put("useNatureCode", useNatureCode); } if (doc.getElementById("prpCitemCar.enrollDate") != null) { String enrollDate = doc.getElementById("prpCitemCar.enrollDate").attr("value"); System.out.println(": " + enrollDate); xubCopyMap.put("enrollDate", enrollDate); // enrollDate2 = enrollDate; try { Date date1 = new SimpleDateFormat("yyyy-MM-dd").parse(enrollDate1); Date date2 = new SimpleDateFormat("yyyy-MM-dd").parse(enrollDate2); if (!date1.equals(date2)) { System.out.println(""); System.out.println(": " + new_frameNo + "\t: " + new_engineNo); return queryBaseData2(new_frameNo, new_engineNo, map); } } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } if (doc.getElementById("prpCitemCar.modelCode") != null) { String modelCode = doc.getElementById("prpCitemCar.modelCode").attr("value"); System.out.println(": " + modelCode); xubCopyMap.put("modelCode", modelCode); } if (doc.getElementById("prpCitemCar.purchasePrice") != null) { String purchasePrice = doc.getElementById("prpCitemCar.purchasePrice").attr("value"); System.out.println(": " + purchasePrice); xubCopyMap.put("purchasePrice", purchasePrice); } if (doc.getElementById("prpCitemCar.seatCount") != null) { String seatCount = doc.getElementById("prpCitemCar.seatCount").attr("value"); System.out.println("(): " + seatCount); xubCopyMap.put("seatCount", seatCount); } if (doc.getElementById("prpCitemCar.exhaustScale") != null) { String exhaustScale = doc.getElementById("prpCitemCar.exhaustScale").attr("value"); System.out.println("/(): " + exhaustScale); xubCopyMap.put("exhaustScale", exhaustScale); } if (doc.getElementById("prpCinsureds[0].insuredName") != null) { String insuredName = doc.getElementById("prpCinsureds[0].insuredName").attr("value"); System.out.println(": " + insuredName); xubCopyMap.put("insuredName", insuredName); } if (doc.getElementById("prpCinsureds[0].identifyNumber") != null) { String identifyNumber = doc.getElementById("prpCinsureds[0].identifyNumber").attr("value"); System.out.println(": " + identifyNumber); xubCopyMap.put("identifyNumber", identifyNumber); } if (doc.getElementById("prpCinsureds[0].insuredAddress") != null) { String insuredAddress = doc.getElementById("prpCinsureds[0].insuredAddress").attr("value"); System.out.println(": " + insuredAddress); xubCopyMap.put("insuredAddress", insuredAddress); } if (doc.getElementById("prpCinsureds[0].mobile") != null) { String mobile = doc.getElementById("prpCinsureds[0].mobile").attr("value"); System.out.println(": " + mobile); xubCopyMap.put("mobile", mobile); } System.out.println(); Element element = null; for (int i = 0; i < 11; i++) { Map<String, String> xianzMap = new HashMap<String, String>(); element = doc.getElementById("prpCitemKindsTemp[" + i + "].chooseFlag"); // System.out.println(element.toString()); String xianz = ""; if (element != null) xianz = element.attr("checked"); // if(xianz.equals("")) // continue; if (i == 0) xianzMap.put("A", xianz); if (i == 1) xianzMap.put("G", xianz); if (i == 2) xianzMap.put("B", xianz); if (i == 3) xianzMap.put("D11", xianz); if (i == 4) xianzMap.put("D12", xianz); if (i == 5) xianzMap.put("L", xianz); if (i == 6) xianzMap.put("F", xianz); if (i == 8) xianzMap.put("Z", xianz); if (i == 9) xianzMap.put("X1", xianz); element = doc.getElementById("prpCitemKindsTemp[" + i + "].specialFlag"); String bujmp = ""; if (element != null) bujmp = element.attr("checked"); xianzMap.put("bujmp", bujmp); element = doc.getElementById("prpCitemKindsTemp[" + i + "].amount"); String amount = ""; if (element != null) amount = element.attr("value"); xianzMap.put("amount", amount); element = doc.getElementById("prpCitemKindsTemp[" + i + "].modeCode"); String modeCode = ""; if (element != null) { Elements tmp = element.select("option"); for (Element et : tmp) { System.out.println(et.toString()); if (et.hasAttr("selected")) { modeCode = tmp.get(0).attr("value"); break; } } } xianzMap.put("modeCode", modeCode); System.out.print(i + ": " + xianz); System.out.print("\t\tbujmp: " + bujmp); System.out.println("\t\tamount: " + amount); System.out.println("\t\tmodeCode: " + modeCode); list.add(xianzMap); } } xubCopyMap.put("xianZDetail", list); outMap.put("xubCopy", xubCopyMap); // map2mapEx(map, outMap); return JSONObject.fromObject(outMap) + ""; }
From source file:de.geeksfactory.opacclient.apis.Pica.java
protected DetailledItem parse_result(String html) { Document doc = Jsoup.parse(html); doc.setBaseUri(opac_url);/* w w w. j av a 2 s . c o m*/ DetailledItem result = new DetailledItem(); for (Element a : doc.select("a[href*=PPN")) { Map<String, String> hrefq = getQueryParamsFirst(a.absUrl("href")); String ppn = hrefq.get("PPN"); result.setId(ppn); break; } // GET COVER if (doc.select("td.preslabel:contains(ISBN) + td.presvalue").size() > 0) { Element isbnElement = doc.select("td.preslabel:contains(ISBN) + td.presvalue").first(); String isbn = ""; for (Node child : isbnElement.childNodes()) { if (child instanceof TextNode) { isbn = ((TextNode) child).text().trim(); break; } } result.setCover(ISBNTools.getAmazonCoverURL(isbn, true)); } // GET TITLE AND SUBTITLE String titleAndSubtitle; Element titleAndSubtitleElem = null; String titleRegex = ".*(Titel|Aufsatz|Zeitschrift|Gesamttitel" + "|Title|Article|Periodical|Collective\\stitle" + "|Titre|Article|P.riodique|Titre\\sg.n.ral).*"; String selector = "td.preslabel:matches(" + titleRegex + ") + td.presvalue"; if (doc.select(selector).size() > 0) { titleAndSubtitleElem = doc.select(selector).first(); titleAndSubtitle = titleAndSubtitleElem.text().trim(); int slashPosition = Math.min(titleAndSubtitle.indexOf("/"), titleAndSubtitle.indexOf(":")); String title; if (slashPosition > 0) { title = titleAndSubtitle.substring(0, slashPosition).trim(); String subtitle = titleAndSubtitle.substring(slashPosition + 1).trim(); result.addDetail(new Detail(stringProvider.getString(StringProvider.SUBTITLE), subtitle)); } else { title = titleAndSubtitle; } result.setTitle(title); } else { result.setTitle(""); } // Details int line = 0; Elements lines = doc.select("td.preslabel + td.presvalue"); if (titleAndSubtitleElem != null) { lines.remove(titleAndSubtitleElem); } for (Element element : lines) { Element titleElem = element.firstElementSibling(); String detail = ""; if (element.select("div").size() > 1 && element.select("div").text().equals(element.text())) { boolean first = true; for (Element div : element.select("div")) { if (!div.text().replace("\u00a0", " ").trim().equals("")) { if (!first) { detail += "\n" + div.text().replace("\u00a0", " ").trim(); } else { detail += div.text().replace("\u00a0", " ").trim(); first = false; } } } } else { detail = element.text().replace("\u00a0", " ").trim(); } String title = titleElem.text().replace("\u00a0", " ").trim(); if (element.select("hr").size() > 0) // after the separator we get the copies { break; } if (detail.length() == 0 && title.length() == 0) { line++; continue; } if (title.contains(":")) { title = title.substring(0, title.indexOf(":")); // remove colon } result.addDetail(new Detail(title, detail)); if (element.select("a").size() == 1 && !element.select("a").get(0).text().trim().equals("")) { String url = element.select("a").first().absUrl("href"); if (!url.startsWith(opac_url)) { result.addDetail(new Detail(stringProvider.getString(StringProvider.LINK), url)); } } line++; } line++; // next line after separator // Copies Copy copy = new Copy(); String location = ""; // reservation info will be stored as JSON JSONArray reservationInfo = new JSONArray(); while (line < lines.size()) { Element element = lines.get(line); if (element.select("hr").size() == 0) { Element titleElem = element.firstElementSibling(); String detail = element.text().trim(); String title = titleElem.text().replace("\u00a0", " ").trim(); if (detail.length() == 0 && title.length() == 0) { line++; continue; } if (title.contains("Standort") || title.contains("Vorhanden in") || title.contains("Location")) { location += detail; } else if (title.contains("Sonderstandort")) { location += " - " + detail; } else if (title.contains("Systemstelle") || title.contains("Subject")) { copy.setDepartment(detail); } else if (title.contains("Fachnummer") || title.contains("locationnumber")) { copy.setLocation(detail); } else if (title.contains("Signatur") || title.contains("Shelf mark")) { copy.setShelfmark(detail); } else if (title.contains("Anmerkung")) { location += " (" + detail + ")"; } else if (title.contains("Link")) { result.addDetail(new Detail(title.replace(":", "").trim(), detail)); } else if (title.contains("Status") || title.contains("Ausleihinfo") || title.contains("Ausleihstatus") || title.contains("Request info")) { // Find return date Pattern pattern = Pattern.compile("(till|bis) (\\d{2}-\\d{2}-\\d{4})"); Matcher matcher = pattern.matcher(detail); if (matcher.find()) { DateTimeFormatter fmt = DateTimeFormat.forPattern("dd-MM-yyyy").withLocale(Locale.GERMAN); try { copy.setStatus(detail.substring(0, matcher.start() - 1).trim()); copy.setReturnDate(fmt.parseLocalDate(matcher.group(2))); } catch (IllegalArgumentException e) { e.printStackTrace(); copy.setStatus(detail); } } else { copy.setStatus(detail); } // Get reservation info if (element.select("a:has(img[src*=inline_arrow])").size() > 0) { Element a = element.select("a:has(img[src*=inline_arrow])").first(); boolean multipleCopies = a.text().matches(".*(Exemplare|Volume list).*"); JSONObject reservation = new JSONObject(); try { reservation.put("multi", multipleCopies); reservation.put("link", _extract_url(a.absUrl("href"))); reservation.put("desc", location); reservationInfo.put(reservation); } catch (JSONException e1) { e1.printStackTrace(); } result.setReservable(true); } } } else { copy.setBranch(location); result.addCopy(copy); location = ""; copy = new Copy(); } line++; } if (copy.notEmpty()) { copy.setBranch(location); result.addCopy(copy); } if (reservationInfo.length() == 0) { // No reservation info found yet, because we didn't find any copies. // If there is a reservation link somewhere in the rows we interpreted // as details, we still want to use it. if (doc.select("td a:has(img[src*=inline_arrow])").size() > 0) { Element a = doc.select("td a:has(img[src*=inline_arrow])").first(); boolean multipleCopies = a.text().matches(".*(Exemplare|Volume list).*"); JSONObject reservation = new JSONObject(); try { reservation.put("multi", multipleCopies); reservation.put("link", _extract_url(a.attr("href"))); reservation.put("desc", location); reservationInfo.put(reservation); } catch (JSONException e1) { e1.printStackTrace(); } result.setReservable(true); } } result.setReservation_info(reservationInfo.toString()); // Volumes if (doc.select("a[href^=FAM?PPN=]").size() > 0) { String href = doc.select("a[href^=FAM?PPN=]").attr("href"); String ppn = getQueryParamsFirst(href).get("PPN"); Map<String, String> data = new HashMap<>(); data.put("ppn", ppn); result.setVolumesearch(data); } return result; }
From source file:com.storm.function.GsxtFunction.java
@SuppressWarnings("unchecked") private Map<String, Object> getHtmlInfoMapOfShanxi(String area, HtmlPage firstInfoPage, String keyword, ChannelLogger LOGGER) throws Exception { Map<String, Object> resultHtmlMap = new LinkedHashMap<String, Object>(); ///* w ww. j ava 2s .c om*/ HtmlElement divByXPath = ((HtmlElement) firstInfoPage.getFirstByXPath("//div[@class='list']")); HtmlElement divByXPath2 = ((HtmlElement) firstInfoPage.getFirstByXPath("//div[@class='list-a']")); if (divByXPath == null && divByXPath2 == null) { //DomElement checkcode = firstInfoPage.getElementById("checkNo"); //String val = checkcode.getAttribute("value"); if (firstInfoPage.asXml().contains("???")) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.IMAGECODE_ERROR); } else { resultHtmlMap.put("statusCodeDef", StatusCodeDef.FAILURE); } } else { List<HtmlAnchor> anchors = (List<HtmlAnchor>) firstInfoPage.getByXPath("//div[@class='list']/ul/li/a"); if (anchors.size() == 0) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.NO_DATA_FOUND); } else { resultHtmlMap.put("statusCodeDef", StatusCodeDef.SCCCESS); } } // List<HtmlAnchor> anchors = (List<HtmlAnchor>) firstInfoPage.getByXPath("//div[@class='list']/ul/li/a"); LOGGER.info(anchors.toString()); if (anchors != null && !anchors.isEmpty()) { boolean matchFlag = false; for (HtmlAnchor anchor : anchors) { String anchorTitle = anchor.getTextContent().toString().trim(); if (anchorTitle.contains(keyword)) { //???? matchFlag = true; //??? HtmlElement target_item_info = (HtmlElement) anchor.getParentNode().getParentNode(); resultHtmlMap.put("target_item_info", target_item_info.asXml()); //?id String idParam = ""; String hrefAttr = anchor.getAttribute("href"); if (!StringUtils.isEmpty(hrefAttr)) { int index = hrefAttr.indexOf("?"); idParam = hrefAttr.substring(index + 1); } //??? HtmlPage gsgsxx_page = anchor.click(); resultHtmlMap.put("gsgsxx", gsgsxx_page.asXml()); //?-->?-->?--> // List<HtmlAnchor> anchors_gdxx_detail = (List<HtmlAnchor>)gsgsxx_page.getByXPath("//div[@id='invDiv']/table[@class='detailsList']/tbody/tr/td/a"); // List<String> gdxx_list = new ArrayList<String>(); // for (HtmlAnchor htmlAnchor : anchors_gdxx_detail) { // HtmlPage gdxx_detail = htmlAnchor.click(); // gdxx_list.add(gdxx_detail.asXml()); // } // resultHtmlMap.put("gsgsxx_gdxx_detail", gdxx_list); //?-->?-->?(??) String mainIdParam = idParam.replace("id", "mainId"); List<String> gdxxList = new ArrayList<String>(); List<String> gdxxDetailList = new ArrayList<String>(); int gdxxPageNo = 1; while (true) { String gsgsxx_djxx_gdxx_url = "http://gsxt.fc12319.com/QueryInvList.jspx?pno=" + gdxxPageNo + "&" + mainIdParam; String gsgsxx_djxx_gdxx_page = firstInfoPage.getWebClient().getPage(gsgsxx_djxx_gdxx_url) .getWebResponse().getContentAsString(); Document gsgsxxGdxxDoc = Jsoup.parse(gsgsxx_djxx_gdxx_page); if (gsgsxxGdxxDoc != null) { Elements aElements = gsgsxxGdxxDoc.select("a"); for (Element aElement : aElements) { String attr = aElement.attr("onclick"); if (attr != null && !"".equals(attr)) { String[] split = attr.split("'"); if (split.length > 2) { String detailUrl = "http://gsxt.fc12319.com" + split[1]; HtmlPage gsgsxx_gdxx_detail_page = null; try { gsgsxx_gdxx_detail_page = firstInfoPage.getWebClient() .getPage(detailUrl); } catch (Exception e) { gsgsxx_gdxx_detail_page = WebClient.getCustomHtmlPage(detailUrl, firstInfoPage.getWebClient().getCurrentWindow()); } gdxxDetailList.add(gsgsxx_gdxx_detail_page.asXml()); } } } } if (!gdxxList.contains(gsgsxx_djxx_gdxx_page)) { gdxxList.add(gsgsxx_djxx_gdxx_page); gdxxPageNo++; } else { break; } } resultHtmlMap.put("gsgsxx_gdxx_page", gdxxList); resultHtmlMap.put("gsgsxx_gdxx_detail", gdxxDetailList); //?-->?-->?? List<String> bgxxList = new ArrayList<String>(); int bgxxPageNo = 1; while (true) { String gsgsxx_bgxx_url = "http://gsxt.fc12319.com/QueryAltList.jspx?pno=" + bgxxPageNo + "&" + mainIdParam; String gsgsxx_bgxx_page = firstInfoPage.getWebClient().getPage(gsgsxx_bgxx_url) .getWebResponse().getContentAsString(); if (!bgxxList.contains(gsgsxx_bgxx_page)) { bgxxList.add(gsgsxx_bgxx_page); bgxxPageNo++; } else { break; } } resultHtmlMap.put("gsgsxx_bgxx_page", bgxxList); //?-->?-->?? List<String> zyryxxList = new ArrayList<String>(); int zyryPageNo = 1; while (true) { DomElement pageNoDom = gsgsxx_page.getElementById("spanmem" + zyryPageNo); if (pageNoDom != null) { String gsgsxx_baxx_zyryxx_url = "http://gsxt.fc12319.com/QueryMemList.jspx?pno=" + zyryPageNo + "&" + mainIdParam; String gsgsxx_baxx_zyryxx_page = firstInfoPage.getWebClient() .getPage(gsgsxx_baxx_zyryxx_url).getWebResponse().getContentAsString(); if (!zyryxxList.contains(gsgsxx_baxx_zyryxx_page)) { zyryxxList.add(gsgsxx_baxx_zyryxx_page); zyryPageNo++; } else { break; } } else { break; } } resultHtmlMap.put("gsgsxx_baxx_zyryxx_page", zyryxxList); //http://gsxt.fc12319.com/QueryMortList.jspx?pno=1&mainId=DB5C6077C0A57243E7C56BA05CD252C0 //?-->?--> List<HtmlAnchor> anchors_dcdyxx_detail = (List<HtmlAnchor>) gsgsxx_page .getByXPath("//div[@id='mortDiv']/table[@class='detailsList']/tbody/tr/td/a"); List<String> gsgs_dcdyxx_list = new ArrayList<String>(); for (HtmlAnchor htmlAnchor : anchors_dcdyxx_detail) { String attribute = htmlAnchor.getAttribute("onclick"); String detailUrl = "http://gsxt.fc12319.com" + attribute.split("'")[1]; HtmlPage dcdyxx_detail_page = null; try { dcdyxx_detail_page = firstInfoPage.getWebClient().getPage(detailUrl); } catch (Exception e) { dcdyxx_detail_page = WebClient.getCustomHtmlPage(detailUrl, firstInfoPage.getWebClient().getCurrentWindow()); } gsgs_dcdyxx_list.add(dcdyxx_detail_page.asXml()); } resultHtmlMap.put("gsgsxx_dcdydjxx_details", gsgs_dcdyxx_list); //?-->??? String gsgsxx_jyycxx_url = "http://gsxt.fc12319.com/QueryExcList.jspx?pno=1&" + mainIdParam; String gsgsxx_jyycxx_page = firstInfoPage.getWebClient().getPage(gsgsxx_jyycxx_url) .getWebResponse().getContentAsString(); resultHtmlMap.put("gsgsxx_jyycxx_page", gsgsxx_jyycxx_page); List<?> liElements = gsgsxx_page.getByXPath("//div[@id='leftTabs']/ul/li"); //???? String qygsxx_url = "http://gsxt.fc12319.com/enterprisePublicity.jspx?" + idParam; HtmlPage qygsxx_page = firstInfoPage.getWebClient().getPage(qygsxx_url); resultHtmlMap.put("qygsxx", qygsxx_page.asXml()); //??-->?--> List<HtmlAnchor> anchors_detail = (List<HtmlAnchor>) qygsxx_page .getByXPath("//div[@id='qiyenianbao']/table[@class='detailsList']/tbody/tr/td/a"); List<String> nbxx_list = new ArrayList<String>(); for (HtmlAnchor htmlAnchor : anchors_detail) { HtmlPage nb_detail = htmlAnchor.click(); nbxx_list.add(nb_detail.asXml()); } resultHtmlMap.put("qygsxx_qynb_detail", nbxx_list); //??-->?? String qygsxx_xzxkxx_url = "http://gsxt.fc12319.com/QueryLicenseRegList.jspx?pno=1&" + mainIdParam; String gsgsxx_xzxkxx_page = firstInfoPage.getWebClient().getPage(qygsxx_xzxkxx_url) .getWebResponse().getContentAsString(); resultHtmlMap.put("gsgsxx_xzxkxx_page", gsgsxx_xzxkxx_page); //??? String qtbmgsxx_url = "http://gsxt.fc12319.com/otherDepartment.jspx?" + mainIdParam; HtmlPage qtbmgsxx_page = firstInfoPage.getWebClient().getPage(qtbmgsxx_url); resultHtmlMap.put("qtbmgsxx", qtbmgsxx_page.asXml()); //?? ???? if (liElements.size() > 3) { String sfxzxx_url = "http://gsxt.fc12319.com/justiceAssistance.jspx?" + idParam; HtmlPage sfxzxx_page = firstInfoPage.getWebClient().getPage(sfxzxx_url); resultHtmlMap.put("sfxzgsxx", sfxzxx_page.asXml()); } break; } } if (!matchFlag) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.NO_DATA_FOUND); LOGGER.info("????"); } } return resultHtmlMap; }
From source file:com.storm.function.GsxtFunction.java
@SuppressWarnings("unchecked") private Map<String, Object> getHtmlInfoMapOfHeilongjiang(String area, HtmlPage firstInfoPage, String keyword, ChannelLogger LOGGER) throws Exception { Map<String, Object> resultHtmlMap = new LinkedHashMap<String, Object>(); WebWindow window = firstInfoPage.getWebClient().getCurrentWindow(); HtmlElement divByXPath = ((HtmlElement) firstInfoPage.getFirstByXPath("//div[@class='list']")); if (divByXPath == null) { DomElement checkcode = firstInfoPage.getElementById("checkNo"); String val = checkcode.getAttribute("value"); if (!StringUtils.isEmpty(val)) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.IMAGECODE_ERROR); } else {/*from ww w.j a v a 2s . c o m*/ resultHtmlMap.put("statusCodeDef", StatusCodeDef.FAILURE); } } else { String textContent = divByXPath.getTextContent(); if (textContent != null) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.SCCCESS); } else { HtmlElement firstByXPath = ((HtmlElement) firstInfoPage.getFirstByXPath("//div[@class='list-a']")); textContent = firstByXPath.getTextContent(); if (textContent.indexOf("??") > 0) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.NO_DATA_FOUND); } } } List<HtmlAnchor> anchors = (List<HtmlAnchor>) firstInfoPage.getByXPath("//div[@class='list']/ul/li/a"); LOGGER.info(anchors.toString()); if (anchors != null && !anchors.isEmpty()) { boolean matchFlag = false; for (HtmlAnchor anchor : anchors) { String anchorTitle = anchor.getTextContent().toString().trim(); if (anchorTitle.contains(keyword)) { //???? matchFlag = true; //??? HtmlElement target_item_info = (HtmlElement) anchor.getParentNode().getParentNode(); resultHtmlMap.put("target_item_info", target_item_info.asXml()); //?id String idParam = ""; String hrefAttr = anchor.getAttribute("href"); if (!StringUtils.isEmpty(hrefAttr)) { int index = hrefAttr.indexOf("?"); idParam = hrefAttr.substring(index + 1); } //??? HtmlPage gsgsxx_page = anchor.click(); resultHtmlMap.put("gsgsxx", gsgsxx_page.asXml()); //?-->?-->?--> // List<HtmlAnchor> anchors_gdxx_detail = (List<HtmlAnchor>)gsgsxx_page.getByXPath("//div[@id='invDiv']/table[@class='detailsList']/tbody/tr/td/a"); // List<String> gdxx_list = new ArrayList<String>(); // for (HtmlAnchor htmlAnchor : anchors_gdxx_detail) { // HtmlPage gdxx_detail = htmlAnchor.click(); // gdxx_list.add(gdxx_detail.asXml()); // } // resultHtmlMap.put("gsgsxx_gdxx_detail", gdxx_list); //?-->?-->?(??) String mainIdParam = idParam.replace("id", "mainId"); List<String> gdxxList = new ArrayList<String>(); List<String> gdxxDetailList = new ArrayList<String>(); int gdxxPageNo = 1; while (true) { String gsgsxx_djxx_gdxx_url = "http://gsxt.hljaic.gov.cn/QueryInvList.jspx?pno=" + gdxxPageNo + "&" + mainIdParam; String gsgsxx_djxx_gdxx_page = firstInfoPage.getWebClient().getPage(gsgsxx_djxx_gdxx_url) .getWebResponse().getContentAsString(); Document gsgsxxGdxxDoc = Jsoup.parse(gsgsxx_djxx_gdxx_page); if (gsgsxxGdxxDoc != null) { Elements aElements = gsgsxxGdxxDoc.select("a"); for (Element aElement : aElements) { String attr = aElement.attr("onclick"); if (attr != null && !"".equals(attr)) { String[] split = attr.split("'"); if (split.length > 2) { String detailUrl = "http://gsxt.hljaic.gov.cn" + split[1]; HtmlPage gsgsxx_gdxx_detail_page = null; try { gsgsxx_gdxx_detail_page = firstInfoPage.getWebClient() .getPage(detailUrl); } catch (Exception e) { gsgsxx_gdxx_detail_page = WebClient.getCustomHtmlPage(detailUrl, firstInfoPage.getWebClient().getCurrentWindow()); } gdxxDetailList.add(gsgsxx_gdxx_detail_page.asXml()); } } } } if (!gdxxList.contains(gsgsxx_djxx_gdxx_page)) { gdxxList.add(gsgsxx_djxx_gdxx_page); gdxxPageNo++; } else { break; } } resultHtmlMap.put("gsgsxx_gdxx_page", gdxxList); resultHtmlMap.put("gsgsxx_gdxx_detail", gdxxDetailList); //?-->?-->?? List<String> bgxxList = new ArrayList<String>(); int bgxxPageNo = 1; while (true) { String gsgsxx_bgxx_url = "http://gsxt.hljaic.gov.cn/QueryAltList.jspx?pno=" + bgxxPageNo + "&" + mainIdParam; String gsgsxx_bgxx_page = firstInfoPage.getWebClient().getPage(gsgsxx_bgxx_url) .getWebResponse().getContentAsString(); if (!bgxxList.contains(gsgsxx_bgxx_page)) { bgxxList.add(gsgsxx_bgxx_page); bgxxPageNo++; } else { break; } } resultHtmlMap.put("gsgsxx_bgxx_page", bgxxList); //?-->?-->?? List<String> zyryxxList = new ArrayList<String>(); int zyryPageNo = 1; while (true) { DomElement pageNoDom = gsgsxx_page.getElementById("spanmem" + zyryPageNo); if (pageNoDom != null) { String gsgsxx_baxx_zyryxx_url = "http://gsxt.hljaic.gov.cn/QueryMemList.jspx?pno=" + zyryPageNo + "&" + mainIdParam; String gsgsxx_baxx_zyryxx_page = firstInfoPage.getWebClient() .getPage(gsgsxx_baxx_zyryxx_url).getWebResponse().getContentAsString(); if (!zyryxxList.contains(gsgsxx_baxx_zyryxx_page)) { zyryxxList.add(gsgsxx_baxx_zyryxx_page); zyryPageNo++; } else { break; } } else { break; } } resultHtmlMap.put("gsgsxx_baxx_zyryxx_page", zyryxxList); //http://gsxt.fc12319.com/QueryMortList.jspx?pno=1&mainId=DB5C6077C0A57243E7C56BA05CD252C0 //?-->?--> List<HtmlAnchor> anchors_dcdyxx_detail = (List<HtmlAnchor>) gsgsxx_page .getByXPath("//div[@id='mortDiv']/table[@class='detailsList']/tbody/tr/td/a"); List<String> gsgs_dcdyxx_list = new ArrayList<String>(); for (HtmlAnchor htmlAnchor : anchors_dcdyxx_detail) { String attribute = htmlAnchor.getAttribute("onclick"); String detailUrl = "http://gsxt.hljaic.gov.cn" + attribute.split("'")[1]; HtmlPage dcdyxx_detail_page = null; try { dcdyxx_detail_page = firstInfoPage.getWebClient().getPage(detailUrl); } catch (Exception e) { dcdyxx_detail_page = WebClient.getCustomHtmlPage(detailUrl, firstInfoPage.getWebClient().getCurrentWindow()); } gsgs_dcdyxx_list.add(dcdyxx_detail_page.asXml()); } resultHtmlMap.put("gsgsxx_dcdydjxx_details", gsgs_dcdyxx_list); //?-->??? String gsgsxx_jyycxx_url = "http://gsxt.hljaic.gov.cn/QueryExcList.jspx?pno=1&" + mainIdParam; String gsgsxx_jyycxx_page = firstInfoPage.getWebClient().getPage(gsgsxx_jyycxx_url) .getWebResponse().getContentAsString(); resultHtmlMap.put("gsgsxx_jyycxx_page", gsgsxx_jyycxx_page); List<?> liElements = gsgsxx_page.getByXPath("//div[@id='leftTabs']/ul/li"); //???? String qygsxx_url = "http://gsxt.hljaic.gov.cn/enterprisePublicity.jspx?" + idParam; HtmlPage qygsxx_page = firstInfoPage.getWebClient().getPage(qygsxx_url); resultHtmlMap.put("qygsxx", qygsxx_page.asXml()); //??-->?--> List<HtmlAnchor> anchors_detail = (List<HtmlAnchor>) qygsxx_page .getByXPath("//div[@id='qiyenianbao']/table[@class='detailsList']/tbody/tr/td/a"); List<String> nbxx_list = new ArrayList<String>(); for (HtmlAnchor htmlAnchor : anchors_detail) { HtmlPage nb_detail = htmlAnchor.click(); nbxx_list.add(nb_detail.asXml()); } resultHtmlMap.put("qygsxx_qynb_detail", nbxx_list); //??-->?? String qygsxx_xzxkxx_url = "http://gsxt.hljaic.gov.cn/QueryLicenseRegList.jspx?pno=1&" + mainIdParam; String gsgsxx_xzxkxx_page = firstInfoPage.getWebClient().getPage(qygsxx_xzxkxx_url) .getWebResponse().getContentAsString(); resultHtmlMap.put("gsgsxx_xzxkxx_page", gsgsxx_xzxkxx_page); //??? // String gsgsxx_sfxzgsxx_url = "http://gsxt.hljaic.gov.cn/otherDepartment.jspx?id="+mystr ; String qtbmgsxx_url = "http://gsxt.hljaic.gov.cn/otherDepartment.jspx?" + mainIdParam; HtmlPage qtbmgsxx_page = firstInfoPage.getWebClient().getPage(qtbmgsxx_url); resultHtmlMap.put("qtbmgsxx", qtbmgsxx_page.asXml()); //?? ???? if (liElements.size() > 3) { String sfxzxx_url = "http://gsxt.hljaic.gov.cn/justiceAssistance.jspx?" + idParam; HtmlPage sfxzxx_page = firstInfoPage.getWebClient().getPage(sfxzxx_url); resultHtmlMap.put("sfxzgsxx", sfxzxx_page.asXml()); } break; } } if (!matchFlag) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.NO_DATA_FOUND); LOGGER.info("????"); } } return resultHtmlMap; }
From source file:com.storm.function.GsxtFunction.java
@SuppressWarnings("unchecked") private Map<String, Object> getHtmlInfoMapOfHainan(String area, HtmlPage firstInfoPage, String keyword, ChannelLogger LOGGER) throws Exception { WebWindow window = firstInfoPage.getWebClient().getCurrentWindow(); Map<String, Object> resultHtmlMap = new LinkedHashMap<String, Object>(); HtmlElement divByXPath = ((HtmlElement) firstInfoPage.getFirstByXPath("//div[@class='list']")); HtmlElement divByXPath2 = ((HtmlElement) firstInfoPage.getFirstByXPath("//div[@class='list-a']")); if (divByXPath == null && divByXPath2 == null) { //DomElement checkcode = firstInfoPage.getElementById("checkNo"); //String val = checkcode.getAttribute("value"); if (firstInfoPage.asXml().contains("???")) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.IMAGECODE_ERROR); } else {/* ww w. j ava 2 s. c om*/ resultHtmlMap.put("statusCodeDef", StatusCodeDef.FAILURE); } } else { List<HtmlAnchor> anchors = (List<HtmlAnchor>) firstInfoPage.getByXPath("//div[@class='list']/ul/li/a"); if (anchors.size() == 0) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.NO_DATA_FOUND); } else { resultHtmlMap.put("statusCodeDef", StatusCodeDef.SCCCESS); } } // List<HtmlAnchor> anchors = (List<HtmlAnchor>) firstInfoPage.getByXPath("//div[@class='list']/ul/li/a"); LOGGER.info(anchors.toString()); if (anchors != null && !anchors.isEmpty()) { boolean matchFlag = false; for (HtmlAnchor anchor : anchors) { String anchorTitle = anchor.getTextContent().toString().trim(); if (anchorTitle.contains(keyword)) { //???? matchFlag = true; //??? HtmlElement target_item_info = (HtmlElement) anchor.getParentNode().getParentNode(); resultHtmlMap.put("target_item_info", target_item_info.asXml()); //?id String idParam = ""; String hrefAttr = anchor.getAttribute("href"); if (!StringUtils.isEmpty(hrefAttr)) { int index = hrefAttr.indexOf("?"); idParam = hrefAttr.substring(index + 1); } //??? HtmlPage gsgsxx_page = anchor.click(); resultHtmlMap.put("gsgsxx", gsgsxx_page.asXml()); //?-->?--> List<HtmlAnchor> anchors_gdxx_detail = (List<HtmlAnchor>) gsgsxx_page .getByXPath("//div[@id='invDiv']/table[@class='detailsList']/tbody/tr/td/a"); List<String> gdxx_list = new ArrayList<String>(); for (HtmlAnchor htmlAnchor : anchors_gdxx_detail) { HtmlPage gdxx_detail = htmlAnchor.click(); gdxx_list.add(gdxx_detail.asXml()); } resultHtmlMap.put("gsgsxx_gdxx_detail", gdxx_list); //?-->?-->?(??) String mainIdParam = idParam.replace("id", "mainId"); List<String> gdxxList = new ArrayList<String>(); List<String> gdxxDetailList = new ArrayList<String>(); int gdxxPageNo = 1; while (true) { //http://aic.hainan.gov.cn:1888/businessPublicity.jspx?id=46C738AE70B9CEF10433B8BCA8443AAD String gsgsxx_djxx_gdxx_url = "http://aic.hainan.gov.cn:1888/QueryInvList.jspx?pno=" + gdxxPageNo + "&" + mainIdParam; String gsgsxx_djxx_gdxx_page = firstInfoPage.getWebClient().getPage(gsgsxx_djxx_gdxx_url) .getWebResponse().getContentAsString(); Document gsgsxxGdxxDoc = Jsoup.parse(gsgsxx_djxx_gdxx_page); if (gsgsxxGdxxDoc != null) { Elements aElements = gsgsxxGdxxDoc.select("a"); for (Element aElement : aElements) { String attr = aElement.attr("onclick"); if (attr != null && !"".equals(attr)) { String[] split = attr.split("'"); if (split.length > 2) { String detailUrl = "http://aic.hainan.gov.cn:1888" + split[1]; HtmlPage gsgsxx_gdxx_detail_page = null; try { gsgsxx_gdxx_detail_page = firstInfoPage.getWebClient() .getPage(detailUrl); } catch (Exception e) { gsgsxx_gdxx_detail_page = WebClient.getCustomHtmlPage(detailUrl, firstInfoPage.getWebClient().getCurrentWindow()); } gdxxDetailList.add(gsgsxx_gdxx_detail_page.asXml()); } } } } if (!gdxxList.contains(gsgsxx_djxx_gdxx_page)) { // gdxxList.add(gsgsxx_djxx_gdxx_page); // gdxxPageNo++; if (gdxxList.size() >= 1) { String march1 = gdxxList.get(0); String matchString = march1.split("width=20%")[1].split("width=23%")[0]; if (gsgsxx_djxx_gdxx_page.contains(matchString)) { break; } } else { gdxxList.add(gsgsxx_djxx_gdxx_page); gdxxPageNo++; } } else { break; } } resultHtmlMap.put("gsgsxx_gdxx_page", gdxxList); resultHtmlMap.put("gsgsxx_gdxx_detail", gdxxDetailList); //?-->?-->?? List<String> bgxxList = new ArrayList<String>(); int bgxxPageNo = 1; while (true) { String gsgsxx_bgxx_url = "http://aic.hainan.gov.cn:1888/QueryAltList.jspx?pno=" + bgxxPageNo + "&" + mainIdParam; String gsgsxx_bgxx_page = firstInfoPage.getWebClient().getPage(gsgsxx_bgxx_url) .getWebResponse().getContentAsString(); if (!bgxxList.contains(gsgsxx_bgxx_page)) { bgxxList.add(gsgsxx_bgxx_page); bgxxPageNo++; } else { break; } } resultHtmlMap.put("gsgsxx_bgxx_page", bgxxList); //?-->?-->?? List<String> zyryxxList = new ArrayList<String>(); int zyryPageNo = 1; while (true) { DomElement pageNoDom = gsgsxx_page.getElementById("spanmem" + zyryPageNo); if (pageNoDom != null) { String gsgsxx_baxx_zyryxx_url = "http://aic.hainan.gov.cn:1888/QueryMemList.jspx?pno=" + zyryPageNo + "&" + mainIdParam; String gsgsxx_baxx_zyryxx_page = firstInfoPage.getWebClient() .getPage(gsgsxx_baxx_zyryxx_url).getWebResponse().getContentAsString(); if (!zyryxxList.contains(gsgsxx_baxx_zyryxx_page)) { zyryxxList.add(gsgsxx_baxx_zyryxx_page); zyryPageNo++; } else { break; } } else { break; } } resultHtmlMap.put("gsgsxx_baxx_zyryxx_page", zyryxxList); //http://gsxt.fc12319.com/QueryMortList.jspx?pno=1&mainId=DB5C6077C0A57243E7C56BA05CD252C0 //?-->?--> List<HtmlAnchor> anchors_dcdyxx_detail = (List<HtmlAnchor>) gsgsxx_page .getByXPath("//div[@id='mortDiv']/table[@class='detailsList']/tbody/tr/td/a"); List<String> gsgs_dcdyxx_list = new ArrayList<String>(); for (HtmlAnchor htmlAnchor : anchors_dcdyxx_detail) { String attribute = htmlAnchor.getAttribute("onclick"); String detailUrl = "http://aic.hainan.gov.cn:1888" + attribute.split("'")[1]; HtmlPage dcdyxx_detail_page = null; try { dcdyxx_detail_page = firstInfoPage.getWebClient().getPage(detailUrl); } catch (Exception e) { dcdyxx_detail_page = WebClient.getCustomHtmlPage(detailUrl, firstInfoPage.getWebClient().getCurrentWindow()); } gsgs_dcdyxx_list.add(dcdyxx_detail_page.asXml()); } resultHtmlMap.put("gsgsxx_dcdydjxx_details", gsgs_dcdyxx_list); //?-->??? String gsgsxx_jyycxx_url = "http://aic.hainan.gov.cn:1888/QueryExcList.jspx?pno=1&" + mainIdParam; String gsgsxx_jyycxx_page = firstInfoPage.getWebClient().getPage(gsgsxx_jyycxx_url) .getWebResponse().getContentAsString(); resultHtmlMap.put("gsgsxx_jyycxx_page", gsgsxx_jyycxx_page); List<?> liElements = gsgsxx_page.getByXPath("//div[@id='leftTabs']/ul/li"); //???? // HtmlElement qygsxx_tab = (HtmlElement)liElements.get(1); // HtmlPage qygsxx_page = (HtmlPage)qygsxx_tab.click(); // resultHtmlMap.put("qygsxx", qygsxx_page.asXml()); String qygsxx_url = "http://aic.hainan.gov.cn:1888/enterprisePublicity.jspx?" + idParam; HtmlPage qygsxx_page = firstInfoPage.getWebClient().getPage(qygsxx_url); resultHtmlMap.put("qygsxx", qygsxx_page.asXml()); //??-->?--> List<HtmlAnchor> anchors_detail = (List<HtmlAnchor>) qygsxx_page .getByXPath("//div[@id='qiyenianbao']/table[@class='detailsList']/tbody/tr/td/a"); List<String> nbxx_list = new ArrayList<String>(); for (HtmlAnchor htmlAnchor : anchors_detail) { HtmlPage nb_detail = htmlAnchor.click(); nbxx_list.add(nb_detail.asXml()); } resultHtmlMap.put("qygsxx_qynb_detail", nbxx_list); //??-->?? String qygsxx_xzxkxx_url = "http://aic.hainan.gov.cn:1888/QueryLicenseRegList.jspx?pno=1&" + mainIdParam; String gsgsxx_xzxkxx_page = firstInfoPage.getWebClient().getPage(qygsxx_xzxkxx_url) .getWebResponse().getContentAsString(); resultHtmlMap.put("gsgsxx_xzxkxx_page", gsgsxx_xzxkxx_page); //??? // HtmlElement qtbmgsxx_tab = (HtmlElement)liElements.get(2); // HtmlPage qtbmgsxx_page = (HtmlPage)qtbmgsxx_tab.click(); // resultHtmlMap.put("qtbmgsxx", qtbmgsxx_page.asXml()); String qtbmgsxx_url = "http://aic.hainan.gov.cn:1888/otherDepartment.jspx?" + mainIdParam; HtmlPage qtbmgsxx_page = firstInfoPage.getWebClient().getPage(qtbmgsxx_url); resultHtmlMap.put("qtbmgsxx", qtbmgsxx_page.asXml()); //?? ???? if (liElements.size() > 3) { //HtmlElement sfxzgsxx_tab = (HtmlElement)liElements.get(3); //HtmlPage sfxzgsxx_page = (HtmlPage)sfxzgsxx_tab.click(); //resultHtmlMap.put("sfxzgsxx", sfxzgsxx_page.asXml()); //http://aic.hainan.gov.cn:1888/ String sfxzxx_url = "http://aic.hainan.gov.cn:1888/justiceAssistance.jspx?" + idParam; HtmlPage sfxzxx_page = firstInfoPage.getWebClient().getPage(window, new WebRequest(new URL(sfxzxx_url))); resultHtmlMap.put("sfxzgsxx", sfxzxx_page.asXml()); } break; } } if (!matchFlag) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.NO_DATA_FOUND); LOGGER.info("????"); } } return resultHtmlMap; }
From source file:com.storm.function.GsxtFunction.java
private Map<String, Object> getHtmlInfoMapOfJilin(String area, String keyword, ChannelLogger LOGGER) throws Exception { Map<String, Object> resultHtmlMap = new LinkedHashMap<String, Object>(); String[] command = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/jilin.js", "--web-security=no", "--keyword=" + keyword }; String casperjsResult = CommandUtil.runCommand(command); Elements divDataItems = Jsoup.parse(casperjsResult).getElementsByClass("list"); Elements divNoDataItems = Jsoup.parse(casperjsResult).getElementsByClass("list-a"); if (divDataItems.isEmpty() && !divNoDataItems.isEmpty()) { // ? resultHtmlMap.put("statusCodeDef", StatusCodeDef.NO_DATA_FOUND); } else if (divDataItems.isEmpty() && divDataItems.isEmpty()) { // ?? // ????/* ww w . ja v a2 s.c o m*/ if (casperjsResult.contains("")) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.IMAGECODE_ERROR); } else { resultHtmlMap.put("statusCodeDef", StatusCodeDef.FAILURE); } } else if (!divDataItems.isEmpty() && divNoDataItems.isEmpty()) { // ? // ??????? Element nowCookies = Jsoup.parse(casperjsResult).getElementById("nextParams"); Elements tokenEts = Jsoup.parse(casperjsResult).getElementsByAttributeValue("name", "_csrf"); if (null == nowCookies || null == tokenEts || tokenEts.isEmpty()) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.COOKIE_ERROR); return resultHtmlMap; } String nowCookiesJson = nowCookies.text().trim(); String nowCookiesStr = ((String) new GsonBuilder().create().fromJson(nowCookiesJson, Map.class) .get("Cookie")).trim(); String tokenStr = tokenEts.get(0).attr("content"); String HOST_OF_JILIN = "http://211.141.74.198:8081/aiccips/pub/"; String HOST_OF_XQ = "http://211.141.74.198:8081/"; String htmlAnchorHref = ""; for (Element divDataItem : divDataItems) { Element htmlAnchor = divDataItem.getElementsByTag("a").get(0); String htmlAnchorText = htmlAnchor.text(); if (htmlAnchorText.contains(keyword)) { htmlAnchorHref = HOST_OF_JILIN + htmlAnchor.attr("href"); break; } } if (StringUtils.isEmpty(htmlAnchorHref)) { htmlAnchorHref = "http://211.141.74.198:8081/aiccips/pub/" + divDataItems.get(0).getElementsByTag("a").get(0).attr("href"); } String commonUrl = htmlAnchorHref.split("gsgsdetail")[1]; String commonUrlZ = htmlAnchorHref.substring(htmlAnchorHref.lastIndexOf("/") + 1, htmlAnchorHref.length()); // ?->? String[] command11 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/getSimpleRequestPage.js", "--web-security=no", "--url=" + htmlAnchorHref }; String casperjsResult11 = CommandUtil.runCommand(command11); resultHtmlMap.put("gsgsxx", casperjsResult11); Thread.sleep(1000); // ?->?->?? String baxxZyryxxUrl = HOST_OF_JILIN + "gsryxx/1151?encrpripid=" + commonUrlZ; String[] command121 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/postJilinSimpleRequestPage.js", "--web-security=no", "--cookieStr=" + nowCookiesStr, "--refererStr=" + htmlAnchorHref, "--tokenStr=" + tokenStr, "--url=" + baxxZyryxxUrl }; String casperjsResult121 = CommandUtil.runCommand(command121); resultHtmlMap.put("gsgsxx_baxx_zyryxx", casperjsResult121); // ?->?->? String baxxFzjgxxUrl = HOST_OF_JILIN + "gsfzjg/1151?encrpripid=" + commonUrlZ; String[] command123 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/postJilinSimpleRequestPage.js", "--web-security=no", "--cookieStr=" + nowCookiesStr, "--refererStr=" + htmlAnchorHref, "--tokenStr=" + tokenStr, "--url=" + baxxFzjgxxUrl }; String casperjsResult123 = CommandUtil.runCommand(command123); resultHtmlMap.put("gsgsxx_baxx_fzjgxx", casperjsResult123); // ?->?->? String dcdydjxxDcdydjxxUrl = HOST_OF_JILIN + "gsdcdy?encrpripid=" + commonUrlZ; String[] command131 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/postJilinSimpleRequestPage.js", "--web-security=no", "--cookieStr=" + nowCookiesStr, "--refererStr=" + htmlAnchorHref, "--tokenStr=" + tokenStr, "--url=" + dcdydjxxDcdydjxxUrl }; String casperjsResult131 = CommandUtil.runCommand(command131); resultHtmlMap.put("gsgsxx_dcdydjxx_dcdydjxx", casperjsResult131); // ?->??->?? String gqczdjxxGqczdjxxUrl = HOST_OF_JILIN + "gsgqcz?encrpripid=" + commonUrlZ; String[] command141 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/postJilinSimpleRequestPage.js", "--web-security=no", "--cookieStr=" + nowCookiesStr, "--refererStr=" + htmlAnchorHref, "--tokenStr=" + tokenStr, "--url=" + gqczdjxxGqczdjxxUrl }; String casperjsResult141 = CommandUtil.runCommand(command141); resultHtmlMap.put("gsgsxx_gqczdjxx_gqczdjxx", casperjsResult141); // ?->?->? String xzcfxxXzcfxxUrl = HOST_OF_JILIN + "gsxzcfxx?encrpripid=" + commonUrlZ; String[] command151 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/postJilinSimpleRequestPage.js", "--web-security=no", "--cookieStr=" + nowCookiesStr, "--refererStr=" + htmlAnchorHref, "--tokenStr=" + tokenStr, "--url=" + xzcfxxXzcfxxUrl }; String casperjsResult151 = CommandUtil.runCommand(command151); resultHtmlMap.put("gsgsxx_xzcfxx_xzcfxx", casperjsResult151); // ?->???->??? String jyycxxJyycxxUrl = HOST_OF_JILIN + "jyyc/1151?encrpripid=" + commonUrlZ; String[] command161 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/postJilinSimpleRequestPage.js", "--web-security=no", "--cookieStr=" + nowCookiesStr, "--refererStr=" + htmlAnchorHref, "--tokenStr=" + tokenStr, "--url=" + jyycxxJyycxxUrl }; String casperjsResult161 = CommandUtil.runCommand(command161); resultHtmlMap.put("gsgsxx_jyycxx_jyycxx", casperjsResult161); // ?->???->??? String yzwfxxYzwfxxUrl = HOST_OF_JILIN + "yzwfqy?encrpripid=" + commonUrlZ; String[] command171 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/postJilinSimpleRequestPage.js", "--web-security=no", "--cookieStr=" + nowCookiesStr, "--refererStr=" + htmlAnchorHref, "--tokenStr=" + tokenStr, "--url=" + yzwfxxYzwfxxUrl }; String casperjsResult171 = CommandUtil.runCommand(command171); resultHtmlMap.put("gsgsxx_yzwfxx_yzwfxx", casperjsResult171); // ?->?->? String ccjcxxCcjcxxUrl = HOST_OF_JILIN + "ccjcxx?encrpripid=" + commonUrlZ; String[] command181 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/postJilinSimpleRequestPage.js", "--web-security=no", "--cookieStr=" + nowCookiesStr, "--refererStr=" + htmlAnchorHref, "--tokenStr=" + tokenStr, "--url=" + ccjcxxCcjcxxUrl }; String casperjsResult181 = CommandUtil.runCommand(command181); resultHtmlMap.put("gsgsxx_ccjcxx_ccjcxx", casperjsResult181); // ?? String qygsUrl = HOST_OF_JILIN + "qygsdetail" + commonUrl; String[] command2 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/getSimpleRequestPage.js", "--web-security=no", "--url=" + qygsUrl }; String casperjsResult2 = CommandUtil.runCommand(command2); resultHtmlMap.put("qygsxx_list", casperjsResult2); // ? ??->?-> Document qygsxxHtml = Jsoup.parseBodyFragment(casperjsResult2); Element qynbDiv = qygsxxHtml.getElementById("qiyenianbao"); if (null != qynbDiv) { Elements qynb_trs = qynbDiv.select("tbody").get(0).select("tr"); if (null != qynb_trs && qynb_trs.size() > 2) { List<Map<String, Object>> qygsxx_qynb_infos = new ArrayList<Map<String, Object>>(); for (int i = 2; i < qynb_trs.size(); i++) { Map<String, Object> qygsxx_qynb_info_map = new LinkedHashMap<String, Object>(); Element wdd = qynb_trs.get(i).select("td").get(1).select("a").get(0); String qygsxx_qynb_list_a_text = wdd.text(); String qygsxx_qynb_list_pubdate = qynb_trs.get(i).select("td").get(2).text(); qygsxx_qynb_info_map.put("qygsxx_qynb_list_a_text", qygsxx_qynb_list_a_text); qygsxx_qynb_info_map.put("qygsxx_qynb_list_pubdate", qygsxx_qynb_list_pubdate); String qynbxqUrl = HOST_OF_XQ + wdd.attr("href"); String[] command21 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/getSimpleRequestPage.js", "--web-security=no", "--url=" + qynbxqUrl }; String casperjsResult21 = CommandUtil.runCommand(command21); qygsxx_qynb_info_map.put("qygsxx_qynb_info_page", casperjsResult21); qygsxx_qynb_infos.add(qygsxx_qynb_info_map); } resultHtmlMap.put("qygsxx_qynb_infos", qygsxx_qynb_infos); } } Thread.sleep(1000); // ??->??->?? String gdjczxxGdjczxxUrl = HOST_OF_JILIN + "qygsjsxxxzczxx?encrpripid=" + commonUrlZ; String[] command221 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/postJilinSimpleRequestPage.js", "--web-security=no", "--cookieStr=" + nowCookiesStr, "--refererStr=" + qygsUrl, "--tokenStr=" + tokenStr, "--url=" + gdjczxxGdjczxxUrl }; String casperjsResult221 = CommandUtil.runCommand(command221); resultHtmlMap.put("qygsxx_gdjczxx_gdjczxx", casperjsResult221); // ??->??->?? String gdjczxxBgxxUrl = HOST_OF_JILIN + "qygsjsxxczxxbgsx?encrpripid=" + commonUrlZ; String[] command222 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/postJilinSimpleRequestPage.js", "--web-security=no", "--cookieStr=" + nowCookiesStr, "--refererStr=" + qygsUrl, "--tokenStr=" + tokenStr, "--url=" + gdjczxxBgxxUrl }; String casperjsResult222 = CommandUtil.runCommand(command222); resultHtmlMap.put("qygsxx_gdjczxx_bgxx", casperjsResult222); // ??->???->??? String gqbgxxGqbgxxUrl = HOST_OF_JILIN + "qygsJsxxgqbg?encrpripid=" + commonUrlZ; String[] command231 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/postJilinSimpleRequestPage.js", "--web-security=no", "--cookieStr=" + nowCookiesStr, "--refererStr=" + qygsUrl, "--tokenStr=" + tokenStr, "--url=" + gqbgxxGqbgxxUrl }; String casperjsResult231 = CommandUtil.runCommand(command231); resultHtmlMap.put("qygsxx_gqbgxx_gqbgxx", casperjsResult231); // ??->??->?? String xzxkxxXzxkxxUrl = HOST_OF_JILIN + "qygsjsxxxzxk?encrpripid=" + commonUrlZ; String[] command241 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/postJilinSimpleRequestPage.js", "--web-security=no", "--cookieStr=" + nowCookiesStr, "--refererStr=" + qygsUrl, "--tokenStr=" + tokenStr, "--url=" + xzxkxxXzxkxxUrl }; String casperjsResult241 = CommandUtil.runCommand(command241); resultHtmlMap.put("qygsxx_xzxkxx_xzxkxx", casperjsResult241); // ??->??->?? String zscqczZscqczUrl = HOST_OF_JILIN + "/qygsjsxxzscqcz?encrpripid=" + commonUrlZ; String[] command251 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/postJilinSimpleRequestPage.js", "--web-security=no", "--cookieStr=" + nowCookiesStr, "--refererStr=" + qygsUrl, "--tokenStr=" + tokenStr, "--url=" + zscqczZscqczUrl }; String casperjsResult251 = CommandUtil.runCommand(command251); resultHtmlMap.put("qygsxx_zscqcz_zscqcz", casperjsResult251); // ??->?->? String qygsxxXzcfxxUrl = HOST_OF_JILIN + "qygsjsxxxzcfxx?encrpripid=" + commonUrlZ; String[] command261 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/postJilinSimpleRequestPage.js", "--web-security=no", "--cookieStr=" + nowCookiesStr, "--refererStr=" + qygsUrl, "--tokenStr=" + tokenStr, "--url=" + qygsxxXzcfxxUrl }; String casperjsResult261 = CommandUtil.runCommand(command261); resultHtmlMap.put("qygsxx_zscqcz_zscqcz", casperjsResult261); // ? String qtbmUrl = HOST_OF_JILIN + "qtgsdetail" + commonUrl; String[] command3 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/getSimpleRequestPage.js", "--web-security=no", "--url=" + qtbmUrl }; String casperjsResult3 = CommandUtil.runCommand(command3); resultHtmlMap.put("qtbmgsxx", casperjsResult3); // ???? String sfxzUrl = HOST_OF_JILIN + "sfgsdetail" + commonUrl; String[] command4 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/getSimpleRequestPage.js", "--web-security=no", "--url=" + sfxzUrl }; String casperjsResult4 = CommandUtil.runCommand(command4); resultHtmlMap.put("sfxzgsxx_list", casperjsResult4); resultHtmlMap.put("statusCodeDef", StatusCodeDef.SCCCESS); } return resultHtmlMap; }
From source file:com.storm.function.GsxtFunction.java
private Map<String, Object> getHtmlInfoMapOfShandong(String area, HtmlPage firstInfoPage, String keyword, ChannelLogger LOGGER) throws Exception { Map<String, Object> resultHtmlMap = new LinkedHashMap<String, Object>(); //DomElement checkcode = firstInfoPage.getElementById("checkNo"); WebWindow window = firstInfoPage.getWebClient().getCurrentWindow(); //???/*from w ww. j av a2s .c o m*/ HtmlElement divByXPathyzm = (HtmlElement) firstInfoPage .getFirstByXPath("//div[@class='input-center3']/font"); if (divByXPathyzm != null) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.IMAGECODE_ERROR); } else { HtmlElement divByXPath = ((HtmlElement) firstInfoPage.getFirstByXPath("//div[@class='list']")); //? if (divByXPath != null) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.SCCCESS); } else { resultHtmlMap.put("statusCodeDef", StatusCodeDef.NO_DATA_FOUND); } } @SuppressWarnings("unchecked") List<HtmlAnchor> anchors = (List<HtmlAnchor>) firstInfoPage.getByXPath("//div[@class='list']/ul/li/a"); LOGGER.info(anchors.toString()); if (anchors != null && !anchors.isEmpty()) { boolean matchFlag = false; for (HtmlAnchor anchor : anchors) { String anchorTitle = anchor.getTextContent().toString().trim(); if (anchorTitle.contains(keyword)) { //???? matchFlag = true; //??? HtmlElement target_item_info = (HtmlElement) anchor.getParentNode().getParentNode(); resultHtmlMap.put("target_item_info", target_item_info.asXml()); Elements e1 = Jsoup.parseBodyFragment(target_item_info.asXml()).getElementsByClass("font16"); Element element1 = e1.get(0); Element aElement = element1.select("a").get(0); //String hrefElement = getElementAttr(aElement, "href"); String hrefElement = aElement.hasAttr("href") ? aElement.attr("href") : ""; String encrpripid = hrefElement.substring(hrefElement.lastIndexOf("/") + 1); String enttype = hrefElement.split("/")[1]; String gsurl = "http://218.57.139.24/pub/" + hrefElement; //System.out.println(hrefElement); String mystr = hrefElement.split("gsgsdetail")[1]; //System.out.println(mystr); //?? ?->? // HtmlPage qyxx_page = anchor.click(); if (!qyxx_page.asXml().contains("?")) { return resultHtmlMap; } resultHtmlMap.put("qyxx_gsgsxx", qyxx_page.asXml()); String[] command = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/getSimpleRequestPage.js", "--web-security=no", "--url=" + gsurl }; String qyxx_gsgsxx_djxx = CommandUtil.runCommand(command); resultHtmlMap.put("qyxx_gsgsxx_djxx", qyxx_gsgsxx_djxx); //? ?->?->??var url = webroot+"pub/gsczxx"; WebRequest czxxWebRequest = new WebRequest( new URL("http://218.57.139.24/pub/gsryxx/" + enttype + "?encrpripid=" + encrpripid), HttpMethod.POST); DomElement csrfMetaEle = qyxx_page.getFirstByXPath("//meta[@name='_csrf']"); String csrfToken = ""; if (csrfMetaEle != null) { csrfToken = csrfMetaEle.getAttribute("content"); } czxxWebRequest.setAdditionalHeader("X-CSRF-TOKEN", csrfToken); //window, new WebRequest(new URL( Page zyryxxPage = qyxx_page.getWebClient().getPage(window, czxxWebRequest); resultHtmlMap.put("qyxx_gsgsxx_baxx_zyryxx", zyryxxPage.getWebResponse().getContentAsString()); //System.out.println(zyryxxPage.getWebResponse().getContentAsString()); //? ?->?-> WebRequest czxxWebRequestfzjg = new WebRequest( new URL("http://218.57.139.24/pub/gsfzjg/" + enttype + "?encrpripid=" + encrpripid), HttpMethod.POST); czxxWebRequestfzjg.setAdditionalHeader("X-CSRF-TOKEN", csrfToken); // Page czxxPage = qyxx_page.getWebClient().getPage(window, czxxWebRequestfzjg); resultHtmlMap.put("qyxx_gsgsxx_baxx_fzjgxx", czxxPage.getWebResponse().getContentAsString()); //System.out.println(czxxPage.getWebResponse().getContentAsString()); //? ?->? WebRequest czxxWebRequestdcdydjxx = new WebRequest( new URL("http://218.57.139.24/pub/gsdcdy?encrpripid=" + encrpripid), HttpMethod.POST); czxxWebRequestdcdydjxx.setAdditionalHeader("X-CSRF-TOKEN", csrfToken); // Page czxxPagedcdydjxx = qyxx_page.getWebClient().getPage(window, czxxWebRequestdcdydjxx); resultHtmlMap.put("qyxx_gsgsxx_dcdydjxx", czxxPagedcdydjxx.getWebResponse().getContentAsString()); //System.out.println(czxxPage.getWebResponse().getContentAsString()); //? ?->?? WebRequest czxxWebRequestgqczdjxx = new WebRequest( new URL("http://218.57.139.24/pub/gsgqcz?encrpripid=" + encrpripid), HttpMethod.POST); czxxWebRequestgqczdjxx.setAdditionalHeader("X-CSRF-TOKEN", csrfToken); // Page czxxPagegqczdjxx = qyxx_page.getWebClient().getPage(window, czxxWebRequestdcdydjxx); resultHtmlMap.put("qyxx_gsgsxx_gqczdjxx", czxxPagegqczdjxx.getWebResponse().getContentAsString()); //System.out.println(czxxPage.getWebResponse().getContentAsString()); //? ?->? WebRequest czxxWebRequestxzcfxx = new WebRequest( new URL("http://218.57.139.24/pub/gsxzcfxx?encrpripid=" + encrpripid), HttpMethod.POST); czxxWebRequestxzcfxx.setAdditionalHeader("X-CSRF-TOKEN", csrfToken); // Page czxxPagegxzcfxx = qyxx_page.getWebClient().getPage(window, czxxWebRequestxzcfxx); resultHtmlMap.put("qyxx_gsgsxx_xzcfxx", czxxPagegxzcfxx.getWebResponse().getContentAsString()); //? ?->???var url = webroot+"pub/jyyc/"+enttype; WebRequest czxxWebRequestjyjcxx = new WebRequest( new URL("http://218.57.139.24/pub/jyyc/" + enttype + "?encrpripid=" + encrpripid), HttpMethod.POST); czxxWebRequestjyjcxx.setAdditionalHeader("X-CSRF-TOKEN", csrfToken); // Page czxxPagegjyjcxx = qyxx_page.getWebClient().getPage(window, czxxWebRequestjyjcxx); resultHtmlMap.put("qyxx_gsgsxx_jyjcxx", czxxPagegjyjcxx.getWebResponse().getContentAsString()); //System.out.println( czxxPagegjyjcxx.getWebResponse().getContentAsString()); //? ?->???var url = webroot+"pub/yzwfqy"; WebRequest czxxWebRequestyzwfxx = new WebRequest( new URL("http://218.57.139.24/pub/yzwfqy?encrpripid=" + encrpripid), HttpMethod.POST); czxxWebRequestyzwfxx.setAdditionalHeader("X-CSRF-TOKEN", csrfToken); // Page czxxPagegyzwfxx = qyxx_page.getWebClient().getPage(window, czxxWebRequestyzwfxx); resultHtmlMap.put("qyxx_gsgsxx_yzwfxx", czxxPagegyzwfxx.getWebResponse().getContentAsString()); //? ?->? var url = webroot+"pub/ccjcxx"; WebRequest czxxWebRequestcxjcxx = new WebRequest( new URL("http://218.57.139.24/pub/ccjcxx?encrpripid=" + encrpripid), HttpMethod.POST); czxxWebRequestcxjcxx.setAdditionalHeader("X-CSRF-TOKEN", csrfToken); // Page czxxPagegcxjcxx = qyxx_page.getWebClient().getPage(window, czxxWebRequestyzwfxx); resultHtmlMap.put("qyxx_gsgsxx_cxjcxx", czxxPagegcxjcxx.getWebResponse().getContentAsString()); //???? // HtmlElement qyxx_qygsxx = (HtmlElement)qyxx_page.getByXPath("//div[@id='leftTabs']/ul/li").get(1); // HtmlPage qygsxx_page = (HtmlPage)qyxx_qygsxx.click(); // resultHtmlMap.put("qygsxx", qygsxx_page.asXml()); // System.out.println(qygsxx_page.asXml()); //http://218.57.139.24/pub/qygsdetail/1100/2396ed6cd3e0e1a30bc8098cadaef458e48f827ea3353ac3b826876e37a1ca6f String gsgsxx_sfxzgsxx_url_hqqygsxx = "http://218.57.139.24/pub/qygsdetail" + mystr; HtmlPage qygsxx_page = firstInfoPage.getWebClient().getPage(window, new WebRequest(new URL(gsgsxx_sfxzgsxx_url_hqqygsxx))); resultHtmlMap.put("qygsxx", qygsxx_page.asXml()); // String[] command2 = {"casperjs", "/home/ubuntu/nfs-images/casperjscode/getSimpleRequestPage.js", "--web-security=no", "--url="+gsgsxx_sfxzgsxx_url_hqqygsxx}; // String qygsxx = CommandUtil.runCommand(command2); // resultHtmlMap.put("qygsxx", qygsxx); //<a href="http://218.57.139.24/pub/nb/detail/1100/0453801D2A010583E05012AC9E011868" target="_blank"> List<HtmlAnchor> anchors_detail = (List<HtmlAnchor>) qygsxx_page .getByXPath("//div[@id='qiyenianbao']/table[@class='detailsList']/tbody/tr/td/a"); List<String> nbxx_list = new ArrayList<String>(); for (HtmlAnchor htmlAnchor : anchors_detail) { String attribute = htmlAnchor.getAttribute("href"); String nburldetail = "http://218.57.139.24" + attribute; String[] command2 = { "casperjs", "/home/ubuntu/nfs-images/casperjscode/getSimpleRequestPage.js", "--web-security=no", "--url=" + nburldetail }; String nianbaodetail = CommandUtil.runCommand(command2); //HtmlPage nb_detail = htmlAnchor.click(); nbxx_list.add(nianbaodetail); } resultHtmlMap.put("qygsxx_qynb_detail", nbxx_list); //??->??->?? var url = webroot+"pub/qygsjsxxxzczxx"; WebRequest czxxWebRequestgdjczxx = new WebRequest( new URL("http://218.57.139.24/pub/qygsjsxxxzczxx?encrpripid=" + encrpripid), HttpMethod.POST); DomElement csrfMetaEle2 = qygsxx_page.getFirstByXPath("//meta[@name='_csrf']"); String csrfToken2 = ""; if (csrfMetaEle2 != null) { csrfToken2 = csrfMetaEle.getAttribute("content"); } czxxWebRequestgdjczxx.setAdditionalHeader("X-CSRF-TOKEN", csrfToken2); // Page czxxPageggdjczxx = qyxx_page.getWebClient().getPage(window, czxxWebRequestgdjczxx); resultHtmlMap.put("qyxx_qygsxx_gdjczxx", czxxPageggdjczxx.getWebResponse().getContentAsString()); //??->??->?? var url = webroot+"pub/qygsjsxxczxxbgsx"; WebRequest czxxWebRequestbgxx = new WebRequest( new URL("http://218.57.139.24/pub/qygsjsxxczxxbgsx?encrpripid=" + encrpripid), HttpMethod.POST); czxxWebRequestbgxx.setAdditionalHeader("X-CSRF-TOKEN", csrfToken2); // Page czxxPagegbgxx = qyxx_page.getWebClient().getPage(window, czxxWebRequestgdjczxx); resultHtmlMap.put("qyxx_qygsxx_gdjczxx_bgxx", czxxPagegbgxx.getWebResponse().getContentAsString()); //??->??? var url = webroot+"pub/qygsJsxxgqbg"; WebRequest czxxWebRequestgqbgxx = new WebRequest( new URL("http://218.57.139.24/pub/qygsJsxxgqbg?encrpripid=" + encrpripid), HttpMethod.POST); czxxWebRequestgqbgxx.setAdditionalHeader("X-CSRF-TOKEN", csrfToken2); // Page czxxPageggqbgxx = qyxx_page.getWebClient().getPage(window, czxxWebRequestgqbgxx); resultHtmlMap.put("qyxx_qygsxx_gqbgxx", czxxPageggqbgxx.getWebResponse().getContentAsString()); //??->?? var url = webroot+"pub/qygsjsxxxzxk"; WebRequest czxxWebRequestxzxkxx = new WebRequest( new URL("http://218.57.139.24/pub/qygsjsxxxzxk?encrpripid=" + encrpripid), HttpMethod.POST); czxxWebRequestxzxkxx.setAdditionalHeader("X-CSRF-TOKEN", csrfToken2); // Page czxxPagegxzxkxx = qyxx_page.getWebClient().getPage(window, czxxWebRequestxzxkxx); resultHtmlMap.put("qyxx_qygsxx_xzxkxx", czxxPagegxzxkxx.getWebResponse().getContentAsString()); //??->?? var url = webroot+"pub/qygsjsxxzscqcz"; WebRequest czxxWebRequestzscqczdjxx = new WebRequest( new URL("http://218.57.139.24/pub/qygsjsxxzscqcz?encrpripid=" + encrpripid), HttpMethod.POST); czxxWebRequestzscqczdjxx.setAdditionalHeader("X-CSRF-TOKEN", csrfToken2); // Page czxxPagegzscqczdjxx = qyxx_page.getWebClient().getPage(window, czxxWebRequestzscqczdjxx); resultHtmlMap.put("qyxx_qygsxx_zscqczdjxx", czxxPagegzscqczdjxx.getWebResponse().getContentAsString()); //??->? var url = webroot+"pub/qygsjsxxxzcfxx"; WebRequest czxxWebRequestxzcfxxx = new WebRequest( new URL("http://218.57.139.24/pub/qygsjsxxxzcfxx?encrpripid=" + encrpripid), HttpMethod.POST); czxxWebRequestxzcfxxx.setAdditionalHeader("X-CSRF-TOKEN", csrfToken2); // Page czxxPagegxzcfxxx = qyxx_page.getWebClient().getPage(window, czxxWebRequestxzcfxxx); resultHtmlMap.put("qyxx_qygsxx_xzcfxxx", czxxPagegxzcfxxx.getWebResponse().getContentAsString()); //??? String gsgsxx_sfxzgsxx_url_hqqtbmgsxx = "http://218.57.139.24/pub/qtgsdetail" + mystr; HtmlPage qtbmgsxx_page = firstInfoPage.getWebClient().getPage(window, new WebRequest(new URL(gsgsxx_sfxzgsxx_url_hqqtbmgsxx))); resultHtmlMap.put("qtbmgsxx", qtbmgsxx_page.asXml()); // String[] command3 = {"casperjs", "/home/ubuntu/nfs-images/casperjscode/getSimpleRequestPage.js", "--web-security=no", "--url="+gsgsxx_sfxzgsxx_url_hqqtbmgsxx}; // String qtbmgsxx = CommandUtil.runCommand(command3); // resultHtmlMap.put("qygsxx", qtbmgsxx); //?????? //http://218.57.139.24/pub/sfgsdetail/1130/95f6c493f094da93009e08daa27616d8 //String gsgsxx_sfxzgsxx_url = "http://218.57.139.24/pub/sfgsdetail"+mystr ; //HtmlPage gsgsxx_baxx_zyryxx_page = firstInfoPage.getWebClient().getPage(gsgsxx_sfxzgsxx_url); //resultHtmlMap.put("gsgsxx_baxx_zyryxx", gsgsxx_baxx_zyryxx_page.asXml()); //?????? //HtmlElement sfxzgsxx_tab = (HtmlElement)qyxx_page.getByXPath("//div[@id='leftTabs']/ul/li").get(3); String gsgsxx_sfxzgsxx_url = "http://218.57.139.24/pub/sfgsdetail" + mystr; HtmlPage sfxzgsxx_page = firstInfoPage.getWebClient().getPage(window, new WebRequest(new URL(gsgsxx_sfxzgsxx_url))); resultHtmlMap.put("sfxzgsxx_page", sfxzgsxx_page.asXml()); // String[] command4 = {"casperjs", "/home/ubuntu/nfs-images/casperjscode/getSimpleRequestPage.js", "--web-security=no", "--url="+gsgsxx_sfxzgsxx_url}; // String sfxzgsxx = CommandUtil.runCommand(command4); // resultHtmlMap.put("sfxzgsxx_page", sfxzgsxx); //http://218.57.139.24/pub/sfgsgqxxdetail/95f6c493f094da93009e08daa27616d8/1130/12D1EA5D6111126BE054/1 String mystrdetil = ""; if (mystr != null && !"".equals(mystr)) { String mystrspill[] = mystr.split("/"); mystrdetil = "/" + mystrspill[2] + "/" + mystrspill[1]; } String mystrdetilurl = "http://218.57.139.24/pub/sfgsgqxxdetail" + mystrdetil; String urlstring = sfxzgsxx_page.asXml(); List<String> sfxzxxlist = new AbstractParser() { }.getSubStringByRegex(urlstring, "var gqxxliststr ='\\[.*\\]"); String sfxzgsxxzzfc = sfxzxxlist.get(0).substring(19, sfxzxxlist.get(0).length() - 1); if (!"".equals(sfxzgsxxzzfc) && null != sfxzgsxxzzfc) { String[] sfxzgsxxzzfcsplil = sfxzgsxxzzfc.split(","); List<String> urllist = new ArrayList<String>(); String num = ""; for (int m = 0; m < sfxzgsxxzzfcsplil.length; m++) { String strname = sfxzgsxxzzfcsplil[m]; String strname1[] = strname.split(":"); String myname = strname1[0]; String strna = strname1[1]; String namesss = ""; if ("\"frozstate\"".equals(myname)) { // ? namesss = strna.substring(1, strna.length() - 1); num = namesss; } if ("\"pid\"".equals(myname)) { // ??? namesss = strna.substring(1, strna.length() - 1); mystrdetilurl = mystrdetilurl + "/" + namesss; mystrdetilurl = mystrdetilurl + "/" + num; urllist.add(mystrdetilurl); } } List<String> gqdjxx_list = new ArrayList<String>(); for (int i = 0; i < urllist.size(); i++) { HtmlPage sfxzgsxx_page_detail = firstInfoPage.getWebClient().getPage(window, new WebRequest(new URL(urllist.get(i)))); gqdjxx_list.add(sfxzgsxx_page_detail.asXml()); } resultHtmlMap.put("sfxzgsxx_gqdjxx_detail", gqdjxx_list); } break; } } if (!matchFlag) { resultHtmlMap.put("statusCodeDef", StatusCodeDef.NO_DATA_FOUND); LOGGER.info("????"); } } return resultHtmlMap; }
From source file:com.weavers.duqhan.business.impl.ProductServiceImpl.java
@Override public void loadTempProducts(List<StatusBean> statusBeans) { boolean isSuccess = true; String startDate = new Date().toString(); Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, "(==I==)DATE: " + startDate + "Store product details in temp product table start....."); try {/* ww w . ja v a 2 s . c o m*/ String status = ""; for (StatusBean statusBean : statusBeans) { status = "Link duplicate"; Temtproductlinklist temtproductlinklist = temtproductlinklistDao.loadById(statusBean.getId()); if (temtproductlinklist != null && temtproductlinklist.getStatus() == 0) { Product testProduct = productDao.getProductByExternelLink(temtproductlinklist.getLink()); if (testProduct == null) { String value = ""; Elements detailMain; Elements detailSub; Elements specifics; double votes = 0.0; double stars = 0.0; double feedback = 0.0; String url = temtproductlinklist.getLink(); try { testProduct = new Product(); Product savedTestProduct; //=================== Random sleep START ===================// // TimeUnit.SECONDS.sleep(30 + (int) (Math.random() * 100)); Random randomObj = new Random(); TimeUnit.SECONDS.sleep(randomObj.ints(30, 60).findFirst().getAsInt()); //=================== Random sleep END =====================// Document doc = Jsoup.connect(url).get(); detailMain = doc.select("#j-detail-page"); if (!detailMain.isEmpty()) { //=================== Criteria Block START==================// detailMain = doc.select(".rantings-num"); if (!detailMain.isEmpty()) { votes = Double.valueOf(detailMain.text().split(" votes")[0].split("\\(")[1]); } detailMain = doc.select(".percent-num"); if (!detailMain.isEmpty()) { stars = Double.valueOf(detailMain.text()); } detailMain = doc.select("ul.ui-tab-nav li[data-trigger='feedback'] a"); if (!detailMain.isEmpty()) { feedback = Double.valueOf(detailMain.text().split("\\(")[1].split("\\)")[0]); } //=================== Criteria Block END==================// if (votes > 10.0 && stars > 4.0 && feedback > 4.0) { detailMain = doc.select(".detail-wrap .product-name"); testProduct.setName(detailMain .text());/*.substring(0, Math.min(detailMain.text().length(), 50))*/ detailMain = doc.select(".detail-wrap .product-name"); testProduct.setDescription(detailMain.text()); testProduct.setExternalLink(url); testProduct.setVendorId(1l);//?????????????????????? //=================== Packaging block START==================// Double weight = 1.0; Double width = 1.0; Double height = 1.0; Double length = 1.0; detailMain = doc.select( "div#j-product-desc div.pnl-packaging-main ul li.packaging-item"); for (Element element : detailMain) { String packagingTitle = element.select("span.packaging-title").text(); String packagingDesc = element.select("span.packaging-des").text(); if (packagingTitle.trim().equals("Package Weight:")) { String str = packagingDesc; str = str.replaceAll("[^.?0-9]+", " "); if (Arrays.asList(str.trim().split(" ")) != null) { if (!Arrays.asList(str.trim().split(" ")).isEmpty()) { try { weight = Double.parseDouble( Arrays.asList(str.trim().split(" ")).get(0)); } catch (Exception e) { weight = 1.0; } } } System.out.println("weight == " + weight); } else if (packagingTitle.trim().equals("Package Size:")) { String str = packagingDesc; str = str.replaceAll("[^.?0-9]+", " "); if (Arrays.asList(str.trim().split(" ")) != null) { if (!Arrays.asList(str.trim().split(" ")).isEmpty()) { try { width = Double.parseDouble( Arrays.asList(str.trim().split(" ")).get(0)); height = Double.parseDouble( Arrays.asList(str.trim().split(" ")).get(1)); length = Double.parseDouble( Arrays.asList(str.trim().split(" ")).get(2)); } catch (Exception e) { width = 1.0; height = 1.0; length = 1.0; } } } System.out.println("width == " + width); System.out.println("height == " + height); System.out.println("length == " + length); } } //=================== Packaging block END==================// //=================== Category block START==================// detailMain = doc.select("div.ui-breadcrumb div.container a"); Long productCategoryId = 0L; String parentPath = ""; String thisCategory = detailMain.last().text().trim(); System.out.println("thisCategory == " + thisCategory); Category parentCategory = new Category(); parentCategory.setId(0L); parentCategory.setParentPath(""); for (Element element : detailMain) { String newCategory; newCategory = element.text().trim(); System.out.println("newCategory======" + newCategory); if (newCategory.equals("Home") || newCategory.equals("All Categories")) { } else { Category category = categoryDao.getCategoryByName(newCategory); if (category != null) { if (category.getName().equals(thisCategory)) { productCategoryId = category.getId(); parentPath = category.getParentPath(); } parentCategory = category; } else { category = new Category(); category.setId(null); category.setName(newCategory); category.setParentId(parentCategory.getId()); category.setParentPath(parentCategory.getParentPath() + parentCategory.getId() + "="); category.setQuantity(0); category.setImgUrl("-"); category.setDisplayText(newCategory); Category category2 = categoryDao.save(category); if (category.getName().equals(thisCategory)) { productCategoryId = category2.getId(); parentPath = category2.getParentPath(); } parentCategory = category2; } } } //=================== Category block END==================// //=============== Specifications block START==============// detailMain = doc.select(".product-property-list .property-item"); String specifications = ""; for (Element element : detailMain) { specifications = specifications + element.select(".propery-title").get(0).text().replace(",", "/") .replace(":", "-") + ":" + element.select(".propery-des").get(0).text() .replace(",", "/").replace(":", "-") + ",";//TODO:, check } //=============== Specifications Block END==============// //=============== Shipping Time Block START==============// String shippingTime = ""; detailMain = doc.select(".shipping-days[data-role='delivery-days']"); System.out.println("value detailMain" + detailMain.toString()); shippingTime = detailMain.text(); //=============== Shipping Time Block END==============// //=============== Shipping Cost Block START==============// detailMain = doc.select(".logistics-cost"); value = detailMain.text(); if (!value.equalsIgnoreCase("Free Shipping")) { // f = 0.00; } else { // f = Double.parseDouble(value.replaceAll(".*?([\\d.]+).*", "$1")); } //=============== Shipping Cost Block END==============// //=================Product save 1st START==============// testProduct.setCategoryId(productCategoryId); testProduct.setLastUpdate(new Date()); testProduct.setParentPath(parentPath); testProduct.setImgurl("-"); testProduct.setProperties("-"); testProduct.setProductWidth(width); testProduct.setProductLength(length); testProduct.setProductWeight(weight); testProduct.setProductHeight(height); testProduct.setShippingRate(0.0); testProduct.setShippingTime("45"); testProduct.setSpecifications(specifications); savedTestProduct = productDao.save(testProduct); //====================Product save 1st END==============// //========= Property, Property Value, Property Product Map Block START ========// double discountPrice = 0.0; double actualPrice = 0.0; double markupPrice = 0.0; String id = ""; String allProperties = ""; //------------------------Read Color css START---------------------// specifics = doc.select("#j-product-info-sku dl.p-property-item"); Elements cssdetailMain = doc.select("link[href]"); Document cssdoc = new Document(""); System.out.println( "====================================================cssdetailMain" + cssdetailMain.size()); for (Element element : cssdetailMain) { String cssurl = element.attr("abs:href"); if (cssurl.contains("??main-detail")) { try { cssdoc = Jsoup.connect(cssurl).get(); } catch (IOException ex) { } break; } } //-----------------------Read Color css END--------------------------// //-----------Product Property, Property Value START--------// Map<String, ProductPropertyvalues> propertyValuesMap = new HashMap<>(); if (!specifics.isEmpty()) { ProductProperties testPorperties; ProductProperties saveTestPorperties; ProductPropertyvalues testPropertyValues; for (Element specific : specifics) { System.out.println("head ==== " + specific.select("dt").text()); testPorperties = productPropertiesDao .loadByName(specific.select("dt").text()); if (testPorperties == null) { testPorperties = new ProductProperties(); testPorperties.setPropertyName(specific.select("dt").text()); saveTestPorperties = productPropertiesDao.save(testPorperties); } else { saveTestPorperties = testPorperties; } allProperties = allProperties + saveTestPorperties.getId().toString() + "-"; detailSub = specific.select("dd ul li"); String valu = "-"; for (Element element : detailSub) { testPropertyValues = new ProductPropertyvalues(); id = element.select("a[data-sku-id]").attr("data-sku-id").trim(); testPropertyValues.setRefId(id); if (element.hasClass("item-sku-image")) { valu = element.select("a img[src]").get(0).absUrl("src") .split(".jpg")[0] + ".jpg"; String title = element.select("a img").get(0).attr("title"); String imgUrl = GoogleBucketFileUploader .uploadProductImage(valu, savedTestProduct.getId()); valu = "<img src='" + imgUrl + "' title='" + title + "' style='height:40px; width:40px;'/>"; } else if (element.hasClass("item-sku-color")) { String style = cssdoc.html().split("sku-color-" + id)[1] .split("}")[0].substring(1); valu = "<span style='" + style + "' ; height:40px; width:40px; display:block;'></span>"; } else { valu = element.select("a span").toString(); } System.out.println("valu === " + valu); testPropertyValues.setProductId(savedTestProduct.getId()); testPropertyValues.setPropertyId(saveTestPorperties.getId()); testPropertyValues.setValueName(valu); propertyValuesMap.put(id, productPropertyvaluesDao.save(testPropertyValues)); } } savedTestProduct.setProperties(allProperties); } //-----------Product Property, Property Value END--------// //----------------------Read json START------------------// List<AxpProductDto> axpProductDtos = new ArrayList<>(); Elements scripts = doc.select("script"); // Get the script part for (Element script : scripts) { if (script.html().contains("var skuProducts=")) { String jsonData = ""; jsonData = script.html().split("var skuProducts=")[1] .split("var GaData")[0].trim(); jsonData = jsonData.substring(0, jsonData.length() - 1); Gson gsonObj = new Gson(); axpProductDtos = Arrays .asList(gsonObj.fromJson(jsonData, AxpProductDto[].class)); break; } } //----------------------Read json END------------------// //-------------Product Properties Map START------------// for (AxpProductDto thisAxpProductDto : axpProductDtos) { SkuVal skuVal = thisAxpProductDto.getSkuVal(); if (skuVal.getActSkuCalPrice() != null) { value = skuVal.getActSkuCalPrice().trim(); discountPrice = CurrencyConverter.usdTOinr( Double.parseDouble(value.replaceAll(".*?([\\d.]+).*", "$1"))); value = skuVal.getSkuCalPrice().trim(); actualPrice = CurrencyConverter.usdTOinr( Double.parseDouble(value.replaceAll(".*?([\\d.]+).*", "$1"))); markupPrice = discountPrice * 0.15 + 100; discountPrice = Math.ceil((discountPrice + markupPrice) / 10) * 10; actualPrice = Math.round(actualPrice + markupPrice); } else { discountPrice = 0.0; value = skuVal.getSkuCalPrice().trim(); actualPrice = CurrencyConverter.usdTOinr( Double.parseDouble(value.replaceAll(".*?([\\d.]+).*", "$1"))); markupPrice = actualPrice * 0.15 + 100; discountPrice = Math.round(actualPrice + markupPrice); actualPrice = Math.round(actualPrice + markupPrice); } ProductPropertiesMap productPropertyMap = new ProductPropertiesMap(); String myPropValueIds = ""; if (thisAxpProductDto.getSkuAttr() != null) { String[] skuPropIds = thisAxpProductDto.getSkuPropIds().split(","); for (String skuPropId : skuPropIds) { myPropValueIds = myPropValueIds + propertyValuesMap.get(skuPropId).getId().toString() + "_"; } productPropertyMap.setPropertyvalueComposition(myPropValueIds); } else { productPropertyMap.setPropertyvalueComposition("_"); } productPropertyMap.setDiscount(discountPrice); productPropertyMap.setPrice(actualPrice); productPropertyMap.setProductId(savedTestProduct); productPropertyMap.setQuantity(5l); productPropertiesMapDao.save(productPropertyMap); } //-------------Product Properties Map START------------// //========= Property, Property Value, Property Product Map Block END ========// //============= Multiple Image Block START =============// detailMain = doc.select("ul.image-thumb-list span.img-thumb-item img[src]"); int flg = 0; String imgUrl = ""; for (Element element : detailMain) { imgUrl = GoogleBucketFileUploader.uploadProductImage( element.absUrl("src").split(".jpg")[0] + ".jpg", savedTestProduct.getId()); if (flg == 0) { flg++; savedTestProduct.setImgurl(imgUrl); } else { ProductImg productImg = new ProductImg(); productImg.setId(null); productImg.setImgUrl(imgUrl); productImg.setProductId(savedTestProduct.getId()); productImgDao.save(productImg); } } //============= Multiple Image Block END =============// //=================Product save final START==============// if (productDao.save(savedTestProduct) != null) { temtproductlinklist.setStatus(1);// temtproductlinklistDao.save(temtproductlinklist); status = "Success"; } //=================Product save final START==============// } else { temtproductlinklist.setStatus(2);// temtproductlinklistDao.save(temtproductlinklist); status = "criteria mismatch"; } } else { status = "Page not found"; } } catch (Exception ex) { System.out.println( "=============================================================Exception1" + ex); temtproductlinklist.setStatus(4);// temtproductlinklistDao.save(temtproductlinklist); System.out.println("Exception === " + ex); status = "Failure"; Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, "(==E==)DATE: " + new Date().toString() + "Store product details in temp product table get error in sub process.....\n Link Id: " + statusBean.getId() + "\n Started on" + startDate, ex); } } else { temtproductlinklist.setStatus(3);// temtproductlinklistDao.save(temtproductlinklist); status = "Product exsist"; } } // String body = "Id: " + temtproductlinklist.getId() + "<br/> Status: " + status; // MailSender.sendEmail("krisanu.nandi@pkweb.in", "Product captured", body, "subhendu.sett@pkweb.in"); statusBean.setStatus(status); } System.out.println("=============================================================status" + status); } catch (Exception e) { System.out.println("=============================================================Exception2" + e); isSuccess = false; String body = "(==E==)DATE: " + new Date().toString() + "Store product details in temp product table get error.....<br/> Started on" + startDate + "<br/>"; Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, body, e); // MailSender.sendEmail("krisanu.nandi@pkweb.in", "Stopped store product details", body + e.getLocalizedMessage(), "subhendu.sett@pkweb.in"); } if (isSuccess) { String body = "(==I==)DATE: " + new Date().toString() + "Store product details in temp product table end.....<br/> Started on" + startDate; Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, body); /*ObjectMapper mapper = new ObjectMapper(); try { MailSender.sendEmail("krisanu.nandi@pkweb.in", "Completed store product details", body + "=============<br/><br/>" + mapper.writeValueAsString(statusBeans), "subhendu.sett@pkweb.in"); } catch (JsonProcessingException ex) { Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, null, ex); }*/ } // return statusBeans; System.out.println("=============================================================end"); }
From source file:ca.zadrox.dota2esportticker.service.UpdateMatchService.java
private void updateMatches(boolean doResults) { if (!checkForConnectivity()) { LocalBroadcastManager.getInstance(this).sendBroadcast(new Intent(UPDATE_NO_CONNECTIVITY)); return;//from w w w .j a va2 s . c o m } LocalBroadcastManager.getInstance(this).sendBroadcast(new Intent(UPDATE_STARTED)); final String BASE_URL = "http://www.gosugamers.net/dota2/gosubet"; final String MATCH_LINK_URL_BASE = "http://www.gosugamers.net"; try { String rawHtml = new OkHttpClient().newCall(new Request.Builder().url(BASE_URL).build()).execute() .body().string(); rawHtml = rawHtml.substring(rawHtml.indexOf("<div id=\"col1\" class=\"rows\">"), rawHtml.indexOf("<div id=\"col2\" class=\"rows\">")); Document doc = Jsoup.parse(rawHtml); Elements tables = doc.getElementsByClass("matches"); ArrayList<ArrayList<String>> matchLinks = new ArrayList<ArrayList<String>>(tables.size()); int numSeries = 0; for (Element table : tables) { Elements links = table.getElementsByClass("match"); if (links.size() != 0) { ArrayList<String> innerMatchLink = new ArrayList<String>(links.size()); for (Element link : links) { String linkHref = link.attr("href"); innerMatchLink.add(MATCH_LINK_URL_BASE + linkHref); numSeries++; } matchLinks.add(innerMatchLink); } } // needed if there are massive reschedules to update content properly. Uri resultsUri = MatchContract.SeriesEntry.buildSeriesUriWithAfterTime(TimeUtils.getUTCTime()); Cursor c = getContentResolver().query(resultsUri, new String[] { MatchContract.SeriesEntry.COLUMN_GG_MATCH_PAGE }, null, null, null); while (c.moveToNext()) { if (!matchLinks.get(0).contains(c.getString(0))) { matchLinks.get(0).add(c.getString(0)); } } Iterator<ArrayList<String>> iterator = matchLinks.iterator(); int numResults = 0; ExecutorService executorService = Executors.newFixedThreadPool(10); ArrayList<Future<BundledMatchItem>> seriesItemFutures = new ArrayList<Future<BundledMatchItem>>( numSeries); LogUtils.LOGD(TAG, "Starting Retrieval, num elements gathered: " + numSeries); int i = 0; while (iterator.hasNext()) { ArrayList<String> matchList = iterator.next(); for (String matchUrl : matchList) { boolean hasResult = !iterator.hasNext(); if (!doResults && hasResult) { continue; } else if (hasResult) { numResults++; } seriesItemFutures.add(executorService.submit(new MatchGetter(matchUrl, hasResult))); i++; } } executorService.shutdown(); executorService.awaitTermination(20L, TimeUnit.SECONDS); LogUtils.LOGD(TAG, "Stopping Retrieval, elements submitted for fetching: " + i); ContentValues[] seriesEntries = new ContentValues[i]; ContentValues[] resultEntries = new ContentValues[numResults]; int seriesEntryWriteIndex = 0; int resultEntryWriteIndex = 0; for (Future<BundledMatchItem> seriesItemFuture : seriesItemFutures) { try { BundledMatchItem seriesItem = seriesItemFuture.get(); if (seriesItem != null) { seriesEntries[seriesEntryWriteIndex] = seriesItem.mMatch; seriesEntryWriteIndex++; if (seriesItem.hasResult) { resultEntries[resultEntryWriteIndex] = seriesItem.mResult; resultEntryWriteIndex++; } } } catch (ExecutionException e) { Log.e(TAG, "Should never get here"); } } this.getContentResolver().bulkInsert(MatchContract.SeriesEntry.CONTENT_URI, seriesEntries); if (doResults) this.getContentResolver().bulkInsert(MatchContract.ResultEntry.CONTENT_URI, resultEntries); PrefUtils.setLastUpdateTime(this, TimeUtils.getUTCTime()); } catch (IOException e) { Log.e(TAG, e.getMessage(), e); e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } LocalBroadcastManager.getInstance(this).sendBroadcast(new Intent(UPDATE_COMPLETE)); PrefUtils.setLastResultsUpdateTime(this, TimeUtils.getUTCTime()); }
From source file:ca.zadrox.dota2esportticker.service.UpdateTeamsService.java
private void updateTopTeams() { LOGD(TAG, "starting update"); // actually, first, check for connectivity: if (!checkForConnectivity()) { LocalBroadcastManager.getInstance(this).sendBroadcast(new Intent(STATUS_NO_CONNECTIVITY)); LOGD(TAG, "returning due to no connectivity"); return;//from w ww. ja v a 2 s . c om } // first, check last update time long lastUpdate = PrefUtils.lastTeamsUpdate(this); long currentTime = TimeUtils.getUTCTime(); // if last update is less than 1 hour old, boot user to cursorloader op. if (currentTime - lastUpdate < 60000 * 60) { LOGD(TAG, "returnning due to too soon"); LocalBroadcastManager.getInstance(this).sendBroadcast(new Intent(STATUS_COMPLETED)); return; } // else // use local broadcast manager to show loading indicator LocalBroadcastManager.getInstance(this).sendBroadcast(new Intent(STATUS_UPDATING)); final String BASE_URL = "http://www.gosugamers.net/dota2/rankings"; final String TEAM_LINK_BASE_URL = "http://www.gosugamers.net/dota2/teams/"; // we see what teams are in top 50. (httpreq -> gosugamers) try { String rawHtml = new OkHttpClient().newCall(new Request.Builder().url(BASE_URL).build()).execute() .body().string(); String processedHtml = rawHtml.substring(rawHtml.indexOf("<div id=\"col1\" class=\"rows\">"), rawHtml.indexOf("<div id=\"col2\" class=\"rows\">")); Elements teamRows = Jsoup.parse(processedHtml).getElementsByClass("ranking-link"); ExecutorService executorService = Executors.newFixedThreadPool(10); ContentValues[] teamRanks = new ContentValues[50]; HashMap<ContentValues, Future<String>> newTeamInfo = new HashMap<ContentValues, Future<String>>(); HashMap<ContentValues, Future<String>> updateTeamInfo = new HashMap<ContentValues, Future<String>>(); int i = 0; for (Element teamRow : teamRows) { ContentValues contentValues = new ContentValues(); String teamId = teamRow.attr("data-id"); contentValues.put(MatchContract.TeamEntry._ID, teamId); String untrimmedTeamName = teamRow.getElementsByTag("h4").first().text(); String teamUrl = TEAM_LINK_BASE_URL + teamId + "-" + untrimmedTeamName.replaceAll("[\\W]?[\\W][\\W]*", "-").toLowerCase(); contentValues.put(MatchContract.TeamEntry.COLUMN_TEAM_URL, teamUrl); String teamName = untrimmedTeamName.replaceAll(" ?\\.?\\-?-?Dot[aA][\\s]?2", ""); contentValues.put(MatchContract.TeamEntry.COLUMN_TEAM_NAME, teamName); if (teamUrl.charAt(teamUrl.length() - 1) == '-') { teamUrl = teamUrl.substring(0, teamUrl.length() - 2); } // then, we query db for id of the team ( Cursor cursor = getContentResolver().query( MatchContract.TeamEntry.buildTeamUri(Long.parseLong(teamId)), new String[] { MatchContract.TeamEntry.COLUMN_TEAM_NAME, MatchContract.TeamEntry.COLUMN_TEAM_URL }, null, null, null); // -> if present, and data remains unchanged, continue. // -> if present, but data is changed, add to update queue. if (cursor.moveToFirst()) { LOGD(TAG, "Have team already?"); if (!cursor.getString(0).contentEquals(teamName) || !cursor.getString(1).contentEquals(teamUrl)) { LOGD(TAG, "Team has updated values."); updateTeamInfo.put(contentValues, executorService.submit(new TeamGetter(teamUrl))); } } // -> if not present, add to update queue. else { LOGD(TAG, "Do team update"); newTeamInfo.put(contentValues, executorService.submit(new TeamGetter(teamUrl))); } // LOGD(TAG, "\n" + // "data-id: " + teamId + "\n" + // "team-name: " + teamName + "\n" + // "team-url: " + teamUrl); teamRanks[i] = new ContentValues(); teamRanks[i].put(MatchContract.TeamRankEntry._ID, i + 1); teamRanks[i].put(MatchContract.TeamRankEntry.COLUMN_TEAM_ID, teamId); cursor.close(); i++; } executorService.shutdown(); executorService.awaitTermination(20, TimeUnit.SECONDS); for (ContentValues contentValues : newTeamInfo.keySet()) { try { String teamLogo = newTeamInfo.get(contentValues).get(); contentValues.put(MatchContract.TeamEntry.COLUMN_TEAM_LOGO_URL, teamLogo); } catch (ExecutionException e) { LocalBroadcastManager.getInstance(this).sendBroadcast(new Intent(STATUS_ERROR)); e.printStackTrace(); } } for (ContentValues contentValues : updateTeamInfo.keySet()) { try { String teamLogo = updateTeamInfo.get(contentValues).get(); contentValues.put(MatchContract.TeamEntry.COLUMN_TEAM_LOGO_URL, teamLogo); String teamId = contentValues.getAsString(MatchContract.TeamEntry._ID); contentValues.remove(MatchContract.TeamEntry._ID); int updatedRows = getContentResolver().update(MatchContract.TeamEntry.CONTENT_URI, contentValues, MatchContract.TeamEntry.TABLE_NAME + "." + MatchContract.TeamEntry._ID + " = ?", new String[] { teamId }); LOGD(TAG, "updatedRows: " + updatedRows); } catch (ExecutionException e) { LocalBroadcastManager.getInstance(this).sendBroadcast(new Intent(STATUS_ERROR)); e.printStackTrace(); } } getContentResolver().bulkInsert(MatchContract.TeamEntry.CONTENT_URI, newTeamInfo.keySet().toArray(new ContentValues[newTeamInfo.size()])); getContentResolver().bulkInsert(MatchContract.TeamRankEntry.CONTENT_URI, teamRanks); } catch (IOException e) { LocalBroadcastManager.getInstance(this).sendBroadcast(new Intent(STATUS_ERROR)); e.printStackTrace(); } catch (InterruptedException e2) { LocalBroadcastManager.getInstance(this).sendBroadcast(new Intent(STATUS_ERROR)); e2.printStackTrace(); } // String[] projection = new String[]{ // MatchContract.TeamEntry.TABLE_NAME + "." + MatchContract.TeamEntry._ID, // MatchContract.TeamEntry.COLUMN_TEAM_NAME, // MatchContract.TeamEntry.COLUMN_TEAM_URL, // MatchContract.TeamEntry.COLUMN_TEAM_LOGO_URL, // MatchContract.TeamEntry.COLUMN_TEAM_STARRED, // MatchContract.TeamRankEntry.TABLE_NAME + "." + MatchContract.TeamRankEntry._ID // }; // // String sortOrder = // MatchContract.TeamRankEntry.TABLE_NAME + "." + // MatchContract.TeamRankEntry._ID + " ASC"; // // Cursor c = getContentResolver().query( // MatchContract.TeamEntry.TOP_50_URI, // projection, // null, // null, // sortOrder // ); // // while (c.moveToNext()) { // String teamPrintOut = // "Rank: " + c.getInt(5) + "\n" + // "teamId: " + c.getInt(0) + " teamName: " + c.getString(1) + "\n" + // "teamUrl: " + c.getString(2) + "\n" + // "teamLogoUrl: " + c.getString(3) + "\n" + // "isFavourited: " + (c.getInt(4) == 0 ? "false" : "true"); // LOGD(TAG + "/UTT", teamPrintOut); // } // // c.close(); // use local broadcast manager to hide loading indicator // and signal that cursorloader for top50 can happen. PrefUtils.setLastTeamUpdate(this, currentTime); LocalBroadcastManager.getInstance(this).sendBroadcast(new Intent(STATUS_COMPLETED)); }