List of usage examples for org.jsoup.select Elements last
public Element last()
From source file:edu.uci.ics.crawler4j.examples.login.LoginCrawlController.java
public static void main(String[] args) throws Exception { // if (args.length != 2) { // System.out.println("Needed parameters: "); // System.out.println("\t rootFolder (it will contain intermediate crawl data)"); // System.out.println("\t numberOfCralwers (number of concurrent threads)"); // return; // }/* w w w .ja va 2 s . c om*/ /* * crawlStorageFolder is a folder where intermediate crawl data is * stored. */ String crawlStorageFolder = "/tmp/test_crawler/"; /* * numberOfCrawlers shows the number of concurrent threads that should * be initiated for crawling. */ int numberOfCrawlers = 1; CrawlConfig config = new CrawlConfig(); config.setCrawlStorageFolder(crawlStorageFolder); /* * Be polite: Make sure that we don't send more than 1 request per * second (1000 milliseconds between requests). */ config.setPolitenessDelay(1000); /* * You can set the maximum crawl depth here. The default value is -1 for * unlimited depth */ config.setMaxDepthOfCrawling(0); /* * You can set the maximum number of pages to crawl. The default value * is -1 for unlimited number of pages */ config.setMaxPagesToFetch(1000); /* * Do you need to set a proxy? If so, you can use: * config.setProxyHost("proxyserver.example.com"); * config.setProxyPort(8080); * * If your proxy also needs authentication: * config.setProxyUsername(username); config.getProxyPassword(password); */ /* * This config parameter can be used to set your crawl to be resumable * (meaning that you can resume the crawl from a previously * interrupted/crashed crawl). Note: if you enable resuming feature and * want to start a fresh crawl, you need to delete the contents of * rootFolder manually. */ config.setResumableCrawling(false); config.setIncludeHttpsPages(true); HttpClient client = new DefaultHttpClient(); HttpResponse response = client.execute(new HttpGet("http://58921.com/user/login")); HttpEntity entity = response.getEntity(); String content = EntityUtils.toString(entity, HTTP.UTF_8); Document doc = Jsoup.parse(content); Elements elements = doc.getElementById("user_login_form").children(); Element tokenEle = elements.last(); String token = tokenEle.val(); System.out.println(token); LoginConfiguration somesite; try { somesite = new LoginConfiguration("58921.com", new URL("http://58921.com/user/login"), new URL("http://58921.com/user/login/ajax?ajax=submit&__q=user/login")); somesite.addParam("form_id", "user_login_form"); somesite.addParam("mail", "paxbeijing@gmail.com"); somesite.addParam("pass", "cetas123"); somesite.addParam("submit", ""); somesite.addParam("form_token", token); config.addLoginConfiguration(somesite); } catch (MalformedURLException e) { e.printStackTrace(); } /* * Instantiate the controller for this crawl. */ PageFetcher pageFetcher = new PageFetcher(config); RobotstxtConfig robotstxtConfig = new RobotstxtConfig(); robotstxtConfig.setEnabled(false); RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher); CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer); /* * For each crawl, you need to add some seed urls. These are the first * URLs that are fetched and then the crawler starts following links * which are found in these pages */ controller.addSeed("http://58921.com/alltime?page=60"); /* * Start the crawl. This is a blocking operation, meaning that your code * will reach the line after this only when crawling is finished. */ controller.start(LoginCrawler.class, numberOfCrawlers); controller.env.close(); }
From source file:popo.defcon.MsgMeCDC.java
void Parse() { String input = readPage();/*w ww .j a v a 2 s . c o m*/ if (input == null) { System.out.println("Error connecting to Internet"); return; } String time; Document cdc = Jsoup.parse(input); Elements notices = cdc.getElementsByTag("tbody"); Elements alerts = notices.get(1).getElementsByTag("tr"); alerts.remove(0); System.out.println("Current Old Time = " + oldtime); for (Element node : alerts) { Elements content = node.getElementsByTag("td"); time = content.last().text(); if (convertTime(time).compareTo(convertTime(oldtime)) <= 0) { MsgMeCDC.oldtime = alerts.get(0).getElementsByTag("td").last().text(); return; } System.out.println("Current notice time :" + convertTime(time)); Logger.getLogger(MsgMeCDC.class.getName()).log(Level.INFO, "Current notice time :" + convertTime(time)); //for (Element text : content) { // System.out.println(text.text()); //} String smsTitle = content.get(1).text(); //String smsCompanyName = content.get(2).text(); String smsNoticeTime = content.get(4).text(); String preSMStext = content.get(3).text(); String randomtext = "Placement/ Internship Form Description Files"; int start = preSMStext.indexOf(randomtext) + randomtext.length() + 1; int twilio = "Sent from your Twilio trial account - ".length(); int end = 150 - (smsTitle.length() + smsNoticeTime.length() + twilio + 2); String smsContent = preSMStext.substring(start, start + end); String sms = smsTitle + '\n' + smsNoticeTime + '\n' + smsContent; System.out.println(sms); sendSMS(sms); Logger.getLogger(MsgMeCDC.class.getName()).log(Level.INFO, "SMS sent: " + sms); Logger.getLogger(MsgMeCDC.class.getName()).log(Level.INFO, "Length of SMS is " + (sms.length() + twilio)); System.out.println("\nLength of SMS is " + (sms.length() + twilio)); System.out.println(""); } //System.out.println(notices.toString()); }
From source file:mergedoc.core.APIDocument.java
/** * ? Javadoc ????/*from w ww . j a v a 2 s . com*/ * @param className ?? * @param docHtml API */ private void parseMethodComment(String className, Document doc) { Elements elements = doc.select("body > div.contentContainer > div.details > ul > li > ul > li > ul > li"); for (Element element : elements) { Element sigElm = element.select("pre").first(); if (sigElm == null) { continue; } String sigStr = sigElm.html(); Signature sig = createSignature(className, sigStr); Comment comment = new Comment(sig); // deprecated String depre = ""; Elements divs = element.select("div"); if (divs.size() == 2) { depre = divs.get(0).html(); } if (divs.size() > 0) { String body = divs.last().html(); body = formatLinkTag(className, body); comment.setDocumentBody(body); } Elements dtTags = element.select("dl dt"); for (Element dtTag : dtTags) { String dtText = dtTag.text(); if (dtText.contains(":")) { Element dd = dtTag; while (true) { dd = dd.nextElementSibling(); if (dd == null || dd.tagName().equalsIgnoreCase("dd") == false) { break; } String name = dd.select("code").first().text(); if (dtText.contains(":")) { name = "<" + name + ">"; } String items = dd.html(); Pattern p = PatternCache .getPattern("(?si)<CODE>(.+?)</CODE>\\s*-\\s*(.*?)(<DD>|</DD>|</DL>|<DT>|$)"); Matcher m = p.matcher(items); if (m.find()) { String desc = formatLinkTag(className, m.group(2)); comment.addParam(name, desc); } } continue; } if (dtText.contains(":")) { Element dd = dtTag.nextElementSibling(); String str = dd.html(); str = formatLinkTag(className, str); comment.addReturn(str); continue; } if (dtText.contains(":")) { Element dd = dtTag; while (true) { dd = dd.nextElementSibling(); if (dd == null || dd.tagName().equalsIgnoreCase("dd") == false) { break; } String name = dd.select("code").first().text(); String items = dd.html(); Pattern p = PatternCache .getPattern("(?si)<CODE>(.+?)</CODE>\\s*-\\s*(.*?)(<DD>|</DD>|</DL>|<DT>|$)"); Matcher m = p.matcher(items); if (m.find()) { String desc = formatLinkTag(className, m.group(2)); String param = name + " " + desc; comment.addThrows(param); } } continue; } } // deprecated parseDeprecatedTag(className, depre, comment); // parseCommonTag(className, element, comment); contextTable.put(sig, comment); } }
From source file:mergedoc.core.APIDocument.java
/** * ? Javadoc ????//from w w w . j a va 2 s . c o m * author, version ? Javadoc ???????????<br> * @param className ?? * @param docHtml API */ private void parseClassComment(String className, Document doc) { Elements elements = doc.select("body > div.contentContainer > div.description > ul > li"); for (Element element : elements) { String sigStr = element.select("pre").first().html(); Signature sig = createSignature(className, sigStr); Comment comment = new Comment(sig); // deprecated String depre = ""; Elements divs = element.select("div"); if (divs.size() == 2) { depre = divs.get(0).html(); } parseDeprecatedTag(className, depre, comment); // if (divs.size() > 0) { String body = divs.last().html(); body = formatLinkTag(className, body); comment.setDocumentBody(body); } // parseCommonTag(className, element, comment); log.debug(sig); contextTable.put(sig, comment); } }
From source file:de.geeksfactory.opacclient.apis.Open.java
@Override public SearchRequestResult searchGetPage(int page) throws IOException, OpacErrorException, JSONException { /*/* w ww . j av a 2 s . com*/ When there are many pages of results, there will only be links to the next 4 and previous 4 pages, so we will click links until it gets to the correct page. */ if (searchResultDoc == null) throw new NotReachableException(); Document doc = searchResultDoc; Elements pageLinks = doc.select("span[id$=DataPager1]").first().select("a[id*=LinkButtonPageN"); int from = Integer.valueOf(pageLinks.first().text()); int to = Integer.valueOf(pageLinks.last().text()); Element linkToClick; boolean willBeCorrectPage; if (page < from) { linkToClick = pageLinks.first(); willBeCorrectPage = false; } else if (page > to) { linkToClick = pageLinks.last(); willBeCorrectPage = false; } else { linkToClick = pageLinks.get(page - from); willBeCorrectPage = true; } Pattern pattern = Pattern.compile("javascript:__doPostBack\\('([^,]*)','([^\\)]*)'\\)"); Matcher matcher = pattern.matcher(linkToClick.attr("href")); if (!matcher.find()) throw new OpacErrorException(StringProvider.INTERNAL_ERROR); FormElement form = (FormElement) doc.select("form").first(); HttpEntity data = formData(form, null).addTextBody("__EVENTTARGET", matcher.group(1)) .addTextBody("__EVENTARGUMENT", matcher.group(2)).build(); ByteArrayOutputStream stream = new ByteArrayOutputStream(); data.writeTo(stream); String postUrl = form.attr("abs:action"); String html = httpPost(postUrl, data, "UTF-8"); if (willBeCorrectPage) { // We clicked on the correct link Document doc2 = Jsoup.parse(html); doc2.setBaseUri(postUrl); return parse_search(doc2, page); } else { // There was no correct link, so try to find one again searchResultDoc = Jsoup.parse(html); searchResultDoc.setBaseUri(postUrl); return searchGetPage(page); } }
From source file:com.weavers.duqhan.business.impl.ProductServiceImpl.java
@Override public void loadTempProducts(List<StatusBean> statusBeans) { boolean isSuccess = true; String startDate = new Date().toString(); Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, "(==I==)DATE: " + startDate + "Store product details in temp product table start....."); try {// w w w . j a v a 2s. co m String status = ""; for (StatusBean statusBean : statusBeans) { status = "Link duplicate"; Temtproductlinklist temtproductlinklist = temtproductlinklistDao.loadById(statusBean.getId()); if (temtproductlinklist != null && temtproductlinklist.getStatus() == 0) { Product testProduct = productDao.getProductByExternelLink(temtproductlinklist.getLink()); if (testProduct == null) { String value = ""; Elements detailMain; Elements detailSub; Elements specifics; double votes = 0.0; double stars = 0.0; double feedback = 0.0; String url = temtproductlinklist.getLink(); try { testProduct = new Product(); Product savedTestProduct; //=================== Random sleep START ===================// // TimeUnit.SECONDS.sleep(30 + (int) (Math.random() * 100)); Random randomObj = new Random(); TimeUnit.SECONDS.sleep(randomObj.ints(30, 60).findFirst().getAsInt()); //=================== Random sleep END =====================// Document doc = Jsoup.connect(url).get(); detailMain = doc.select("#j-detail-page"); if (!detailMain.isEmpty()) { //=================== Criteria Block START==================// detailMain = doc.select(".rantings-num"); if (!detailMain.isEmpty()) { votes = Double.valueOf(detailMain.text().split(" votes")[0].split("\\(")[1]); } detailMain = doc.select(".percent-num"); if (!detailMain.isEmpty()) { stars = Double.valueOf(detailMain.text()); } detailMain = doc.select("ul.ui-tab-nav li[data-trigger='feedback'] a"); if (!detailMain.isEmpty()) { feedback = Double.valueOf(detailMain.text().split("\\(")[1].split("\\)")[0]); } //=================== Criteria Block END==================// if (votes > 10.0 && stars > 4.0 && feedback > 4.0) { detailMain = doc.select(".detail-wrap .product-name"); testProduct.setName(detailMain .text());/*.substring(0, Math.min(detailMain.text().length(), 50))*/ detailMain = doc.select(".detail-wrap .product-name"); testProduct.setDescription(detailMain.text()); testProduct.setExternalLink(url); testProduct.setVendorId(1l);//?????????????????????? //=================== Packaging block START==================// Double weight = 1.0; Double width = 1.0; Double height = 1.0; Double length = 1.0; detailMain = doc.select( "div#j-product-desc div.pnl-packaging-main ul li.packaging-item"); for (Element element : detailMain) { String packagingTitle = element.select("span.packaging-title").text(); String packagingDesc = element.select("span.packaging-des").text(); if (packagingTitle.trim().equals("Package Weight:")) { String str = packagingDesc; str = str.replaceAll("[^.?0-9]+", " "); if (Arrays.asList(str.trim().split(" ")) != null) { if (!Arrays.asList(str.trim().split(" ")).isEmpty()) { try { weight = Double.parseDouble( Arrays.asList(str.trim().split(" ")).get(0)); } catch (Exception e) { weight = 1.0; } } } System.out.println("weight == " + weight); } else if (packagingTitle.trim().equals("Package Size:")) { String str = packagingDesc; str = str.replaceAll("[^.?0-9]+", " "); if (Arrays.asList(str.trim().split(" ")) != null) { if (!Arrays.asList(str.trim().split(" ")).isEmpty()) { try { width = Double.parseDouble( Arrays.asList(str.trim().split(" ")).get(0)); height = Double.parseDouble( Arrays.asList(str.trim().split(" ")).get(1)); length = Double.parseDouble( Arrays.asList(str.trim().split(" ")).get(2)); } catch (Exception e) { width = 1.0; height = 1.0; length = 1.0; } } } System.out.println("width == " + width); System.out.println("height == " + height); System.out.println("length == " + length); } } //=================== Packaging block END==================// //=================== Category block START==================// detailMain = doc.select("div.ui-breadcrumb div.container a"); Long productCategoryId = 0L; String parentPath = ""; String thisCategory = detailMain.last().text().trim(); System.out.println("thisCategory == " + thisCategory); Category parentCategory = new Category(); parentCategory.setId(0L); parentCategory.setParentPath(""); for (Element element : detailMain) { String newCategory; newCategory = element.text().trim(); System.out.println("newCategory======" + newCategory); if (newCategory.equals("Home") || newCategory.equals("All Categories")) { } else { Category category = categoryDao.getCategoryByName(newCategory); if (category != null) { if (category.getName().equals(thisCategory)) { productCategoryId = category.getId(); parentPath = category.getParentPath(); } parentCategory = category; } else { category = new Category(); category.setId(null); category.setName(newCategory); category.setParentId(parentCategory.getId()); category.setParentPath(parentCategory.getParentPath() + parentCategory.getId() + "="); category.setQuantity(0); category.setImgUrl("-"); category.setDisplayText(newCategory); Category category2 = categoryDao.save(category); if (category.getName().equals(thisCategory)) { productCategoryId = category2.getId(); parentPath = category2.getParentPath(); } parentCategory = category2; } } } //=================== Category block END==================// //=============== Specifications block START==============// detailMain = doc.select(".product-property-list .property-item"); String specifications = ""; for (Element element : detailMain) { specifications = specifications + element.select(".propery-title").get(0).text().replace(",", "/") .replace(":", "-") + ":" + element.select(".propery-des").get(0).text() .replace(",", "/").replace(":", "-") + ",";//TODO:, check } //=============== Specifications Block END==============// //=============== Shipping Time Block START==============// String shippingTime = ""; detailMain = doc.select(".shipping-days[data-role='delivery-days']"); System.out.println("value detailMain" + detailMain.toString()); shippingTime = detailMain.text(); //=============== Shipping Time Block END==============// //=============== Shipping Cost Block START==============// detailMain = doc.select(".logistics-cost"); value = detailMain.text(); if (!value.equalsIgnoreCase("Free Shipping")) { // f = 0.00; } else { // f = Double.parseDouble(value.replaceAll(".*?([\\d.]+).*", "$1")); } //=============== Shipping Cost Block END==============// //=================Product save 1st START==============// testProduct.setCategoryId(productCategoryId); testProduct.setLastUpdate(new Date()); testProduct.setParentPath(parentPath); testProduct.setImgurl("-"); testProduct.setProperties("-"); testProduct.setProductWidth(width); testProduct.setProductLength(length); testProduct.setProductWeight(weight); testProduct.setProductHeight(height); testProduct.setShippingRate(0.0); testProduct.setShippingTime("45"); testProduct.setSpecifications(specifications); savedTestProduct = productDao.save(testProduct); //====================Product save 1st END==============// //========= Property, Property Value, Property Product Map Block START ========// double discountPrice = 0.0; double actualPrice = 0.0; double markupPrice = 0.0; String id = ""; String allProperties = ""; //------------------------Read Color css START---------------------// specifics = doc.select("#j-product-info-sku dl.p-property-item"); Elements cssdetailMain = doc.select("link[href]"); Document cssdoc = new Document(""); System.out.println( "====================================================cssdetailMain" + cssdetailMain.size()); for (Element element : cssdetailMain) { String cssurl = element.attr("abs:href"); if (cssurl.contains("??main-detail")) { try { cssdoc = Jsoup.connect(cssurl).get(); } catch (IOException ex) { } break; } } //-----------------------Read Color css END--------------------------// //-----------Product Property, Property Value START--------// Map<String, ProductPropertyvalues> propertyValuesMap = new HashMap<>(); if (!specifics.isEmpty()) { ProductProperties testPorperties; ProductProperties saveTestPorperties; ProductPropertyvalues testPropertyValues; for (Element specific : specifics) { System.out.println("head ==== " + specific.select("dt").text()); testPorperties = productPropertiesDao .loadByName(specific.select("dt").text()); if (testPorperties == null) { testPorperties = new ProductProperties(); testPorperties.setPropertyName(specific.select("dt").text()); saveTestPorperties = productPropertiesDao.save(testPorperties); } else { saveTestPorperties = testPorperties; } allProperties = allProperties + saveTestPorperties.getId().toString() + "-"; detailSub = specific.select("dd ul li"); String valu = "-"; for (Element element : detailSub) { testPropertyValues = new ProductPropertyvalues(); id = element.select("a[data-sku-id]").attr("data-sku-id").trim(); testPropertyValues.setRefId(id); if (element.hasClass("item-sku-image")) { valu = element.select("a img[src]").get(0).absUrl("src") .split(".jpg")[0] + ".jpg"; String title = element.select("a img").get(0).attr("title"); String imgUrl = GoogleBucketFileUploader .uploadProductImage(valu, savedTestProduct.getId()); valu = "<img src='" + imgUrl + "' title='" + title + "' style='height:40px; width:40px;'/>"; } else if (element.hasClass("item-sku-color")) { String style = cssdoc.html().split("sku-color-" + id)[1] .split("}")[0].substring(1); valu = "<span style='" + style + "' ; height:40px; width:40px; display:block;'></span>"; } else { valu = element.select("a span").toString(); } System.out.println("valu === " + valu); testPropertyValues.setProductId(savedTestProduct.getId()); testPropertyValues.setPropertyId(saveTestPorperties.getId()); testPropertyValues.setValueName(valu); propertyValuesMap.put(id, productPropertyvaluesDao.save(testPropertyValues)); } } savedTestProduct.setProperties(allProperties); } //-----------Product Property, Property Value END--------// //----------------------Read json START------------------// List<AxpProductDto> axpProductDtos = new ArrayList<>(); Elements scripts = doc.select("script"); // Get the script part for (Element script : scripts) { if (script.html().contains("var skuProducts=")) { String jsonData = ""; jsonData = script.html().split("var skuProducts=")[1] .split("var GaData")[0].trim(); jsonData = jsonData.substring(0, jsonData.length() - 1); Gson gsonObj = new Gson(); axpProductDtos = Arrays .asList(gsonObj.fromJson(jsonData, AxpProductDto[].class)); break; } } //----------------------Read json END------------------// //-------------Product Properties Map START------------// for (AxpProductDto thisAxpProductDto : axpProductDtos) { SkuVal skuVal = thisAxpProductDto.getSkuVal(); if (skuVal.getActSkuCalPrice() != null) { value = skuVal.getActSkuCalPrice().trim(); discountPrice = CurrencyConverter.usdTOinr( Double.parseDouble(value.replaceAll(".*?([\\d.]+).*", "$1"))); value = skuVal.getSkuCalPrice().trim(); actualPrice = CurrencyConverter.usdTOinr( Double.parseDouble(value.replaceAll(".*?([\\d.]+).*", "$1"))); markupPrice = discountPrice * 0.15 + 100; discountPrice = Math.ceil((discountPrice + markupPrice) / 10) * 10; actualPrice = Math.round(actualPrice + markupPrice); } else { discountPrice = 0.0; value = skuVal.getSkuCalPrice().trim(); actualPrice = CurrencyConverter.usdTOinr( Double.parseDouble(value.replaceAll(".*?([\\d.]+).*", "$1"))); markupPrice = actualPrice * 0.15 + 100; discountPrice = Math.round(actualPrice + markupPrice); actualPrice = Math.round(actualPrice + markupPrice); } ProductPropertiesMap productPropertyMap = new ProductPropertiesMap(); String myPropValueIds = ""; if (thisAxpProductDto.getSkuAttr() != null) { String[] skuPropIds = thisAxpProductDto.getSkuPropIds().split(","); for (String skuPropId : skuPropIds) { myPropValueIds = myPropValueIds + propertyValuesMap.get(skuPropId).getId().toString() + "_"; } productPropertyMap.setPropertyvalueComposition(myPropValueIds); } else { productPropertyMap.setPropertyvalueComposition("_"); } productPropertyMap.setDiscount(discountPrice); productPropertyMap.setPrice(actualPrice); productPropertyMap.setProductId(savedTestProduct); productPropertyMap.setQuantity(5l); productPropertiesMapDao.save(productPropertyMap); } //-------------Product Properties Map START------------// //========= Property, Property Value, Property Product Map Block END ========// //============= Multiple Image Block START =============// detailMain = doc.select("ul.image-thumb-list span.img-thumb-item img[src]"); int flg = 0; String imgUrl = ""; for (Element element : detailMain) { imgUrl = GoogleBucketFileUploader.uploadProductImage( element.absUrl("src").split(".jpg")[0] + ".jpg", savedTestProduct.getId()); if (flg == 0) { flg++; savedTestProduct.setImgurl(imgUrl); } else { ProductImg productImg = new ProductImg(); productImg.setId(null); productImg.setImgUrl(imgUrl); productImg.setProductId(savedTestProduct.getId()); productImgDao.save(productImg); } } //============= Multiple Image Block END =============// //=================Product save final START==============// if (productDao.save(savedTestProduct) != null) { temtproductlinklist.setStatus(1);// temtproductlinklistDao.save(temtproductlinklist); status = "Success"; } //=================Product save final START==============// } else { temtproductlinklist.setStatus(2);// temtproductlinklistDao.save(temtproductlinklist); status = "criteria mismatch"; } } else { status = "Page not found"; } } catch (Exception ex) { System.out.println( "=============================================================Exception1" + ex); temtproductlinklist.setStatus(4);// temtproductlinklistDao.save(temtproductlinklist); System.out.println("Exception === " + ex); status = "Failure"; Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, "(==E==)DATE: " + new Date().toString() + "Store product details in temp product table get error in sub process.....\n Link Id: " + statusBean.getId() + "\n Started on" + startDate, ex); } } else { temtproductlinklist.setStatus(3);// temtproductlinklistDao.save(temtproductlinklist); status = "Product exsist"; } } // String body = "Id: " + temtproductlinklist.getId() + "<br/> Status: " + status; // MailSender.sendEmail("krisanu.nandi@pkweb.in", "Product captured", body, "subhendu.sett@pkweb.in"); statusBean.setStatus(status); } System.out.println("=============================================================status" + status); } catch (Exception e) { System.out.println("=============================================================Exception2" + e); isSuccess = false; String body = "(==E==)DATE: " + new Date().toString() + "Store product details in temp product table get error.....<br/> Started on" + startDate + "<br/>"; Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, body, e); // MailSender.sendEmail("krisanu.nandi@pkweb.in", "Stopped store product details", body + e.getLocalizedMessage(), "subhendu.sett@pkweb.in"); } if (isSuccess) { String body = "(==I==)DATE: " + new Date().toString() + "Store product details in temp product table end.....<br/> Started on" + startDate; Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, body); /*ObjectMapper mapper = new ObjectMapper(); try { MailSender.sendEmail("krisanu.nandi@pkweb.in", "Completed store product details", body + "=============<br/><br/>" + mapper.writeValueAsString(statusBeans), "subhendu.sett@pkweb.in"); } catch (JsonProcessingException ex) { Logger.getLogger(ProductServiceImpl.class.getName()).log(Level.SEVERE, null, ex); }*/ } // return statusBeans; System.out.println("=============================================================end"); }
From source file:com.normalexception.app.rx8club.fragment.category.CategoryFragment.java
/** * Grab contents from the forum that the user clicked on * @param doc The document parsed from the link * @param id The id number of the link * @param isMarket True if the link is from a marketplace category *///w ww.jav a2 s.c om public void getCategoryContents(Document doc, String id, boolean isMarket) { // Update pagination try { Elements pageNumbers = doc.select("div[class=pagenav]"); Elements pageLinks = pageNumbers.first().select("td[class^=vbmenu_control]"); thisPage = pageLinks.text().split(" ")[1]; finalPage = pageLinks.text().split(" ")[3]; } catch (Exception e) { } // Make sure id contains only numbers if (!isNewTopicActivity) id = Utils.parseInts(id); // Grab each thread Elements threadListing = doc.select("table[id=threadslist] > tbody > tr"); for (Element thread : threadListing) { try { boolean isSticky = false, isLocked = false, hasAttachment = false, isAnnounce = false, isPoll = false; String formattedTitle = "", postCount = "0", views = "0", forum = "", threadUser = "", lastUser = "", threadLink = "", lastPage = "", totalPosts = "0", threadDate = ""; Elements announcementContainer = thread.select("td[colspan=5]"); Elements threadTitleContainer = thread.select("a[id^=thread_title]"); // We could have two different types of threads. Announcement threads are // completely different than the other types of threads (sticky, locked, etc) // so we need to play some games here if (announcementContainer != null && !announcementContainer.isEmpty()) { Log.d(TAG, "Announcement Thread Found"); Elements annThread = announcementContainer.select("div > a"); Elements annUser = announcementContainer.select("div > span[class=smallfont]"); formattedTitle = "Announcement: " + annThread.first().text(); threadUser = annUser.last().text(); threadLink = annThread.attr("href"); isAnnounce = true; } else if (threadTitleContainer != null && !threadTitleContainer.isEmpty()) { Element threadLinkEl = thread.select("a[id^=thread_title]").first(); Element repliesText = thread.select("td[title^=Replies]").first(); Element threaduser = thread.select("td[id^=td_threadtitle_] div.smallfont").first(); Element threadicon = thread.select("img[id^=thread_statusicon_]").first(); Element threadDiv = thread.select("td[id^=td_threadtitle_] > div").first(); Element threadDateFull = thread.select("td[title^=Replies:] > div").first(); try { isSticky = threadDiv.text().contains("Sticky:"); } catch (Exception e) { } try { isPoll = threadDiv.text().contains("Poll:"); } catch (Exception e) { } try { String icSt = threadicon.attr("src"); isLocked = (icSt.contains("lock") && icSt.endsWith(".gif")); } catch (Exception e) { } String preString = ""; try { preString = threadDiv.select("span > b").text(); } catch (Exception e) { } try { hasAttachment = !threadDiv.select("a[onclick^=attachments]").isEmpty(); } catch (Exception e) { } // Find the last page if it exists try { lastPage = threadDiv.select("span").last().select("a").last().attr("href"); } catch (Exception e) { } threadDate = threadDateFull.text(); int findAMPM = threadDate.indexOf("M") + 1; threadDate = threadDate.substring(0, findAMPM); String totalPostsInThreadTitle = threadicon.attr("alt"); if (totalPostsInThreadTitle != null && totalPostsInThreadTitle.length() > 0) totalPosts = totalPostsInThreadTitle.split(" ")[2]; // Remove page from the link String realLink = Utils.removePageFromLink(link); if (threadLinkEl.attr("href").contains(realLink) || (isNewTopicActivity || isMarket)) { String txt = repliesText.getElementsByClass("alt2").attr("title"); String splitter[] = txt.split(" ", 4); postCount = splitter[1].substring(0, splitter[1].length() - 1); views = splitter[3]; try { if (this.isNewTopicActivity) forum = thread.select("td[class=alt1]").last().text(); } catch (Exception e) { } formattedTitle = String.format("%s%s%s", isSticky ? "Sticky: " : isPoll ? "Poll: " : "", preString.length() == 0 ? "" : preString + " ", threadLinkEl.text()); } threadUser = threaduser.text(); lastUser = repliesText.select("a[href*=members]").text(); threadLink = threadLinkEl.attr("href"); } // Add our thread to our list as long as the thread // contains a title if (!formattedTitle.equals("")) { ThreadModel tv = new ThreadModel(); tv.setTitle(formattedTitle); tv.setStartUser(threadUser); tv.setLastUser(lastUser); tv.setLink(threadLink); tv.setLastLink(lastPage); tv.setPostCount(postCount); tv.setMyPosts(totalPosts); tv.setViewCount(views); tv.setLocked(isLocked); tv.setSticky(isSticky); tv.setAnnouncement(isAnnounce); tv.setPoll(isPoll); tv.setHasAttachment(hasAttachment); tv.setForum(forum); tv.setLastPostTime(threadDate); threadlist.add(tv); } else if (thread.text() .contains(MainApplication.getAppContext().getString(R.string.constantNoUpdate))) { Log.d(TAG, String.format("Found End of New Threads after %d threads...", threadlist.size())); if (threadlist.size() > 0) { ThreadModel ltv = threadlist.get(threadlist.size() - 1); Log.d(TAG, String.format("Last New Thread '%s'", ltv.getTitle())); } if (!PreferenceHelper.hideOldPosts(MainApplication.getAppContext())) threadlist.add(new ThreadModel(true)); else { Log.d(TAG, "User Chose To Hide Old Threads"); break; } } } catch (Exception e) { Log.e(TAG, "Error Parsing That Thread...", e); Log.d(TAG, "Thread may have moved"); } } }
From source file:com.normalexception.app.rx8club.fragment.pm.PrivateMessageViewFragment.java
/** * Construct the view elements/* w w w .ja v a2 s.c o m*/ */ private void constructView() { AsyncTask<Void, String, Void> updaterTask = new AsyncTask<Void, String, Void>() { @Override protected void onPreExecute() { loadingDialog = ProgressDialog.show(getActivity(), getString(R.string.loading), getString(R.string.pleaseWait), true); } @Override protected Void doInBackground(Void... params) { String link = getArguments().getString("link"); Document doc = VBForumFactory.getInstance().get(getActivity(), VBForumFactory.getRootAddress() + "/" + link); if (doc != null) { securityToken = HtmlFormUtils.getInputElementValueByName(doc, "securitytoken"); pmid = HtmlFormUtils.getInputElementValueByName(doc, "pmid"); title = HtmlFormUtils.getInputElementValueByName(doc, "title"); Elements userPm = doc.select("table[id^=post]"); publishProgress(getString(R.string.asyncDialogLoadingPM)); // User Control Panel Elements userCp = userPm.select("td[class=alt2]"); Elements userDetail = userCp.select("div[class=smallfont]"); Elements userSubDetail = userDetail.last().select("div"); Elements userAvatar = userDetail.select("img[alt$=Avatar]"); Elements postMessage = doc.select("div[id=post_message_]"); PMPostModel pv = new PMPostModel(); pv.setUserName(userCp.select("div[id^=postmenu]").text()); pv.setIsLoggedInUser(LoginFactory.getInstance().isLoggedIn() ? UserProfile.getInstance().getUsername().equals(pv.getUserName()) : false); pv.setUserTitle(userDetail.first().text()); pv.setUserImageUrl(Utils.resolveUrl(userAvatar.attr("src"))); pv.setPostDate(userPm.select("td[class=thead]").first().text()); // userSubDetail // 0 - full container , full container // 1 - Trader Score , Trader Score // 2 - Join Date , Join Date // 3 - Post Count , Location // 4 - Blank , Post Count // 5 - , Blank || Social // Iterator<Element> itr = userSubDetail.listIterator(); while (itr.hasNext()) { String txt = itr.next().text(); if (txt.contains("Location:")) pv.setUserLocation(txt); else if (txt.contains("Posts:")) pv.setUserPostCount(txt); else if (txt.contains("Join Date:")) pv.setJoinDate(txt); } // User Post Content pv.setUserPost(formatUserPost(postMessage)); pmlist.add(pv); TextView comment = (TextView) getView().findViewById(R.id.pmitem_comment); Elements textarea = doc.select("textarea[id=vB_Editor_QR_textarea]"); if (textarea != null) { comment.setText(textarea.first().text()); } updateList(); } return null; } @Override protected void onProgressUpdate(String... progress) { if (loadingDialog != null) loadingDialog.setMessage(progress[0]); } @Override protected void onPostExecute(Void result) { try { loadingDialog.dismiss(); loadingDialog = null; } catch (Exception e) { Log.w(TAG, e.getMessage()); } } }; updaterTask.execute(); }
From source file:com.normalexception.app.rx8club.fragment.thread.ThreadFragment.java
/** * Grab contents from the forum that the user clicked on * @param doc The document parsed from the link * @param id The id number of the link * @return An arraylist of forum contents *///www. j ava 2 s . com public void getThreadContents(Document doc) { // Update pagination try { Elements pageNumbers = doc.select("div[class=pagenav]"); if (pageNumbers.first() != null) { Elements pageLinks = pageNumbers.first().select("td[class^=vbmenu_control]"); thisPage = pageLinks.text().split(" ")[1]; finalPage = pageLinks.text().split(" ")[3]; Log.d(TAG, String.format("This Page: %s, Final Page: %s", thisPage, finalPage)); } else { Log.d(TAG, "Thread only contains one page"); } } catch (Exception e) { Log.e(TAG, "We had an error with pagination", e); } // Is user thread admin?? Elements threadTools = doc.select("div[id=threadtools_menu] > form > table"); if (threadTools.text().contains(MODERATION_TOOLS)) { Log.d(TAG, "<><> User has administrative rights here! <><>"); } else { //adminContent.setVisibility(View.GONE); lv.removeHeaderView(adminContent); } // Get the user's actual ID, there is a chance they never got it // before UserProfile.getInstance().setUserId(HtmlFormUtils.getInputElementValueByName(doc, "loggedinuser")); // Get Post Number and security token securityToken = HtmlFormUtils.getInputElementValueByName(doc, "securitytoken"); Elements pNumber = doc.select("a[href^=http://www.rx8club.com/newreply.php?do=newreply&noquote=1&p=]"); String pNumberHref = pNumber.attr("href"); postNumber = pNumberHref.substring(pNumberHref.lastIndexOf("=") + 1); threadNumber = doc.select("input[name=searchthreadid]").attr("value"); Elements posts = doc.select("div[id=posts]").select("div[id^=edit]"); Log.v(TAG, String.format("Parsing through %d posts", posts.size())); for (Element post : posts) { try { Elements innerPost = post.select("table[id^=post]"); // User Control Panel Elements userCp = innerPost.select("td[class=alt2]"); Elements userDetail = userCp.select("div[class=smallfont]"); Elements userSubDetail = userDetail.last().select("div"); Elements userAvatar = userDetail.select("img[alt$=Avatar]"); // User Information PostModel pv = new PostModel(); pv.setUserName(userCp.select("div[id^=postmenu]").text()); pv.setIsLoggedInUser(LoginFactory.getInstance().isLoggedIn() ? UserProfile.getInstance().getUsername().equals(pv.getUserName()) : false); pv.setUserTitle(userDetail.first().text()); pv.setUserImageUrl(userAvatar.attr("src")); pv.setPostDate(innerPost.select("td[class=thead]").first().text()); pv.setPostId(Utils.parseInts(post.attr("id"))); pv.setRootThreadUrl(currentPageLink); // get Likes if any exist Elements eLikes = innerPost.select("div[class*=vbseo_liked] > a"); List<String> likes = new ArrayList<String>(); for (Element eLike : eLikes) likes.add(eLike.text()); pv.setLikes(likes); Iterator<Element> itr = userSubDetail.listIterator(); while (itr.hasNext()) { String txt = itr.next().text(); if (txt.contains("Location:")) pv.setUserLocation(txt); else if (txt.contains("Posts:")) pv.setUserPostCount(txt); else if (txt.contains("Join Date:")) pv.setJoinDate(txt); } // User Post Content pv.setUserPost(formatUserPost(innerPost)); // User signature try { Element userSig = innerPost.select("div[class=konafilter]").first(); pv.setUserSignature(userSig.html()); } catch (NullPointerException npe) { } Elements postAttachments = innerPost.select("a[id^=attachment]"); if (postAttachments != null && !postAttachments.isEmpty()) { ArrayList<String> attachments = new ArrayList<String>(); for (Element postAttachment : postAttachments) { attachments.add(postAttachment.attr("href")); } pv.setAttachments(attachments); } pv.setSecurityToken(securityToken); // Make sure we aren't adding a blank user post if (pv.getUserPost() != null) postlist.add(pv); } catch (Exception e) { Log.w(TAG, "Error Parsing Post...Probably Deleted"); } } }
From source file:org.bonitasoft.web.designer.visitors.HtmlBuilderVisitorTest.java
@Test public void should_add_elements_to_the_container_rows() throws Exception { // we should have two div.col-xs-12 with two div.row containing added components Elements rows = toBody( visitor.visit(aContainer().with(aRow().with(aComponent().withWidgetId("pbLabel").build()), aRow().with(aComponent().withWidgetId("customLabel").build())).build())).select(".row"); assertThat(rows.size()).isEqualTo(2); assertThat(rows.first().select("pb-label").outerHtml()).isEqualTo("<pb-label></pb-label>"); assertThat(rows.last().select("custom-label").outerHtml()).isEqualTo("<custom-label></custom-label>"); }