lolthx.autohome.buy.AutohomePriceListFetch.java Source code

Introduction

Here is the source code for lolthx.autohome.buy.AutohomePriceListFetch.java

Source

package lolthx.autohome.buy;

import java.text.ParseException;
import java.util.Date;

import lakenono.base.DistributedParser;
import lakenono.base.Task;
import lolthx.autohome.buy.bean.AutohomePriceInfoBean;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class AutohomePriceListFetch extends DistributedParser {

    @Override
    public String getQueueName() {
        return "autohome_price_list";
    }

    @Override
    public void parse(String result, Task task) throws Exception {
        if (StringUtils.isBlank(result)) {
            return;
        }

        Date start = task.getStartDate();
        Date end = task.getEndDate();

        Document doc = Jsoup.parse(result);
        Elements lis = doc.select("li.price-item");

        AutohomePriceInfoBean bean = new AutohomePriceInfoBean();

        for (Element li : lis) {

            try {
                Elements postTimeEl = li.select("div.user-name span");
                String postTime = "";
                if (!postTimeEl.isEmpty()) {
                    postTime = StringUtils.trim(
                            StringUtils.substringBefore(postTimeEl.first().text(), "?").replaceAll("", ""));

                    if (!isTime(postTime, start, end)) {
                        continue;
                    }
                }
                bean.setPostTime(postTime);
                bean.setUrl(task.getUrl());
                bean.setForumId(StringUtils.substringBefore(task.getExtra(), ":"));
                bean.setProjectName(task.getProjectName());
                bean.setKeyword(StringUtils.substringAfter(task.getExtra(), ":"));

                // post id
                Elements id = li.select("div.price-share a.share");
                if (!id.isEmpty()) {
                    String idStr = id.first().attr("data-target");
                    idStr = StringUtils.substringAfterLast(idStr, "_");
                    if (StringUtils.isBlank(idStr)) {
                        continue;
                    }

                    bean.setId(idStr);
                }

                // 
                Elements user = li.select("div.user-name a");
                if (!user.isEmpty()) {
                    String userUrl = user.first().absUrl("href");
                    String userId = StringUtils.substringAfterLast(userUrl, "/");
                    String userName = user.first().text();

                    bean.setUserId(userId);
                    bean.setUserUrl(userUrl);
                    bean.setUserName(userName);
                }

                Elements dataLis = li.select("div.price-item-bd li");
                for (Element dataLi : dataLis) {
                    String data = dataLi.text();

                    if (StringUtils.startsWith(data, "")) {
                        bean.setCar(StringUtils.trim(StringUtils.substringAfter(data, "")));
                    }

                    if (StringUtils.startsWith(data, "")) {
                        bean.setPrice(StringUtils.trim(StringUtils.substringAfter(data, "")));
                    }

                    if (StringUtils.startsWith(data, "")) {
                        bean.setGuidePrice(StringUtils.trim(StringUtils.substringAfter(data, "")));
                    }

                    if (StringUtils.startsWith(data, "?")) {
                        bean.setTotalPrice(StringUtils.trim(StringUtils.substringAfter(data, "")));
                    }

                    if (StringUtils.startsWith(data, "")) {
                        bean.setPurchaseTax(StringUtils.trim(StringUtils.substringAfter(data, "")));
                    }

                    if (StringUtils.startsWith(data, "?")) {
                        bean.setCommercialInsurance(StringUtils.trim(StringUtils.substringAfter(data, "")));
                    }

                    if (StringUtils.startsWith(data, "")) {
                        bean.setVehicleUseTax(StringUtils.trim(StringUtils.substringAfter(data, "")));
                    }
                    if (StringUtils.startsWith(data, "")) {
                        bean.setCompulsoryInsurance(StringUtils.trim(StringUtils.substringAfter(data, "")));
                    }
                    if (StringUtils.startsWith(data, "")) {
                        bean.setLicenseFee(StringUtils.trim(StringUtils.substringAfter(data, "")));
                    }
                    if (StringUtils.startsWith(data, "?")) {
                        bean.setPromotion(StringUtils.trim(StringUtils.substringAfter(data, "")));
                    }
                    if (StringUtils.startsWith(data, "")) {
                        bean.setBuyTime(StringUtils.trim(StringUtils.substringAfter(data, "")));
                    }
                    if (StringUtils.startsWith(data, "")) {
                        String area = StringUtils.trim(StringUtils.substringAfter(data, ""));
                        String[] pAndC = StringUtils.splitByWholeSeparator(area, ",", 2);

                        if (pAndC.length == 1) {
                            bean.setBuyProvince(pAndC[0]);
                            bean.setBuyCity(pAndC[0]);
                        }

                        if (pAndC.length == 2) {
                            bean.setBuyProvince(pAndC[0]);
                            bean.setBuyCity(pAndC[1]);
                        }

                    }
                    if (StringUtils.startsWith(data, "")) {
                        Elements level = dataLi.select("span.level");
                        // 
                        if (!level.isEmpty()) {
                            bean.setSellerComment(level.first().text());
                        }

                        // ?
                        Elements seller = dataLi.select("a.title");
                        if (!seller.isEmpty()) {
                            String sellerUrl = seller.first().absUrl("href");
                            String sellerName = seller.first().text();
                            String sellerId = StringUtils.substringAfterLast(sellerUrl, "/");

                            bean.setSellerId(sellerId);
                            bean.setSellerName(sellerName);
                            bean.setSellerUrl(sellerUrl);
                        }

                        // ?
                        Elements sellerPhone = dataLi.select("em.phone-num");
                        if (!sellerPhone.isEmpty()) {
                            bean.setSellerPhone(sellerPhone.first().text());
                        }

                        // ?
                        // Elements sellerAddress =
                        // dataLi.select("em.phone-num");

                    }
                    if (StringUtils.startsWith(data, "?")) {
                        bean.setBuyFeeling(StringUtils.trim(StringUtils.substringAfter(data, "")));
                    }
                }
                bean.saveOnNotExist();
            } catch (Exception e) {
                e.printStackTrace();
                continue;
            }
        }
    }

    private boolean isTime(String time, Date start, Date end) {
        try {
            Date srcDate = DateUtils.parseDate(time.trim(), "yyyy-MM-dd");
            return between(start, end, srcDate);
        } catch (ParseException e) {
            e.printStackTrace();
        }
        return false;
    }

    private boolean between(Date beginDate, Date endDate, Date src) {
        return beginDate.before(src) && endDate.after(src);
    }

    public static void main(String args[]) {
        for (int i = 1; i <= 20; i++) {
            new AutohomePriceListFetch().run();
        }
    }

}