com.weibo.datasys.crawler.impl.crawlUnit.fetcher.wxcs.WXCSFetcher.java Source code

Introduction

Here is the source code for com.weibo.datasys.crawler.impl.crawlUnit.fetcher.wxcs.WXCSFetcher.java

Source

/**
 *  Copyright (c)  2016-2020 Weibo, Inc.
 *  All rights reserved.
 *
 *  This software is the confidential and proprietary information of Weibo, 
 *  Inc. ("Confidential Information"). You shall not
 *  disclose such Confidential Information and shall use it only in
 *  accordance with the terms of the license agreement you entered into with Weibo.
 */
package com.weibo.datasys.crawler.impl.crawlUnit.fetcher.wxcs;

import java.text.SimpleDateFormat;
import java.util.Date;

import org.dom4j.Document;
import org.dom4j.DocumentHelper;
import org.dom4j.Element;

import com.weibo.datasys.crawler.commonDownloader.entity.DownRequest;
import com.weibo.datasys.common.util.StringUtils;
import com.weibo.datasys.crawler.base.entity.SeedData;
import com.weibo.datasys.crawler.impl.crawlUnit.fetcher.DefaultFetcher;

/**
 * Fetcher
 * 
 * 
 */
public class WXCSFetcher extends DefaultFetcher {

    private static final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");

    @Override
    protected void configPostReq(SeedData seedData, DownRequest req) {
        Document doc = DocumentHelper.createDocument();
        doc.setXMLEncoding("utf-8");
        // 
        Element root = doc.addElement("root");
        // transactionid
        root.addElement("transactionid").addText("");
        // timestamp
        root.addElement("timestamp").addText(sdf.format(new Date()));
        // msgname
        String msgname = seedData.getExtendField(POST_FIELD_PREFIX + "msgname");
        msgname = StringUtils.nullToEmpty(msgname);
        root.addElement("msgname").addText(msgname);
        // result
        root.addElement("result").addText("");
        // resultdesc
        root.addElement("resultdesc").addText("");
        // body
        Element body = root.addElement("body");
        // dircode ?, root
        String dircode = seedData.getExtendField(POST_FIELD_PREFIX + "dircode");
        dircode = StringUtils.nullToEmpty(dircode);
        body.addElement("dircode").addText(dircode);
        // areacode ???
        String areacode = seedData.getExtendField(POST_FIELD_PREFIX + "areacode");
        areacode = StringUtils.nullToEmpty(areacode);
        body.addElement("areacode").addText(areacode);
        // nodeversion ??
        String nodeversion = seedData.getExtendField(POST_FIELD_PREFIX + "nodeversion");
        nodeversion = StringUtils.nullToEmpty(nodeversion);
        body.addElement("nodeversion").addText(nodeversion);
        // depth ?[0..10]?10
        // ?0?????
        // ?>0????
        String depth = seedData.getExtendField(POST_FIELD_PREFIX + "depth");
        depth = StringUtils.nullToEmpty(depth);
        body.addElement("depth").addText(depth);
        // returenres ???
        // 0
        // 1?
        // ?
        String returenres = seedData.getExtendField(POST_FIELD_PREFIX + "returenres");
        if (StringUtils.isEmptyString(returenres)) {
            returenres = "1";
        }
        body.addElement("returenres").addText(returenres);

        req.setContentType("text/xml");
        req.setPostString(doc.asXML());
    }
}