Java动态代理实现动态爬虫

全栈程序员站长

发布于 2022-09-14 09:23:51

7830

发布于 2022-09-14 09:23:51

大家好，又见面了，我是你们的朋友全栈君。

笔者公司是一家区块链门户网站，该网站的很多资讯，快讯，视频等数据都是通过爬虫爬取得第三方网站获得的，需要从很多网站要爬取数据，如果每个数据源网站都需要单独写个接口去爬的话，工作量无疑是巨大的，因为笔者想到了通过动态代理实现一套爬虫机制，每次要爬取新的数据源，只要在数据库里增加一条数据源即可，无需修改代码。废话不多说，下贴出数据库表结构

DROP TABLE IF EXISTS `yiyi_crawler_website`;
CREATE TABLE `yiyi_crawler_website` ( `id` bigint(16) NOT NULL AUTO_INCREMENT COMMENT '自增ID', `gmt_create` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `gmt_modified` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', `url` varchar(255) DEFAULT NULL COMMENT '网站链接（抓取内容的接口）', `interval` bigint(16) DEFAULT '0' COMMENT '抓取时间间隔（以毫秒为单位）', `website_type` tinyint(20) DEFAULT NULL COMMENT '网站类型(1、快讯)', `website_name` varchar(32) DEFAULT NULL COMMENT '网站名', `source_link` varchar(255) DEFAULT NULL COMMENT '来源链接', `data_field` varchar(32) DEFAULT NULL COMMENT '数据所在字段，如果没有，为空则直接取数(多级以.连接，如果：data.items表示data下面的items为内容列表)', PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;
DROP TABLE IF EXISTS `yiyi_crawler_website_content`;
CREATE TABLE `yiyi_crawler_website_content` ( `id` bigint(16) NOT NULL AUTO_INCREMENT, `gmt_create` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `gmt_modified` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', `website_id` bigint(16) DEFAULT NULL COMMENT '网站ID', `content_name` varchar(16) DEFAULT NULL COMMENT '内容名', `table_name` varchar(32) DEFAULT NULL COMMENT '所属表名', `column_name` varchar(32) DEFAULT NULL COMMENT '所属字段名', `return_field` varchar(32) DEFAULT NULL COMMENT '当前要抓取的字段所返回的字段', `field_type` tinyint(2) DEFAULT '2' COMMENT '字段类型（1、日期2、数值0、其他）', PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;

下面贴出爬虫的动态代理实现，基于cglib框架实现的

/** * 爬虫任务代理接口 * * @author liyi * @create 2018-03-17 16:58 **/
public interface CrawlerProxy { 
   

    /** * 任务开始 * @param website */
    void start(CrawlerWebsiteModelOut website);
}
/** * 爬虫任务类 * * @author liyi * @create 2018-03-17 18:21 **/
public class CrawlerTask implements CrawlerProxy { 
   

    @Override
    public void start(CrawlerWebsiteModelOut website) {
        System.out.println("爬虫任务开始");
    }
}
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.TypeReference;
import com.lynn.yiyi.http.Request;
import com.lynn.yiyi.http.WebUtils;
import com.lynn.yiyi.model.out.CrawlerWebsiteModelOut;

import java.util.*;

/** * 爬虫定时任务 * * @author liyi * @create 2018-03-17 18:35 **/
public class CrawlerTimerTask extends TimerTask { 
   
    private CrawlerWebsiteModelOut website = null;

    @Override
    public void run() {
        String json = WebUtils.executeHttp(Request.options().setMethod(com.lynn.yiyi.http.Method.GET).setUrl(website.getUrl()).build()).getJsonString();
        String strs[] = website.getDataField().split(".");
        List<String> dataList = new ArrayList<>();
        Arrays.stream(strs).forEach(s -> {
            dataList.clear();
            String data = JSON.parseObject(json, new TypeReference<Map<String, List<Map<String, String>>>>() {}.getType());
            dataList.add(data);
        });
    }

    public void setWebsite(CrawlerWebsiteModelOut website) {
        this.website = website;
    }

    public CrawlerWebsiteModelOut getWebsite() {
        return website;
    }
}
import com.alibaba.fastjson.JSONObject;
import com.lynn.yiyi.http.Request;
import com.lynn.yiyi.http.WebUtils;
import com.lynn.yiyi.model.out.CrawlerWebsiteModelOut;
import com.lynn.yiyi.service.CrawlerService;
import com.lynn.yiyi.utils.SpringUtils;
import net.sf.cglib.proxy.Enhancer;
import net.sf.cglib.proxy.MethodInterceptor;
import net.sf.cglib.proxy.MethodProxy;

import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/** * 爬虫动态代理类 * * @author liyi * @create 2018-03-17 18:22 **/
public class CrawlerCglibProxy implements MethodInterceptor { 
   

    private Map<Long,Timer> timerMap = new HashMap<>();

    private Enhancer enhancer = new Enhancer();

    Object getProxy(Class<?> clazz){
        enhancer.setSuperclass(clazz);
        enhancer.setCallback(this);
        return enhancer.create();
    }

    @Override
    public Object intercept(Object obj, Method method, Object[] args, MethodProxy proxy) throws Throwable {
        Object o = proxy.invokeSuper(obj,args);
        if("start".equals(method.getName())){
            if(args[0] instanceof CrawlerWebsiteModelOut){
                CrawlerWebsiteModelOut website = (CrawlerWebsiteModelOut)args[0];
                if(timerMap.get(website.getId()) == null){
                    Timer timer = new Timer();
                    timer.schedule(new TimerTask() {
                        @Override
                        public void run() {
                            String data = WebUtils.executeHttp(Request.options().setMethod(com.lynn.yiyi.http.Method.GET).setUrl(website.getUrl()).build()).getJsonString();
                            String strs[] = website.getDataField().split("\\.");
                            for (String s : strs) {
                                JSONObject object = JSONObject.parseObject(data);
                                data = object.getString(s);
                            }
                            //TODO 这里将爬取到的数据写到数据库对应的表中
                        }
                    }, 0, website.getInterval());
                    timerMap.put(website.getId(),timer);
                }
            }
        }
        return o;
    }

    public static <T> T create(Class<T> cls){
        CrawlerCglibProxy proxy = new CrawlerCglibProxy();
        return (T)proxy.getProxy(cls);
    }

}
import java.util.ArrayList;
import java.util.List;

/** * 网站爬虫输出参数 * * @author liyi * @create 2018-03-17 17:04 **/
public class CrawlerWebsiteModelOut extends BaseModelOut { 
   

    private String url;

    private Long interval;

    private Integer websiteType;

    private String websiteName;

    private String sourceLink;

    private String dataField;

    private List<CrawlerWebsiteContentModelOut> contentList = new ArrayList<>();

    public String getUrl() {
        return url;
    }

    public void setUrl(String url) {
        this.url = url;
    }

    public Long getInterval() {
        return interval;
    }

    public void setInterval(Long interval) {
        this.interval = interval;
    }

    public Integer getWebsiteType() {
        return websiteType;
    }

    public void setWebsiteType(Integer websiteType) {
        this.websiteType = websiteType;
    }

    public String getWebsiteName() {
        return websiteName;
    }

    public void setWebsiteName(String websiteName) {
        this.websiteName = websiteName;
    }

    public String getSourceLink() {
        return sourceLink;
    }

    public void setSourceLink(String sourceLink) {
        this.sourceLink = sourceLink;
    }

    public String getDataField() {
        return dataField;
    }

    public void setDataField(String dataField) {
        this.dataField = dataField;
    }

    public List<CrawlerWebsiteContentModelOut> getContentList() {
        return contentList;
    }

    public void setContentList(List<CrawlerWebsiteContentModelOut> contentList) {
        this.contentList = contentList;
    }
}
/** * 爬虫网站内容输出参数 */
public class CrawlerWebsiteContentModelOut extends BaseModelOut{ 
   

    private Long websiteId;

    private String contentName;

    private String tableName;

    private String columnName;

    private String returnField;

    private Integer fieldType;

    public Long getWebsiteId() {
        return websiteId;
    }

    public void setWebsiteId(Long websiteId) {
        this.websiteId = websiteId;
    }

    public String getContentName() {
        return contentName;
    }

    public void setContentName(String contentName) {
        this.contentName = contentName;
    }

    public String getTableName() {
        return tableName;
    }

    public void setTableName(String tableName) {
        this.tableName = tableName;
    }

    public String getColumnName() {
        return columnName;
    }

    public void setColumnName(String columnName) {
        this.columnName = columnName;
    }

    public String getReturnField() {
        return returnField;
    }

    public void setReturnField(String returnField) {
        this.returnField = returnField;
    }

    public Integer getFieldType() {
        return fieldType;
    }

    public void setFieldType(Integer fieldType) {
        this.fieldType = fieldType;
    }

}
import java.util.Date;
/** * 基础输出参数 * * @author liyi * @create 2018-03-17 17:02 **/
public abstract class BaseModelOut{ 
   

    private Long id;

    private Date create;

    private Date modified;

    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public Date getCreate() {
        return create;
    }

    public void setCreate(Date create) {
        this.create = create;
    }

    public Date getModified() {
        return modified;
    }

    public void setModified(Date modified) {
        this.modified = modified;
    }
}

下面给出测试的main方法

public static void main(String[] args) {
        CrawlerProxy proxy = CrawlerFactory.create();
        proxy.start(website);//website即当前要爬取的网站对象，可以从数据库中读取
    }