如何运用Python的AI爬虫技术，采集比天眼查更完整的数据？我的经验分享

用Python的AI爬虫技术，采集到比天眼查更完整和更及时的数据，提高python的挖掘性能，其实天眼查、查查查这些网站的数据还是具有延迟性，很多数据还是不够完整，而且很多数据源都是通国各种网站数据采集整合过来的，最近在研究Python的多进程和多线程AI智能引擎技术实现一个全网数据挖掘和分析的引擎，搜索引擎技术就是能够扫描全网，然后收录一些有用的价值，像百度搜索引擎、google搜索引擎，他们就是采用爬虫技术，对全网进行24小时的扫描，然后把各种网站的数据进行采集做快照，基于这种搜索引擎技术，我们如果要爬取比天眼查、查查查更完整的数据，只要在我们搜索引擎里面植入我们自己的代理IP池技术，采用python的多进程技术进对目标网站进行分任务，每个进程负责一个目标网站，然后采用python的多线程技术，每个来负责翻页爬取，每个线程负责10页的翻页，然后再启动守护进程24小时进行全网扫描。下面是我自己写的代码，结果效率和爬取得到的数据结果还是超出满意：

#AI大数据系统-配置参数

ConfigData=[]

ThreadNum=20 #k配置开启线程，开启越多就越快，但是需要看电脑CPU 配置来

#按省份-分任务全完深度挖掘

ConfigData.append({'DataType':'anhui|beijing|chongqing|fujian|gansu|guangdong|

guangxi|guizhou|hebei|heilongjiang|hubei|hunan|jiangsu|jiangxi|

jilin|liaoning|neimenggu|ningxia|qinghai|shandong|shanghai|shanxi|

shanxii|ichuan|tianjin|xinjiang|xizang|yunnan|zhejiang'})

do = Spider()

pageUrls=ConfigData[0]['DataType'].split("|")

BaseConfig = {

'Urls': pageUrls, # 获取任务列表

'ThreadNum': ThreadNum , # 开启进程数量

'Type': 2, # 1 采用多线程模式， 2采用多进程模式

'do': do,

}

lock = multiprocessing.Lock() # 创建进程锁

# 计算任务总量

# 切分N段，每个进程负责一段

taskNUm=len(pageUrls)=

average = math.ceil(taskNUm / (BaseConfig['ThreadNum']))

print('本次总任务数量{}'.format(taskNUm))

for i in range(0, BaseConfig['ThreadNum']):

# time.sleep(2)

pstart = average * i

pend = pstart + average

p = multiprocessing.Process(target=runProcessing, args=(BaseConfig, pstart, pend, i, lock))

print('进程完成结束'.format(p))

p.start()

p.join()

print("进程创建完成，开始采集...")

下面是跑下来得到的数据结构：*（感觉还是不赖的）

{

    "url": "http://www.xxx.com/firm_0a6ca0be1266656b93d03390abf4e40b.html",

    "基本信息": {

        "人员规模：": "-",

        "企业地址：": "温州市鹿城区蒲鞋市街道车站大道733号",

        "公司名": "温州均瑶集团有限公司",

        "公司类型：": "有限责任公司(自然人投资或控股)",

        "地址": "温州市鹿城区蒲鞋市街道车站大道733号",

        "官网": "http://www.juneyao.com",

        "成立日期：": "1995-08-23",

        "所属地区": "浙江省",

        "所属行业": "住宿和餐饮业",

        "曾用名：": "均瑶集团有限公司",

        "核准日期：": "2016-12-30",

        "法定代表人信息": {

            "pl_url": "http://www.xxx.com/pl_p808d5b5f8c083541230a8951a8866aa.html",

            "他关联公司数": "39",

            "姓名": "王均豪"

        },

        "注册号：": "330305000013113",

        "注册资本：": "8850万人民币",

        "登记机关：": "温州市市场监督管理局",

        "登记状态": "存续",

        "纳税人识别号：": "91330300145067169Y",

        "组织机构代码：": "14506716-9",

        "经营状态：": "存续",

        "经营范围：": "住宿服务（限下设分支机构经营）。  对实业项目的投资。",

        "统一社会信用代码：": "91330300145067169Y",

        "联系方式": "0577-88098816",

        "英文名：": "-",

        "营业期限：": "1995-08-23 至 2045-08-22",

        "邮箱": "暂无"

    },

    "主要成员信息": [

        {

            "pl_url": "http://www.xxx.com/pl_p7663e2e7b8bcd23cfa9e1d85eac902a.html",

            "他关联公司数": "15",

            "姓名": "裴学龙",

            "职务": "监事"

        },

        {

            "pl_url": "http://www.xxx.com/pl_p75b867211714a967474c4ac4706e918.html",

            "他关联公司数": "11",

            "姓名": "王瀚",

            "职务": "董事"

        },

        {

            "pl_url": "http://www.xxx.com/pl_pa2912161d6acaed1baea9e26e08e062.html",

            "他关联公司数": "12",

            "姓名": "钱克流",

            "职务": "董事"

        },

        {

            "pl_url": "http://www.xxx.com/pl_pad4f49d512d91e830c1fcf76235e986.html",

            "他关联公司数": "5",

            "姓名": "陈体理",

            "职务": "监事"

        },

        {

            "pl_url": "http://www.xxx.com/pl_p808d5b5f8c083541230a8951a8866aa.html",

            "他关联公司数": "39",

            "姓名": "王均豪",

            "职务": "-"

        },

        {

            "pl_url": "http://www.xxx.com/pl_pc97b89c41134307548eb11de6430258.html",

            "他关联公司数": "23",

            "姓名": "蒋海龙",

            "职务": "监事"

        },

        {

            "pl_url": "http://www.xxx.com/pl_pd1515f7945aecbfe2bdafdf098bc819.html",

            "他关联公司数": "32",

            "姓名": "王均金",

            "职务": "-"

        },

        {

            "pl_url": "http://www.xxx.com/pl_p2a05f8d97284529e0ad9f176a603db8.html",

            "他关联公司数": "14",

            "姓名": "林乃机",

            "职务": "董事"

        }

    ],

    "公司名": "温州均瑶集团有限公司",

    "其他链接": {

        "企业年报": {

            "count": "4",

            "url": "http://www.xxx.com/creport_0a6ca0be1266656b93d03390abf4e40b"

        },

        "对外投资": {

            "count": "12",

            "url": "http://www.xxx.com/cinvestment_0a6ca0be1266656b93d03390abf4e40b"

        },

        "法律诉讼": {

            "count": "0",

            "url": "http://www.xxx.com/clawsuit_0a6ca0be1266656b93d03390abf4e40b"

        },

        "知识产权": {

            "count": "106",

            "url": "http://www.xxx.com/cassets_0a6ca0be1266656b93d03390abf4e40b"

        },

        "经营状况": {

            "count": "7",

            "url": "http://www.xxx.com/crun_0a6ca0be1266656b93d03390abf4e40b"

        }

    },

    "分支机构信息": [

        {

            "company": "温州均瑶集团有限公司均瑶宾馆",

            "url": "http://www.xxx.com/firm_2fd416c499d26a28b934e2c7af7b4c47.html"

        },

        {

            "company": "温州均瑶集团有限公司均瑶宾馆",

            "url": "http://www.xxx.com/firm_2fd416c499d26a28b934e2c7af7b4c47.html"

        },

        {

            "company": "温州均瑶集团有限公司兰州分公司",

            "url": "http://www.xxx.com/firm_cfeba3d00d1cfc9483791d1db6ba9b2b.html"

        },

        {

            "company": "温州均瑶集团有限公司武汉分公司",

            "url": "http://www.xxx.com/firm_7254f781af1c076cfa181050349c75c7.html"

        },

        {

            "company": "温州均瑶集团有限公司杭州分公司",

            "url": "http://www.xxx.com/firm_73a2dc6bc97566f04bb31224ea99b9a2.html"

        },

        {

            "company": "温州均瑶集团有限公司西安分公司",

            "url": "http://www.xxx.com/firm_9112b5ad54de9b2a192351e97a17bfda.html"

        },

        {

            "company": "温州均瑶集团有限公司无锡分公司",

            "url": "http://www.xxx.com/firm_b5522b2f6f333f4fe080184344d68181.html"

        }

    ],

    "变更信息": [

        {

            "变更项目": "历史法定代表人",

            "姓名": "王均金"

        },

        {

            "变更前": "王均豪*,钱克流,王均金,许彪,陈体理,蒋海龙,裴学龙",

            "变更后": "王均豪*,钱克流,王均金,陈体理,蒋海龙,裴学龙,王瀚,林乃机",

            "变更日期": "2016-12-30",

            "变更项目": "高级管理人员备案 带有*标记的为法定代表人",

            "序号": "1"

        },

        {

            "变更前": "原章程",

            "变更后": "2016年6月23日章程修订本",

            "变更日期": "2016-12-30",

            "变更项目": "章程修正案备案",

            "序号": "1"

        },

        {

            "变更前": "注册号:330305000013113组织机构代码证:145067169",

            "变更后": "统一社会信用代码:91330300145067169Y",

            "变更日期": "2016-04-08",

            "变更项目": "换发统一社会信用代码执照",

            "序号": "1"

        },

        {

            "变更前": "一般经营项目:工艺美术品、其他日用品、装饰材料、食品机械设备的销售 机票代理服务。",

            "变更后": "一般经营项目:对实业项目的投资。",

            "变更日期": "2013-08-23",

            "变更项目": "一般经营项目变更",

            "序号": "1"

        },

        {

            "变更前": "许可经营项目:",

            "变更后": "许可经营项目:住宿服务(限下设分支机构经营)。",

            "变更日期": "2012-06-18",

            "变更项目": "许可经营项目变更",

            "序号": "1"

        },

        {

            "变更前": "住所:****号\n                                        邮政编码:325011\n                                        电话:********\n                                        住所:****",

            "变更后": "住所:****号\n                                        邮政编码:325011\n                                        电话:********\n                                        住所:****",

            "变更日期": "2008-11-03",

            "变更项目": "住所(营业场所、地址)变更",

            "序号": "1"

        },

        {

            "变更前": "王均金",

            "变更后": "王均豪*",

            "变更日期": "2005-04-14",

            "变更项目": "法定代表人(负责人、董事长、首席代表)变更 带有*标记的为法定代表人",

            "序号": "1"

        },

        {

            "变更前": "姓名:倪细伦\n                                        证件名称:\n                                        证件号码:********\n                                        性别:男\n                                        职务:董事",

            "变更后": "姓名:裴学龙\n                                        证件名称:\n                                        证件号码:********\n                                        性别:男\n                                        职务:监事",

            "变更日期": "2004-11-22",

            "变更项目": "法定代表人(负责人、董事长、首席代表)变更 带有*标记的为法定代表人",

            "序号": "2"

        },

        {

            "变更前": "均瑶集团有限公司",

            "变更后": "温州均瑶集团有限公司",

            "变更日期": "2004-03-02",

            "变更项目": "名称变更",

            "序号": "1"

        },

        {

            "变更前": "温州市均瑶集团有限公司",

            "变更后": "均瑶集团有限公司",

            "变更日期": "1998-07-22",

            "变更项目": "名称变更",

            "序号": "2"

        },

        {

            "变更前": "经营范围:食品,非酒精饮料,机票代理,塑料制品、塑料包装,工艺品,航空用品,日用百货,五金家电,化工原料(不含危险品),礼品,装璜材料,家具,庆上用品,针织品,服装,游艺活动(须持许可证方可经营)食品机械,餐饮(须持卫生许可证方可经营)。行业代码:1439",

            "变更后": "经营范围:生产销售食品,非酒精饮料,机票代理,塑料制品、塑料包装,工艺品,航空用品,日用百货,五金家电,化工原料(不含危险品),礼品,装璜材料,家具,庆上用品,针织品,服装,游艺活动(须持许可证方可经营)食品机械,餐饮(须持卫生许可证方可经营)。附设分公司经营范围:餐饮、住宿、KTV、歌舞厅、台球、美容美发、电子游戏 保龄球、健美、健美操、运动保健按摩。行业代码:1439",

            "变更日期": "1999-07-01",

            "变更项目": "经营范围(业务范围)变更",

            "序号": "1"

        },

        {

            "变更前": "经营范围:食品,非酒精饮料,机票代理,塑料制品、塑料包装,工艺品,航空用品。行业代码:1439",

            "变更后": "经营范围:食品,非酒精饮料,机票代理,塑料制品、塑料包装,工艺品,航空用品,日用百货,五金家电,化工原料(不含危险品),礼品,装璜材料,家具,庆上用品,针织品,服装,游艺活动(须持许可证方可经营)食品机械,餐饮(须持卫生许可证方可经营)。行业代码:1439",

            "变更日期": "1997-04-29",

            "变更项目": "经营范围(业务范围)变更",

            "序号": "2"

        },

        {

            "变更前": "2050",

            "变更后": "8850",

            "变更日期": "1998-06-01",

            "变更项目": "注册资本(金)变更",

            "序号": "1"

        }

    ],

    "对外投资信息": [

        {

            "url": "http://www.xxx.com/firm_766b85e5c8b26c022f981bdd528fd4e9.html",

            "投资公司": "均瑶集团出租汽车有限公司",

            "持股比例": "90.0%",

            "股东类型": "企业法人",

            "认缴出资日期": "1998-11-03",

            "认缴出资额（万元）": "900"

        },

        {

            "url": "http://www.xxx.com/firm_756f1439563684c178c87bcdc4a7e84e.html",

            "投资公司": "均瑶集团上海有限公司",

            "持股比例": "60.0%",

            "股东类型": "企业法人",

            "认缴出资日期": "-",

            "认缴出资额（万元）": "120"

        },

        {

            "url": "http://www.xxx.com/firm_f62ffa20abdd72a393801d9b75958bdb.html",

            "投资公司": "均瑶集团航空服务有限公司",

            "持股比例": "10.0%",

            "股东类型": "企业法人",

            "认缴出资日期": "-",

            "认缴出资额（万元）": "100"

        },

        {

            "url": "http://www.xxx.com/firm_498d2af5c593b5ca293e46c0f50d97e7.html",

            "投资公司": "均瑶集团乳业股份有限公司",

            "持股比例": "-",

            "股东类型": "企业法人",

            "认缴出资日期": "2010-07-22",

            "认缴出资额（万元）": "-"

        },

        {

            "url": "http://www.xxx.com/firm_fe5ea392f0f4477b90c3eb70be7cb7fc.html",

            "投资公司": "上海均瑶航空投资有限公司",

            "持股比例": "50.0%",

            "股东类型": "企业法人",

            "认缴出资日期": "2002年8月12日",

            "认缴出资额（万元）": "5400"

        },

        {

            "url": "http://www.xxx.com/firm_b52f27fda2943d818c6d2ffbde8b9640.html",

            "投资公司": "温州均瑶商务服务有限公司",

            "持股比例": "20.0%",

            "股东类型": "法人股东",

            "认缴出资日期": "1997-05-23",

            "认缴出资额（万元）": "10 万元"

        },

        {

            "url": "http://www.xxx.com/firm_75d3cea813c7487dc2dc476934f8045d.html",

            "投资公司": "温州瑶住酒店管理有限公司",

            "持股比例": "40.0%",

            "股东类型": "企业法人",

            "认缴出资日期": "-",

            "认缴出资额（万元）": "20"

        },

        {

            "url": "http://www.xxx.com/firm_52c347f6739232f68430a61f349d0e0f.html",

            "投资公司": "温州均瑶物业服务有限公司",

            "持股比例": "100%",

            "股东类型": "企业法人",

            "认缴出资日期": "-",

            "认缴出资额（万元）": "-"

        },

        {

            "url": "http://www.xxx.com/firm_1d75185b284c1bde2663c75e8f7ec65b.html",

            "投资公司": "温州市天龙包机实业有限公司",

            "持股比例": "51.00%",

            "股东类型": "企业法人",

            "认缴出资日期": "1991-12-29",

            "认缴出资额（万元）": "306"

        },

        {

            "url": "http://www.xxx.com/firm_ed495f1f911e0602a215c686b2f44c7a.html",

            "投资公司": "宜昌均瑞房地产开发有限公司",

            "持股比例": "77.00%",

            "股东类型": "企业法人",

            "认缴出资日期": "2012/10/16 0:00:00",

            "认缴出资额（万元）": "7700"

        },

        {

            "url": "http://www.xxx.com/firm_8a82d0cdc7d8b401c3b8c80811dfcb77.html",

            "投资公司": "宜昌均瑶贸易有限公司",

            "持股比例": "100%",

            "股东类型": "企业法人",

            "认缴出资日期": "2025年12月31日",

            "认缴出资额（万元）": "46000"

        },

        {

            "url": "http://www.xxx.com/firm_032731387625750f57f84067e3cfea41.html",

            "投资公司": "深圳市均瑶奶品有限公司",

            "持股比例": "70.0%",

            "股东类型": "法人股东",

            "认缴出资日期": "-",

            "认缴出资额（万元）": "140万元"

        }

    ],

    "股东信息": [

        {

            "pl_url": "http://www.xxx.com/pl_p75b867211714a967474c4ac4706e918.html",

            "他关联公司数": "11",

            "持股比例": "38.500%",

            "股东(发起人)": "王瀚",

            "股东类型": "自然人股东",

            "认缴出资日期": "1997-04-14",

            "认缴出资额（万元）": "3407.25"

        },

        {

            "pl_url": "http://www.xxx.com/pl_pd1515f7945aecbfe2bdafdf098bc819.html",

            "他关联公司数": "32",

            "持股比例": "35.00%",

            "股东(发起人)": "王均金",

            "股东类型": "自然人股东",

            "认缴出资日期": "1998-03-31",

            "认缴出资额（万元）": "3097.5"

        },

        {

            "pl_url": "http://www.xxx.com/pl_p808d5b5f8c083541230a8951a8866aa.html",

            "他关联公司数": "39",

            "持股比例": "25.00%",

            "股东(发起人)": "王均豪",

            "股东类型": "自然人股东",

            "认缴出资日期": "1998-03-31",

            "认缴出资额（万元）": "2212.5"

        },

        {

            "pl_url": "http://www.xxx.com/pl_p68ed1b7c5c7f645c5ac7a1e5dec44dd.html",

            "他关联公司数": "3",

            "持股比例": "0.500%",

            "股东(发起人)": "王超",

            "股东类型": "自然人股东",

            "认缴出资日期": "1997-04-14",

            "认缴出资额（万元）": "44.25"

        },

        {

            "pl_url": "http://www.xxx.com/pl_pe3c8177ddbd2ed2f2080fddb9d9c806.html",

            "他关联公司数": "3",

            "持股比例": "0.500%",

            "股东(发起人)": "王宝弟",

            "股东类型": "自然人股东",

            "认缴出资日期": "1997-04-14",

            "认缴出资额（万元）": "44.25"

        },

        {

            "pl_url": "http://www.xxx.com/pl_p6269be8f67e18ab3bc1403e941d86ba.html",

            "他关联公司数": "5",

            "持股比例": "0.500%",

            "股东(发起人)": "王滢滢",

            "股东类型": "自然人股东",

            "认缴出资日期": "1997-04-14",

            "认缴出资额（万元）": "44.25"

        }

    ]

}