上代码:
#***************************************************
#这份代码用于从911网站上爬取信息
#其中的IP.txt文件为我本地存IP的文件
#
#***************************************************
import requests,csv
import pandas as pd
import time,random
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
#利用pandas读取csv文件
def getNames(csvfile):
data = pd.read_csv(csvfile) # 1--读取的文件编码问题有待考虑
names = data['name']
return names
'''
获取IP列表
事先已经在网上爬取了大量ip地址存入IP.txt中
IP地址的获取方法参照我的另一篇博客( Python--代理IP)
'''
def get_ip_list():
f=open('IP.txt','r')
ip_list=f.readlines()
f.close()
return ip_list
#从IP列表中获取随机IP
def get_random_ip(ip_list):
proxy_ip = random.choice(ip_list)
proxy_ip=proxy_ip.strip('\n')
proxies = {'http': proxy_ip}
return proxies
#爬取网页并返回所需信息以及状态码
def parsePage(url,ip_list):
fuck_header= {'User-Agent':str(UserAgent().random)}
proxies = get_random_ip(ip_list)
try:
r = requests.get(url, proxies=proxies, headers=fuck_header, timeout=10,verify=False) #verify设置为False,Requests也能忽略对SSL证书的验证。
except: #2--verify的说明有待补充
print('运行错误,程序暂停20秒')
time.sleep(20)
fuck_header= {'User-Agent':str(UserAgent().random)}
proxies = get_random_ip(ip_list)
r = requests.get(url, proxies=proxies, headers=fuck_header, timeout=10, verify=False)
if r.status_code == 200: #状态码status_code为200代表爬取成功,为404则为未爬取到相关信息
soup = BeautifulSoup(r.text, 'lxml')
body = soup.find("div", class_="pp")
contents = body.find_all('p')
return r.status_code, contents
else:
return r.status_code, None
def getDict(contents):
namesChineseTransliteration = [] #中文音译
namesGender = [] #名字性别
namesFromLanguage = [] #来源语种
namesMoral = [] #名字寓意
namesImpression = [] #名字印象
namesMeaning = [] #名字含义
#get_text() 方法,这个方法获取到tag中包含的所有文版内容包括子孙tag中的内容,并将结果作为Unicode字符串返回
#[4:]:读取第五的字符以后的字符
namesChineseTransliteration.append(contents[1].get_text()[4:])
namesGender.append(contents[-5].get_text()[4:])
namesFromLanguage.append(contents[-4].get_text()[4:])
namesMoral.append(contents[-3].get_text()[4:])
namesImpression.append(contents[-2].get_text()[4:])
namesMeaning.append(contents[-1].get_text()[4:])
str_row=namesChineseTransliteration+namesGender+namesFromLanguage+namesMoral+namesImpression+namesMeaning
return str_row
#功能:将信息写入文件
def write_file(filePath,row):
with open(filePath,'a+',encoding='utf-8',newline='') as csvfile:
spanreader = csv.writer(csvfile,delimiter='|',quoting=csv.QUOTE_MINIMAL)
spanreader.writerow(row)
if __name__ == "__main__":
for i in range(26,27):
names = getNames("{}.csv".format(i)) #获取需要爬取文件的名字
base_url = "https://myingwenming.911cha.com/" #kzb--在地址后面追加了xm_
ip_list = get_ip_list()
for j in range(len(names)):
url = base_url + names[j] + ".html"
status_code, contents = parsePage(url,ip_list)
print(names[j], status_code)
if status_code == 200: #状态码为200爬取成功,状态码为404爬取失败
str_row = getDict(contents)
# row = names[j]+str_row
temp = []
temp.append(names[j])
row = temp + str_row
write_file("爬取成功的人名/new{}.csv".format(i),row)
else:
continue
欢迎留言交流!