实战项目四：爬取911网站

def get_ip_list():
	'''
	读取IP.txt中的数据
	'''
	f=open('IP.txt','r')
	ip_list=f.readlines()
	f.close()
	return ip_list

def get_random_ip(ip_list):
	'''
	从IP列表中获取随机IP
	'''
	proxy_ip = random.choice(ip_list)
	proxy_ip=proxy_ip.strip('\n')
	proxies = {'http': proxy_ip}
	return proxies

from fake_useragent					import UserAgent
from bs4							import	BeautifulSoup
import pandas						as pd
import requests,csv,time,random

def get_ip_list():
	'''
	读取IP.txt中的数据
	'''
	f=open('IP.txt','r')
	ip_list=f.readlines()
	f.close()
	return ip_list

def get_random_ip(ip_list):
	'''
	从IP列表中获取随机IP
	'''
	proxy_ip = random.choice(ip_list)
	proxy_ip=proxy_ip.strip('\n')
	proxies = {'http': proxy_ip}
	return proxies

def parsePage(url,ip_list):
	'''
	爬取网页并返回所需信息以及状态码
	'''
	headers= {'User-Agent':str(UserAgent().random)}
	proxies = get_random_ip(ip_list)
	try:
		#verify设置为False,Requests也能忽略对SSL证书的验证。
		r = requests.get(url, proxies=proxies, headers=headers, timeout=10,verify=False)
	except:
		print('运行错误，程序暂停20秒')
		time.sleep(20)
		headers= {'User-Agent':str(UserAgent().random)}
		proxies = get_random_ip(ip_list)
		r = requests.get(url, proxies=proxies, headers=headers, timeout=10, verify=False)
	#状态码status_code为200代表爬取成功，为404则为未爬取到相关信息
	if r.status_code == 200:
		soup = BeautifulSoup(r.text, 'lxml')
		body = soup.find("div", class_="pp")
		contents = body.find_all('p')
		return r.status_code, contents
	else:
		return r.status_code, None

def getDict(contents):
	namesChineseTransliteration = []        #中文音译
	namesGender = []        #名字性别
	namesFromLanguage = []      #来源语种
	namesMoral = []         #名字寓意
	namesImpression = []        #名字印象
	namesMeaning = []           #名字含义

	namesChineseTransliteration.append(contents[1].get_text()[4:])
	namesGender.append(contents[-5].get_text()[4:])
	namesFromLanguage.append(contents[-4].get_text()[4:])
	namesMoral.append(contents[-3].get_text()[4:])  
	namesImpression.append(contents[-2].get_text()[4:])
	namesMeaning.append(contents[-1].get_text()[4:])
	str_row=namesChineseTransliteration+namesGender+namesFromLanguage+namesMoral+namesImpression+namesMeaning
 
	return str_row
 
def write_file(filePath, row):
	with open(filePath,'a+',encoding='utf-8',newline='') as csvfile:
		spanreader = csv.writer(csvfile,delimiter='|',quoting=csv.QUOTE_MINIMAL)
		spanreader.writerow(row)

if __name__ == "__main__":
	names = pd.read_csv("name_data.csv")['name']					#获取需要爬取文件的名字
	base_url = "https://myingwenming.911cha.com/"
	ip_list = get_ip_list()
	
	for name in names:
		url = base_url + name + ".html"

		status_code, contents = parsePage(url,ip_list)

		print("{}检索完成".format(name), "状态码为：{}".format(status_code))
		#状态码为200爬取成功，状态码为404爬取失败
		if status_code == 200:
			str_row = getDict(contents)
			row = ["{}".format(name)] + str_row
			write_file("new.csv",row)
		else:
			continue