Python 系列文章 —— 爬虫小案例

# 采用 HTTP GET 请求的方法模拟谷歌浏览器访问网站，输出响应上下文
from urllib import request,parse
url = 'http://www.python.org'
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'
}
req = request.Request(url, headers = headers, method = 'GET')
response = request.urlopen(req) 
print(response.read())

# 采用 HTTP POST 请求的方法模拟谷歌浏览器访问网站，输出响应上下文
from urllib import request
from urllib import parse
url = 'https://fanyi.baidu.com/sug'
data = {'kw' : 'python'}
data = bytes(parse.urlencode(data), encoding = 'utf-8')
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'
}
req = request.Request(url, data, headers, method = 'POST')
response = request.urlopen(req) 
print(response.read().decode('unicode_escape'))

# 创建一个 HTTP GET 请求，通过 add_header 添加一个 UserAgent
import urllib.request
import random
url = 'http://www.python.org'
headerUserAgentList = ['Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0']
randomHeaderUserAgent = random.choice(headerUserAgentList) # 随机选取一个 UserAgent
req = urllib.request.Request(url) 
req.add_header('User-Agent', randomHeaderUserAgent) # 添加 UserAgent
response=urllib.request.urlopen(req)
print(req.get_header('User-agent'))
print(req.headers) # 打印请求的 header 信息

# 返回401未授权错误
from urllib import request,error
try:
    response=request.urlopen('http://pythonscraping.com/pages/auth/login.php')
    print(response.getcode())
except error.HTTPError as e:
    print('1.错误原因：\n%s\n2.状态码：\n%s\n3.响应头信息：\n%s' %(e.reason, e.code, e.headers))
except error.URLError as e:
    print(e.reason)

# 采用 quote 对 url 中的汉字进行编码，输出编码后的结果
import urllib
url = 'http://www.baidu.com/爬虫'
result = urllib.parse.quote(url)
print(result)
url = 'http://www.baidu.com/+爬虫'
result = urllib.parse.quote(url, '+') # 更改 safe 参数
print(result)

# 使用两种爬虫代理分别查看是否可以对 'http://www.baidu.com' 网站进行爬取
from urllib.robotparser import RobotFileParser
rp = RobotFileParser()
rp.set_url("http://www.baidu.com/robots.txt")
rp.read()
print(rp.can_fetch('Baiduspider', 'http://www.baidu.com')) 
print(rp.can_fetch('*', 'http://www.baidu.com'))