python_爬虫基础学习

 1 r = requests.get("http://www.baidu.com")
 2 print(r.status_code)        #返回状态码（内容值为‘200’表示访问成功）
 3 #结果：    200
 4 print(r.text)   #返回响应内容的字符串形式
 5 #结果：【即-baidu主页的源代码内容（存在乱码字符）】
 6 print(r.encoding)   #从HTTP header中猜测的响应内容编码方式
 7 #结果：    ISO-8859-1      即baidu页面的编码标准（方式 ）
 8 print(r.apparent_encoding)  #从内容中分析出的响应内容编码方式[备选编码方式]
 9 #结果：    utf-8
10 print(r.content)    #内容的二进制形式
11 #结果：    总之看不懂
12 
13 r.encoding = 'utf-8'    #(utf-8为r.apparent_encoding得到的结果)
14 print(r.text)
15 #结果就为人可看懂的响应内容  主要是通过r.encoding的重新定义类型，而r.text是参照encoding的编码方式

 1 def getHTMLText(url):
 2     try:
 3         r = requests.get(url,timeout = 30)
 4         r.raise_for_status() #如果状态不是200 则抛出异常：requests.HTTPError(r.status_code)
 5         r.encoding = r.apparent_encoding
 6         return r.text
 7     except:
 8         return "产生异常"
 9 
10 if __name__ == "__main__":
11     url = 'http://www.baidu.com'
12     print(getHTMLText(url))

 1 def getHTMLText(url):
 2     try:
 3         r = requests.get(url,timeout = 30)
 4         r.raise_for_status() #如果状态不是200 则抛出异常：requests.HTTPError(r.status_code)
 5         r.encoding = r.apparent_encoding
 6         return r.text
 7     except:
 8         return "产生异常"
 9 
10 if __name__ == "__main__":
11     url = 'http://www.baidu.com'
12     print(getHTMLText(url))

 1 #requests库的head()方法 （请求获取URL位置资源的响应信息报告，即获得该资源的头部信息)
 2 r = requests.head('http://httpbin.org/get')
 3 print(r.headers)
 4 '''
 5 结果：
 6 {'Connection': 'keep-alive', 'Server': 'gunicorn/19.9.0',
 7  'Date': 'Thu, 22 Nov 2018 03:52:22 GMT', 'Content-Type':
 8   'application/json', 'Content-Length': '268',
 9    'Access-Control-Allow-Origin': '*',
10     'Access-Control-Allow-Credentials': 'true', 'Via': '1.1 vegur'}
11 '''
12 #requests库的post()方法（请求向URL位置的资源后附加新的数据）
13 payload = {'key1':'value1','key2':'value2'}
14 r = requests.post('http://httpbin.org/post',data = payload)
15 #向URL POST一个字典，自动编码未form（表单）
16 print(r.text)
17 '''
18 结果：
19 {
20   "args": {},
21   "data": "",
22   "files": {},
23   "form": {
24     "key1": "value1",
25     "key2": "value2"
26   },
27   "headers": {
28     "Accept": "*/*",
29     "Accept-Encoding": "gzip, deflate",
30     "Connection": "close",
31     "Content-Length": "23",
32     "Content-Type": "application/x-www-form-urlencoded",
33     "Host": "httpbin.org",
34     "User-Agent": "python-requests/2.20.1"
35   },
36   "json": null,
37   "origin": "106.111.147.213",
38   "url": "http://httpbin.org/post"
39 }
40 '''
41 r = requests.post('http://httpbin.org/post',data = 'abc')
42 #向URL POST一个字符串自动编码为 data
43 print(r.text)
44 '''
45 结果：
46 {
47   "args": {},
48   "data": "abc",
49   "files": {},
50   "form": {},
51   "headers": {
52     "Accept": "*/*",
53     "Accept-Encoding": "gzip, deflate",
54     "Connection": "close",
55     "Content-Length": "3",
56     "Host": "httpbin.org",
57     "User-Agent": "python-requests/2.20.1"
58   },
59   "json": null,
60   "origin": "106.111.147.213",
61   "url": "http://httpbin.org/post"
62 }
63 '''
64 #requests库的put()方法  （请求向URL位置存储一个资源，覆盖原URL位置的资源）
65 payload = {'key1':'value1','key2':'value2'}
66 r = requests.put('http://httpbin.org/put',data = payload)
67 #向URL PUT一个字典，自动编码未form（表单）[和post的区别是会覆盖原有内容]
68 print(r.text)
69 '''
70 结果:
71 {
72   "args": {},
73   "data": "",
74   "files": {},
75   "form": {
76     "key1": "value1",
77     "key2": "value2"
78   },
79   "headers": {
80     "Accept": "*/*",
81     "Accept-Encoding": "gzip, deflate",
82     "Connection": "close",
83     "Content-Length": "23",
84     "Content-Type": "application/x-www-form-urlencoded",
85     "Host": "httpbin.org",
86     "User-Agent": "python-requests/2.20.1"
87   },
88   "json": null,
89   "origin": "106.111.147.213",
90   "url": "http://httpbin.org/put"
91 }
92 '''

requests.request(method,url,**kwargs)

 1 #实例：爬取京东商品
 2 import requests
 3 import os
 4 '''
 5 r = requests.get('https://item.jd.com/36826348085.html?jd_pop=14982c1c-64d9-4bab-ac5c-e40af7ce62a2&abt=0')
 6 print(r.status_code)
 7 print(r.encoding)
 8 print(r.text[:1000])
 9 #实例：爬取亚马逊
10 
11 kv = {'User-Agent':'mozilla/5.0'}   #打造访问头（伪造成一个浏览器）
12 r=requests.get('https://www.amazon.cn/dp/B073LJR2JF/ref=cngwdyfloorv2_recs_0/459-8865541-1481366?pf_rd_m=A1AJ19PSB66TGU&pf_rd_s=desktop-2&pf_rd_r=AT12APC0BDJ216Y55VSR&pf_rd_r=AT12APC0BDJ216Y55VSR&pf_rd_t=36701&pf_rd_p=d2aa3428-dc2b-4cfe-bca6-5e3a33f2342e&pf_rd_p=d2aa3428-dc2b-4cfe-bca6-5e3a33f2342e&pf_rd_i=desktop',headers=kv)
13 print(r.status_code)
14 print(r.text[:1000])
15 print(r.request.headers)
16 '''
17 #实例：百度搜索
18 #百度搜索的搜索url：http://www.baidu.com/s?wd:{搜索内容}
19 '''
20 keyword = 'python'
21 try:
22     kv = {'wd':keyword}
23     r = requests.get('http://www.baidu.com/s',params = kv)
24     print(r.request.url)
25     r.raise_for_status()
26     print(len(r.text))
27 except:
28     print("爬取失败")
29     '''
30 #案例：网络图片爬取和存储
31 '''
32 url = "http://img0.dili360.com/pic/2018/09/21/5ba4a6af2af103q29157813_t.jpg"
33 rot = "E://"
34 path = rot + url.split('/')[-1] #指定图片保存名为url地址中的后缀名（url）
35 try:
36     if not os.path.exists(rot): #查看有没有rot目录，没有则创建
37         os.mkdir(rot)       #创建rot目录
38     if not os.path.exists(path):    #查看path目录下有没有path文件
39         r = requests.get(url)   #链接url地址
40         #with open(path,'wb') as f:  #另一种文件打开和存储方式(？？？)
41         f = open(path,'wb') #打开文件
42         f.write(r.content)  #r.content 响应url的二进制内容
43         f.close()
44         print("保存成功")
45     else:
46         print("文件已存在")
47 except:
48     print("爬取失败")
49 '''
50 #爬单个视频
51 '''
52 url = "http://video.pearvideo.com/mp4/adshort/20181121/cont-1480150-13269168_adpkg-ad_hd.mp4"
53 rot = "E://"
54 path = rot + url.split('/')[-1] #指定视频保存名为url地址中的后缀名（url）
55 try:
56     if not os.path.exists(rot): #查看有没有rot目录，没有则创建
57         os.mkdir(rot)       #创建rot目录
58     if not os.path.exists(path):    #查看path目录下有没有path文件
59         r = requests.get(url)   #链接url地址
60         #with open(path,'wb') as f:  #另一种文件打开和存储方式(？？？)
61         f = open(path,'wb') #打开文件
62         f.write(r.content)  #r.content 响应url的二进制内容
63         f.close()
64         print("保存成功")
65     else:
66         print("文件已存在")
67 except:
68     print("爬取失败")
69 '''
70 #IP地址归属地的自动查询
71 #IP168查询链接url：http://ip138.com/ips138.asp?ip=123.125.6.1&action=2
72 #由上得出：IP查询url：http://ip138.com/ips138.asp?ip=ip地址
73 
74 url = "http://ip138.com/ips138.asp"
75 ip = {"ip":"106.111.147.213"}
76 try:
77     r = requests.get(url,params=ip)
78     r.raise_for_status()
79     print(r)
80     #print(r.status_code)
81     r.encoding = r.apparent_encoding
82     print(r.text)
83 except:
84     print("爬取失败")

1 form bs4 import BeautifulSoup
2 
3 soup = BeautifulSoup( ‘ <p>data</p> ‘ , ‘ html.parser ‘ )

 1 r = requests.get('http://python123.io/ws/demo.html')
 2 print(r.text)
 3 demo = r.text   #demo == url网页代码
 4 soup = BeautifulSoup(demo,'html.parser')
 5 print(soup.prettify())
 6 r = requests.get('http://python123.io/ws/demo.html')
 7 demo = r.text
 8 soup = BeautifulSoup(demo,"html.parser")
 9 print(soup.title)   #查找 title 标签
10 print(soup.a.string)  #打印a标签的字符串内容
11 print(soup.a.name)  #打印a标签的名字
12 print(soup.a.parent.name)   #打印a标签的父标签（上一层标签）
13 print(soup.a.parent.parent.name)    #打印a标签的父标签的上一层标签
14 tag = soup.a    #soup:查找标签a
15 print(soup.a.attrs)    #打印标签的属性
16 print(tag.attrs['class'])  #打印标签属性中的‘class’的内容
17 print(tag.attrs['href'])   #打印标签属性中的‘href’的内容
18 print(tag)  #打印a标签的内容

 1 r = requests.get('http://python123.io/ws/demo.html')
 2 demo = r.text
 3 soup = BeautifulSoup(demo,'html.parser')
 4 print(soup.a.prettify())    #输出a标签（HTML格式）
 5 '''
 6 <a class="py1" href="http://www.icourse163.org/course/BIT-268001" id="link1">
 7  Basic Python
 8 </a>
 9 '''
10 soup = BeautifulSoup('<p>中文</p>','html.parser')
11 print(soup.p.string)    #以[获取<tag>标签的非属性字符串/注释]输出标签
12 '''
13 中文
14 '''
15 print(soup.p.prettify())    #以HTML格式输出标签
16 '''
17 <p>
18  中文
19 </p>
20 '''

 1 import requests
 2 from bs4 import BeautifulSoup
 3 '''#提取HTML中所有的URL链接
 4 1、搜索到所有的<a>标签（a标签的内容即url）
 5 2、解析<a>标签格式，提取href后的链接内容
 6 '''
 7 
 8 
 9 r = requests.get('http://python123.io/ws/demo.html')
10 demo = r.text
11 soup = BeautifulSoup(demo,'html.parser')
12 for link in soup.find_all('a'): #查找demo中的<a>标签
13     print(link.get('href')) #在<a>标签中查找“href”函数
14 
15 
16 '''
17 结果:
18   http://www.icourse163.org/course/BIT-268001
19   http://www.icourse163.org/course/BIT-1001870001
20 '''

 1 r = requests.get('http://python123.io/ws/demo.html')
 2 demo = r.text
 3 soup = BeautifulSoup(demo,"html.parser")
 4 print(soup.find_all('a'))   #list形式返回soup（文本）中的标签字符串（检索a标签）
 5 print(soup.find_all(['a','b']))
 6 for tag in soup.find_all(True):     #遍历soup中的所有标签
 7     print(tag.name) #打印（获得）所有标签的name（名称）！！
 8 '''
 9 [<a class="py1" href="http://www.icourse163.org/course/BIT-268001"
10 id="link1">Basic Python</a>, <a class="py2" href="http://www.icourse163.org
11 /course/BIT-1001870001" id="link2">Advanced Python</a>]
12 [<b>The demo python introduces several python courses.</b>, <a class="py1"
13  href="http://www.icourse163.org/course/BIT-268001" id="link1">Basic
14  Python</a>, <a class="py2" href="http://www.icourse163.org/course/
15  BIT-1001870001" id="link2">Advanced Python</a>]
16 html
17 head
18 title
19 body
20 p
21 b
22 p
23 a
24 a
25 '''#结果

 1 r = requests.get('http://python123.io/ws/demo.html')
 2 demo = r.text
 3 soup = BeautifulSoup(demo,'html.parser')
 4 print(soup.find_all('p', 'course'))  #返回带有course属性的P标签
 5 print(soup.find_all(id='link1')) #返回ID属性==link1的内容
 6 
 7 ‘’’
 8 
 9 [<p class="course">Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:
10 
11 <a class="py1" href="http://www.icourse163.org/course/BIT-268001" id="link1">Basic Python</a> and <a class="py2" href="http://www.icourse163.org/course/BIT-1001870001" id="link2">Advanced Python</a>.</p>]
12 
13 [<a class="py1" href="http://www.icourse163.org/course/BIT-268001" id="link1">Basic Python</a>]
14 
15 ‘’’

1 r = requests.get('http://python123.io/ws/demo.html')
2 demo = r.text
3 soup = BeautifulSoup(demo,'html.parser')
4 print(soup.find_all('a'))   #检索a标签
5 print(soup.find_all('a',recursive=False))#返回一个空列表，说明a的下层节点没有a标签（False下只检索一层（儿子层））

1 r = requests.get('http://python123.io/ws/demo.html')
2 demo = r.text
3 soup = BeautifulSoup(demo,'html.parser')
4 print(soup) #输出经过解析器解析的完整HTML代码
5 print(soup.find_all(string = 'Basic Python'))   #列表返回检索到的字符串信息

 1 import requests
 2 import bs4
 3 from bs4 import BeautifulSoup
 4 #soup = BeautifulSoup(demo,'html.parser')
 5 def getHTML(url):#标准框架
 6     try:            
 7         r = requests.get(url)
 8         r.raise_for_status()
 9         r.encoding = r.apparent_encoding
10         return (r.text)
11     except:
12         print("链接爬取失败")
13 
14 def uitHTML(ulist,demo):
15     soup = BeautifulSoup(demo,'html.parser')
16     for tr in soup.find('tbody').children:  
17 
18 #.find(..)搜索标签结果返回 .children同时遍历儿子（下层）节点
19         if isinstance(tr,bs4.element.Tag):  
20 
21 #检测tr的类型，如果不是标签类型，则过滤（避免遍历到字符串内容）isinstance判断变量类型
22             tds = tr('td') #等价于tr.find(...)>>>查找 td 标签  （返回的对象是list）
23             #print(tds)         #检验使用：查看tds内容
24             #tds此时得到了url的HTML中所有tr中td标签的HTML格式（list格式返回）
25             ulist.append([tds[0].string, tds[1].string, tds[3].string])
26             #将tds中需要的数据(只提取.string>>字符串 区域)存入ulist列表中
27 
28 def uitUlist(ulist,num):
29     print("{:^16}\t{:^16}\t{:^16}".format("排名","学校名称","总分"))
30     for i in range(num):
31         u = ulist[i]
32         print("{:^16}\t{:^16}\t{:^16}".format(u[0],u[1],u[2]))
33 
34 def main():
35     ulist = []
36     url = "http://www.zuihaodaxue.cn/zuihaodaxuepaiming2016.html"
37     demo = getHTML(url)
38     uitHTML(ulist,demo)
39     uitUlist(ulist,20)
40 
41 main()

 1 import re
 2 match = re.search(r'[1-9]\d{5}','BIT 100081')
 3 #Match对象的属性
 4 print(match.string) #返回匹配时的待匹配字符串
 5 print(match.re)     #返回匹配时的re表达式
 6 print(match.pos)    #返回匹配的搜索文本开始的位置
 7 print(match.endpos) #返回匹配的搜索文本结束的位置
 8 #Match对象的方法
 9 print(match.group(0))   #返回匹配后的字符串（第一次匹配结果）
10 print(match.start())    #匹配原始字符串开始的位置
11 print(match.end())      #匹配原始字符串结束的位置
12 print(match.span())     #元组形式返回 .start()和.end()

1 match = re.search(r'PY.*N','PYANBNCNDN') #贪婪匹配（re库默认贪婪匹配，即输出最长的子串匹配）
2 print(match.group(0))
3 match = re.search(r'PY.*?N','PYANBNCNDN') #最小匹配
4 print(match.group(0))