《手把手带你学爬虫──初级篇》第2课 Requests库讲解

pip install requests

import requests


def get_html():
    response = requests.get("http://www.baidu.com")
    print(response.status_code)
    print(response.encoding)
    html_text = response.text
    with open("baidu.html", "w") as file:
        file.write(html_text)
        print("write finished.")


if __name__ == '__main__':
    get_html()

# HTTP请求状态码
200
# 网页编码
ISO-8859-1
# 响应头信息(headers)
{'Content-Length': '7610', 'Cache-Control': 'private, no-cache, no-store, proxy-revalidate, no-transform', 'Connection': 'keep-alive', 'Content-Type': 'text/html', 'Date': 'Sat, 16 Jun 2018 07:00:11 GMT', 'Keep-Alive': 'timeout=4', 'Last-Modified': 'Mon, 23 Jan 2017 13:27:29 GMT', 'Pragma': 'no-cache', 'Proxy-Connection': 'keep-alive', 'Server': 'bfe/1.0.8.18', 'Set-Cookie': 'BDORZ=27315; max-age=86400; domain=.baidu.com; path=/'}
# 文件写入完毕
write finished.

200
UTF-8
{'Content-Length': '12886', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive', 'Content-Type': 'text/html;charset=UTF-8', 'Keep-Alive': 'timeout=4', 'Pragma': 'no-cache', 'Proxy-Connection': 'keep-alive', 'Server': 'Apache-Coyote/1.1', 'Set-Cookie': 'andr_zz=7;domain=.baidu.com;path=/;max-age=600'}
write finished.

GET / HTTP/1.1
Host: www.baidu.com
Connection: keep-alive
Cache-Control: max-age=0
Upgrade-Insecure-Requests: 1
DNT: 1
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
Accept-Encoding: gzip, deflate, br
Accept-Language: zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7
Cookie: BAIDUID=2D5F6595F31C411DF5DADEA1C10D8A81:FG=1; BIDUPSID=2D5F6595F31C411DF5DADEA1C10D8A81; PSTM=1529210459; BD_HOME=0; H_PS_PSSID=1446_21122_26350_26432; BD_UPN=123253

In [20]: r = requests.post('http://httpbin.org/post', data = {'username':'zhangsan','password':'123456'})

In [21]: r.request.method
Out[21]: 'POST'

In [22]: r.request.url
Out[22]: 'http://httpbin.org/post'

In [23]: r.request.headers
Out[23]: {'User-Agent': 'python-requests/2.18.4', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Length': '33', 'Content-Type': 'application/x-www-form-urlencoded'}

In [24]: r.request.body
Out[24]: 'username=zhangsan&password=123456'

$ ipython
Python 3.6.5 (default, Mar 30 2018, 06:42:10)
Type 'copyright', 'credits' or 'license' for more information
IPython 6.4.0 -- An enhanced Interactive Python. Type '?' for help.

In [1]: import requests

In [2]: r = requests.head('http://httpbin.org/get')

In [3]: r.headers
Out[3]: {'Content-Length': '208', 'Access-Control-Allow-Credentials': 'true', 'Access-Control-Allow-Origin': '*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Date': 'Sat, 16 Jun 2018 10:44:44 GMT', 'Keep-Alive': 'timeout=4', 'Proxy-Connection': 'keep-alive', 'Server': 'gunicorn/19.8.1', 'Via': '1.1 vegur'}

In [4]: payload = {'key1': 'value1','key2': 'value2'}

In [5]: r = requests.post('http://httpbin.org/post',data = payload)

In [6]: print(r.text)
{"args":{},"data":"","files":{},"form":{"key1":"value1","key2":"value2"},"headers":{"Accept":"*/*","Accept-Encoding":"gzip, deflate","Connection":"close","Content-Length":"23","Content-Type":"application/x-www-form-urlencoded","Host":"httpbin.org","User-Agent":"python-requests/2.18.4"},"json":null,"origin":"45.77.28.136","url":"http://httpbin.org/post"}

"form":{"key1":"value1","key2":"value2"}

In [7]: abc = 'ABC'

In [8]: r = requests.post('http://httpbin.org/post',data = abc)

In [9]: print(r.text)
{"args":{},"data":"ABC","files":{},"form":{},"headers":{"Accept":"*/*","Accept-Encoding":"gzip, deflate","Connection":"close","Content-Length":"3","Host":"httpbin.org","User-Agent":"python-requests/2.18.4"},"json":null,"origin":"45.77.28.136","url":"http://httpbin.org/post"}

"data":"ABC"

   In 10: kv = {'key1': 'value1','key2': 'value2'}

   In 11: r = requests.request('GET','[http://python123.io/ws](http://python123.io/ws)', params = kv)

   In 12: print(r.url)

   [https://python123.io/ws?key1=value1&key2=value2](https://python123.io/ws?key1=value1&key2=value2)

   In 21: kv = {'key1': 'value1','key2': 'value2'}

   In 22: r = requests.request('POST','[http://python123.io/ws](http://python123.io/ws)', data = kv)

   In 23: body = '主体内容'

   In 24: r = requests.request('POST','[http://python123.io/ws](http://python123.io/ws)', data = body.encode('utf-8'))

   ln 25: kv = {'key1': 'value1','key2': 'value2'}

   In 26: r = requests.request('POST','[http://python123.io/ws](http://python123.io/ws)', json = kv)

   In 27: hd = {'User-Agent': 'Chrome/10'}

   In 28: r = requests.request('POST','[http://python123.io/ws](http://python123.io/ws)', headers = hd)

   In 35: fs = {'file': open('baidu.html','rb')}

   In 36: r = requests.request('POST','[http://python123.io/ws](http://python123.io/ws)', files = fs)

   In 37: r = requests.request('GET','[http://python123.io/ws](http://python123.io/ws)', timeout = 10)

   In 40: pxs = {
                ...: 'http': 'http://user:pass@10.10.10.10:1234',
                ...: 'https': 'https://10.10.10.10:4321'}
   In 41: r = requests.request('GET', '[http://www.baidu.com](http://www.baidu.com)', proxies = pxs)

出现异常，类型为：<class 'requests.exceptions.MissingSchema'>，内容为：Invalid URL 'www.baidu.com': No schema supplied. Perhaps you meant http://www.baidu.com?

import requests


def get_html(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except Exception as e:
        print("出现异常，类型为：{}，内容为：{}", type(e), str(e))


if __name__ == '__main__':
    url = "http://www.baidu.com"
    result = get_html(url)
    print(result)

In [1]: import requests

In [2]: r = requests.get("https://item.jd.com/3446665.html")

In [3]: r.status_code
Out[3]: 200

In [4]: r.encoding
Out[4]: 'gbk'

In [5]: r.text[:500]
Out[5]: '<!DOCTYPE HTML>\n<html lang="zh-CN">\n<head>\n    <!-- shouji -->\n    <meta http-equiv="Content-Type" content="text/html; charset=gbk" />\n    <title>【LG27UD58】LG 27UD58-B 27英寸4K IPS硬屏 低闪屏滤蓝光LED背光液晶显示器【行情 报价 价格 评测】-京东</title>\n    <meta name="keywords" content="LG27UD58,LG27UD58,LG27UD58报价,LG27UD58报价"/>\n    <meta name="description" content="【LG27UD58】京东JD.COM提供LG27UD58正品行货，并包括LG27UD58网购指南，以及LG27UD58图片、27UD58参数、27UD58评论、27UD58心得、27UD58技巧等信息，网购LG27UD58上京东,放心又轻松" />\n    <meta name="format-detection" con'

import requests
import os


def get_pic(url):
    # Linux、Unix系统路径
    # 如果是Windows系统应该写 file_dir = ".//pics//"
    file_dir = './pics/'
    filename = url.split("/")[-1]
    file_path = file_dir + filename
    try:
        if not os.path.exists(file_dir):
            os.mkdir(file_dir)
        if not os.path.exists(file_path):
            r = requests.get(url, timeout=30)
            r.raise_for_status()
            with open(file_path, 'wb') as f:
                f.write(r.content)
            print('文件保存成功')
        else:
            print('图片已经存在')
    except Exception as e:
        print("出现异常，类型为：{}，内容为：{}".format(type(e), str(e)))


if __name__ == '__main__':
    url = "http://image.ngchina.com.cn/2018/0616/20180616123038195.jpg"
    get_pic(url)