Python协程理解、基于爬虫举例

import requests
from time import sleep
from lxml import etree
​
def setUp(fun):
    """
    启动生成器
    :param fun: 生成器函数
    """
    def inner(**kwargs):
        k = fun(**kwargs)  #调用生成器
        next(k) #执行一次，效果类似初始化
        return k
    return inner
​
@setUp
def getSrc():
     """
         根据内容获取链接
     """
     a = 1
     while True:
         content = yield a
         html = etree.HTML(content)
         imglist = html.xpath('//div[@id="content-left"]//img')
         for img in imglist:
             src = "http"+img.attrib.get("src")
             print(src)
@setUp
def getData():
    """
    根据地址获取内容
    :return:
    """
    while True:
        url = "https://www.qiushibaike.com/pic/page/%s/?s=5144861"
        page = yield url
        if page == 1:
            Referer = "https://www.qiushibaike.com"
            url = "https://www.qiushibaike.com/pic/"
        else:
            Referer = url%page
            url = url % page
        print(url)
        headers = {
            "Referer": Referer,
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36",
        }
        request_data = requests.get(url = url,headers = headers)
        content = request_data.content
        img = getSrc()
        img.send(content)
        sleep(1)
​
def getPage():
    """
    启动，获取所有的页码
    """
    d = getData()
    page = 0
    while page < 35:
        page += 1
        d.send(page)
​
if __name__ == "__main__":
    getPage()