爬取壁纸

import requests
import time
#import _thread
# 爬虫结果输出路径
out_dir = './pictures/'
root_url = 'https://www.4kbizhi.com/meinv/'
headers = {
    'Connection': 'keep-alive',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37',
    #'Cookie': 'csrfToken=fxB64yKN6YmKp2x6IBImOond; global_cookie=4qsim60u3xw9srizbptt3nh3q1yl5377g9v; engine_source_cookie=bing; sf_source=bing; city=xian; lastscanpage=0; g_sourcepage=esf_fy%5Exq_pc; unique_cookie=U_4qsim60u3xw9srizbptt3nh3q1yl5377g9v*62',
}
# 记录匹配失败网页
fail_url=[]
def get_url(url):
    """
    获取html文件
    """
    response = requests.get(url,   headers=headers, verify=True)
    print(f"开始爬取{url}")
    return response
def parse_page(response):
    """
    提取当页中所有所需数据的存储位置以及下一爬取网页
    """
    from bs4 import BeautifulSoup
    # 解析页面提取页面内的房源url
    html = response.content.decode('gbk')
    img_url_dict={}
    next_url=None
    bs = BeautifulSoup(html, 'html.parser')
    #bs.find_all('a',attrs={'target':'_blank'})
    result = bs.find_all('img')
    for i in result:
        img_name = i.get('alt')
        img_url = 'https://www.4kbizhi.com/'+i.get('src')
        # 去除一些干扰图片(.gif)
        if img_url.split('.')[-1] in ['jpg','png']:
            img_url_dict[img_name]=img_url
    next_url = bs.find_all(attrs={'class': 'next'})
    if next_url:
        return img_url_dict, 'https://www.4kbizhi.com/'+next_url[0].get('href')
    #elif temp_url[-3:-1]%25==0:
       # return img_url_dict,None
    else:
        return img_url_dict,next_url
def save(response, out_dir,img_name):
    """
    保存爬取结果
    """
    import os
    img_url=response.url
    if not(os.path.exists(out_dir)):
        os.mkdir(out_dir)
    with open(out_dir+img_name+'.'+img_url.split('.')[-1], 'wb') as fp:
        fp.write(response.content)
def parse_data_page(response,img_name):
    """
    获得所需数据
    """
    try:
        save(response, out_dir,img_name)
        print(f"壁纸{response.url}保存成功")
    except:
        fail_url.append(response.url)
        #save([response.url], 'fail_url.csv',['链接地址'])  # 字符串写入时会默认拆开
        print(f"{response.url}解析失败，请稍后查看实际情况，当前解析失败网页总数为{len(fail_url)}")
def  main(url):
    response = get_url(url)
    img_url_dict, next_url = parse_page(response)
    while(next_url) and (next_url.split('.')[0][-1]>'3'):
        for img_name,img_url in img_url_dict.items():
            response = get_url(img_url)
            parse_data_page(response,img_name)
        response=get_url(next_url)
        img_url_dict, next_url = parse_page(response)
        if not(next_url):#将最后一页数据进行解析
            for img_name, img_url in img_url_dict.items():
                response = get_url(img_url)
                parse_data_page(response, img_name)
if __name__ == '__main__':
    start = time.time()
    main(root_url)
    with open('fail_url.csv', 'w') as f:
        for i in fail_url:
            f.write(i+'\n')
    end = time.time()
    print("运行程序花费了%s秒" % (end-start))