爬虫 | 百行代码爬取14.5W条豆瓣图书信息

import requests
from bs4 import BeautifulSoup
from requests.exceptions import RequestException
from urllib.parse import urlencode
import pymongo
import numpy as np
import time
from faker import Faker

tag_url = 'https://book.douban.com' +　标签页中a标签括起来的内容

# 解析总标签页面,并拼接获得所有标签页页面链接
def splice_tags_indexhtml(html):
    url = 'https://book.douban.com'
    book_tags = []
    tags_url = []
    soup = BeautifulSoup(html, 'lxml')
    tagurl_lists = soup.select('#content > div > div.article > div > div > table > tbody > tr > td > a')
    for tag_url in tagurl_lists:
        # 获取全部标签的a标签内容，并拼接到一起
        book_tags += [tag_url.attrs["href"]]
    for book_tag in book_tags:
        tags_url.append([url + book_tag])
    return tags_url

# 解析单个tag页面下单页的信息
def parse_tag_page(html):
    try:
        soup = BeautifulSoup(html,"lxml")
        tag_name = soup.select('title')[0].get_text().strip()
        list_soup = soup.find('ul', {'class': 'subject-list'})
        if list_soup == None:
            print('获取信息列表失败')
        else:
            for book_info in list_soup.findAll('div', {'class': 'info'}):
                # 书名
                title = book_info.find('a').get('title').strip()
                # 评价人数
                people_num = book_info.find('span', {'class': 'pl'}).get_text().strip()
                # 出版信息,作者
                pub = book_info.find('div', {'class': 'pub'}).get_text().strip()
                pub_list = pub.split('/')
                try:
                    author_info = '作者/译者： ' + '/'.join(pub_list[0:-3])
                except:
                    author_info = '作者/译者： 暂无'
                try:
                    pub_info = '出版信息： ' + '/'.join(pub_list[-3:-1])
                except:
                    pub_info = '出版信息： 暂无'
                try:
                    price_info = '价格： ' + '/'.join(pub_list[-1:])
                except:
                    price_info = '价格： 暂无'
                try:
                    rating_num= book_info.find('span', {'class': 'rating_nums'}).get_text().strip()
                except:
                    rating_num = '0.0'
                book_data = {
                    'title': title,
                    'people_num': people_num,
                    'author_info': author_info,
                    'pub_info': pub_info,
                    'price_info': price_info,
                    'rating_num': rating_num
                }
                # return book_data
                if book_data:
                    save_to_mongo(book_data,tag_name)
    except:
        print('解析错误')
        return None

# 请求tag下的每个页面
def get_tag_page(tag_url,page):
        formdata = {
            'start': page,
            'type': 'T'
        }
        url = tag_url[0]+'?'+ urlencode(formdata)
        try:
            reponse = requests.get(url, headers=headers)
            if reponse.status_code == 200:
                return reponse.text
            return None
        except RequestException:
            print('请求列表页错误')
            return None

#使用Faker库随机生成虚假header
from faker import Faker
fake = Faker()
headers ={'User-Agent':fake.user_agent()}