爬虫实例一：小说

import requests
import bs4
import os
headers = {
     'user - agent': 'Mozilla / 5.0(WindowsNT10.0;Win64;x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 80.0.3987.116Safari / 537.36'
}
# 创建文件夹
if not os.path.exists('D:/爬虫--笔趣阁'):
    os.mkdir('D:/爬虫--笔趣阁')


def get_name_lists(url):  # 得到小说名字和章节链接列表
    response = requests.get(url=url, headers=headers)
    html = response.text
    soup = bs4.BeautifulSoup(html, 'html.parser')
    novel_lists = soup.select('#list dd a')  # 获取小说章节
    novel_name = soup.select('#info h1')[0].string  # 获得小说名
    novel_lists = novel_lists[12:]  # 去掉前面12节重复内容链接
    return novel_name, novel_lists


def get_content(url):  # 得到章节名和章节内容
    response = requests.get(url=url, headers=headers)
    html = response.text
    soup = bs4.BeautifulSoup(html, 'html.parser')
    name = soup.select(".bookname h1")[0].get_text()  # 得到章节名
    text = soup.select("#content")[0].get_text().replace('', "").replace('     ', '')
    text = text.replace('笔趣阁 www.52bqg.net，最快更新万古第一神最新章节！    ', '')  # 得到章节内容，并利用替换，去掉广告和空格
    return name, text


def text_save(filename, data):  # filename为写入的文件，data为要写入数据列表.
    file = open(filename, 'w', encoding='utf-8')
    for i in range(len(data)):
        s = str(data[i]).replace('[', '').replace(']', '')  # 去除[]
        s = s.replace("'", '').replace(',', '') + '\n'  # 去除单引号，逗号，每行末尾追加换行符
        file.write(s)  # 将列表中数据依次写入文件中
    file.close()


def main():
    list_all = list()  # 先定义一个空列表，方便之后把内容放在里面
    base_url = 'https://www.52bqg.net/book_126836/'
    novel_name, novel_lists = get_name_lists(base_url)  # 调用函数
    text_name = 'D:/爬虫--笔趣阁/' + '{}.txt'.format(novel_name)
    # for i in range(len(novel_lists)):   # 这个循环是爬取整本小说
    for i in range(0, 2):  # 学习笔记，所以只爬了前两节
        novel_url = base_url + novel_lists[i].get("href")
        name, novel = get_content(novel_url)  # 调用函数
        list_all.append(name)
        list_all.append(novel)
        print(name, '下载成功啦！！！')
    text_save(text_name, list_all)  # 调用函数
    print('本小说所有章节全部下载完毕！！！')


if __name__ == '__main__':
    main()