用 Python 爬取小说

# _*_ coding:UTF-8 _*_
from bs4 import BeautifulSoup
import requests
import time
import sys
"""
download 《武动乾坤》 from www.biqukan.com
Parameters:
    None
Returns:
    None
"""


class downloader(object):
    def __init__(self):
        self.server = "http://www.biqukan.com"
        self.target = "http://www.biqukan.com/3_3012"
        self.names = []
        self.urls = []
        self.nums = 0
    """
    To get Urls for download
    """

    def get_download_url(self):
        req = requests.get(url=self.target)
        html = req.text
        div_bf = BeautifulSoup(html, "html.parser")
        div = div_bf.find_all("div", class_="listmain")
        a_bf = BeautifulSoup(str(div[0]), "html.parser")
        a = a_bf.find_all("a")
        self.nums = len(a[12:])
        with open('武动乾坤目录.txt', 'a', encoding='utf-8') as f:
            for each in a[12:]:
                f.write(each.string)
                self.names.append(each.string)
                f.write(self.server + each.get("href"))
                self.urls.append(self.server + each.get("href"))
    """
    To get content
    Parameters:
        target - 下载链接
    Returns:
        content - 章节内容
    """

    def get_contents(self, target):
        headers = requests.utils.default_headers()
        headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:61.0) Gecko/20100101 Firefox/61.0'
        req = requests.get(url=target, headers=headers)
        html = req.text
        bf = BeautifulSoup(html, "html.parser")
        texts = bf.find_all("div", class_="showtxt")
        content = texts[0].text.replace('\xa0' * 8, '\n\n')
        return content
    """
    To save to text
    Parameters:
        name - 章节名称
        path - 当前路径 + 小说名
        text - 章节内容
    Returns:
        None
    """

    def writer(self, name, path, text):
       # writer_flag = True
        with open(path, 'a', encoding='utf-8') as f:
            f.write(name + '\n')
            f.writelines(text)
            f.write('\n\n')


if __name__ == "__main__":
    dl = downloader()
    dl.get_download_url()
    print("第", dl.nums, "章")
    print("开始下载:")
    for i in range(dl.nums):
        time.sleep(1)
        dl.writer(dl.names[i], '武动乾坤.txt', dl.get_contents(dl.urls[i]))
        sys.stdout.write("  已下载:%.3f%%" % float(i/dl.nums*100) + '\r')
        sys.stdout.flush()
    print("下载完成")