python下载小说

# -*- coding: utf-8 -*-
"""
Created on Fri May 11 08:39:52 2018

@author: dflx
"""

import requests
import re
import time
import os
import requests.packages.urllib3.util.ssl_
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = 'ALL'
import urllib3.contrib.pyopenssl
urllib3.contrib.pyopenssl.inject_into_urllib3()

def get_html(url):
    header ={
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.90 Safari/537.36 2345Explorer/9.3.2.17331',
}
# 使用request库传参时不需要使用urlencode函数去编码成byte类型数据，库自动去编码。get方法参数是params=,post方法是data=
    response = requests.get(url,headers = header,verify=False)
    response.encoding=response.apparent_encoding
    html=response.text
    return html

def novel_information(html,reg):
    novel_text=re.findall(reg,html)
    #print(novel_text)
    return novel_text

def get_regular_text(text):
    global count
    print("------get rgulax text----")
    #print(type(text[0]),'length=',len(text))
  
    new_word=text
    if ' ' in text[0]:
        new_word=re.sub(r' ',' ',text[0])
    print(type(new_word),'length=',len(new_word))
    if '<br />' in new_word:
        new_word=re.sub(r'<br />','\n',new_word)
    count+=len(new_word)
    return new_word

def get_onechapter(url):
    html = get_html(url)
    re1 = r'<h1>(.*?)</h1>'
    re2 = r'<div id="content">(.*?)</div>'
    novel_chapter_name = novel_information(html, re1)
    text = novel_information(html, re2)
    print("下载：------",count)
    print(novel_chapter_name)
    chapt_text="                "+novel_chapter_name[0]+'\n'+get_regular_text(text)
    return chapt_text
    
def download_onechapter(words):
#下面是文件存目录，修改你的路径
    path="/home/dflx/下载/novel/永夜君王.txt"
    with open(path,'a+',encoding="utf-8") as f:
        for word in words:
            f.write(word)
        f.write('\n')
        
def chapter_all(url,re3,host):
    html=get_html(url)
    #print(html)
    chapter_list=novel_information(html,re3)
    print("length=",len(chapter_list),type(chapter_list))
    chapters=[host+ i for i in chapter_list]
    chapters=[i+'.html' for i in chapters]
    print(chapters)
    #print(chapters.index("https://www.xxbiquge.com/68_68479/4010677.html"))
    return chapters


def download_novel(url_all):
    #for index in range(472,len(url_all)):
    for chapt in url_all:
        try:
            words=get_onechapter(chapt)
            download_onechapter(words)
        except Exception:
            continue
        finally:
            print("GOTO ANOTHER CHAPTER")

    
count=0;
def  main():
    host="https://www.xxbiquge.com"
#下面是小说目录界面，可以打开参考一下，修改成你需要的．
    url0="https://www.xxbiquge.com/1_1797/"
    re3=r'<dd><a href="(.*?).html"'
    start=time.clock();
    try:
        all=chapter_all(url0,re3,host)
        download_novel(all)
    except Exception as e:
        print("has error"+str(e))
    finally:
        end=time.clock()
        print("总计用时：%.2f s"%(end-start))