python 爬虫示例--基金查询demo

"http://quotes.money.163.com/fund/jzzs_{code}_{page}.html?start={start}&end={end}&sort=TDATE&order=desc".format(
            code=code,page="0",start=start,end=end)
如 "http://quotes.money.163.com/fund/jzzs_001630_0.html?start=2009-02-22&end=2019-10-29&sort=TDATE&order=desc" 

import sys
from PyQt5.QtWidgets import *
from PyQt5.QtGui import QColor, QFont, QIcon,QPixmap,QRegExpValidator
from PyQt5.QtCore import Qt, QSize,QDate,QRegExp
import pickle
import requests
import re
from bs4 import BeautifulSoup
from matplotlib import pyplot as plt
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg,NavigationToolbar2QT
from matplotlib.figure import Figure
#import numpy as np

class Canvas(FigureCanvasQTAgg):
    def __init__(self, parent=None, width=5, height=4, dpi=100):
        fig = Figure(figsize=(width, height), dpi=dpi) #创建画布,设置宽高，每英寸像素点数
        fig.set_tight_layout(True)
        self.axes = fig.add_subplot(111)#
        self.axes.tick_params(axis='x',rotation =90,direction="in")#日期旋转90度显示
        FigureCanvasQTAgg.__init__(self, fig)#调用基类的初始化函数
        self.setParent(parent)
        #self.update_figure(1,1)
        FigureCanvasQTAgg.updateGeometry(self)
    
    def update_figure(self, x ,y,title):
        #x = [4,3,2,1]
        #y=[1,2,3,5]
        x.reverse()
        y.reverse()
        self.axes.cla()#清除已绘的图形
        self.axes.set_title(title,fontsize=18)
        self.axes.plot(x,y)
        self.axes.scatter(x,y,  marker ='o')
        self.axes.set_ylabel("基金净值[元]")
        self.axes.grid(lw=0.5,ls="--",alpha=0.5)
        self.draw()#重新绘制
 
class MainWindow(QMainWindow):
    def __init__(self, parent = None):
        super().__init__(parent)
        self.funds = pickle.load(open("info.obj","rb")) # 基金代码和名称信息存在字典中保存到文件了，pickle加载
        self.setWindowTitle("A股基金查询工具【数据来源于网易财经，python爬虫demo】")
        self.create_table()
        self.create_canvas()
        self.setup_centralWidget()
        #self.setWindowIcon(QIcon(":ICON/ICON/retest.png"))
        self.createActions()
        self.setup_toolBar()
        self.setup_menuBar()
        self.statusBar().showMessage("ready")
        self.code = None
       
        #self.resize(800,500)
    def create_table(self):
        self.table = QTableWidget()
        self.table.setEditTriggers(QAbstractItemView.NoEditTriggers)
        HorizontalHeaderLabels = ["公布日期", "单位净值","累计净值","增长率"]
        columns = len(HorizontalHeaderLabels)
        self.table.setColumnCount(columns)
        self.rows=100
        self.table.setRowCount(self.rows)#
        self.headerWidth = (100,80,80,80)
       
        self.table.setSortingEnabled (True)
        self.table.horizontalHeader().setStyleSheet("QHeaderView::section{background-color:rgb(180,180,250);}")
        for i in range(columns-1):
            self.table.setColumnWidth (i,self.headerWidth[i])
       
        self.table.setHorizontalHeaderLabels(HorizontalHeaderLabels)
    
    def update_table(self):
        self.table.clearContents()#清除内容
        rows = len(self.rate)
        if rows> self.rows:
            self.table.setRowCount(rows)
        for i in range(len(self.rate)):
            item = QTableWidgetItem(self.date[i])
            item.setTextAlignment(Qt.AlignHCenter |Qt.AlignVCenter)
            self.table.setItem(i, 0, item)
            item = QTableWidgetItem(str(self.net[i]))
            item.setTextAlignment(Qt.AlignHCenter |Qt.AlignVCenter)
            self.table.setItem(i, 1, item)
            item = QTableWidgetItem(str(self.acc_net[i]))
            item.setTextAlignment(Qt.AlignHCenter |Qt.AlignVCenter)
            self.table.setItem(i, 2, item)
            rate = self.rate[i]
            item = QTableWidgetItem(rate)#rate用的是文本
            item.setTextAlignment(Qt.AlignHCenter |Qt.AlignVCenter)
            if rate[0] == "-":
                item.setForeground(QColor("green"))
            else:
                item.setForeground(QColor("red"))
            self.table.setItem(i, 3, item)
            
    def create_canvas(self):
        self.canvas = Canvas(self)
       
    def setup_centralWidget(self):
        #设置主窗口中心部件
        self.tabWidget = QTabWidget()
        self.tabWidget.addTab(self.table,"Table ")
       
        vlayout = QVBoxLayout()
        Navigation_toolbar = NavigationToolbar2QT(self.canvas, self)
        vlayout.addWidget(self.canvas)
        vlayout.addWidget(Navigation_toolbar)
        plotWidget = QWidget()
        plotWidget.setLayout(vlayout)
       
        self.tabWidget.addTab(plotWidget,"Plot")
        self.tabWidget.setCurrentIndex(1)
        self.setCentralWidget(self.tabWidget)#指定主窗口中心部件 
    
    def createActions(self):
        #self.newAction = QAction("New record", self)
        #self.newAction.setIcon(QIcon(":new.png"))
        #self.newAction.triggered.connect(self.newRecord)
        #self.newAction.setStatusTip("###")
        self.exitAction = QAction("E&xit",self)
        self.exitAction.triggered.connect(self.close)
        self.queryAction = QAction("查询",self)
        self.queryAction.triggered.connect(self.query)
       
        self.helpAboutAction = QAction("About",self)
        self.helpAboutAction.setShortcut("Ctrl+H")
        self.helpAboutAction.triggered.connect(self.showAboutDlg)  
    
    def setup_menuBar(self):
        fileMenu = self.menuBar().addMenu("&File")
        fileMenu.addAction(self.exitAction)
       
        helpMenu = self.menuBar().addMenu("&Help")
        helpMenu.addAction(self.helpAboutAction)
    
    def showAboutDlg(self):
        QMessageBox.about(self,u"title",
                          u"Version:  0.1\n"
                          u"author:  wsp")
       
    def name_selected(self):
        self.name = self.comboName.currentText()
        self.code = self.name.split(" ")[0]
        print(self.name,self.code)
    
    def closeEvent(self, event):
        reply = QMessageBox.question(self, '提示',"是否要退出程序？",
                                               QMessageBox.Yes | QMessageBox.No,QMessageBox.No)
        if reply == QMessageBox.Yes:
            pickle.dump(self.funds, open("info.obj","wb")) # 基金代码和名称信息存在字典中保存到文件
            event.accept()
        else:
            event.ignore()

    @staticmethod
    def download(url,user_agent='wswp',num_retries=2,proxies=None):
        print("Downloading: ", url)
        headers = {'User-Agent' : user_agent}
        resp = requests.get(url, headers=headers, proxies=proxies)
        html = None
        try:
            resp = requests.get(url, headers=headers, proxies=proxies)
            #print("status: ",resp.status_code)
            html = resp.text
            if resp.status_code >= 400:
                print("Download error: ", html)
                html = None
                if num_retries>0 and 500 < resp.status_code <600:
                    #递归调用，遇到5xx错误，最多重试 2 次
                    return download(url, user_agent, num_retries-1, proxies)
        except requests.exceptions.RequestException as e:
            print('Download error: ' ,e.reason)
            html = None
        finally:
            return html
   
    def query(self):
        if self.code is None:
            QMessageBox.critical(self, "错误", "基金代码为空或格式错误！")
            self.codeInput.setFocus()
            return
           
        code = self.code
        start = self.start.text()
        end = self.end.text()
        #url0 ="http://quotes.money.163.com/fund/jzzs_001630_0.html?start=2009-02-22&end=2019-10-29&sort=TDATE&order=desc"
        #url0="http://quotes.money.163.com/fund/jzzs_001630_0.html?start=2019-10-29&end=2019-10-29&sort=TDATE&order=desc"
        #url0="http://quotes.money.163.com/fund/jzzs_001630_0.html?start=2019-07-01&end=2019-10-29&sort=TDATE&order=desc"
        url0 = "http://quotes.money.163.com/fund/jzzs_{code}_{page}.html?start={start}&end={end}&sort=TDATE&order=desc".format(
            code=code,page="0",start=start,end=end)
        #print(url0)
        html = self.download(url0)
        if html is None:
            QMessageBox.critical(self, "错误", "爬不到有效信息，请检查基金代码是否有误！")
            return
        #print(html[:100])
        soup = BeautifulSoup(html, 'html.parser')
        html = soup.prettify() #修正可能存在的Html错误
        #提取基金名称
        fundInfo =soup.find(name="title")
        #print(fundInfo.text)
        self.name = fundInfo.text.split("_")[0]
        #提取总的页数
        matched =soup.find(name="div", attrs = {"class": "mod_pages"})
        a_founds = matched.find_all(name="a")
        if len(a_founds) ==0:
            pages =1
        else:
            pages = int(matched.find_all(name="a")[-2].text)
        print("pages:", pages)
        self.date, self.net, self.acc_net, self.rate = [], [], [],[]
        i = 0
        for matched in soup.find_all("td"): #提取
            text = matched.text
            if i %4 == 0:
                    self.date.append(text)# datetime string
            elif i%4 ==1:
                    self.net.append(float(text)) # 单位净值
            elif i%4 ==2:
                    self.acc_net.append(float(text)) #累计净值
            else:
                self.rate.append(text)
            i += 1

        if pages>1:
            for page in range(1,pages):
                url = "http://quotes.money.163.com/fund/jzzs_{code}_{page}.html?start={start}&end={end}&sort=TDATE&order=desc".format(code=code,page=str(page),start=start,end=end)
                html = self.download(url)
                #print(html[:100])
                soup = BeautifulSoup(html, 'html.parser')
                html = soup.prettify() #修正可能存在的Html错误
                i = 0
                for matched in soup.find_all("td"): #提取
                    text = matched.text
                    if i %4 == 0:
                            self.date.append(text)# datetime string
                    elif i%4 ==1:
                            self.net.append(float(text)) # 单位净值
                    elif i%4 ==2:
                            self.acc_net.append(float(text)) #累计净值
                    else:
                        self.rate.append(text)
                    i += 1
        self.update_table()
        self.canvas.update_figure(x=self.date ,y =self.net,title="%s (%s) 净值走势"%(self.name,self.code))
        itemText = self.code+" "+self.name
        if self.code not in self.funds:
            self.funds[self.code] = self.name
            self.comboName.addItem(itemText)
        self.comboName.setCurrentText(itemText)
       
    def codeInputFinished(self):
        self.code = self.codeInput.text()
    
    def setup_toolBar(self):      
        label0 = QLabel("选择基金：")
        self.comboName = QComboBox()
        fundItems = list(self.funds.items())
        fundItems.sort()
        for i, fund in enumerate(fundItems):
            self.comboName.addItem(fund[0]+" "+fund[1])
        self.comboName.currentIndexChanged[int].connect(self.name_selected)
        self.comboName.setStatusTip("选择基金")
        label_ = QLabel("   基金代码：")
        self.codeInput = QLineEdit()
        regExp = QRegExp("^\d{6}$")
        validator = QRegExpValidator(regExp)
        self.codeInput.setValidator(validator)
        self.codeInput.setFixedWidth(50)
        self.codeInput.editingFinished.connect(self.codeInputFinished)
       
        label1 = QLabel("   起始日期")
        self.start= QDateEdit()
        self.start.setCalendarPopup(True)
        self.start.setDisplayFormat("yyyy-MM-dd")
        label2 = QLabel("   截止日期")
        self.end= QDateEdit()
        self.end.setCalendarPopup(True)
        self.end.setDisplayFormat("yyyy-MM-dd")
        today = QDate.currentDate()#当前时间
        self.start.setMaximumDate(today) #不超过今天
        self.start.setDate(today.addMonths (-3)) #3月前此时
        self.end.setDate(today)
        self.end.setMaximumDate(today)
       
        toolbar0 = self.addToolBar("选择")#添加工具条      
        toolbar0.addWidget(label0)
        toolbar0.addWidget(self.comboName)
        toolbar0.addWidget(label_)
        toolbar0.addWidget(self.codeInput)
        toolbar0.addWidget(label1)
        toolbar0.addWidget(self.start)
        toolbar0.addWidget(label2)
        toolbar0.addWidget(self.end)
        toolbar0.addSeparator()
        #toolbar0.addAction(self.queryAction)
        self.queryButton = QPushButton("查询")
        self.queryButton.clicked.connect(self.query)
        toolbar0.addWidget(self.queryButton)
        #help(toolbar0)
        toolbar0.addSeparator()
    
       
if __name__ == '__main__':
    app = QApplication(sys.argv)
    mw = MainWindow()
    mw.show()
    sys.exit(app.exec_())