Python——量化分析常用命令介绍（四）

├── README
├── MyQuant_v1 #量化分析程序目录
   ├── __init__.py
   ├── data #数据处理目录
   │   ├── __init__.py
   │   └── data_crawler.py  #爬取指数、股票数据

   ├──util # 公用程序
   │   ├── __init__.py

   │   └── database.py #链接数据库

   ├── backtest #回测
   │   ├── __init__.py

   │   └── _backtest_ #计划写一下回测走势图

   ├── factor #因子
   │   ├── __init__.py
   │   └──_ factor_.py #不准备开发
   ├── strategy #策略
   │   ├── __init__.py
   │   └── _strategy_ #计划简单写个，主要用于回测
   ├── trading #交易
   │   ├── __init__.py

   │   └── _trading_ #不准备开发

   └── log #日志目录
          ├── __init__.py
          ├── backtest.log #不准备开发

           └── transactions.log#不准备开发

#!/usr/bin/env python3.6
# -*- coding: utf-8 -*-
# @Time    : 2019-07-06 20:14
# @Author  : Ed Frey
# @File    : database.py
# @Software: PyCharm
from pymongo import MongoClient

DB_CONN = MongoClient('mongodb://127.0.0.1:27017')['my_quant']

#!/usr/bin/env python3.6
# -*- coding: utf-8 -*-
# @Time    : 2019-07-01 22:11
# @Author  : Ed Frey
# @File    : data_crawler.py
# @Software: PyCharm
import tushare as ts
from util.database import DB_CONN
from pymongo import UpdateOne
from datetime import datetime

class DataCrawler():
    def __init__(self):
        self.daily = DB_CONN['daily']
        self.daily_hfq = DB_CONN['daily_hfq']
        self.daily_qfq = DB_CONN['daily_qfq']


    def crawl_index(self,begin_date=None,end_date=None):
        '''
        to crawl the index's trading information and set up our own database
        :param begin_date: YYYY-MM-DD
        :param end_date: YYYY-MM-DD
        :return:
        '''
        codes = ['000001','000300','399001','399005','399006']

        if begin_date is None:
            begin_date = '2008-01-01'
        if end_date is None:
            end_date = datetime.now().strftime('%Y-%m-%d')

        for code in codes:
            df_daily = ts.get_k_data(code,index=True,start=begin_date,end=end_date)
            self.save_data(code, df_daily, self.daily, {'index': True})


    def save_data(self, code, df_daily, collection, extra_fields=None):
        """
        save the data into MongoDB
        :param code: stock's code
        :param df_daily: the DataFrame including k line
        :param collection: saving collection
        :param extra_fields: the other fields that will be used one day
        """
        update_requests = []
        for df_index in df_daily.index:
            daily_obj = df_daily.loc[df_index]
            doc = self.daily_obj_2_doc(code, daily_obj)

            if extra_fields is not None:
                doc.update(extra_fields)

            update_requests.append(
                UpdateOne(
                    {'code': doc['code'], 'date': doc['date'], 'index': doc['index']},
                    {'$set': doc},
                    upsert=True)
            )

        if len(update_requests) > 0:
            update_result = collection.bulk_write(update_requests, ordered =False)
            print('Saving index daily, code: %s, inserted: %4d, modified: %4d'
                  %(code, update_result.upserted_count, update_result.modified_count),
                  flush=True)


    def crawl(self, begin_date=None, end_date=None):
        '''
        to crawl the stock's trading information and set up our own database,including 前复权、后复权、不复权
        :param begin_date: YYYY-MM-DD
        :param end_date: YYYY-MM-DD
        :return:
        '''
        df_stock = ts.get_stock_basics()
        codes = list(df_stock.index)

        if begin_date is None:
            begin_date = '2008-01-01'
        if end_date is None:
            end_date = datetime.now().strftime('%Y-%m-%d')

        for code in codes:
            df_daily = ts.get_k_data(code, autype=None, start=begin_date, end=end_date)
            self.save_data(code, df_daily, self.daily, {'index': False})

            df_daily_hfq = ts.get_k_data(code, autype='hfq', start=begin_date, end=end_date)
            self.save_data(code, df_daily_hfq, self.daily_hfq, {'index': False})

            df_daily_qfq = ts.get_k_data(code, autype='qfq', start=begin_date, end=end_date)
            self.save_data(code, df_daily_qfq, self.daily_qfq, {'index': False})


    @staticmethod
    def daily_obj_2_doc(code, daily_obj):
        doc = dict(daily_obj)
        doc['code'] = code

        return doc


if __name__ == '__main__':

    dc = DataCrawler()
    dc.crawl_index(begin_date='2019-07-01',end_date='2019-07-03')
    dc.crawl(begin_date='2019-07-01', end_date='2019-07-03')

测试结果随便贴一点：

前面几行是已经爬取过的数据，没有插入或修改操作，后面几行新的数据进行了插入。