Python——量化分析介绍（七）

├── README
├── MyQuant_v1 #量化分析程序目录
   ├── __init__.py
   ├── data #数据处理目录
   │   ├── __init__.py

   │   ├── basic_crawler.py# 爬取股票基础信息存入MongoDB数据库.

   │   └── data_crawler.py  #爬取指数、股票数据

   ├──util # 公用程序
   │   ├── __init__.py

   │   ├── stock_util.py#获取股票交易日期，所有股票代码

   │   └── database.py #链接数据库

   ├── backtest #回测
   │   ├── __init__.py

   │   └── _backtest_ #计划写一下回测走势图

   ├── factor #因子
   │   ├── __init__.py
   │   └──_ factor_.py #不准备开发
   ├── strategy #策略
   │   ├── __init__.py
   │   └── _strategy_ #计划简单写个，主要用于回测
   ├── trading #交易
   │   ├── __init__.py

   │   └── _trading_ #不准备开发

   └── log #日志目录
          ├── __init__.py
          ├── backtest.log #不准备开发

           └── transactions.log#不准备开发

#!/usr/bin/env python3.6
# -*- coding: utf-8 -*-
# @Time    : 2019-07-31 21:12
# @Author  : Ed Frey
# @File    : basic_crawler.py.py
# @Software: PyCharm
from pymongo import UpdateOne, DESCENDING
from util.database import DB_CONN
from util.stock_util import get_trading_dates
import tushare as ts
from datetime import datetime, timedelta
import numpy

"""
to get basic data from tushare and save it to MongoDB.
"""
class BasicCrawler:
    def __init__(self):
        self.basic = DB_CONN['basic']
        self.daily = DB_CONN['daily']

    def crawl_basic(self, begin_date=None, end_date=None):
        '''
        to get the basic infomation of stocks on "date"
        :param begin_date:
        :param end_date:
        :return:
        '''
        if begin_date is None:
            current_date = datetime.now().strftime('%Y-%m-%d')

            begin_date = self.daily.find_one(
                {'code': '000001', 'index': True, 'date': {'$lt': current_date}},
                sort=[('date', DESCENDING)],
                projection={'date': True})['date']

        dates = get_trading_dates(begin_date=begin_date, end_date=end_date)

        for date in dates:
            try:
                df_basic = ts.get_stock_basics(date)
                update_requests = []
                for index in df_basic.index:
                    doc = dict(df_basic.loc[index])
                    doc['code'] = index
                    doc['date'] = date

                    outstanding = doc['outstanding']
                    if outstanding == 0:
                        continue
                    # 解决流通股本和总股本单位不一致的情况,有些单位是股，目前a股股本规模的最大的工商银行，是3564亿股，最小的德方纳米4274万股
                    if outstanding > 4000:
                        outstanding *= 1e4
                    else:
                        outstanding *= 1e8
                    doc['outstanding'] = outstanding

                    totals = doc['totals']
                    if totals > 4000:
                        totals *= 1e4
                    else:
                        totals *= 1e8
                    doc['totals'] = totals

                    for key in doc:
                        field_type = type(doc[key])
                        if type(doc[key]) == numpy.float64:
                            doc[key] = float(doc[key])
                        elif field_type == numpy.int64:
                            doc[key] = int(doc[key])

                    # format"20180101'turned to format"2018-01-01'
                    timeToMarket = doc['timeToMarket']
                    doc['timeToMarket'] = datetime.strptime(
                        str(timeToMarket), '%Y%m%d').strftime('%Y-%m-%d')

                    update_requests.append(
                        UpdateOne(
                            {'code': doc['code'], 'date': doc['date']},
                            {'$set': doc},
                            upsert=True
                        ))

                if len(update_requests) > 0:
                    update_result = self.basic.bulk_write(update_requests, ordered=False)
                    print('更新基本数据, 日期：%s, 插入：%4d，更新：%4d' %
                          (date, update_result.upserted_count, update_result.modified_count),
                          flush=True)
            except:
                print('获取基本信息出错，日期 %s' % date, flush=True)

if __name__ == '__main__':
    bc = BasicCrawler()
    bc.crawl_basic('2018-01-01', '2019-07-28')