第六篇 爬虫技术之天天基金网（4） 基金档案信息获取篇

# -*- encoding: utf-8 -*-
# !/usr/bin/python
"""
@File    : day_day_funding_detail_scrapy.py
@Time    : 2019/10/20 15:18
@Author  : haishiniu
@Software: PyCharm
"""
# -*- encoding: utf-8 -*-
# !/usr/bin/python
"""
@File    : day_day_funding_scrapy.py
@Time    : 2019/10/13 17:34
@Author  : haishiniu
@Software: PyCharm
"""

import json
import requests
import logging
import random
from pyquery import PyQuery as pq
import sys
reload(sys)
sys.setdefaultencoding('utf8')


class FundScrapy(object):
    """
    天天基金网
    """

    def __init__(self):
        """
        初始化信息
        """
        self.session = requests.session()
        self.timeout = 60

    def get_main_info(self):
        """
        获取首页的信息
        :return: 
        """
        try:
            main_url = 'http://fund.eastmoney.com/'
            self.session.headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
            self.session.headers["Accept-Encoding"] = "gzip, deflate, br"
            self.session.headers["Accept-Language"] = "zh-CN,zh;q=0.9"
            self.session.headers["Connection"] = "keep-alive"
            self.session.headers["Host"] = "fund.eastmoney.com"
            self.session.headers["Upgrade-Insecure-Requests"] = "1"
            self.session.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"

            main_response = self.session.get(main_url, verify=False, timeout=self.timeout).content
            self.session.headers["Referer"] = 'http://fund.eastmoney.com/data/fundranking.html'
            # 获取前10% 倒数5% order数据 -->总样本量:750
            page_list = [1]
            order_list = []
            hander_list = ['fund_code', 'fund_name', 'date', 'asset_value', 'asset_value', 'day_rate', 'wek_rate', 'one_m_rate',
                           'three_m_rate', 'six_m_rate', 'one_y_rate', 'two_y_rate', 'three_y_rate', 'this_y_rate', 'complate_day_rate']
            for i in page_list:
                v_value = random.random()
                fund_ranking_url = "http://fund.eastmoney.com/data/rankhandler.aspx?op=ph&dt=kf&ft=all&rs=&gs=0&sc=zzf&st=desc&sd=2018-08-03&ed=2019-08-03&qdii=&tabSubtype=,,,,,&pi=%s&pn=50&dx=1&v=%s" %(i, v_value)
                # print fund_ranking_url
                fund_ranking_response = self.session.get(fund_ranking_url, verify=False, timeout=self.timeout)
                fund_ranking_response.encode = 'utf8'
                fund_ranking_response = fund_ranking_response.text

                response_list_json = fund_ranking_response[fund_ranking_response.find('var rankData = {datas:['):fund_ranking_response.find(']')].replace("var rankData = {datas:[", "")
                response_list_json = response_list_json
                for i in response_list_json.split('"'):
                    if len(i.split(',')) < 10:
                        continue
                    order = i.split(',')
                    order_list.append(order)

            for i in order_list:
                try:
                    order = i[:16]
                    order_detail_url = 'http://fund.eastmoney.com/%s.html' % str(order[0])
                    # print order_detail_url
                    order_detail_response = self.session.get(order_detail_url, verify=False, timeout=self.timeout)
                    html_pq = pq(order_detail_response.content)
                    scale_funds = html_pq('.infoOfFund table tr').eq(0)('td').eq(1).text()
                    print scale_funds
                    # print html_pq('.infoOfFund table tr').eq(0)('td').eq(1).text().find("基金规模：")
                    # print html_pq('.infoOfFund table tr').eq(0)('td').eq(1).text().find("亿元")
                    print scale_funds[scale_funds.find("基金规模："):scale_funds.find("亿元")].replace('基金规模：', '')

                except Exception as ex:
                    logging.exception(str(ex))

        except Exception as ex:
            logging.exception(str(ex))


if __name__ == "__main__":
    CRZY_SCRAPY = FundScrapy().get_main_info()