Python3爬虫实战(一)：新闻标题及其URL

import requests
from lxml import etree
import re
import json
import time

class BtcSpider(object):
    def __init__(self):
        self.base_url = 'https://www.chainnode.com/forum/61-'
        self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/73.0.3683.86 Chrome/73.0.3683.86 Safari/537.36'}
        self.news_name = []
        self.news_url = []

    # 1,发请求：
    def get_response(self, url):
        response = requests.get(url, headers = self.headers)
        data = response.content.decode()
        return data

    # 2,解析数据：
    def parse_data(self, data):
        # 1,转类型; 使用xpath解析网页
        x_data = etree.HTML(data)

        # 2,xpath路径解析：        
        result2 = x_data.xpath('//a[@class ="link-dark-major font-bold bbt-block"]/text()')  # 返回新闻标题
        result3 = x_data.xpath('//a[@class ="link-dark-major font-bold bbt-block"]/@href')   # 返回新闻url

        for i in result2:                         # 使用re规范化新闻标题
            result = re.sub(r'\s+', '', i)
            self.news_name.append(result)

        url_head = 'https://www.chainnode.com'
        for i in range(len(result3)):             # 合成新闻完整的url
            url_element = url_head + result3[i]
            self.news_url.append(url_element)
        # print(news_url)

        news_name_url = dict(zip(self.news_name, self.news_url))  # 将新闻标题与新闻url一一对应起来,组成一个字典
        #print(news_name_url)

        return news_name_url

    # 3,保存数据：
    def save_data(self, data):

        news_data = json.dumps(data)
        with open('dict_of_news_name_url.json', 'w') as f:  # 将字典以json格式保存到本地
            f.write(news_data)

    # 4,爬虫走起：
    def run(self):

        for i in range(1, 100):  # 爬取前100页新闻标题与其对应的url
            # 1),拼接完整的url:
            url = self.base_url + str(i)
            print(url)
            # 2),发请求：
            data = self.get_response(url)
            # 3),做解析：
            data_news = self.parse_data(data)

        # 4),保存数据：
        self.save_data(data_news)


start = time.time()
BtcSpider().run()
# 读取保存的json文件,检验是否成功：
with open('dict_of_news_name_url.json', 'r') as f:
    data_dict = json.load(f)
print(data_dict)  # 成功打印
print(len(data_dict))
end = time.time()
print('cost time: ', end-start, 's')