python获取百度热榜链接的实例方法

https://www.baidu.com/

import requests

import pprint

import re

import urllib.parse

 

url = 'https://www.baidu.com/'

 

headers = {

    'Host': 'www.baidu.com',

    'Referer': 'https://www.baidu.com/',

    'User-Agent': 你的User-Agent,

    'Cookie': 你的Cookie

}

 

response = requests.get(url, headers=headers).content.decode('utf-8')

# 获取关键字

pat = '"pure_title": "(.*?)"'

keyword = re.findall(pat, response, re.S)

print(len(keyword))

 

for hot_word in keyword:

    # 汉字不符合url标准，所以这里需要进行url编码

    i = urllib.parse.quote(hot_word, encoding='utf-8', errors='replace')

    # url构建

    link = f'https://www.baidu.com/s?cl=3&tn=baidutop10&fr=top1000&wd={i}&rsv_idx=2&rsv_dl=fyb_n_homepage&hisfilter=1'

    print(link)

from lxml import etree

import re

import requests

import string

import json

headers = {

"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"

}

response = requests.get('https://www.baidu.com/s?wd=腾讯视频优惠&lm=1',headers=headers)

r = response.text

html = etree.HTML(r,etree.HTMLParser())

r1 = html.xpath('//h3')

r2 = html.xpath('//*[@class="c-abstract"]')

r3 = html.xpath('//a[@class="c-showurl"]')

for i in range(10) :

r11 = r1[i].xpath('string(.)')

r22 = r2[i].xpath('string(.)')

r33 = r3[i].xpath('string(.)')

# with open('test.txt', 'a', encoding='utf-8') as f:

# f.write(json.dumps(r11,ensure_ascii=False) + '\n')

# f.write(json.dumps(r22, ensure_ascii=False) + '\n')

# f.write(json.dumps(r33, ensure_ascii=False) + '\n')

print(r11,end='\n')

print(r22,end='\n')

print(r33)

print()