基于bs4+requests爬取世界赛艇男运动员信息(进阶篇)

from bs4 import BeautifulSoup as bs
import requests

response = requests.get('http://www.worldrowing.com/events/2018-world-rowing-under-23-championships/u23-mens-eight/')
soup = bs(response.text, 'html.parser')
athlete_list = soup.select('tr.resultsDetails li')
item_list = []
for athlete in athlete_list:
    item = {}
    item['name'] = athlete.select('h4 a')[0].text
    item['position'] = athlete.select('p.yPadding')[0].text.strip()
    item['img_url'] = 'http://www.worldrowing.com' + athlete.select('img')[0]['src']
    detail_url = 'http://www.worldrowing.com' + athlete.select('h4 a')[0]['href']
    response = requests.get(detail_url)
    soup = bs(response.text, 'html.parser')
    li_list = soup.select('div.athleteInfoBody li')
    for li in li_list:
        key = li.select('div.dt')[0].text
        value = li.select('div.dd')[0].text
        item[key] = value
    item_list.append(item)

longest_keys = item_list[0].keys()
for item in item_list:
    if len(item.keys()) > len(longest_keys):
        longest_keys = item.keys()
        
import pandas as pd
df = pd.DataFrame(item_list, columns=longest_keys)
df.to_excel('athleteRecord3.xlsx')