Python etree解析豆瓣电影TOP250

#coding: utf-8
import requests
from lxml import etree
import xlwt
import os

class Solution:
    def __init__(self):
        pass

    def getItems(self, url):
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.72 Safari/537.36 Edg/89.0.774.45"
        }
        html = requests.get(url, headers=headers).text
        element = etree.HTML(html)
        tree = element.xpath('//*[@id="content"]/div/div[1]/ol/li')
        return tree

    def parseItem(self, tree):
        results = []
        for item in tree:
            resMap = {}
            node = etree.HTML(etree.tostring(item[0], encoding="utf-8").decode())
            title = ''.join(node.xpath('//div/div[2]/div[1]/a/span/text()'))
            actor = node.xpath('//div/div[2]/div[2]/p[1]/text()')[0].replace(" ", "").strip('\n')
            year = node.xpath('//div/div[2]/div[2]/p[1]/text()')[1].replace(" ", "").strip('\n')
            desc = node.xpath('//div/div[2]/div[2]/p[2]/span/text()')[0]
            score = node.xpath('//div/div[2]/div[2]/div/span[2]/text()')[0]
            comment = node.xpath('//div/div[2]/div[2]/div/span[4]/text()')[0]
            mvlink = node.xpath('//div/div[2]/div[1]/a/@href')[0]
            piclink = node.xpath('//div/div[1]/a/img/@src')[0]
            resMap['title'] = title
            resMap['score'] = score
            resMap['comment'] = comment
            resMap['actor'] = actor
            resMap['year'] = year
            resMap['desc'] = desc
            resMap['mvlink'] = mvlink
            resMap['piclink'] = piclink
            results.append(resMap.copy())
        return results

    def saveXlwt(self, datalist, savepath):
        book = xlwt.Workbook(encoding="utf-8", style_compression=0)
        sheet = book.add_sheet('豆瓣电影top250', cell_overwrite_ok=True)
        col = ('影片名', '演员', '年份', '评分', '评价数', '概况', '电影链接详情', '图片链接')
        for i in range(0, len(col)):
            sheet.write(0, i, col[i])
        for i in range(0, len(datalist)):
            print("第%d条" % (i+1))
            data = datalist[i]
            sheet.write(i + 1, 0, data["title"])
            sheet.write(i + 1, 1, data["actor"])
            sheet.write(i + 1, 2, data["year"])
            sheet.write(i + 1, 3, data["score"])
            sheet.write(i + 1, 4, data["comment"])
            sheet.write(i + 1, 5, data["desc"])
            sheet.write(i + 1, 6, data["mvlink"])
            sheet.write(i + 1, 7, data["piclink"])
        book.save(savepath)


if __name__ == '__main__':
    baseurl = 'https://movie.douban.com/top250?start='
    sl = Solution()
    tree = sl.getItems(baseurl)
    results = sl.parseItem(tree)
    sl.saveXlwt(results, os.path.join(os.getcwd(), "movie.xls"))