爬取豆瓣电影排行top250

"""
    作者：李舵
    日期：2019-4-27
    功能：抓取豆瓣电影top250
    版本：V1.0
"""

import time
import json
import requests
from bs4 import BeautifulSoup
from requests.exceptions import RequestException


def get_one_page(url):
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36'}
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        return None


def parse_one_page(html):
    soup = BeautifulSoup(html, 'lxml')
    ol_list = soup.find('ol', {'class': 'grid_view'})
    li_list = ol_list.find_all('li')
    for i in range(25):
        move_value = li_list[i]
        yield {
            'index': move_value.find('em', {'class': ''}).text.strip(),
            'title': move_value.find('span', {'class': 'title'}).text.strip(),
            'actor': move_value.find('p', {'class': ''}).text.strip(),
            'score': move_value.find('span', {'class': 'rating_num'}).text.strip()
        }


def write_to_file(content):
    with open('result.txt', 'a', encoding='utf-8') as f:
        print(type(json.dumps(content)))
        f.write(json.dumps(content, ensure_ascii=False)+'\n')


def main(start):
    url = 'https://movie.douban.com/top250?start=' + str(start)
    html = get_one_page(url)
    for item in parse_one_page(html):
        print(item)
        write_to_file(item)


if __name__ == '__main__':
    for i in range(0,250,25):
        main(start=i)
        time.sleep(1)

"""
作者：李舵
日期：2019 - 4 - 8
功能：抓取豆瓣电影top250
版本：V2.0
"""

import re
import time
import requests
from requests.exceptions import RequestException


def get_one_page(url):
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'}
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        return None


def parse_one_page(html):
    pattern = re.compile(u'<div.*?class="item">.*?'
                            + u'<div.*?class="pic">.*?'
                            + u'<em.*?class="">(.*?)</em>.*?'
                            + u'<div.*?class="info">.*?'
                            + u'<span.*?class="title">(.*?)</span>.*?'
                            + u'<span.*?class="other">(.*?)</span>.*?'
                            + u'<div.*?class="bd">.*?'
                            + u'<p.*?class="">.*?'
                            + u'导演:\s(.*?)\s.*?<br>'
                            + u'(.*?) / '
                            + u'(.*?) / (.*?)</p>.*?'
                            + u'<div.*?class="star">.*?'
                            + u'<span.*?class="rating_num".*?property="v:average">'
                            + u'(.*?)</span>.*?'
                            + u'<span>(.*?)人评价</span>.*?'
                            + u'<span.*?class="inq">(.*?)</span>', re.S)
    movies = re.findall(pattern, html)
    movie_list = []
    for movie in movies:
        movie_list.append([movie[0],
                           movie[1],
                           movie[2].lstrip(' / '),
                           movie[3],
                           movie[4].lstrip(),
                           movie[5],
                           movie[6].strip(),
                           movie[7],
                           movie[8],
                           movie[9]])
    return movie_list


def write_to_file(movie_list):
    with open('top_250.txt', 'w', encoding='utf-8',) as f:
        for movie in movie_list:
            f.write('电影排名：' + movie[0] + '\n')
            f.write('电影名称：' + movie[1] + '\n')
            f.write('电影别名：' + movie[2] + '\n')
            f.write('导演：' + movie[3] + '\n')
            f.write('上映年份：' + movie[4] + '\n')
            f.write('制作国家/地区：' + movie[5] + '\n')
            f.write('电影类别：' + movie[6] + '\n')
            f.write('评分：' + movie[7] + '\n')
            f.write('参评人数：' + movie[8] + '\n')
            f.write('简短影评：' + movie[9] + '\n')
            f.write('\n')
        print('成功写入文件，共有%d条记录……' % len(movie_list))
        f.close()


def main(start):
    url = 'https://movie.douban.com/top250?start=' + str(start)
    html = get_one_page(url)
    movie_list = parse_one_page(html)
    write_to_file(movie_list)


if __name__ == '__main__':
    for i in range(0, 250, 25):
        main(start=i)
        time.sleep(1)