手把手教你用Scrapy爬虫框架爬取食品论坛数据并存入数据库

http://bbs.foodmate.net

def parse(self, response):
    self.logger.info("已进入网页！")
    self.logger.info("正在获取版块列表！")
    column_path_list = response.css('#ct > div.mn > div:nth-child(2) > div')[:-1]
    for column_path in column_path_list:
        col_paths = column_path.css('div > table > tbody > tr > td > div > a').xpath('@href').extract()
        for path in col_paths:
            block_url = response.urljoin(path)
            yield scrapy.Request(
                url=block_url,
                callback=self.get_next_path,
            )

def get_next_path(self, response):
    self.logger.info("已进入版块！")
    self.logger.info("正在获取文章列表！")
    if response.url == 'http://www.foodmate.net/know/':
        pass
    else:
        try:
            nums = response.css('#fd_page_bottom > div > label > span::text').extract_first().split(' ')[-2]
        except:
            nums = 1
        for num in range(1, int(nums) + 1):
            tbody_list = response.css('#threadlisttableid > tbody')
            for tbody in tbody_list:
                if 'normalthread' in str(tbody):
                    item = LunTanItem()
                    item['article_url'] = response.urljoin(
                        tbody.css('* > tr > th > a.s.xst').xpath('@href').extract_first())
                    item['type'] = response.css(
                        '#ct > div > div.bm.bml.pbn > div.bm_h.cl > h1 > a::text').extract_first()
                    item['title'] = tbody.css('* > tr > th > a.s.xst::text').extract_first()
                    item['spider_type'] = "论坛"
                    item['source'] = "食品论坛"
                    if item['article_url'] != 'http://bbs.foodmate.net/':
                        yield scrapy.Request(
                            url=item['article_url'],
                            callback=self.get_data,
                            meta={'item': item, 'content_info': []}
                        )
        try:
            callback_url = response.css('#fd_page_bottom > div > a.nxt').xpath('@href').extract_first()
            callback_url = response.urljoin(callback_url)
            yield scrapy.Request(
                url=callback_url,
                callback=self.get_next_path,
            )
        except IndexError:
            pass

def get_data(self, response):
    self.logger.info("正在爬取论坛数据！")
    item = response.meta['item']
    content_list = []
    divs = response.xpath('//*[@id="postlist"]/div')
    user_name = response.css('div > div.pi > div:nth-child(1) > a::text').extract()
    publish_time = response.css('div.authi > em::text').extract()
    floor = divs.css('* strong> a> em::text').extract()
    s_id = divs.xpath('@id').extract()
    for i in range(len(divs) - 1):
        content = ''
        try:

            strong = response.css('#postmessage_' + s_id[i].split('_')[-1] + '').xpath('string(.)').extract()
            for s in strong:
                content += s.split(';')[-1].lstrip('\r\n')
            datas = dict(content=content,  # 内容
                         reply_id=0,  # 回复的楼层,默认0
                         user_name=user_name[i],  # ⽤户名
                         publish_time=publish_time[i].split('于 ')[-1],  # %Y-%m-%d %H:%M:%S'
                         id='#' + floor[i],  # 楼层
                         )
            content_list.append(datas)
        except IndexError:
            pass
    item['content_info'] = response.meta['content_info']
    item['scrawl_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    item['content_info'] += content_list

    data_url = response.css('#ct > div.pgbtn > a').xpath('@href').extract_first()
    if data_url != None:
        data_url = response.urljoin(data_url)
        yield scrapy.Request(
            url=data_url,
            callback=self.get_data,
            meta={'item': item, 'content_info': item['content_info']}
        )
    else:
        item['scrawl_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        self.logger.info("正在存储！")
        print('储存成功')
        yield item

def process_request(self, request, spider):
    chrome_options = Options()
    chrome_options.add_argument('--headless')  # 使用无头谷歌浏览器模式
    chrome_options.add_argument('--disable-gpu')
    chrome_options.add_argument('--no-sandbox')
    # 指定谷歌浏览器路径
    self.driver = webdriver.Chrome(chrome_options=chrome_options,
                                   executable_path='E:/pycharm/workspace/爬虫/scrapy/chromedriver')
    if request.url != 'http://bbs.foodmate.net/':
        self.driver.get(request.url)
        html = self.driver.page_source
        time.sleep(1)
        self.driver.quit()
        return scrapy.http.HtmlResponse(url=request.url, body=html.encode('utf-8'), encoding='utf-8',
                                        request=request)

class LunTanItem(scrapy.Item):
    """
        论坛字段
    """
    title = Field()  # str: 字符类型 | 论坛标题
    content_info = Field()  # str: list类型 | 类型list: [LunTanContentInfoItem1, LunTanContentInfoItem2]
    article_url = Field()  # str: url | 文章链接
    scrawl_time = Field()  # str: 时间格式 参照如下格式 2019-08-01 10:20:00 | 数据爬取时间
    source = Field()  # str: 字符类型 | 论坛名称 eg: 未名BBS, 水木社区, 天涯论坛
    type = Field()  # str: 字符类型 | 板块类型 eg: '财经', '体育', '社会'
    spider_type = Field()  # str: forum | 只能写 'forum'

import pymongo

class FMPipeline():
    def __init__(self):
        super(FMPipeline, self).__init__()
        # client = pymongo.MongoClient('139.217.92.75')
        client = pymongo.MongoClient('localhost')
        db = client.scrapy_FM
        self.collection = db.FM

    def process_item(self, item, spider):
        query = {
            'article_url': item['article_url']
        }
        self.collection.update_one(query, {"$set": dict(item)}, upsert=True)
        return item

query = {
    'article_url': item['article_url']
}
self.collection.update_one(query, {"$set": dict(item)}, upsert=True)