手把手教你抓取链家二手房首页的全部数据

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Created by iFantastic on 2022/6/7

import scrapy
from scrapy.cmdline import execute



class LJ_spiders(scrapy.Spider):
    name = 'lj'
    start_urls = [f'https://cs.lianjia.com/ershoufang/pg{page}/' for page in range(1, 101)]

    def parse(self, response, **kwargs):
        for info in response.xpath("//div[@class='info clear']"):
            yield {
                'title': info.xpath('./div[@class="title"]/a/text()').get(),
                'positionInfo': info.xpath('./div[@class="flood"]//text()').extract(),
                'houseInfo': info.xpath('./div[@class="address"]/div/text()').get(),
                'followInfo': info.xpath('./div[@class="followInfo"]//text()').get(),
                'tag': info.xpath('./div[@class="tag"]//text()').extract(),
                'totalPrice': info.xpath('//div[@class="priceInfo"]/div[1]/span/text()').get() + '万',
                'unitPrice': info.xpath('//div[@class="priceInfo"]/div[2]/span/text()').get()
            }


if __name__ == '__main__':
    execute('scrapy crawl lj -o 长沙二手房.csv'.split())