使用Scrapy框架爬取Google搜索结果

pip install scrapy

scrapy startproject google_search

cd google_search

scrapy genspider google_spider www.google.com

edit google_search/spiders/google_spider.py

import scrapy
from scrapy.http import Request
from urllib.parse import quote
class GoogleSpider(scrapy.Spider):
   name = 'google_spider'
   allowed_domains = ['www.google.com']
   start_urls = ['https://www.google.com/search?q=so&source=hp&ei=clWpZMHEArze4-EP_tGusA8&iflsig=AD69kcEAAAAAZKljgpV7rjBRGIpmk1S12wEUcYes8Kk-&ved=0ahUKEwjBxa2gjf__AhU87zgGHf6oC_YQ4dUDCAk&uact=5&oq=so&gs_lcp=Cgdnd3Mtd2l6EAMyBwgAEAQQgAQyBwgAEAQQgAQyBwgAEAQQgAQyBwgAEAQQgAQyBwgAEAQQgAQyBQgAEIAEMgUIABCABDIFCAAQgAQyBwgAEAQQgAQyBwgAEAQQgARQnxdY_lFgllhoBXAAeACAAYEBiAHgApIBAzAuM5gBAKABAbABAA&sclient=gws-wiz']
    def parse(self, response):
        # 提取搜索结果中的链接
        search_results = response.css('div.g')
        for result in search_results:
            title = result.css('h3 a::text').get()
            link = result.css('h3 a::attr(href)').get()
            yield {'title': title, 'link': link}            
        # 提取下一页的链接
        next_page = response.css('a[rel="next"]::attr(href)').get()
        if next_page is not None:
            yield response.follow(next_page, callback=self.parse)

scrapy crawl google_spider

settings = {
    'CONCURRENT_REQUESTS': 10,
}

USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'