【Python爬虫实战】多进程结合 BeautifulSoup 与 Scrapy 构建爬虫项目

import requests
from multiprocessing import Pool

def fetch_url(url):
    try:
        response = requests.get(url)
        print(f"Fetched {url}: {response.status_code}")
        return response.text
    except Exception as e:
        print(f"Failed to fetch {url}: {e}")

def main(urls):
    with Pool(processes=4) as pool:  # 创建4个进程
        results = pool.map(fetch_url, urls)  # 并发抓取
    return results

if __name__ == '__main__':
    urls = [
        'https://www.example.com',
        'https://www.example.org',
        'https://www.example.net',
        # 添加更多网址
    ]
    main(urls)

import requests
from bs4 import BeautifulSoup
from multiprocessing import Process, JoinableQueue
import time

# 定义爬虫工作函数
def worker(queue):
    while True:
        url = queue.get()  # 从队列中获取 URL
        if url is None:  # 用于退出条件
            queue.task_done()
            break
        print(f"Fetching: {url}")
        try:
            response = requests.get(url, timeout=5)
            if response.status_code == 200:
                soup = BeautifulSoup(response.text, 'html.parser')
                # 假设提取文章标题
                titles = soup.find_all('h2')
                for title in titles:
                    print(f"Title: {title.get_text()}")
            else:
                print(f"Failed to fetch {url}: {response.status_code}")
        except Exception as e:
            print(f"Error fetching {url}: {e}")
        finally:
            queue.task_done()  # 标记任务完成

# 主函数
def main(urls):
    queue = JoinableQueue()
    processes = []

    # 启动多个工作进程
    for _ in range(4):  # 根据需要启动的进程数
        p = Process(target=worker, args=(queue,))
        p.start()
        processes.append(p)

    # 向队列中添加 URL
    for url in urls:
        queue.put(url)

    # 等待队列中所有任务完成
    queue.join()  # 阻塞直到所有任务调用 task_done()

    # 发送退出信号给每个进程
    for _ in processes:
        queue.put(None)

    # 等待所有进程完成
    for p in processes:
        p.join()

if __name__ == '__main__':
    urls = [
        'https://www.example.com/page1',
        'https://www.example.com/page2',
        'https://www.example.com/page3',
        # 添加更多网址
    ]
    start_time = time.time()
    main(urls)
    print(f"All tasks completed in {time.time() - start_time:.2f} seconds.")

import requests
from bs4 import BeautifulSoup
from multiprocessing import Process, JoinableQueue
import time

# 定义爬虫工作函数
def worker(queue):
    while True:
        url = queue.get()
        if url is None:  # 退出条件
            queue.task_done()
            break

        try:
            response = requests.get(url, timeout=5)
            if response.status_code == 200:
                soup = BeautifulSoup(response.text, 'html.parser')
                # 假设需要提取所有标题和链接
                titles = soup.find_all('h2')  # 根据实际结构调整
                for title in titles:
                    print(f"Title: {title.get_text()} - URL: {url}")
            else:
                print(f"Failed to fetch {url}: {response.status_code}")
        except Exception as e:
            print(f"Error fetching {url}: {e}")
        finally:
            queue.task_done()

# 主函数
def main(urls):
    queue = JoinableQueue()
    processes = []

    # 启动多个进程
    for _ in range(4):  # 启动4个工作进程
        p = Process(target=worker, args=(queue,))
        p.start()
        processes.append(p)

    # 添加 URL 到队列
    for url in urls:
        queue.put(url)

    # 等待队列中所有任务完成
    queue.join()

    # 发送退出信号给每个进程
    for _ in processes:
        queue.put(None)

    # 等待所有进程完成
    for p in processes:
        p.join()

if __name__ == '__main__':
    urls = [
        'https://www.example.com/page1',
        'https://www.example.com/page2',
        'https://www.example.com/page3',
        # 添加更多网址
    ]
    start_time = time.time()
    main(urls)
    print(f"All tasks completed in {time.time() - start_time:.2f} seconds.")

import scrapy

class MySpider(scrapy.Spider):
    name = 'myspider'
    start_urls = ['https://www.example.com']

    def parse(self, response):
        titles = response.css('h2::text').getall()  # 假设要提取的内容为 <h2> 标签的文本
        for title in titles:
            yield {'title': title}

from multiprocessing import Process
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from myproject.spiders.my_spider import MySpider

def run_spider():
    process = CrawlerProcess(get_project_settings())
    process.crawl(MySpider)
    process.start()

if __name__ == '__main__':
    processes = []

    # 启动多个爬虫进程
    for _ in range(3):  # 根据需求启动多个爬虫实例
        p = Process(target=run_spider)
        p.start()
        processes.append(p)

    # 等待所有爬虫进程完成
    for p in processes:
        p.join()