Python实现多线程PDF抓取与下载脚本

import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import os
import threading
from queue import Queue
import time
import argparse
​
class PDFDownloader:
    def __init__(self, base_url, max_threads=5, download_dir="downloads"):
        self.base_url = base_url
        self.max_threads = max_threads
        self.download_dir = download_dir
        self.pdf_links = set()
        self.visited_links = set()
        self.lock = threading.Lock()
        self.queue = Queue()
        
        # 创建下载目录
        if not os.path.exists(download_dir):
            os.makedirs(download_dir)
    
    def is_valid_url(self, url):
        """
        检查URL是否有效且属于同一域名
        """
        parsed = urlparse(url)
        base_parsed = urlparse(self.base_url)
        return bool(parsed.netloc) and parsed.netloc == base_parsed.netloc
    
    def get_absolute_url(self, url):
        """
        将相对URL转换为绝对URL
        """
        return urljoin(self.base_url, url)
    
    def extract_pdf_links(self, url):
        """
        从页面提取所有PDF链接
        """
        try:
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
            }
            response = requests.get(url, headers=headers, timeout=10)
            response.raise_for_status()
            
            soup = BeautifulSoup(response.content, 'html.parser')
            
            # 查找所有链接
            for link in soup.find_all('a', href=True):
                href = link['href']
                
                # 检查是否是PDF链接
                if href.lower().endswith('.pdf'):
                    absolute_url = self.get_absolute_url(href)
                    with self.lock:
                        self.pdf_links.add(absolute_url)
                
                # 如果是普通链接，添加到队列中继续爬取
                else:
                    absolute_url = self.get_absolute_url(href)
                    if self.is_valid_url(absolute_url) and absolute_url not in self.visited_links:
                        self.queue.put(absolute_url)
                        self.visited_links.add(absolute_url)
                        
        except Exception as e:
            print(f"Error processing {url}: {str(e)}")
    
    def download_pdf(self, url):
        """
        下载PDF文件
        """
        try:
            # 从URL提取文件名
            filename = os.path.basename(urlparse(url).path)
            if not filename:
                filename = f"document_{int(time.time())}.pdf"
            
            filepath = os.path.join(self.download_dir, filename)
            
            # 如果文件已存在，跳过下载
            if os.path.exists(filepath):
                print(f"File already exists: {filename}")
                return
            
            print(f"Downloading: {filename}")
            
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
            }
            response = requests.get(url, headers=headers, stream=True, timeout=30)
            response.raise_for_status()
            
            # 写入文件
            with open(filepath, 'wb') as f:
                for chunk in response.iter_content(chunk_size=8192):
                    if chunk:
                        f.write(chunk)
            
            print(f"Downloaded: {filename}")
            
        except Exception as e:
            print(f"Error downloading {url}: {str(e)}")
    
    def crawl_worker(self):
        """
        爬取工作线程函数
        """
        while True:
            url = self.queue.get()
            if url is None:
                break
            
            self.extract_pdf_links(url)
            self.queue.task_done()
    
    def download_worker(self):
        """
        下载工作线程函数
        """
        while True:
            url = self.queue.get()
            if url is None:
                break
            
            self.download_pdf(url)
            self.queue.task_done()
    
    def find_all_pdfs(self, max_pages=50):
        """
        查找网站中的所有PDF链接
        """
        print("Starting to crawl for PDF links...")
        
        # 初始URL加入队列
        self.queue.put(self.base_url)
        self.visited_links.add(self.base_url)
        
        # 创建爬取线程
        threads = []
        for _ in range(self.max_threads):
            t = threading.Thread(target=self.crawl_worker)
            t.start()
            threads.append(t)
        
        # 等待队列完成或达到最大页面数
        pages_processed = 0
        while not self.queue.empty() and pages_processed < max_pages:
            time.sleep(0.1)
            pages_processed += 1
        
        # 停止所有线程
        for _ in range(self.max_threads):
            self.queue.put(None)
        
        for t in threads:
            t.join()
        
        print(f"Found {len(self.pdf_links)} PDF documents")
        return self.pdf_links
    
    def download_all_pdfs(self):
        """
        下载所有找到的PDF文件
        """
        print("Starting download process...")
        
        # 将PDF链接加入队列
        for pdf_url in self.pdf_links:
            self.queue.put(pdf_url)
        
        # 创建下载线程
        threads = []
        for _ in range(self.max_threads):
            t = threading.Thread(target=self.download_worker)
            t.start()
            threads.append(t)
        
        # 等待所有下载完成
        self.queue.join()
        
        # 停止所有线程
        for _ in range(self.max_threads):
            self.queue.put(None)
        
        for t in threads:
            t.join()
        
        print("All downloads completed!")
​
def main():
    parser = argparse.ArgumentParser(description='PDF Downloader from Website')
    parser.add_argument('url', help='Base URL to start crawling from')
    parser.add_argument('-t', '--threads', type=int, default=5, help='Number of threads (default: 5)')
    parser.add_argument('-d', '--directory', default='downloads', help='Download directory (default: downloads)')
    parser.add_argument('-m', '--max-pages', type=int, default=50, help='Maximum pages to crawl (default: 50)')
    
    args = parser.parse_args()
    
    # 创建下载器实例
    downloader = PDFDownloader(
        base_url=args.url,
        max_threads=args.threads,
        download_dir=args.directory
    )
    
    # 查找所有PDF链接
    pdf_links = downloader.find_all_pdfs(max_pages=args.max_pages)
    
    if pdf_links:
        # 下载所有PDF文件
        downloader.download_all_pdfs()
    else:
        print("No PDF documents found on the website.")
​
if __name__ == "__main__":
    main()

pip install requests beautifulsoup4

python pdf_downloader.py https://example.com -t 10 -d my_pdfs -m 100