Python抓取亚马逊指定商品的所有页面

import undetected_chromedriver
from bs4 import BeautifulSoup
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support import expected_conditions as ExpectedConditions
import pandas as pd
import time
from fake_useragent import UserAgent

from selenium.common import NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait


def get_url(search_term):
    # 根据搜索词生成亚马逊的搜索链接
    template = 'https://www.amazon.com/s?k={}'
    search_term = search_term.replace(' ', '+')
    url = template.format(search_term)
    return url


def scrape_records(item):
    # 从商品元素中提取商品信息
    atag = item.h2.a
    description = atag.text.strip()
    url = 'https://amazon.com' + atag.get('href')

    price_parent = item.find('span', 'a-price')
    price = price_parent.find('span', 'a-offscreen').text.strip() if price_parent and price_parent.find('span', 'a-offscreen') else ''

    rating_element = item.find('span', {'class': 'a-icon-alt'})
    rating = rating_element.text.strip() if rating_element else ''

    review_count_element = item.find('span', {'class': 'a-size-base s-underline-text'})
    review_count = review_count_element.text.strip() if review_count_element else ''

    result = (description, price, rating, review_count, url)
    return result


def scrape_amazon(search_term):
    ua = UserAgent()
    
    # 创建Options对象
    options = Options()
    
    
    # 设置 亿牛云 爬虫代理加强版 用户名、密码、IP和端口号
    options.add_argument('--proxy-server=http://16YUN:16IP@www.16yun.cn:31000')
    
    # 设置随机User-Agent
    options.add_argument(f"user-agent={ua.random}")
    
    driver = undetected_chromedriver.Chrome(options=options)
    url = get_url(search_term)
    driver.get(url)
    time.sleep(5)
    records = []

    while True:
        # 滚动到页面底部加载更多商品
        time.sleep(5)
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

        soup = BeautifulSoup(driver.page_source, 'html.parser')
        results = soup.find_all('div', {'data-component-type': 's-search-result'})

        for item in results:
            try:
                record = scrape_records(item)
                records.append(record)
            except Exception as e:
                print(f"Error scraping item: {e}")

        # 检查页面是否有"Next"按钮
        try:
            nextButton = driver.find_element(By.XPATH, '//a[text()="Next"]')
            driver.execute_script("arguments[0].scrollIntoView();", nextButton)
            WebDriverWait(driver, 10).until(ExpectedConditions.element_to_be_clickable(nextButton))
            nextButton.click()
        except NoSuchElementException:
            print("Breaking as Last page Reached")
            break

    driver.close()

    # 处理商品信息并转换为DataFrame对象
    df = pd.DataFrame(records, columns=['Description', 'Price', 'Rating', 'Review Count', 'URL'])
    return df


# 获取用户输入的搜索词
search_term = 'washing machine'

# 爬取亚马逊的搜索结果
df = scrape_amazon(search_term)

# 将DataFrame导出为Excel文件
df.to_excel('output.xlsx', index=False)