使用 Playwright 和 C# 自动化采集亚马逊网站的商品信息和评论

using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Playwright;

class Program
{
    static async Task Main(string[] args)
    {
        // 初始化 Playwright
        using var playwright = await Playwright.CreateAsync();
        //亿牛云爬虫代理加强版的代理服务器地址和端口号
            var proxyServer = "http://www.16yun.cn:3100";  
        var browser = await playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
        {
            Proxy = new ProxySettings
            {
                Server = proxyServer,
                //亿牛云爬虫代理加强版的用户名
                Username = "your_username", 
                //亿牛云爬虫代理加强版的密码
                Password = "your_password",
            }
        });
        
        // 创建一个新的上下文
        var context = await browser.NewContextAsync(new BrowserNewContextOptions
        {
            ViewportSize = new ViewportSize
            {
                Width = 1280,
                Height = 720,
            },
        });
        
        // 创建一个新的页面
        var page = await context.NewPageAsync();
        
        // 导航到亚马逊网站
        await page.GotoAsync("https://www.amazon.com/");
        
        // 输入关键字搜索
        await page.FillAsync("#twotabsearchtextbox", "laptop");
        await page.ClickAsync("#nav-search-submit-button");
        
        // 等待搜索结果页面加载完成
        await page.WaitForLoadStateAsync(LoadState.NetworkIdle);
        
        // 获取商品链接列表
        var links = await page.EvaluateAsync<IEnumerable<string>>(@"
            Array.from(document.querySelectorAll('.s-result-item h2 a'))
            .map(a => a.href)
        ");
        
        // 创建任务列表
        var tasks = new List<Task>();
        
        // 遍历商品链接列表，采集评论数据
        foreach (var link in links)
        {
            tasks.Add(Task.Run(async () =>
            {
                // 创建一个新的页面
                var page = await context.NewPageAsync();
                
                // 设置页面的 User-Agent
                await page.SetUserAgentAsync("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36");
                
                // 导航到商品页面
                await page.GotoAsync(link);
                
                // 等待商品页面加载完成
                await page.WaitForLoadStateAsync(LoadState.NetworkIdle);
                
                // 获取商品名称
                var title = await page.InnerTextAsync("#productTitle");
                
                // 获取商品评价信息
                var rating = await page.InnerTextAsync("#averageCustomerReviews .a-icon-star-small .a-icon-alt");
                var reviewCount = await page.InnerTextAsync("#acrCustomerReviewText");
                
                // 输出采集的数据
                Console.WriteLine($"{title}: {rating} ({reviewCount})");
                
                // 关闭页面
                await page.CloseAsync();
            }));
        }
        
        // 等待所有任务完成
        await Task.WhenAll(tasks);
        
        // 关闭浏览器
        await browser.CloseAsync();
    }
}