C#和HttpClient结合示例：微博热点数据分析

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;

namespace WeiboCrawler
{
    class Program
    {
        // 创建一个HTTP客户端对象，用于发送请求和接收响应
        static HttpClient httpClient = new HttpClient();

        // 创建一个数据结构，用于保存热点话题的信息
        static List<Topic> topics = new List<Topic>();

        // 定义一个锁对象，用于同步多线程操作
        static object locker = new object();

        static void Main(string[] args)
        {
            // 亿牛云爬虫标准版，使用代理服务设置代理域名、端口、用户名和密码
            var proxy = new WebProxy("http://wwww.16yun.cn:8080");
            proxy.Credentials = new NetworkCredential("16YUNXXX", "16IPXXX");
            httpClient.DefaultRequestHeaders.Add("Proxy-Authorization", "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes("username:password")));
            httpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36");
            httpClient.Timeout = TimeSpan.FromSeconds(10);

            // 定义一个热点话题的URL列表，每个URL对应一个热点话题的页面
            var urls = new List<string>
            {
                "https://s.weibo.com/top/summary?cate=realtimehot",
                "https://s.weibo.com/top/summary?cate=socialevent",
                "https://s.weibo.com/top/summary?cate=entertainment",
                "https://s.weibo.com/top/summary?cate=sports",
                "https://s.weibo.com/top/summary?cate=tech",
                "https://s.weibo.com/top/summary?cate=finance"
            };

            // 使用多线程技术，创建多个线程，每个线程负责爬取一个热点话题的数据
            var tasks = new List<Task>();
            foreach (var url in urls)
            {
                tasks.Add(Task.Run(() => Crawl(url)));
            }

            // 等待所有线程完成任务
            Task.WaitAll(tasks.ToArray());

            // 使用LINQ或者其他方法，简单的对数据进行排序、分组操作，得到一些有意义的统计结果
            Console.WriteLine("爬取完成，共获取了{0}个热点话题的信息。", topics.Count);
            Console.WriteLine("按阅读量降序排列的前10个热点话题如下：");
            foreach (var topic in topics.OrderByDescending(t => t.ReadCount).Take(10))
            {
                Console.WriteLine("{0} {1} {2} {3}", topic.Title, topic.Link, topic.ReadCount, topic.DiscussCount);
            }
            Console.WriteLine("按讨论量降序排列的前10个热点话题如下：");
            foreach (var topic in topics.OrderByDescending(t => t.DiscussCount).Take(10))
            {
                Console.WriteLine("{0} {1} {2} {3}", topic.Title, topic.Link, topic.ReadCount, topic.DiscussCount);
            }
            Console.WriteLine("按类别分组的热点话题数量如下：");
            foreach (var group in topics.GroupBy(t => t.Category))
            {
                Console.WriteLine("{0} {1}", group.Key, group.Count());
            }
        }

        // 定义一个方法，用于爬取一个热点话题的数据
        static void Crawl(string url)
        {
            try
            {
                // 发送GET请求，获取响应内容
                var response = httpClient.GetAsync(url).Result;
                var content = response.Content.ReadAsStringAsync().Result;

                // 使用正则表达式或者HTML解析器，从响应内容中提取热点话题的标题、链接、阅读量、讨论量等信息，并保存到一个数据结构中
                var regex = new Regex(@"<td class=""td-02""><a href=""(?<link>.+?)"" target=""_blank"" title=""(?<title>.+?)"">.+?</a><span>(?<readcount>\d+)</span></td>\s*<td class=""td-03""><i class=""icon-txt"">.+?</i><span>(?<discusscount>\d+)</span></td>");
                var matches = regex.Matches(content);
                foreach (Match match in matches)
                {
                    var topic = new Topic
                    {
                        Title = match.Groups["title"].Value,
                        Link = "https://s.weibo.com" + match.Groups["link"].Value,
                        ReadCount = int.Parse(match.Groups["readcount"].Value),
                        DiscussCount = int.Parse(match.Groups["discusscount"].Value),
                        Category = url.Split('=')[1]
                    };
                    // 使用锁对象，避免多线程操作数据结构时发生冲突
                    lock (locker)
                    {
                        topics.Add(topic);
                    }
                }
            }
            catch (Exception ex)
            {
                // 处理异常情况，如网络超时、响应格式错误等
                Console.WriteLine("爬取{0}时发生错误：{1}", url, ex.Message);
            }
        }
    }

    // 定义一个类，用于表示一个热点话题的信息
    class Topic
    {
        public string Title { get; set; } // 标题
        public string Link { get; set; } // 链接
        public int ReadCount { get; set; } // 阅读量
        public int DiscussCount { get; set; } // 讨论量
        public string Category { get; set; } // 类别
    }
}