C#高效的单个数据表列数据的重复数据消除

using System;
using System.Collections.Generic;
using System.Data.SqlClient;

namespace DataDeduplication
{
    class Program
    {
        static void Main(string[] args)
        {
            string connectionString = "YourConnectionString"; // 替换为你的数据库连接字符串
            string tableName = "YourTableName"; // 替换为你的数据表名
            string columnName = "YourColumnName"; // 替换为你的列名

            List<string> dataList = new List<string>();

            using (SqlConnection connection = new SqlConnection(connectionString))
            {
                string query = $"SELECT {columnName} FROM {tableName}";

                SqlCommand command = new SqlCommand(query, connection);
                connection.Open();

                SqlDataReader reader = command.ExecuteReader();

                while (reader.Read())
                {
                    string data = reader[columnName].ToString();
                    dataList.Add(data);
                }

                reader.Close();
            }

            HashSet<string> deduplicatedData = new HashSet<string>();

            foreach (string data in dataList)
            {
                if (!deduplicatedData.Contains(data))
                {
                    deduplicatedData.Add(data);
                }
            }

            // 处理去重后的数据，例如写入数据库或进行其他操作
            foreach (string data in deduplicatedData)
            {
                Console.WriteLine(data);
            }
        }
    }
}