用 WorkBuddy 写 Python 脚本，半小时搞定几万份文件批量去重

帮我写一个 Python 脚本，遍历指定目录下的所有 .txt 文件，按内容做去重。
要求：
1) 先按文件内容 MD5 精确去重；
2) 再做一层"近似去重"：把全角/半角、空白、中英文标点统一归一化后再哈希，
   识别格式/标点变体造成的重复；
3) 输出一份 CSV，每行 = 重复组哈希 + 保留路径 + 待删除路径；
4) 不要直接删除文件，只生成清单供人工核对；
5) 遇到编码错误跳过，不要崩。
只用 Python 标准库。

import hashlib, os, re, csv, unicodedata

def normalize(text):
    """归一化：NFKC 统一全角半角 -> 转小写 -> 去所有空白 -> 去常见中英文标点"""
    text = unicodedata.normalize("NFKC", text)
    text = text.lower()
    text = re.sub(r"\s+", "", text)
    text = re.sub(r"[，。！？、；：""''（）【】《》—…·.,!?:;\"'()\[\]{}<>/\\|]", "", text)
    return text

def read_text(path):
    # 先按 utf-8，失败再按 gbk；再不行忽略错误，保证不崩
    for enc in ("utf-8", "gbk"):
        try:
            with open(path, encoding=enc) as f:
                return f.read()
        except (UnicodeDecodeError, OSError):
            continue
    with open(path, encoding="utf-8", errors="ignore") as f:
        return f.read()

def content_keys(path):
    txt = read_text(path)
    exact = hashlib.md5(txt.encode("utf-8", "ignore")).hexdigest()
    near = hashlib.md5(normalize(txt).encode("utf-8", "ignore")).hexdigest()
    return exact, near

def find_dupes(root, ext=".txt"):
    exact_groups, near_groups = {}, {}
    for dirpath, _, files in os.walk(root):
        for name in files:
            if not name.lower().endswith(ext):
                continue
            p = os.path.join(dirpath, name)
            try:
                e, n = content_keys(p)
            except Exception:
                continue
            exact_groups.setdefault(e, []).append(p)
            near_groups.setdefault(n, []).append(p)
    return exact_groups, near_groups

def write_report(root):
    exact, near = find_dupes(root)
    rows = []
    # 精确重复
    for h, paths in exact.items():
        if len(paths) > 1:
            keep, *dels = sorted(paths, key=lambda x: (len(x), x))
            for d in dels:
                rows.append([h, "exact", keep, d])
    # 近似重复（同一精确组内不再重复报）
    seen_exact = {frozenset(v) for v in exact.values() if len(v) > 1}
    for h, paths in near.items():
        if len(paths) > 1 and frozenset(paths) not in seen_exact:
            keep, *dels = sorted(paths, key=lambda x: (len(x), x))
            for d in dels:
                rows.append([h, "near", keep, d])
    out = os.path.join(root, "dup_report.csv")
    with open(out, "w", newline="", encoding="utf-8-sig") as f:
        w = csv.writer(f)
        w.writerow(["hash", "type", "keep", "delete"])
        w.writerows(rows)
    print(f"生成清单：{out}")
    print(f"精确重复待删 {sum(len(v)-1 for v in exact.values() if len(v)>1)} 个")
    print(f"近似重复待删 {len(rows) - sum(len(v)-1 for v in exact.values() if len(v)>1)} 个")

if __name__ == "__main__":
    write_report(r"D:\小说库")   # 改成你自己的目录

import shutil, csv, os

def to_trash(csv_path, batch=200):
    with open(csv_path, encoding="utf-8-sig") as f:
        for i, row in enumerate(csv.DictReader(f)):
            if i >= batch:
                print("已处理一批，确认无误后再跑下一批")
                break
            p = row["delete"]
            if os.path.exists(p):
                shutil.move(p, os.path.join(os.environ["RecycleBin"], os.path.basename(p)))
                # 没有回收站变量就改用 shutil.move 到备份文件夹
    print("本批完成")

# to_trash("D:\\小说库\\dup_report.csv")