#WorkBuddy# 383 封邮件归档：朴素脚本主题还原只有 35.25%，我用一条七阶段流水线做到 100%

CANDIDATES = ["utf-8", "gb18030", "big5", "gbk", "shift_jis", "euc-kr", "latin-1"]

def _score(t):
    """给解码结果打分：汉字占比高、控制字符/替换符/高位拉丁少 = 越像人话"""
    if not t:
        return -9.0
    n = len(t)
    repl = t.count("\ufffd")                              # 解码替换符
    cjk = sum(1 for c in t if "\u4e00" <= c <= "\u9fff")  # 汉字
    latinsup = sum(1 for c in t if "\x80" <= c <= "\xff")  # 高位拉丁（乱码特征）
    ctrl = sum(1 for c in t if ord(c) < 32 and c not in "\r\n\t")
    return (1.5 * (cjk / n) + (n - latinsup - ctrl - repl) / n
            - 3.0 * (repl / n) - 1.2 * (latinsup / n))

def smart_decode(raw, declared=None):
    order = []
    if declared:                                  # 1) 声明字符集优先试
        cs = declared.lower().strip().replace('"', "")
        cs = "latin-1" if cs == "iso-8859-1" else cs
        if cs in CANDIDATES:
            order.append(cs)
    for c in CANDIDATES:                          # 2) 候选池补齐
        if c not in order:
            order.append(c)
    best, best_cs, best_sc = None, None, -9.0
    for cs in order:                              # 3) 严格解码 + 打分取最优
        try:
            t = raw.decode(cs, errors="strict")
        except Exception:
            try:
                t = raw.decode(cs, errors="replace")
            except Exception:
                continue
        sc = _score(t)
        if sc > best_sc:
            best, best_cs, best_sc = t, cs, sc
    if declared:                                  # 4) 声明可信度足够高则直接采用
        cs0 = "latin-1" if declared.lower().strip() == "iso-8859-1" else declared.lower().strip()
        try:
            t0 = raw.decode(cs0, errors="replace")
            if _score(t0) >= 0.80:
                return t0, cs0
        except Exception:
            pass
    return (best or raw.decode("utf-8", errors="replace")), best_cs

from email.header import decode_header
import re

def dec_header(v):
    if not v:
        return ""
    out = []
    for s, cs in decode_header(v):
        out.append(smart_decode(s, cs)[0] if isinstance(s, bytes) else s)
    return re.sub(r"\s+", " ", "".join(out)).strip()

used = {}
for part in msg.walk():
    fn = part.get_filename()
    if not fn:
        continue
    fn = dec_header(fn)
    used[fn] = used.get(fn, 0) + 1
    if used[fn] > 1:
        stem, ext = os.path.splitext(fn)
        fn = "%s(%d)%s" % (stem, used[fn] - 1, ext)
    payload = part.get_payload(decode=True) or b""
    with open(os.path.join(attdir, fn), "wb") as f:
        f.write(payload)

DATE_FMTS = ["%Y/%m/%d %H:%M", "%Y-%m-%d %H:%M:%S", "%Y年%m月%d日 %H:%M", "%Y-%m-%d %H:%M"]

def parse_date(msg):
    v = msg.get("Date")
    if v:
        try:                                        # 1) 标准 RFC 2822
            dt = parsedate_to_datetime(v)
            if dt:
                return int(dt.timestamp()), "rfc2822"
        except Exception:
            pass
        for fm in DATE_FMTS:                        # 2) 显式格式表
            try:
                dt = datetime.datetime.strptime(v.strip(), fm)
                return int(dt.replace(tzinfo=datetime.timezone.utc).timestamp()), "fmt"
            except Exception:
                continue
    rv = msg.get("Received")                        # 3) Received 头链
    if rv:
        m = re.search(r"(\d{1,2}\s+\w{3}\s+\d{4}\s+\d{2}:\d{2}:\d{2})", rv)
        if m:
            try:
                return int(parsedate_to_datetime(m.group(1)).timestamp()), "received"
            except Exception:
                pass
    return None, "fail"                             # 4) 标记归零，交上层决策

seen_mid, seen_bh = set(), set()
uniq, dup_primary, dup_secondary = [], 0, 0
for r in recs:
    if r["mid"]:                       # 一级：Message-ID（强键）
        if r["mid"] in seen_mid:
            dup_primary += 1
            continue
        seen_mid.add(r["mid"])
    if r["body_hash"] in seen_bh:      # 二级：内容指纹（补捞无主键者）
        dup_secondary += 1
        continue
    seen_bh.add(r["body_hash"])
    uniq.append(r)

# 1) 生成合成语料与标准答案
python scripts/mailbox_gen.py
#   -> outputs/mailbox_corpus/eml/*.eml (323)
#   -> outputs/mailbox_corpus/archive.mbox (60)
#   -> outputs/mailbox_gt.json

# 2) 跑三档对比与评估
python scripts/mailbox_run.py
#   -> outputs/mailbox_report.json（本文全部数字来源）
#   -> outputs/mailbox_out/index.jsonl（去重后索引）