别急着给 AI 编程助手打分：先搭一套能重跑的评测任务集

id: fix-001
category: 单文件缺陷修复
fixture: fixtures/repo-a
allow_paths:
  - src/rate_limit.py
prompt: |
  src/rate_limit.py 的 allow() 在窗口边界会把计数算错。
  请只修改这个文件，让 test_rate_limit.py 全部通过，不要改测试文件。
assertions:
  - name: 单测通过
    type: exit0
    cmd: python -m pytest -q test_rate_limit.py
  - name: 未改测试文件
    type: exit0
    cmd: test -z "$(git diff --name-only | grep test_)"

#!/usr/bin/env python3
import hashlib, json, os, shutil, subprocess, time
from pathlib import Path
import yaml

ROOT = Path(__file__).resolve().parent
TASKS, FIXTURES, OUT = ROOT / "tasks", ROOT / "fixtures", ROOT / "runs"

def tree_sha(path):
    h = hashlib.sha256()
    for p in sorted(path.rglob("*")):
        if p.is_file():
            h.update(str(p.relative_to(path)).encode())
            h.update(p.read_bytes())
    return h.hexdigest()[:16]

def prepare(task, run_dir):
    work = run_dir / task["id"]
    shutil.copytree(FIXTURES / task["fixture"], work)
    for cmd in (["git", "init", "-q"],
                ["git", "add", "."],
                ["git", "-c", "user.email=e@local", "-c",
                 "user.name=eval", "commit", "-qm", "baseline"]):
        subprocess.run(cmd, cwd=work, check=True, capture_output=True)
    return work

def invoke(work, prompt_file):
    cmd = os.environ["AGENT_CLI"].format(prompt=str(prompt_file))
    t0 = time.monotonic()
    p = subprocess.run(cmd, shell=True, cwd=work, capture_output=True,
                       text=True, timeout=int(os.environ.get("AGENT_TIMEOUT", "300")))
    return {"elapsed_s": round(time.monotonic() - t0, 2),
            "exit_code": p.returncode,
            "stderr_tail": p.stderr[-1000:]}

def diff_stat(work):
    r = subprocess.run(["git", "diff", "--numstat"], cwd=work,
                       capture_output=True, text=True, check=True)
    files = added = deleted = 0
    for line in r.stdout.splitlines():
        parts = line.split("\t")
        a, d = parts[0], parts[1]
        files += 1
        added += 0 if a == "-" else int(a)
        deleted += 0 if d == "-" else int(d)
    return {"files": files, "added": added, "deleted": deleted}

def run_assertions(work, assertions):
    out = []
    for a in assertions:
        p = subprocess.run(a["cmd"], shell=True, cwd=work,
                           capture_output=True, text=True)
        ok = p.returncode == 0 if a["type"] == "exit0" else a["expect"] in p.stdout
        out.append({"name": a["name"], "pass": ok,
                    "detail": (p.stdout + p.stderr)[-300:]})
    return out

def main():
    run_id = time.strftime("%Y%m%d-%H%M%S")
    run_dir = OUT / run_id
    run_dir.mkdir(parents=True, exist_ok=True)
    with (run_dir / "results.jsonl").open("w") as f:
        for path in sorted(TASKS.glob("*.yaml")):
            task = yaml.safe_load(path.read_text())
            work = prepare(task, run_dir)
            prompt = work / ".eval-prompt.md"
            prompt.write_text(task["prompt"])
            inv = invoke(work, prompt)
            asserts = run_assertions(work, task["assertions"]) if inv["exit_code"] == 0 else []
            rec = {
                "run_id": run_id,
                "task_id": task["id"],
                "category": task["category"],
                "fixture_sha": tree_sha(FIXTURES / task["fixture"]),
                "agent_build": os.environ.get("AGENT_BUILD", "unset"),
                "elapsed_s": inv["elapsed_s"],
                "diff": diff_stat(work),
                "assertions": asserts,
                "pass_rate": sum(1 for a in asserts if a["pass"]) / max(len(asserts), 1),
            }
            f.write(json.dumps(rec, ensure_ascii=False) + "\n")
            print(task["id"], rec["pass_rate"])
    print("results:", run_dir / "results.jsonl")

if __name__ == "__main__":
    main()

export AGENT_CLI='your-agent-cli --non-interactive --prompt-file {prompt}'
export AGENT_BUILD='记录模型版本与客户端版本，自行填写'
python3 eval.py