assistant/scripts/eval_run.py

"""同一任务连续跑 N 次，统计成功率 / 步数 / 时长，结果写入 data/eval/*.jsonl。

用法（后端需已启动；端口取 .env 的 API_PORT，可用环境变量 API_PORT 覆盖）：
    .venv\\Scripts\\python.exe scripts\\eval_run.py --n 3 --message "帮我填今天的查课日志，班级计科2301班，听课记录还好" --supplement "课程是高等数学，授课教师是张老师"

默认是干跑（dry run）：Agent 走到「提交前确认」时脚本自动回复取消，不会真正提交，
用来测「到确认为止」的步数和时长。加 --submit 才会在确认时回复「确认提交」。
其他中途提问（如问听课地点）自动回复 --auto-answer。
"""

import argparse
import json
import os
import sys
import time
import urllib.error
import urllib.request
from datetime import datetime
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))

from dotenv import load_dotenv  # noqa: E402

load_dotenv(ROOT / ".env")
PORT = int(os.getenv("API_PORT", "8788"))
BASE = f"http://127.0.0.1:{PORT}/api"
TERMINAL = {"completed", "submitted", "failed"}
CANCEL_ANSWER = "取消，不要提交。这次是评测干跑，请保存 cancelled 后 done 说明表单已填未提交。"
CONFIRM_ANSWER = "确认提交"


def api(method: str, path: str, body: dict | None = None) -> dict:
    data = json.dumps(body, ensure_ascii=False).encode("utf-8") if body is not None else None
    req = urllib.request.Request(BASE + path, data=data, method=method, headers={"Content-Type": "application/json"})
    with urllib.request.urlopen(req, timeout=30) as r:
        return json.loads(r.read().decode("utf-8"))


def count_steps(task_id: str) -> int | None:
    try:
        doc = api("GET", f"/tasks/{task_id}/progress-doc")["content"]
    except Exception:  # noqa: BLE001
        return None
    return sum(1 for line in doc.splitlines() if line.startswith("### ") and "步骤 " in line)


def is_confirm_question(q: str) -> bool:
    return ("确认" in q and "提交" in q) or "即将提交" in q


def run_once(args, message: str) -> dict:
    t0 = time.time()
    task = api("POST", "/tasks/start", {"knowledge_id": "xuegong_demo", "message": message, "student_id": args.student})
    tid = task["id"]
    print(f"    task {tid} | {task.get('business_name') or ''} | {task['status']} | {task['reply'][:80]}")
    if task["status"] == "collecting":
        if not args.supplement:
            return {"task_id": tid, "status": "collecting", "needs": task.get("needs_input"), "duration_s": 0}
        task = api("POST", f"/tasks/{tid}/inputs", {"message": args.supplement})
        print(f"    supplement -> {task['status']} | {task['reply'][:80]}")

    pauses = 0
    confirm_at = None
    steps_at_confirm = None
    status = task["status"]
    while time.time() - t0 < args.timeout:
        time.sleep(3)
        t = api("GET", f"/tasks/{tid}")
        status = t["status"]
        if status == "paused":
            pauses += 1
            q = t.get("pause_reason") or ""
            print(f"    [paused #{pauses} @ {round(time.time() - t0)}s] {q[:160].replace(chr(10), ' ')}")
            if pauses > 3:
                break
            if is_confirm_question(q) and confirm_at is None:
                confirm_at = round(time.time() - t0, 1)
                steps_at_confirm = count_steps(tid)
                api("POST", f"/tasks/{tid}/human", {"answer": CONFIRM_ANSWER if args.submit else CANCEL_ANSWER})
            else:
                api("POST", f"/tasks/{tid}/human", {"answer": args.auto_answer})
            continue
        if status in TERMINAL:
            break
    else:
        status = f"timeout({status})"

    t = api("GET", f"/tasks/{tid}")
    artifacts = t.get("artifacts") or {}
    drafts = []
    try:
        drafts = api("GET", f"/learned?source_task_id={tid}&status=draft")
    except Exception:  # noqa: BLE001
        pass
    return {
        "ts": datetime.now().isoformat(timespec="seconds"),
        "task_id": tid,
        "status": status,
        "dry_run": not args.submit,
        "duration_s": round(time.time() - t0, 1),
        "steps": count_steps(tid),
        "confirm_at_s": confirm_at,
        "steps_to_confirm": steps_at_confirm,
        "pauses": pauses,
        "artifact_keys": sorted(k for k in artifacts if k != "summary"),
        "learned_drafts": [d["key"] for d in drafts],
        "summary": (artifacts.get("summary") or t.get("pause_reason") or "")[:300],
    }


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--n", type=int, default=3)
    ap.add_argument("--message", default="帮我填今天的查课日志，班级计科2301班，听课记录还好")
    ap.add_argument("--supplement", default="", help="任务进入 collecting 时自动补充的一句话")
    ap.add_argument("--student", default="default")
    ap.add_argument("--timeout", type=float, default=900)
    ap.add_argument("--auto-answer", default="不需要，留空继续")
    ap.add_argument("--submit", action="store_true", help="到确认时真正确认提交（默认干跑：取消）")
    args = ap.parse_args()

    try:
        meta = api("GET", "/meta")
    except urllib.error.URLError as e:
        print(f"后端未启动（{BASE}）：{e}")
        return 1
    print(f"后端 pid={meta['pid']} version={meta['version']} db={meta['db']} school={meta.get('school_id')} | {'真实提交' if args.submit else '干跑（到确认即取消）'}")

    out_dir = ROOT / "data" / "eval"
    out_dir.mkdir(parents=True, exist_ok=True)
    out_path = out_dir / f"eval_{datetime.now().strftime('%Y%m%d_%H%M%S')}.jsonl"
    results: list[dict] = []
    for i in range(1, args.n + 1):
        print(f"[{i}/{args.n}] 开始：{args.message}")
        r = run_once(args, args.message)
        results.append(r)
        with out_path.open("a", encoding="utf-8") as f:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
        print(
            f"    -> {r['status']} | {r['duration_s']}s | steps={r.get('steps')} | "
            f"到确认 {r.get('steps_to_confirm')} 步 / {r.get('confirm_at_s')}s | drafts={r.get('learned_drafts')}"
        )

    ok = [r for r in results if r["status"] in ("completed", "submitted")]
    durs = [r["duration_s"] for r in results if r["duration_s"]]
    steps = [r["steps"] for r in results if r.get("steps") is not None]
    to_confirm = [r["steps_to_confirm"] for r in results if r.get("steps_to_confirm") is not None]
    confirm_s = [r["confirm_at_s"] for r in results if r.get("confirm_at_s") is not None]
    print()
    print(f"成功 {len(ok)}/{len(results)}")
    if durs:
        print(f"平均总时长 {sum(durs) / len(durs):.1f}s（最短 {min(durs)}s，最长 {max(durs)}s）")
    if steps:
        print(f"平均总步数 {sum(steps) / len(steps):.1f}")
    if to_confirm:
        print(f"到确认平均 {sum(to_confirm) / len(to_confirm):.1f} 步 / {sum(confirm_s) / len(confirm_s):.0f}s")
    print(f"明细：{out_path}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())