"""Read program ledgers and write a task-local P1 baseline, without running it. Usage: node scripts/python.mjs scripts/measure-prospecting-baseline.py TASK_ID No model summary, source text, account identity or signed URL is exported. """ from __future__ import annotations import argparse from collections import Counter from datetime import datetime import hashlib import json from pathlib import Path import re import sys from zoneinfo import ZoneInfo ROOT = Path(__file__).resolve().parent.parent if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from backend.prospecting.people import group_candidates, target_progress from backend.prospecting import usage TZ = ZoneInfo("Asia/Shanghai") TOKENS = ("input_tokens", "output_tokens", "total_tokens", "cache_hit_tokens", "cache_miss_tokens") PHASES = ("intent", "browser", "compression", "analysis") NOTE_PATH = re.compile(r"/(?:explore|discovery/item|search_result)/([a-fA-F0-9]{24})(?:[/?#\s]|$)") def timestamp(value): if not isinstance(value, str): return None try: result = datetime.fromisoformat(value) return result.replace(tzinfo=TZ) if result.tzinfo is None else result except ValueError: return None def read_json(path: Path, default): if not path.exists(): return default return json.loads(path.read_text(encoding="utf-8")) def usage_report(events: list[dict], damaged: bool) -> dict: calls = {} for event in events: cid = event.get("call_id") if not cid: continue row = calls.setdefault(cid, {"phase": event["phase"], "model": event["model"]}) if event["kind"] == "response": row["usage"] = event["usage"] elif event["kind"] == "finished": row["status"] = event["status"] complete = bool(events and events[0].get("kind") == "task_start" and not damaged) def aggregate(rows): result = usage._aggregate(rows) known = {key: result.pop(key) for key in TOKENS} if not rows and complete: known = {key: 0 for key in TOKENS} totals = {key: known[key] if complete and all( type((row.get("usage") or {}).get(key)) is int for row in rows) else None for key in TOKENS} return {"calls_observed": len(rows), "calls_total": len(rows) if complete else None, "known_tokens": known, "tokens_total": totals, **{key: value for key, value in result.items() if key != "requests"}} grouped = {key: [] for key in PHASES} for row in calls.values(): phase = "analysis" if row["phase"] == "analysis_incremental" else row["phase"] grouped.setdefault(phase, []).append(row) return {"ledger_complete_from_task_start": complete, "ledger_damaged": damaged, "models": sorted({row["model"] for row in calls.values()}), "total": aggregate(list(calls.values())), "phases": {key: aggregate(rows) for key, rows in grouped.items()}, "phase_mapping": {"analysis": ["analysis", "analysis_incremental"], "review": "reported separately; included in total cost"}} def callback_records(progress: str): for block in re.split(r"(?m)^### ", progress)[1:]: if not re.match(r"[^\n]*\] 步骤 \d+\s*\n", block): continue matches = re.findall(r" \| 动作：([^|\n]*) \| 页面：([^|\n]*)", block) if matches: yield matches[-1] def step_report(progress: str, history: dict, budget: dict, screenshot_count: int) -> dict: # These are program-written fields, but the actions are model proposals; # a subsequent action boundary can reject them before execution. proposals = Counter() callback_count = 0 for names, _page in callback_records(progress): callback_count += 1 for name in names.split("、"): if re.fullmatch(r"[a-z][a-z0-9_]*", name): proposals[name] += 1 total = budget.get("total_steps_used") total = total if type(total) is int and total >= 0 else None return {"total_steps_attempted": total, "budget_version": budget.get("version"), "last_run_steps": budget.get("last_run_steps"), "step_callback_records": callback_count, "screenshots_saved": screenshot_count, "proposed_tool_call_counts": dict(sorted(proposals.items())), "proposed_tool_calls": sum(proposals.values()), "tool_distribution_basis": "progress step callback proposals, not proof of execution", "compact_history_action_count": len(history.get("actions") or []), "compact_history_may_be_truncated": len(history.get("actions") or []) >= 200} def executed_tools(path: Path) -> dict: """Optional safe boundary ledger; absence never becomes zero executions.""" if not path.exists(): return {"available": False, "invoked": None, "invoked_total": None, "status_counts": None, "limitation": "No per-invocation execution ledger; proposed actions cannot prove execution."} calls, damaged = {}, False for line in path.read_text(encoding="utf-8").splitlines(): if not line.strip(): continue try: row = json.loads(line) cid, name, status = row["action_id"], row["action_name"], row["status"] if (not re.fullmatch(r"[a-f0-9]{32}", cid) or not re.fullmatch(r"[a-z][a-z0-9_]*", name) or row.get("version") != 1 or row.get("kind") != "browser_action" or status not in {"invoked", "blocked", "returned", "error"} or not timestamp(row.get("at"))): raise ValueError("invalid action event") call = calls.setdefault(cid, {"action": name, "statuses": set()}) if call["action"] != name: raise ValueError("action identity changed") call["statuses"].add(status) except (ValueError, TypeError, KeyError): damaged = True counts = Counter(row["action"] for row in calls.values() if "invoked" in row["statuses"]) statuses = {status: dict(sorted(Counter(row["action"] for row in calls.values() if status in row["statuses"]).items())) for status in ("invoked", "blocked", "returned", "error")} return {"available": True, "ledger_damaged": damaged, "invoked": dict(sorted(counts.items())), "invoked_total": sum(counts.values()), "status_counts": statuses, "pending_calls": sum("invoked" in row["statuses"] and not row["statuses"] & {"returned", "error"} for row in calls.values()), "basis": "distinct action_id with invoked; blocked was not invoked; returned is not proof of business success"} def request_contracts(path: Path, events: list[dict]) -> dict: if not path.exists(): return {"available": False, "requests_prepared": None, "requests_with_response": None, "disabled_tools_exposed": None} contracts, damaged = {}, False for line in path.read_text(encoding="utf-8").splitlines(): if not line.strip(): continue try: row = json.loads(line) if row.get("kind") != "request_prepared" or not re.fullmatch(r"[a-f0-9]{32}", row["call_id"]): raise ValueError("invalid contract event") names = row["action_names"] if not isinstance(names, list) or any(not isinstance(name, str) or not re.fullmatch(r"[a-z][a-z0-9_]*", name) for name in names): raise ValueError("invalid action names") contracts[row["call_id"]] = names except (ValueError, TypeError, KeyError): damaged = True responses = {row["call_id"] for row in events if row["kind"] == "response"} names = set(name for items in contracts.values() for name in items) return {"available": True, "ledger_damaged": damaged, "requests_prepared": len(contracts), "requests_with_response": len(set(contracts) & responses), "disabled_tools_exposed": sorted(names & {"inspect_reply_relations", "resolve_reply_target"}), "basis": "prepared schema joined to provider usage response by call_id; no messages or raw schema included"} def build_baseline(task: dict, directory: Path) -> dict: evidence = read_json(directory / "evidence.json", {}) artifacts = dict(task.get("artifacts") or {}) # Add checkpoint-only records without silently replacing the DB's current # same-ID observations. No source or task is rewritten by measurement. for key, identity in (("sources", "source_id"), ("profiles", "source_id"), ("operation_evidence", "id")): merged = {row[identity]: row for row in artifacts.get(key, []) if isinstance(row, dict) and row.get(identity)} for row in evidence.get(key, []): if isinstance(row, dict) and row.get(identity): merged.setdefault(row[identity], row) artifacts[key] = list(merged.values()) current = {**task, "artifacts": artifacts} groups = group_candidates(current) target = target_progress(current) sources = artifacts.get("sources", []) observations = [*(artifacts.get("source_observations") or []), *(evidence.get("source_observations") or [])] note_ids = {row.get("note_id") for row in sources if row.get("note_id")} inspected_ids = {row.get("note_id") for row in observations if isinstance(row, dict) and row.get("note_id")} progress_path = directory / "progress.md" progress = progress_path.read_text(encoding="utf-8") if progress_path.exists() else "" history = artifacts.get("browser_history") or {} actual_pages = [page for _names, page in callback_records(progress)] observed_visits = {match.group(1).lower() for value in [*actual_pages, *(history.get("urls") or [])] if isinstance(value, str) for match in NOTE_PATH.finditer(value)} events, damaged = usage._read_events(directory) llm = usage_report(events, damaged) budget = read_json(directory / "browser-budget.json", {}) times = [value for raw in [task.get("created_at"), task.get("updated_at"), budget.get("updated_at"), *[row.get("at") for row in events]] if (value := timestamp(raw))] origin = next((timestamp(row["at"]) for row in events if row["kind"] == "task_start"), timestamp(task.get("created_at"))) end = max(times) if times else None seconds = round((end - origin).total_seconds(), 3) if end and origin and end >= origin else None delivered = target["current"] total = llm["total"] per_candidate = {"denominator": "eligible stable-identity candidates under current requirements", "count": delivered, "calls": total["calls_total"] / delivered if delivered and total["calls_total"] is not None else None, "tokens": total["tokens_total"]["total_tokens"] / delivered if delivered and total["tokens_total"]["total_tokens"] is not None else None, "wall_clock_seconds": seconds / delivered if delivered and seconds is not None else None} facts = {row["id"]: row for row in artifacts.get("operation_evidence", []) if row.get("id")} counts = Counter(row.get("source_type", "unknown") for row in sources) ledger_names = ["model-usage.jsonl", "browser-budget.json", "progress.md", "evidence.json", "browser-actions.jsonl", "model-request-contracts.jsonl"] return {"version": 1, "stage": "P1", "task_id": task["id"], "measured_at": datetime.now(TZ).isoformat(timespec="seconds"), "task_status": task.get("status"), "scope": {key: task.get("inputs", {}).get(key) for key in ("intent", "target_candidates", "max_notes", "max_comments_per_note", "_goal_version")}, "candidates": {"delivered_stable_identity": delivered, "stable_identity_all_kept": groups["counts"]["candidates"], "unknown_identity_groups": sum(row["identity_status"] == "unresolved" for row in groups["groups"]), "unknown_identity_sources": groups["counts"]["unresolved_sources"], "supplemental": target["supplemental"], "target": target["target"], "remaining": target["remaining"], "target_reached": target["reached"]}, "reading": {"notes_with_saved_quotes": len(note_ids), "notes_dom_inspected": len(inspected_ids), "notes_visited_observed": len(observed_visits), "note_visit_coverage": "observed callback/history URLs only; may be incomplete", "saved_sources": len(sources), "saved_notes": counts["note"], "saved_comments": counts["comment"], "saved_replies": counts["reply"]}, "llm": llm, "model_tool_contracts": request_contracts(directory / "model-request-contracts.jsonl", events), "browser": {**step_report(progress, history, budget, len(list((directory / "steps").glob("step_*.png")))), "executed_tools": executed_tools(directory / "browser-actions.jsonl"), "verified_distinct_operation_facts": dict(sorted(Counter(row.get("capability", "unknown") for row in facts.values()).items()))}, "timing": {"start": origin.isoformat() if origin else None, "end_last_program_record": end.isoformat() if end else None, "wall_clock_seconds": seconds, "includes_pauses_between_runs": True, "basis": "task_start (fallback DB created_at) to latest DB update / usage event / browser checkpoint; excludes idle after recorded stop", "final": task.get("status") in {"completed", "blocked", "failed", "cancelled"}}, "stop": {"reason": artifacts.get("stop_reason"), "browser_verdict": artifacts.get("browser_verdict"), "basis": "task artifacts program stop_reason; model final_result/summary not used"}, "per_candidate": per_candidate, "source_ledgers": {name: {"present": (directory / name).exists(), "sha256": hashlib.sha256((directory / name).read_bytes()).hexdigest() if (directory / name).exists() else None} for name in ledger_names}, "limitations": ["Visited or DOM-inspected notes are not a claim that every comment was read.", "Unknown identity is not counted toward the target.", "Known token sums exclude unavailable usage; complete totals and per-candidate tokens remain null when incomplete.", "Tool proposals and verified distinct facts are not exact execution counts; use executed_tools only when its ledger is available."]} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("task_id") args = parser.parse_args() if not re.fullmatch(r"task_[a-f0-9]{12}", args.task_id): parser.error("Expected an existing task_<12 hex characters> identifier") from backend.config import TASKS_DIR from backend.db import database as db from backend.prospecting.contracts import KNOWLEDGE_ID task = db.get_task(args.task_id) if not task or task.get("knowledge_id") != KNOWLEDGE_ID: parser.error("No prospecting task found in the current workspace") directory = TASKS_DIR / args.task_id if not directory.is_dir(): parser.error("Task ledger directory does not exist") report = build_baseline(task, directory) destination = directory / "baseline.json" temporary = directory / ".baseline.json.tmp" temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") temporary.replace(destination) print(json.dumps({"task_id": task["id"], "baseline_path": str(destination.resolve()), "candidates": report["candidates"], "llm": report["llm"]["total"], "browser_steps": report["browser"]["total_steps_attempted"], "wall_clock_seconds": report["timing"]["wall_clock_seconds"], "stop": report["stop"]}, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()