274 lines
16 KiB
Python
274 lines
16 KiB
Python
"""Read program ledgers and write a task-local P1 baseline, without running it.
|
|
|
|
Usage: node scripts/python.mjs scripts/measure-prospecting-baseline.py TASK_ID
|
|
No model summary, source text, account identity or signed URL is exported.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter
|
|
from datetime import datetime
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
from zoneinfo import ZoneInfo
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from backend.prospecting.people import group_candidates, target_progress
|
|
from backend.prospecting import usage
|
|
|
|
TZ = ZoneInfo("Asia/Shanghai")
|
|
TOKENS = ("input_tokens", "output_tokens", "total_tokens", "cache_hit_tokens", "cache_miss_tokens")
|
|
PHASES = ("intent", "browser", "compression", "analysis")
|
|
NOTE_PATH = re.compile(r"/(?:explore|discovery/item|search_result)/([a-fA-F0-9]{24})(?:[/?#\s]|$)")
|
|
|
|
|
|
def timestamp(value):
|
|
if not isinstance(value, str):
|
|
return None
|
|
try:
|
|
result = datetime.fromisoformat(value)
|
|
return result.replace(tzinfo=TZ) if result.tzinfo is None else result
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def read_json(path: Path, default):
|
|
if not path.exists():
|
|
return default
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def usage_report(events: list[dict], damaged: bool) -> dict:
|
|
calls = {}
|
|
for event in events:
|
|
cid = event.get("call_id")
|
|
if not cid:
|
|
continue
|
|
row = calls.setdefault(cid, {"phase": event["phase"], "model": event["model"]})
|
|
if event["kind"] == "response":
|
|
row["usage"] = event["usage"]
|
|
elif event["kind"] == "finished":
|
|
row["status"] = event["status"]
|
|
complete = bool(events and events[0].get("kind") == "task_start" and not damaged)
|
|
|
|
def aggregate(rows):
|
|
result = usage._aggregate(rows)
|
|
known = {key: result.pop(key) for key in TOKENS}
|
|
if not rows and complete:
|
|
known = {key: 0 for key in TOKENS}
|
|
totals = {key: known[key] if complete and all(
|
|
type((row.get("usage") or {}).get(key)) is int for row in rows) else None for key in TOKENS}
|
|
return {"calls_observed": len(rows), "calls_total": len(rows) if complete else None,
|
|
"known_tokens": known, "tokens_total": totals,
|
|
**{key: value for key, value in result.items() if key != "requests"}}
|
|
|
|
grouped = {key: [] for key in PHASES}
|
|
for row in calls.values():
|
|
phase = "analysis" if row["phase"] == "analysis_incremental" else row["phase"]
|
|
grouped.setdefault(phase, []).append(row)
|
|
return {"ledger_complete_from_task_start": complete, "ledger_damaged": damaged,
|
|
"models": sorted({row["model"] for row in calls.values()}),
|
|
"total": aggregate(list(calls.values())),
|
|
"phases": {key: aggregate(rows) for key, rows in grouped.items()},
|
|
"phase_mapping": {"analysis": ["analysis", "analysis_incremental"],
|
|
"review": "reported separately; included in total cost"}}
|
|
|
|
|
|
def callback_records(progress: str):
|
|
for block in re.split(r"(?m)^### ", progress)[1:]:
|
|
if not re.match(r"[^\n]*\] 步骤 \d+\s*\n", block):
|
|
continue
|
|
matches = re.findall(r" \| 动作：([^|\n]*) \| 页面：([^|\n]*)", block)
|
|
if matches:
|
|
yield matches[-1]
|
|
|
|
|
|
def step_report(progress: str, history: dict, budget: dict, screenshot_count: int) -> dict:
|
|
# These are program-written fields, but the actions are model proposals;
|
|
# a subsequent action boundary can reject them before execution.
|
|
proposals = Counter()
|
|
callback_count = 0
|
|
for names, _page in callback_records(progress):
|
|
callback_count += 1
|
|
for name in names.split("、"):
|
|
if re.fullmatch(r"[a-z][a-z0-9_]*", name):
|
|
proposals[name] += 1
|
|
total = budget.get("total_steps_used")
|
|
total = total if type(total) is int and total >= 0 else None
|
|
return {"total_steps_attempted": total, "budget_version": budget.get("version"),
|
|
"last_run_steps": budget.get("last_run_steps"),
|
|
"step_callback_records": callback_count, "screenshots_saved": screenshot_count,
|
|
"proposed_tool_call_counts": dict(sorted(proposals.items())),
|
|
"proposed_tool_calls": sum(proposals.values()),
|
|
"tool_distribution_basis": "progress step callback proposals, not proof of execution",
|
|
"compact_history_action_count": len(history.get("actions") or []),
|
|
"compact_history_may_be_truncated": len(history.get("actions") or []) >= 200}
|
|
|
|
|
|
def executed_tools(path: Path) -> dict:
|
|
"""Optional safe boundary ledger; absence never becomes zero executions."""
|
|
if not path.exists():
|
|
return {"available": False, "invoked": None, "invoked_total": None, "status_counts": None,
|
|
"limitation": "No per-invocation execution ledger; proposed actions cannot prove execution."}
|
|
calls, damaged = {}, False
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
if not line.strip():
|
|
continue
|
|
try:
|
|
row = json.loads(line)
|
|
cid, name, status = row["action_id"], row["action_name"], row["status"]
|
|
if (not re.fullmatch(r"[a-f0-9]{32}", cid) or not re.fullmatch(r"[a-z][a-z0-9_]*", name)
|
|
or row.get("version") != 1 or row.get("kind") != "browser_action"
|
|
or status not in {"invoked", "blocked", "returned", "error"} or not timestamp(row.get("at"))):
|
|
raise ValueError("invalid action event")
|
|
call = calls.setdefault(cid, {"action": name, "statuses": set()})
|
|
if call["action"] != name:
|
|
raise ValueError("action identity changed")
|
|
call["statuses"].add(status)
|
|
except (ValueError, TypeError, KeyError):
|
|
damaged = True
|
|
counts = Counter(row["action"] for row in calls.values() if "invoked" in row["statuses"])
|
|
statuses = {status: dict(sorted(Counter(row["action"] for row in calls.values()
|
|
if status in row["statuses"]).items())) for status in ("invoked", "blocked", "returned", "error")}
|
|
return {"available": True, "ledger_damaged": damaged, "invoked": dict(sorted(counts.items())),
|
|
"invoked_total": sum(counts.values()),
|
|
"status_counts": statuses,
|
|
"pending_calls": sum("invoked" in row["statuses"] and not row["statuses"] & {"returned", "error"} for row in calls.values()),
|
|
"basis": "distinct action_id with invoked; blocked was not invoked; returned is not proof of business success"}
|
|
|
|
|
|
def request_contracts(path: Path, events: list[dict]) -> dict:
|
|
if not path.exists():
|
|
return {"available": False, "requests_prepared": None, "requests_with_response": None,
|
|
"disabled_tools_exposed": None}
|
|
contracts, damaged = {}, False
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
if not line.strip():
|
|
continue
|
|
try:
|
|
row = json.loads(line)
|
|
if row.get("kind") != "request_prepared" or not re.fullmatch(r"[a-f0-9]{32}", row["call_id"]):
|
|
raise ValueError("invalid contract event")
|
|
names = row["action_names"]
|
|
if not isinstance(names, list) or any(not isinstance(name, str) or not re.fullmatch(r"[a-z][a-z0-9_]*", name) for name in names):
|
|
raise ValueError("invalid action names")
|
|
contracts[row["call_id"]] = names
|
|
except (ValueError, TypeError, KeyError):
|
|
damaged = True
|
|
responses = {row["call_id"] for row in events if row["kind"] == "response"}
|
|
names = set(name for items in contracts.values() for name in items)
|
|
return {"available": True, "ledger_damaged": damaged, "requests_prepared": len(contracts),
|
|
"requests_with_response": len(set(contracts) & responses),
|
|
"disabled_tools_exposed": sorted(names & {"inspect_reply_relations", "resolve_reply_target"}),
|
|
"basis": "prepared schema joined to provider usage response by call_id; no messages or raw schema included"}
|
|
|
|
|
|
def build_baseline(task: dict, directory: Path) -> dict:
|
|
evidence = read_json(directory / "evidence.json", {})
|
|
artifacts = dict(task.get("artifacts") or {})
|
|
# Add checkpoint-only records without silently replacing the DB's current
|
|
# same-ID observations. No source or task is rewritten by measurement.
|
|
for key, identity in (("sources", "source_id"), ("profiles", "source_id"),
|
|
("operation_evidence", "id")):
|
|
merged = {row[identity]: row for row in artifacts.get(key, []) if isinstance(row, dict) and row.get(identity)}
|
|
for row in evidence.get(key, []):
|
|
if isinstance(row, dict) and row.get(identity):
|
|
merged.setdefault(row[identity], row)
|
|
artifacts[key] = list(merged.values())
|
|
current = {**task, "artifacts": artifacts}
|
|
groups = group_candidates(current)
|
|
target = target_progress(current)
|
|
sources = artifacts.get("sources", [])
|
|
observations = [*(artifacts.get("source_observations") or []), *(evidence.get("source_observations") or [])]
|
|
note_ids = {row.get("note_id") for row in sources if row.get("note_id")}
|
|
inspected_ids = {row.get("note_id") for row in observations if isinstance(row, dict) and row.get("note_id")}
|
|
progress_path = directory / "progress.md"
|
|
progress = progress_path.read_text(encoding="utf-8") if progress_path.exists() else ""
|
|
history = artifacts.get("browser_history") or {}
|
|
actual_pages = [page for _names, page in callback_records(progress)]
|
|
observed_visits = {match.group(1).lower() for value in [*actual_pages, *(history.get("urls") or [])]
|
|
if isinstance(value, str) for match in NOTE_PATH.finditer(value)}
|
|
events, damaged = usage._read_events(directory)
|
|
llm = usage_report(events, damaged)
|
|
budget = read_json(directory / "browser-budget.json", {})
|
|
times = [value for raw in [task.get("created_at"), task.get("updated_at"), budget.get("updated_at"),
|
|
*[row.get("at") for row in events]] if (value := timestamp(raw))]
|
|
origin = next((timestamp(row["at"]) for row in events if row["kind"] == "task_start"), timestamp(task.get("created_at")))
|
|
end = max(times) if times else None
|
|
seconds = round((end - origin).total_seconds(), 3) if end and origin and end >= origin else None
|
|
delivered = target["current"]
|
|
total = llm["total"]
|
|
per_candidate = {"denominator": "eligible stable-identity candidates under current requirements", "count": delivered,
|
|
"calls": total["calls_total"] / delivered if delivered and total["calls_total"] is not None else None,
|
|
"tokens": total["tokens_total"]["total_tokens"] / delivered if delivered and total["tokens_total"]["total_tokens"] is not None else None,
|
|
"wall_clock_seconds": seconds / delivered if delivered and seconds is not None else None}
|
|
facts = {row["id"]: row for row in artifacts.get("operation_evidence", []) if row.get("id")}
|
|
counts = Counter(row.get("source_type", "unknown") for row in sources)
|
|
ledger_names = ["model-usage.jsonl", "browser-budget.json", "progress.md", "evidence.json", "browser-actions.jsonl", "model-request-contracts.jsonl"]
|
|
return {"version": 1, "stage": "P1", "task_id": task["id"],
|
|
"measured_at": datetime.now(TZ).isoformat(timespec="seconds"), "task_status": task.get("status"),
|
|
"scope": {key: task.get("inputs", {}).get(key) for key in ("intent", "target_candidates", "max_notes", "max_comments_per_note", "_goal_version")},
|
|
"candidates": {"delivered_stable_identity": delivered, "stable_identity_all_kept": groups["counts"]["candidates"],
|
|
"unknown_identity_groups": sum(row["identity_status"] == "unresolved" for row in groups["groups"]),
|
|
"unknown_identity_sources": groups["counts"]["unresolved_sources"], "supplemental": target["supplemental"],
|
|
"target": target["target"], "remaining": target["remaining"], "target_reached": target["reached"]},
|
|
"reading": {"notes_with_saved_quotes": len(note_ids), "notes_dom_inspected": len(inspected_ids),
|
|
"notes_visited_observed": len(observed_visits), "note_visit_coverage": "observed callback/history URLs only; may be incomplete",
|
|
"saved_sources": len(sources), "saved_notes": counts["note"], "saved_comments": counts["comment"], "saved_replies": counts["reply"]},
|
|
"llm": llm,
|
|
"model_tool_contracts": request_contracts(directory / "model-request-contracts.jsonl", events),
|
|
"browser": {**step_report(progress, history, budget, len(list((directory / "steps").glob("step_*.png")))),
|
|
"executed_tools": executed_tools(directory / "browser-actions.jsonl"),
|
|
"verified_distinct_operation_facts": dict(sorted(Counter(row.get("capability", "unknown") for row in facts.values()).items()))},
|
|
"timing": {"start": origin.isoformat() if origin else None, "end_last_program_record": end.isoformat() if end else None,
|
|
"wall_clock_seconds": seconds, "includes_pauses_between_runs": True,
|
|
"basis": "task_start (fallback DB created_at) to latest DB update / usage event / browser checkpoint; excludes idle after recorded stop",
|
|
"final": task.get("status") in {"completed", "blocked", "failed", "cancelled"}},
|
|
"stop": {"reason": artifacts.get("stop_reason"), "browser_verdict": artifacts.get("browser_verdict"),
|
|
"basis": "task artifacts program stop_reason; model final_result/summary not used"},
|
|
"per_candidate": per_candidate,
|
|
"source_ledgers": {name: {"present": (directory / name).exists(),
|
|
"sha256": hashlib.sha256((directory / name).read_bytes()).hexdigest() if (directory / name).exists() else None}
|
|
for name in ledger_names},
|
|
"limitations": ["Visited or DOM-inspected notes are not a claim that every comment was read.",
|
|
"Unknown identity is not counted toward the target.",
|
|
"Known token sums exclude unavailable usage; complete totals and per-candidate tokens remain null when incomplete.",
|
|
"Tool proposals and verified distinct facts are not exact execution counts; use executed_tools only when its ledger is available."]}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("task_id")
|
|
args = parser.parse_args()
|
|
if not re.fullmatch(r"task_[a-f0-9]{12}", args.task_id):
|
|
parser.error("Expected an existing task_<12 hex characters> identifier")
|
|
from backend.config import TASKS_DIR
|
|
from backend.db import database as db
|
|
from backend.prospecting.contracts import KNOWLEDGE_ID
|
|
task = db.get_task(args.task_id)
|
|
if not task or task.get("knowledge_id") != KNOWLEDGE_ID:
|
|
parser.error("No prospecting task found in the current workspace")
|
|
directory = TASKS_DIR / args.task_id
|
|
if not directory.is_dir():
|
|
parser.error("Task ledger directory does not exist")
|
|
report = build_baseline(task, directory)
|
|
destination = directory / "baseline.json"
|
|
temporary = directory / ".baseline.json.tmp"
|
|
temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
temporary.replace(destination)
|
|
print(json.dumps({"task_id": task["id"], "baseline_path": str(destination.resolve()),
|
|
"candidates": report["candidates"], "llm": report["llm"]["total"],
|
|
"browser_steps": report["browser"]["total_steps_attempted"],
|
|
"wall_clock_seconds": report["timing"]["wall_clock_seconds"],
|
|
"stop": report["stop"]}, ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|