477 lines
33 KiB
Python
477 lines
33 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import platform
|
|
import re
|
|
import time
|
|
from copy import deepcopy
|
|
from pathlib import Path
|
|
from urllib.parse import parse_qsl, urlsplit
|
|
|
|
from fastapi import HTTPException
|
|
|
|
from .db import Store, digest, encoded, identifier, now
|
|
from .learning_checks import evidence_graph_errors, output_assertion_checks
|
|
from .worker import WorkerError
|
|
|
|
SENSITIVE_KEY = re.compile(r"(^|[_-])(password|passwd|phone|mobile|otp|sms|api_key|access_token|refresh_token)($|[_-])|密码|验证码|手机号", re.I)
|
|
NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,63}")
|
|
|
|
|
|
def check_entry_url(value: str) -> str:
|
|
try:
|
|
parsed = urlsplit(value)
|
|
if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password:
|
|
raise ValueError
|
|
if any(SENSITIVE_KEY.search(key) for key, _ in parse_qsl(parsed.query)):
|
|
raise ValueError
|
|
parsed.port
|
|
except ValueError:
|
|
raise HTTPException(422, "入口需为不含凭据的 http 或 https 页面地址") from None
|
|
return value
|
|
|
|
|
|
def check_parameters(value: dict) -> dict:
|
|
if len(value) > 20:
|
|
raise HTTPException(422, "首版每条链路最多 20 个参数")
|
|
for key, item in value.items():
|
|
if not NAME.fullmatch(key) or SENSITIVE_KEY.search(key):
|
|
raise HTTPException(422, "参数名需为普通业务字段，不能用于登录凭据")
|
|
if not isinstance(item, str) or len(item) > 2000:
|
|
raise HTTPException(422, "首版参数值需为不超过 2000 字符的文本")
|
|
return value
|
|
|
|
|
|
def os_name(value: str) -> str:
|
|
return {"win32": "windows", "windows": "windows", "darwin": "macos", "macos": "macos", "linux": "linux"}.get(value.lower(), value.lower())
|
|
|
|
|
|
def compare_outputs(actual: dict, expected: dict) -> list[str]:
|
|
return [item["message"] for item in output_assertion_checks(actual, expected) if not item["passed"]]
|
|
|
|
|
|
class LearningService:
|
|
def __init__(self, store: Store, worker):
|
|
self.store, self.worker = store, worker
|
|
self.sessions: dict[str, dict] = {}
|
|
# Browser contexts are process-owned. Persisted learning evidence survives,
|
|
# but a new server cannot inherit a predecessor's ready confirmation.
|
|
self.store.execute("UPDATE learning_tasks SET status='paused',updated_at=$1 WHERE status IN ('awaiting_user','observing')", (now(),))
|
|
|
|
def browser_readiness(self, task_id: str) -> dict:
|
|
status = self.sessions.get(task_id, {}).get("status", "closed")
|
|
if status == "ready":
|
|
return {"ready": True, "code": "ready", "message": "本次浏览器会话已由维护者确认，可以观察或试跑", "next_action": "none"}
|
|
if status == "awaiting_user":
|
|
return {"ready": False, "code": "confirmation_required", "message": "请在官方页面完成登录，再人工确认当前账号；确认前不采集页面", "next_action": "confirm_ready"}
|
|
return {"ready": False, "code": "session_closed", "message": "请打开学习浏览器并人工确认登录状态；历史证据和验证记录已保留", "next_action": "open_browser"}
|
|
|
|
def pause_active_task(self, task_id: str) -> None:
|
|
self.store.execute("UPDATE learning_tasks SET status='paused',updated_at=$1 WHERE id=$2 AND status IN ('awaiting_user','observing')", (now(), task_id))
|
|
|
|
async def refresh_browser(self, task_id: str, owner_id: str) -> dict:
|
|
self.task(task_id, owner_id)
|
|
if task_id in self.sessions:
|
|
try:
|
|
state = await self.call(task_id, "status", {})
|
|
except HTTPException:
|
|
self.sessions.pop(task_id, None)
|
|
self.pause_active_task(task_id)
|
|
else:
|
|
if state.get("status") == "closed":
|
|
self.sessions.pop(task_id, None)
|
|
self.pause_active_task(task_id)
|
|
else:
|
|
self.sessions[task_id] = {**state, "status": "ready" if state.get("manual_confirmation") else "awaiting_user"}
|
|
return self.browser_readiness(task_id)
|
|
|
|
async def require_ready(self, task_id: str, owner_id: str) -> None:
|
|
readiness = await self.refresh_browser(task_id, owner_id)
|
|
if not readiness["ready"]:
|
|
code = "browser_confirmation_required" if readiness["code"] == "confirmation_required" else "browser_session_required"
|
|
raise HTTPException(409, {"code": code, "message": readiness["message"], "next_action": readiness["next_action"]})
|
|
|
|
def task(self, task_id: str, owner_id: str) -> dict:
|
|
row = self.store.one("SELECT * FROM learning_tasks WHERE id=$1 AND owner_id=$2", (task_id, owner_id))
|
|
if not row:
|
|
raise HTTPException(404, "学习任务不存在")
|
|
row["parameters"] = json.loads(row.pop("parameters_json"))
|
|
row.pop("owner_id", None)
|
|
row.pop("ordinal", None)
|
|
row["browser_status"] = self.sessions.get(task_id, {}).get("status", "closed")
|
|
row["browser_readiness"] = self.browser_readiness(task_id)
|
|
return row
|
|
|
|
def candidate(self, candidate_id: str, owner_id: str) -> dict:
|
|
row = self.store.one("SELECT c.* FROM candidates c JOIN learning_tasks t ON t.id=c.task_id WHERE c.id=$1 AND t.owner_id=$2", (candidate_id, owner_id))
|
|
if not row:
|
|
raise HTTPException(404, "候选能力不存在")
|
|
row["manifest"] = json.loads(row.pop("manifest_json"))
|
|
row.pop("ordinal", None)
|
|
validation = self.store.one(f"SELECT kind,status FROM validations WHERE candidate_id=$1 ORDER BY {self.store.order_column()} DESC LIMIT 1", (candidate_id,))
|
|
row["status"] = "validated" if validation and validation["kind"] == "browser" and validation["status"] == "passed" else "checked" if validation and validation["kind"] == "static" and validation["status"] == "passed" else "candidate"
|
|
return row
|
|
|
|
def tasks(self, owner_id: str) -> list[dict]:
|
|
return [self.task(row["id"], owner_id) for row in self.store.rows(f"SELECT id FROM learning_tasks WHERE owner_id=$1 ORDER BY {self.store.order_column()} DESC", (owner_id,))]
|
|
|
|
def create_task(self, owner_id: str, values: dict) -> dict:
|
|
check_entry_url(values["entry_url"])
|
|
parameters = check_parameters(values.get("parameters") or {})
|
|
if not values["name"].strip():
|
|
raise HTTPException(422, "学习任务名称不能为空")
|
|
timestamp, task_id = now(), identifier("learn")
|
|
account_key = values["account_key"].strip()
|
|
if not re.fullmatch(r"[A-Za-z0-9_.-]{2,80}", account_key) or re.fullmatch(r"1\d{10}", account_key):
|
|
raise HTTPException(422, "本地账号标识请使用 2 至 80 位字母数字编号，不使用平台手机号")
|
|
self.store.execute("INSERT INTO learning_tasks(id,owner_id,name,platform,entry_url,goal,account_key,parameters_json,status,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", (task_id, owner_id, values["name"].strip(), values["platform"], values["entry_url"], values["goal"].strip(), account_key, encoded(parameters), "draft", timestamp, timestamp))
|
|
return self.task(task_id, owner_id)
|
|
|
|
def set_status(self, task_id: str, status: str) -> None:
|
|
self.store.execute("UPDATE learning_tasks SET status=$1,updated_at=$2 WHERE id=$3", (status, now(), task_id))
|
|
|
|
def overview(self, owner_id: str) -> dict:
|
|
task_ids = "SELECT id FROM learning_tasks WHERE owner_id=$1"
|
|
def count(table, where="", args=()):
|
|
scope = "owner_id=$1" if table == "learning_tasks" else f"task_id IN ({task_ids})"
|
|
return self.store.one(f"SELECT count(*) AS n FROM {table} WHERE {scope}{where}", (owner_id, *args))["n"]
|
|
return {"tasks": count("learning_tasks"), "candidates": count("candidates"), "validations": count("validations"), "passed": count("validations", " AND status='passed'"), "failed": count("validations", " AND status='failed'"), "published": 0}
|
|
|
|
def observations(self, task_id: str) -> list[dict]:
|
|
return [{"id": row["id"], "task_id": row["task_id"], "created_at": row["created_at"], **json.loads(row["data_json"])} for row in self.store.rows(f"SELECT * FROM observations WHERE task_id=$1 ORDER BY {self.store.order_column()} ASC", (task_id,))]
|
|
|
|
def steps(self, task_id: str) -> list[dict]:
|
|
return [{"id": row["id"], "position": row["position"], "created_at": row["created_at"], **json.loads(row["data_json"])} for row in self.store.rows("SELECT * FROM learning_steps WHERE task_id=$1 ORDER BY position", (task_id,))]
|
|
|
|
def validations(self, owner_id: str, candidate_id: str | None = None, task_id: str | None = None) -> list[dict]:
|
|
query = "SELECT v.* FROM validations v JOIN learning_tasks t ON t.id=v.task_id WHERE t.owner_id=$1"
|
|
args = [owner_id]
|
|
if candidate_id:
|
|
args.append(candidate_id)
|
|
query += f" AND v.candidate_id=${len(args)}"
|
|
if task_id:
|
|
args.append(task_id)
|
|
query += f" AND v.task_id=${len(args)}"
|
|
return [{"id": row["id"], "candidate_id": row["candidate_id"], "task_id": row["task_id"], "kind": row["kind"], "status": row["status"], "created_at": row["created_at"], **json.loads(row["data_json"])} for row in self.store.rows(query + f" ORDER BY {self.store.order_column('v')} DESC", tuple(args))]
|
|
|
|
def detail(self, task_id: str, owner_id: str) -> dict:
|
|
task = self.task(task_id, owner_id)
|
|
task.update(observations=self.observations(task_id), steps=self.steps(task_id), candidates=[self.candidate(row["id"], owner_id) for row in self.store.rows("SELECT id FROM candidates WHERE task_id=$1 ORDER BY version DESC", (task_id,))], validations=self.validations(owner_id, task_id=task_id))
|
|
return task
|
|
|
|
def candidates(self, owner_id: str) -> list[dict]:
|
|
return [self.candidate(row["id"], owner_id) for row in self.store.rows(f"SELECT c.id FROM candidates c JOIN learning_tasks t ON t.id=c.task_id WHERE t.owner_id=$1 ORDER BY {self.store.order_column('c')} DESC", (owner_id,))]
|
|
|
|
async def call(self, task_id: str, method: str, params: dict) -> dict:
|
|
try:
|
|
return await self.worker.call(method, {"task_id": task_id, **params})
|
|
except WorkerError as error:
|
|
if error.code.lower() in ("worker_closed", "worker_timeout", "browser_closed", "session_not_found", "session_closed"):
|
|
self.sessions.pop(task_id, None)
|
|
self.pause_active_task(task_id)
|
|
raise HTTPException(409, {"message": str(error), "code": error.code}) from None
|
|
|
|
async def open(self, task_id: str, owner_id: str) -> dict:
|
|
task = self.task(task_id, owner_id)
|
|
scoped_account = hashlib.sha256(f"{owner_id}|{task['platform']}|{task['account_key']}".encode()).hexdigest()
|
|
result = await self.call(task_id, "open", {"account_key": scoped_account, "entry_url": task["entry_url"]})
|
|
self.sessions[task_id] = {**result, "status": "awaiting_user"}
|
|
self.set_status(task_id, "awaiting_user")
|
|
return result
|
|
|
|
async def ready(self, task_id: str, owner_id: str, identity_label: str) -> dict:
|
|
self.task(task_id, owner_id)
|
|
if task_id not in self.sessions:
|
|
raise HTTPException(409, {"code": "browser_session_required", "message": self.browser_readiness(task_id)["message"], "next_action": "open_browser"})
|
|
if re.search(r"(?<!\d)1\d{10}(?!\d)", identity_label):
|
|
raise HTTPException(422, "请填写账号显示名或本地别名，不填写手机号")
|
|
result = await self.call(task_id, "ready", {"identity_label": identity_label})
|
|
self.sessions[task_id] = {**result, "status": "ready"}
|
|
self.store.execute("UPDATE learning_tasks SET identity_label=$1,status='observing',updated_at=$2 WHERE id=$3", (identity_label, now(), task_id))
|
|
return {**result, "manual_confirmation": True, "identity_verification": "operator_confirmed"}
|
|
|
|
def register_artifact(self, task_id: str, raw_path: str) -> str:
|
|
target = Path(raw_path).resolve()
|
|
if not target.is_relative_to(self.store.home) or target.suffix.lower() != ".png" or not target.is_file():
|
|
raise HTTPException(409, "执行器证据文件无效，未登记到验证记录")
|
|
content = target.read_bytes()
|
|
if not content.startswith(b"\x89PNG\r\n\x1a\n"):
|
|
raise HTTPException(409, "执行器证据不是有效 PNG")
|
|
artifact_id = identifier("evidence")
|
|
self.store.execute("INSERT INTO artifacts(id,task_id,relative_path,sha256,bytes,created_at) VALUES($1,$2,$3,$4,$5,$6)", (artifact_id, task_id, target.relative_to(self.store.home).as_posix(), hashlib.sha256(content).hexdigest(), len(content), now()))
|
|
return artifact_id
|
|
|
|
def save_observation(self, task_id: str, capture: dict, note: str = "", source: str = "learning") -> dict:
|
|
data = deepcopy(capture)
|
|
raw_path = data.pop("screenshot_path", None)
|
|
data["source"], data["note"] = source, note
|
|
if raw_path:
|
|
artifact_id = self.register_artifact(task_id, raw_path)
|
|
data["artifact_id"] = artifact_id
|
|
data["screenshot_url"] = "/api/learning/artifacts/" + artifact_id
|
|
data["text"] = str(data.get("text", ""))[:30000]
|
|
data["elements"] = list(data.get("elements") or [])[:300]
|
|
observation_id, timestamp = identifier("observation"), now()
|
|
self.store.execute("INSERT INTO observations(id,task_id,data_json,created_at) VALUES($1,$2,$3,$4)", (observation_id, task_id, encoded(data), timestamp))
|
|
return {"id": observation_id, "task_id": task_id, "created_at": timestamp, **data}
|
|
|
|
async def capture(self, task_id: str, owner_id: str, note: str = "") -> dict:
|
|
await self.require_ready(task_id, owner_id)
|
|
capture = await self.call(task_id, "capture", {"note": note})
|
|
return self.save_observation(task_id, capture, note)
|
|
|
|
async def action(self, task_id: str, owner_id: str, values: dict) -> dict:
|
|
task = self.task(task_id, owner_id)
|
|
await self.require_ready(task_id, owner_id)
|
|
observations = self.observations(task_id)
|
|
if not observations:
|
|
raise HTTPException(409, "请先采集当前页面，再记录链路动作")
|
|
if len(self.steps(task_id)) >= 60:
|
|
raise HTTPException(409, "本次学习已达到 60 步上限，请拆分能力")
|
|
last = observations[-1]
|
|
element = next((item for item in last.get("elements", []) if item.get("selector") == values["selector"]), None)
|
|
if not element or element.get("sensitive"):
|
|
raise HTTPException(422, "请选择最近页面证据中的非敏感元素")
|
|
kind, parameter, value = values["kind"], values.get("parameter"), values.get("value", "")
|
|
if parameter:
|
|
if parameter not in task["parameters"]:
|
|
raise HTTPException(422, "参数未在学习任务中声明")
|
|
value = task["parameters"][parameter]
|
|
if kind == "read" and not NAME.fullmatch(values.get("output_name") or ""):
|
|
raise HTTPException(422, "读取步骤需要输出字段名，例如 results")
|
|
result = await self.call(task_id, "action", {"kind": kind, "selector": values["selector"], "value": value, "output_name": values.get("output_name")})
|
|
before = self.save_observation(task_id, result["before"], "动作前", "action") if result.get("before") else last
|
|
after = self.save_observation(task_id, result["after"], "动作后", "action") if result.get("after") else None
|
|
step = {"kind": kind, "selector": values["selector"], "evidence_id": before["id"], "after_evidence_id": after["id"] if after else None}
|
|
if kind == "fill":
|
|
step.update({"parameter": parameter} if parameter else {"value": value})
|
|
if kind == "read":
|
|
step["output_name"] = values["output_name"]
|
|
with self.store.connect(immediate=True) as conn:
|
|
position = conn.execute("SELECT COALESCE(MAX(position),0)+1 AS position FROM learning_steps WHERE task_id=$1", (task_id,)).fetchone()["position"]
|
|
step_id, timestamp = identifier("step"), now()
|
|
conn.execute("INSERT INTO learning_steps(id,task_id,position,data_json,created_at) VALUES($1,$2,$3,$4,$5)", (step_id, task_id, position, encoded(step), timestamp))
|
|
return {"step": {"id": step_id, "position": position, "created_at": timestamp, **step}, "result": result.get("result"), "before": before, "after": after}
|
|
|
|
async def close(self, task_id: str, owner_id: str) -> dict:
|
|
self.task(task_id, owner_id)
|
|
result = await self.call(task_id, "close", {})
|
|
self.sessions.pop(task_id, None)
|
|
self.pause_active_task(task_id)
|
|
return result
|
|
|
|
def create_candidate(self, task_id: str, owner_id: str, name: str | None = None) -> dict:
|
|
task = self.task(task_id, owner_id)
|
|
steps = [{k: v for k, v in step.items() if k not in ("id", "created_at", "position")} for step in self.steps(task_id)]
|
|
references = sorted({step[key] for step in steps for key in ("evidence_id", "after_evidence_id") if step.get(key)})
|
|
bindings = []
|
|
for reference in references:
|
|
evidence = self.evidence_snapshot(task_id, reference)
|
|
bindings.append({"observation_id": reference, "content_digest": digest(evidence) if evidence else None})
|
|
manifest = {"schema_version": 1, "integrity_version": 1, "entry_url": task["entry_url"], "parameters": deepcopy(task["parameters"]), "steps": steps, "evidence_bindings": bindings, "platform": task["platform"], "identity_requirement": "operator_confirmed_session", "scope": "readonly_learning"}
|
|
candidate_id = identifier("candidate")
|
|
with self.store.connect(immediate=True) as conn:
|
|
version = conn.execute("SELECT COALESCE(MAX(version),0)+1 AS version FROM candidates WHERE task_id=$1", (task_id,)).fetchone()["version"]
|
|
conn.execute("INSERT INTO candidates(id,task_id,name,version,digest,manifest_json,created_at) VALUES($1,$2,$3,$4,$5,$6,$7)", (candidate_id, task_id, name or task["name"], version, digest(manifest), encoded(manifest), now()))
|
|
self.set_status(task_id, "candidate")
|
|
return self.candidate(candidate_id, owner_id)
|
|
|
|
def evidence_snapshot(self, task_id: str, observation_id: str) -> dict | None:
|
|
row = self.store.one("SELECT * FROM observations WHERE id=$1 AND task_id=$2", (observation_id, task_id))
|
|
if not row:
|
|
return None
|
|
data = json.loads(row["data_json"])
|
|
artifact = self.store.one("SELECT id,sha256,bytes FROM artifacts WHERE id=$1 AND task_id=$2", (data.get("artifact_id", ""), task_id))
|
|
return {"observation_id": row["id"], "task_id": task_id, "created_at": row["created_at"], "data": data, "artifact": artifact}
|
|
|
|
def source_integrity_errors(self, candidate: dict) -> list[str]:
|
|
manifest, task_id = candidate["manifest"], candidate["task_id"]
|
|
if manifest.get("integrity_version") != 1:
|
|
return ["候选尚未冻结动作前后证据，请基于原任务生成新版本"]
|
|
bindings = manifest.get("evidence_bindings") or []
|
|
references = {item.get("observation_id") for item in bindings}
|
|
current = {reference: snapshot for reference in references if (snapshot := self.evidence_snapshot(task_id, reference))}
|
|
errors = evidence_graph_errors(manifest.get("steps") or [], bindings, current)
|
|
for snapshot in current.values():
|
|
artifact = snapshot["artifact"]
|
|
if not artifact:
|
|
errors.append("冻结来源证据缺少关联截图")
|
|
continue
|
|
stored = self.store.one("SELECT relative_path FROM artifacts WHERE id=$1 AND task_id=$2", (artifact["id"], task_id))
|
|
target = (self.store.home / stored["relative_path"]).resolve()
|
|
if not target.is_relative_to(self.store.home) or not target.is_file():
|
|
errors.append("冻结来源截图不存在或越出证据目录")
|
|
elif hashlib.sha256(target.read_bytes()).hexdigest() != artifact["sha256"]:
|
|
errors.append("冻结来源截图摘要不匹配")
|
|
return list(dict.fromkeys(errors))
|
|
|
|
def add_validation(self, candidate: dict, kind: str, errors: list[str], **data) -> dict:
|
|
record_id, timestamp = identifier("validation"), now()
|
|
content = {"candidate_digest": candidate["digest"], "errors": errors, "parameters": {}, "expected_outputs": {}, "actual_outputs": {}, "environment": {}, "evidence": [], "sample_name": "", "duration_ms": 0, "model_calls": 0, "model_usage_status": "not_used", **data}
|
|
status = "failed" if errors else "passed"
|
|
self.store.execute("INSERT INTO validations(id,candidate_id,task_id,kind,status,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6,$7)", (record_id, candidate["id"], candidate["task_id"], kind, status, encoded(content), timestamp))
|
|
return {"id": record_id, "candidate_id": candidate["id"], "task_id": candidate["task_id"], "kind": kind, "status": status, "created_at": timestamp, **content}
|
|
|
|
def static_errors(self, candidate: dict) -> list[str]:
|
|
manifest = candidate["manifest"]
|
|
errors = []
|
|
if digest(manifest) != candidate["digest"]:
|
|
errors.append("能力清单校验值不匹配")
|
|
if manifest.get("schema_version") != 1:
|
|
errors.append("不支持的能力契约版本")
|
|
errors.extend(self.source_integrity_errors(candidate))
|
|
steps = manifest.get("steps") or []
|
|
if not steps:
|
|
errors.append("候选没有实际记录的步骤")
|
|
if len(steps) > 60:
|
|
errors.append("候选超过步骤预算")
|
|
output_names = set()
|
|
for position, step in enumerate(steps, 1):
|
|
prefix = f"第 {position} 步"
|
|
if step.get("kind") not in ("fill", "click", "read"):
|
|
errors.append(prefix + "动作不在首版只读范围内")
|
|
if not isinstance(step.get("selector"), str) or not step["selector"].strip():
|
|
errors.append(prefix + "缺少定位规则")
|
|
observation = self.store.one("SELECT data_json FROM observations WHERE id=$1 AND task_id=$2", (step.get("evidence_id", ""), candidate["task_id"]))
|
|
if not observation:
|
|
errors.append(prefix + "没有同任务来源证据")
|
|
else:
|
|
observed = json.loads(observation["data_json"])
|
|
element = next((item for item in observed.get("elements", []) if item.get("selector") == step.get("selector")), None)
|
|
if not element or element.get("sensitive"):
|
|
errors.append(prefix + "定位规则缺少非敏感页面依据")
|
|
elif step.get("kind") != "read" and not element.get("unique"):
|
|
errors.append(prefix + "操作目标不唯一")
|
|
if step.get("kind") == "fill" and step.get("parameter") and step["parameter"] not in manifest.get("parameters", {}):
|
|
errors.append(prefix + "参数未声明")
|
|
if step.get("kind") == "fill" and not step.get("parameter") and not isinstance(step.get("value"), str):
|
|
errors.append(prefix + "缺少明确输入值或参数绑定")
|
|
if step.get("kind") == "read":
|
|
name = step.get("output_name", "")
|
|
if not NAME.fullmatch(name) or name in output_names:
|
|
errors.append(prefix + "输出名缺失、非法或重复")
|
|
output_names.add(name)
|
|
if not output_names:
|
|
errors.append("首版候选至少需要一个可核对的读取输出")
|
|
return errors
|
|
|
|
def check(self, candidate_id: str, owner_id: str) -> dict:
|
|
candidate = self.candidate(candidate_id, owner_id)
|
|
return self.add_validation(candidate, "static", self.static_errors(candidate), environment={"validator": "learning-v1", "os": os_name(platform.system())})
|
|
|
|
async def verify(self, candidate_id: str, owner_id: str, values: dict) -> dict:
|
|
candidate = self.candidate(candidate_id, owner_id)
|
|
manifest = candidate["manifest"]
|
|
static = self.store.one(f"SELECT status FROM validations WHERE candidate_id=$1 AND kind='static' ORDER BY {self.store.order_column()} DESC LIMIT 1", (candidate_id,))
|
|
if not static or static["status"] != "passed" or self.static_errors(candidate):
|
|
raise HTTPException(409, "请先通过候选固定校验，再进行浏览器试跑")
|
|
expected = values.get("expected_outputs") or {}
|
|
output_names = {step.get("output_name") for step in manifest["steps"] if step["kind"] == "read"}
|
|
if not expected or not set(expected).issubset(output_names):
|
|
raise HTTPException(422, "请为候选的读取输出填写至少一项预期文本")
|
|
for expectation in expected.values():
|
|
items = [expectation] if isinstance(expectation, str) else expectation
|
|
if not isinstance(items, list) or not items or any(not isinstance(item, str) or not item.strip() or len(item) > 2000 for item in items):
|
|
raise HTTPException(422, "预期结果需为非空文本或文本列表")
|
|
overrides = check_parameters(values.get("parameters") or {})
|
|
if not set(overrides).issubset(manifest["parameters"]):
|
|
raise HTTPException(422, "试跑包含候选未声明的参数")
|
|
parameters = {**manifest["parameters"], **overrides}
|
|
await self.require_ready(candidate["task_id"], owner_id)
|
|
started = time.monotonic()
|
|
environment = {"os": os_name(platform.system()), "python_version": platform.python_version(), "runner": "playwright-learning-v1", "identity_verification": "operator_confirmed"}
|
|
evidence, errors, outputs = [], [], {}
|
|
try:
|
|
result = await self.call(candidate["task_id"], "verify", {"entry_url": manifest["entry_url"], "steps": manifest["steps"], "parameters": parameters, "expected_outputs": expected})
|
|
outputs = result.get("outputs") or {}
|
|
errors = [str(error) for error in result.get("errors", [])]
|
|
environment["browser_version"] = str(result.get("browser_version") or "unknown")
|
|
if environment["browser_version"] == "unknown":
|
|
errors.append("执行器没有提供实际浏览器版本")
|
|
if result.get("os") and os_name(result["os"]) != environment["os"]:
|
|
errors.append("执行器返回的系统与本机实测系统不一致")
|
|
for capture in result.get("evidence", []):
|
|
if isinstance(capture, dict):
|
|
observation = self.save_observation(candidate["task_id"], capture, values.get("sample_name", "变参试跑"), "verification")
|
|
evidence.append({"observation_id": observation["id"], "artifact_id": observation.get("artifact_id"), "screenshot_url": observation.get("screenshot_url"), "url": observation.get("url"), "title": observation.get("title")})
|
|
errors.extend(compare_outputs(outputs, expected))
|
|
if not result.get("passed") and not errors:
|
|
errors.append("执行器未确认试跑成功")
|
|
if not evidence:
|
|
errors.append("试跑缺少页面证据")
|
|
except HTTPException as error:
|
|
detail = error.detail
|
|
# Missing process-owned session is an operational precondition, not a
|
|
# failed business assertion. Do not fabricate a new validation result.
|
|
if isinstance(detail, dict) and detail.get("code", "").lower() in ("session_closed", "session_not_found", "worker_closed", "worker_timeout", "awaiting_user"):
|
|
raise
|
|
errors.append(detail.get("message", "试跑未完成") if isinstance(detail, dict) else str(detail))
|
|
record = self.add_validation(candidate, "browser", list(dict.fromkeys(errors)), parameters=parameters, expected_outputs=expected, actual_outputs=outputs, environment=environment, evidence=evidence, sample_name=values.get("sample_name", "变参试跑"), duration_ms=round((time.monotonic() - started) * 1000), assertion_checks=output_assertion_checks(outputs, expected), verification_basis="synthetic_browser_fixture" if manifest.get("platform") == "demo" else "browser_execution_with_fixed_assertions", independent_readback={"status": "not_configured", "independent_identity": False, "message": "当前保存浏览器执行输出并由后台固定比较；尚未接入独立身份与来源读回，不能等同 A2A 的独立业务核验"})
|
|
self.set_status(candidate["task_id"], "validated" if record["status"] == "passed" else "validation_failed")
|
|
return record
|
|
|
|
def publish_check(self, candidate_id: str, owner_id: str) -> dict:
|
|
candidate = self.candidate(candidate_id, owner_id)
|
|
rows = self.validations(owner_id, candidate_id=candidate_id)
|
|
static = next((row for row in rows if row["kind"] == "static"), None)
|
|
passed = [row for row in rows if row["kind"] == "browser" and row["status"] == "passed" and row.get("candidate_digest") == candidate["digest"]]
|
|
samples = {encoded(row["parameters"]) for row in passed}
|
|
systems = {row.get("environment", {}).get("os") for row in passed}
|
|
evidence_ok = bool(passed)
|
|
for row in passed:
|
|
if not row.get("evidence"):
|
|
evidence_ok = False
|
|
for item in row.get("evidence", []):
|
|
if not item.get("artifact_id"):
|
|
evidence_ok = False
|
|
continue
|
|
try:
|
|
self.artifact(item["artifact_id"], owner_id)
|
|
except HTTPException:
|
|
evidence_ok = False
|
|
checks = [
|
|
{"name": "候选结构与来源校验", "passed": bool(static and static["status"] == "passed" and not self.static_errors(candidate)), "detail": "引用同版本固定校验与原始页面证据"},
|
|
{"name": "至少两组不同参数试跑", "passed": len(samples) >= 2, "detail": f"已通过 {len(samples)} 组不同参数"},
|
|
{"name": "Windows 实际试跑", "passed": "windows" in systems, "detail": "只认可当前执行主机记录的真实系统"},
|
|
{"name": "macOS 实际试跑", "passed": "macos" in systems, "detail": "未在 macOS 执行不能计为通过"},
|
|
{"name": "试跑证据完整", "passed": evidence_ok, "detail": "各通过记录需具备关联截图证据"},
|
|
{"name": "独立来源读回", "passed": False, "detail": "本轮仅完成浏览器执行与固定断言，尚未接入独立身份的来源业务读回"},
|
|
]
|
|
return {"ready": all(item["passed"] for item in checks), "checks": checks, "candidate_id": candidate_id, "digest": candidate["digest"], "published": False, "message": "当前阶段提供发布预检，尚未接入商家能力分发"}
|
|
|
|
def artifact(self, artifact_id: str, owner_id: str) -> Path:
|
|
row = self.store.one("SELECT a.* FROM artifacts a JOIN learning_tasks t ON t.id=a.task_id WHERE a.id=$1 AND t.owner_id=$2", (artifact_id, owner_id))
|
|
if not row:
|
|
raise HTTPException(404, "证据不存在")
|
|
target = (self.store.home / row["relative_path"]).resolve()
|
|
if not target.is_relative_to(self.store.home) or not target.is_file():
|
|
raise HTTPException(404, "证据文件不存在")
|
|
if hashlib.sha256(target.read_bytes()).hexdigest() != row["sha256"]:
|
|
raise HTTPException(409, "证据内容已变化，不能当作原验证依据")
|
|
return target
|
|
|
|
async def demo(self, owner_id: str, base_url: str) -> dict:
|
|
task = self.create_task(owner_id, {"name": "本机搜索详情评论示例", "platform": "demo", "entry_url": base_url.rstrip("/") + "/api/learning/demo-page", "goal": "记录搜索、打开详情与读取评论，通过两组参数核对当前对象与输出", "account_key": "local-demo", "parameters": {"keyword": "建材"}})
|
|
task_id = task["id"]
|
|
try:
|
|
await self.open(task_id, owner_id)
|
|
await self.ready(task_id, owner_id, "本机演示页面 无平台账号")
|
|
await self.capture(task_id, owner_id, "示例初始页面")
|
|
await self.action(task_id, owner_id, {"kind": "fill", "selector": '[data-testid="search-input"]', "parameter": "keyword"})
|
|
await self.action(task_id, owner_id, {"kind": "click", "selector": '[data-testid="search-button"]'})
|
|
await self.action(task_id, owner_id, {"kind": "read", "selector": '[data-testid="result"]', "output_name": "results"})
|
|
await self.action(task_id, owner_id, {"kind": "click", "selector": '[data-testid="detail-link"]'})
|
|
await self.action(task_id, owner_id, {"kind": "read", "selector": '[data-testid="comment"]', "output_name": "comments"})
|
|
candidate = self.create_candidate(task_id, owner_id)
|
|
self.check(candidate["id"], owner_id)
|
|
for keyword in ("建材", "装修"):
|
|
await self.verify(candidate["id"], owner_id, {"parameters": {"keyword": keyword}, "expected_outputs": {"results": keyword, "comments": keyword}, "sample_name": f"关键词 {keyword}"})
|
|
return {"task_id": task_id, "candidate_id": candidate["id"]}
|
|
finally:
|
|
if task_id in self.sessions:
|
|
try:
|
|
await self.close(task_id, owner_id)
|
|
except HTTPException:
|
|
pass
|