383 lines
21 KiB
Python
383 lines
21 KiB
Python
"""Industry-neutral learned-route recall and replay over isolated SQLite / PG.
|
|
|
|
The browser is a FakeWorker; these tests never open a platform or call a model.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
import asyncio
|
|
import threading
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
import pytest
|
|
from fastapi import HTTPException
|
|
from fastapi.testclient import TestClient
|
|
|
|
from apps.server.db import Store, StoreConflict
|
|
from apps.server.learning_contracts import EXECUTION_PROFILE, execution_metadata
|
|
from apps.server.routes import RouteService
|
|
from apps.server.tests.test_discovery import BASE, EXTRACT, item, output
|
|
from apps.server.tests.test_execution_profile import install_profile_worker
|
|
from apps.server.tests.test_learning import client, initialize
|
|
from apps.server.tests.test_postgres import pg_client
|
|
from apps.server.tests.test_reading import NOTE_ID, NOTE_URL, learned_detail
|
|
|
|
|
|
@pytest.fixture(params=["client", "pg_client"])
|
|
def route_client(request):
|
|
return request.getfixturevalue(request.param)
|
|
|
|
|
|
def route_fixture(client):
|
|
"""Record two separately frozen contracts from one task and one session."""
|
|
initialize(client)
|
|
worker = client.worker_reference
|
|
previous_capture = worker.capture
|
|
|
|
def capture(task_id):
|
|
result = previous_capture(task_id)
|
|
for element in result["elements"]:
|
|
if element["selector"] == '[data-testid="search-input"]':
|
|
element["search_input"] = True
|
|
return result
|
|
|
|
worker.capture = capture
|
|
task, detail, detail_state = learned_detail(client)
|
|
state = {"items": [item("咖啡手冲滤杯怎么选")], "search_mode": "normal"}
|
|
previous_call = worker.call
|
|
|
|
async def call(method, params):
|
|
result = await previous_call(method, params)
|
|
if method == "action" and params.get("output_name") == "search_results":
|
|
result["result"] = output(state["items"])
|
|
if method == "verify" and not params.get("note_binding"):
|
|
result["outputs"] = {"search_results": output(state["items"])}
|
|
if state["search_mode"] == "wrong_source":
|
|
result["outputs"]["search_results"]["source_url"] = "https://elsewhere.test/search"
|
|
elif state["search_mode"] == "failed":
|
|
result.update(passed=False, errors=["合成搜索失败"])
|
|
elif method == "verify" and params.get("note_binding"):
|
|
title = params["parameters"]["note_title"]
|
|
if detail_state["mode"] != "wrong_title":
|
|
result["outputs"]["note_detail"]["items"][0]["title"] = title
|
|
if result["note_binding_result"]:
|
|
result["note_binding_result"]["title"] = title
|
|
if detail_state["mode"] == "normal":
|
|
note_id = params["note_binding"]["note_id"]
|
|
location = BASE + "/explore/" + note_id
|
|
result["outputs"]["note_detail"]["source_url"] = location
|
|
result["outputs"]["comments"]["source_url"] = location
|
|
result["note_binding_result"].update(note_id=note_id, source_url=location)
|
|
return result
|
|
|
|
worker.call = call
|
|
prefix = "/api/learning/tasks/" + task["id"]
|
|
response = client.post(prefix + "/browser/action", json=EXTRACT)
|
|
assert response.status_code == 200, response.text
|
|
steps = client.get(prefix).json()["steps"]
|
|
selected = [row["id"] for row in steps[:3]] + [steps[-1]["id"]]
|
|
response = client.post(prefix + "/candidates", json={"name": "通用搜索", "step_ids": selected})
|
|
assert response.status_code == 201, response.text
|
|
search = response.json()
|
|
assert client.post("/api/learning/candidates/" + search["id"] + "/check").json()["status"] == "passed"
|
|
return task, search, detail, state, detail_state
|
|
|
|
|
|
def verify_calls(client):
|
|
return [params for method, params in client.worker_reference.calls if method == "verify"]
|
|
|
|
|
|
def resolve_input(search=None, detail=None):
|
|
data = {"platform": "demo", "task_type": "search_note_comments"}
|
|
if search:
|
|
data["search_candidate_id"] = search["id"]
|
|
if detail:
|
|
data["detail_candidate_id"] = detail["id"]
|
|
return data
|
|
|
|
|
|
def replay_input(search, detail, **extra):
|
|
return {**resolve_input(search, detail), "search_digest": search["digest"],
|
|
"detail_digest": detail["digest"], "keyword": "咖啡滤杯", **extra}
|
|
|
|
|
|
def replay(client, search, detail, **extra):
|
|
response = client.post("/api/learning/routes/verify", json=replay_input(search, detail, **extra))
|
|
assert response.status_code == 201, response.text
|
|
return response.json()
|
|
|
|
|
|
def facts(store):
|
|
return {table: store.rows(f"SELECT * FROM {table} ORDER BY id") for table in
|
|
("candidates", "learning_steps", "discovery_runs", "reading_runs", "analysis_runs", "business_briefs", "analysis_reviews")}
|
|
|
|
|
|
def test_routes_catalog_resolve_and_new_parameters_are_industry_neutral(route_client):
|
|
client = route_client
|
|
task, search, detail, _, _ = route_fixture(client)
|
|
before_calls = len(client.worker_reference.calls)
|
|
catalog = client.get("/api/learning/routes").json()
|
|
assert catalog["total"] == 2
|
|
assert {row["ability"] for row in catalog["items"]} == {"search_notes", "read_note_comments"}
|
|
assert all(row["static_status"] == "passed" and row["trial_status"] == "not_run" and row["published"] is False for row in catalog["items"])
|
|
response = client.post("/api/learning/routes/resolve", json=resolve_input())
|
|
assert response.status_code == 200, response.text
|
|
route = response.json()
|
|
assert route["route_version"] == "learned-read-route-v1" and route["mode"] == "exploration"
|
|
assert route["task_id"] == task["id"] and route["execution_profile"] is None
|
|
assert route["search"]["candidate_id"] == search["id"] and route["detail"]["candidate_id"] == detail["id"]
|
|
assert route["search"]["digest"] == search["digest"] and route["detail"]["digest"] == detail["digest"]
|
|
assert route["model_calls"] == 0 and route["published"] is False
|
|
assert len(client.worker_reference.calls) == before_calls
|
|
|
|
store = client.app_reference.state.store
|
|
snapshot = facts(store)
|
|
result = replay(client, search, detail)
|
|
assert result["kind"] == "route" and result["status"] == "passed" and result["route_status"] == "completed"
|
|
assert result["keyword"] == "咖啡滤杯" and result["selected"]["note_title"] == "咖啡手冲滤杯怎么选"
|
|
assert result["selected"]["note_id"] == NOTE_ID and result["selected"]["note_url"] == NOTE_URL
|
|
assert result["model_calls"] == 0 and result["published"] is False
|
|
assert set(result["actual_outputs"]) == {"search_results", "note_detail", "comments"}
|
|
assert set(result["usable_outputs"]) == {"search_results", "note_detail", "comments"}
|
|
assert result["search_validation_id"] != result["detail_validation_id"]
|
|
assert all(row["mode"] == "exploration" for row in client.get("/api/learning/validations").json()["items"] if row["kind"] == "browser")
|
|
calls = verify_calls(client)
|
|
assert len(calls) == 2 and all(call["task_id"] == task["id"] for call in calls)
|
|
assert calls[0]["parameters"]["keyword"] == calls[1]["parameters"]["keyword"] == "咖啡滤杯"
|
|
assert calls[1]["parameters"]["note_title"] == "咖啡手冲滤杯怎么选" and calls[1]["note_binding"] == {"note_id": NOTE_ID}
|
|
assert facts(store) == snapshot
|
|
assert client.get("/api/learning/candidates/" + search["id"]).json()["status"] == "validated"
|
|
assert client.get("/api/learning/candidates/" + detail["id"]).json()["status"] == "validated"
|
|
assert client.get("/api/learning/candidates/" + search["id"]).json()["quality"]["state"] == "experimental"
|
|
assert not client.post("/api/learning/candidates/" + detail["id"] + "/publish-check").json()["ready"]
|
|
assert result == next(row for row in client.get("/api/learning/validations").json()["items"] if row["id"] == result["id"])
|
|
|
|
|
|
def test_routes_ambiguity_requires_explicit_version_and_stale_digest_is_rejected(route_client):
|
|
client = route_client
|
|
task, search, detail, _, _ = route_fixture(client)
|
|
step_ids = [step["id"] for step in client.get("/api/learning/tasks/" + task["id"]).json()["steps"][:7]]
|
|
duplicate = client.post("/api/learning/tasks/" + task["id"] + "/candidates", json={"step_ids": step_ids}).json()
|
|
assert client.post("/api/learning/candidates/" + duplicate["id"] + "/check").json()["status"] == "passed"
|
|
before = len(verify_calls(client))
|
|
ambiguous = client.post("/api/learning/routes/resolve", json=resolve_input())
|
|
assert ambiguous.status_code == 409 and ambiguous.json()["detail"]["code"] == "ROUTE_AMBIGUOUS"
|
|
explicit = client.post("/api/learning/routes/resolve", json=resolve_input(search, detail))
|
|
assert explicit.status_code == 200 and explicit.json()["detail"]["candidate_id"] == detail["id"]
|
|
for key in ("search_digest", "detail_digest"):
|
|
stale = client.post("/api/learning/routes/verify", json=replay_input(search, detail, **{key: "0" * 64}))
|
|
assert stale.status_code == 409 and stale.json()["detail"]["code"] == "ROUTE_DIGEST_MISMATCH"
|
|
assert len(verify_calls(client)) == before
|
|
# A historical passed check is metadata only. Replay must still recheck the
|
|
# exact frozen evidence bytes before calling the browser.
|
|
service = client.app_reference.state.learning
|
|
reference = search["manifest"]["evidence_bindings"][0]["observation_id"]
|
|
snapshot = service.evidence_snapshot(task["id"], reference)
|
|
artifact = service.store.one("SELECT relative_path FROM artifacts WHERE id=$1", (snapshot["artifact"]["id"],))
|
|
target = (service.store.home / artifact["relative_path"]).resolve()
|
|
assert target.is_relative_to(service.store.home)
|
|
original = target.read_bytes()
|
|
try:
|
|
target.write_bytes(original + b"fixture-corruption")
|
|
response = client.post("/api/learning/routes/resolve", json=resolve_input(search, detail))
|
|
assert response.status_code == 409 and response.json()["detail"]["code"] == "ROUTE_PAIR_INCOMPATIBLE"
|
|
assert len(verify_calls(client)) == before
|
|
finally:
|
|
target.write_bytes(original)
|
|
|
|
|
|
def test_routes_cross_task_pairs_do_not_share_account_confirmation_or_execute(route_client):
|
|
client = route_client
|
|
_, search, _, _, _ = route_fixture(client)
|
|
_, another_detail, _ = learned_detail(client)
|
|
before = len(verify_calls(client))
|
|
response = client.post("/api/learning/routes/resolve", json=resolve_input(search, another_detail))
|
|
assert response.status_code == 409 and response.json()["detail"]["code"] == "ROUTE_PAIR_INCOMPATIBLE"
|
|
response = client.post("/api/learning/routes/verify", json=replay_input(search, another_detail))
|
|
assert response.status_code == 409 and response.json()["detail"]["code"] == "ROUTE_PAIR_INCOMPATIBLE"
|
|
assert len(verify_calls(client)) == before
|
|
|
|
|
|
def test_routes_no_session_saves_failure_without_browser_actions(route_client):
|
|
client = route_client
|
|
task, search, detail, _, _ = route_fixture(client)
|
|
client.post("/api/learning/tasks/" + task["id"] + "/browser/close")
|
|
before = len(verify_calls(client))
|
|
result = replay(client, search, detail)
|
|
assert result["status"] == "failed" and result["route_status"] == "failed"
|
|
assert result["search_validation_id"] is result["detail_validation_id"] is None
|
|
assert result["actual_outputs"] == {} and result["selected"] is None
|
|
assert "SEARCH_NOT_READY" in result["error_codes"] and len(verify_calls(client)) == before
|
|
|
|
|
|
def test_routes_selection_is_bound_to_current_search_and_skips_ambiguous_titles(route_client):
|
|
client = route_client
|
|
_, search, detail, state, _ = route_fixture(client)
|
|
state["items"] = [item(None, "b"), item("相同标题", "c"), item("相同标题", "d"), item("露营帐篷选购", "e")]
|
|
result = replay(client, search, detail)
|
|
assert result["route_status"] == "completed" and result["selected"]["note_id"] == "e" * 24
|
|
assert result["selected"]["note_title"] == "露营帐篷选购"
|
|
state["items"] = [item("第一篇", "a"), item("第二篇", "b")]
|
|
chosen = replay(client, search, detail, note_id="b" * 24)
|
|
assert chosen["route_status"] == "completed" and chosen["selected"]["note_id"] == "b" * 24
|
|
before = len(verify_calls(client))
|
|
missing = replay(client, search, detail, note_id="c" * 24)
|
|
assert missing["route_status"] == "failed" and missing["selected"] is None
|
|
assert missing["search_validation_id"] and missing["detail_validation_id"] is None
|
|
assert len(verify_calls(client)) == before + 1
|
|
|
|
|
|
def test_routes_all_unsafe_or_ambiguous_titles_never_open_detail(route_client):
|
|
client = route_client
|
|
_, search, detail, state, _ = route_fixture(client)
|
|
cases = ([item(None)], [item("重复", "a"), item("重复", "b")],
|
|
[item("被脱敏[手机号已隐藏]")], [item("站外", url="https://elsewhere.test/explore/" + NOTE_ID)],
|
|
[item("同ID标题一"), item("同ID标题二")])
|
|
for rows in cases:
|
|
state["items"] = rows
|
|
before = len(verify_calls(client))
|
|
result = replay(client, search, detail)
|
|
assert result["route_status"] == "failed" and result["selected"] is None
|
|
assert result["detail_validation_id"] is None and len(verify_calls(client)) == before + 1
|
|
|
|
|
|
def test_routes_search_failure_and_wrong_source_stop_before_detail(route_client):
|
|
client = route_client
|
|
_, search, detail, state, _ = route_fixture(client)
|
|
for mode in ("wrong_source", "failed"):
|
|
state["search_mode"] = mode
|
|
before = len(verify_calls(client))
|
|
result = replay(client, search, detail)
|
|
assert result["status"] == "failed" and result["route_status"] == "failed"
|
|
assert result["search_validation_id"] and result["detail_validation_id"] is None
|
|
assert result["selected"] is None and result["errors"] and result["error_codes"]
|
|
assert not result["usable_outputs"]
|
|
assert len(verify_calls(client)) == before + 1
|
|
|
|
|
|
def test_routes_partial_detail_and_source_mismatch_are_not_completed(route_client):
|
|
client = route_client
|
|
_, search, detail, _, detail_state = route_fixture(client)
|
|
for mode in ("missing_comments", "empty_comments"):
|
|
detail_state["mode"] = mode
|
|
result = replay(client, search, detail)
|
|
assert result["status"] == "failed" and result["route_status"] == "partial"
|
|
assert result["actual_outputs"]["note_detail"]["count"] == 1
|
|
assert result["detail_validation_id"] and result["errors"]
|
|
assert set(result["usable_outputs"]) == {"search_results", "note_detail"}
|
|
for mode in ("wrong_id", "wrong_title", "wrong_comments_source", "no_proof"):
|
|
detail_state["mode"] = mode
|
|
result = replay(client, search, detail)
|
|
assert result["status"] == "failed" and result["route_status"] == "failed"
|
|
assert "NOTE_BINDING_MISMATCH" in result["error_codes"]
|
|
assert not next(check for check in result["assertion_checks"] if check["name"] == "note_source_binding")["passed"]
|
|
assert set(result["usable_outputs"]) == {"search_results"}
|
|
|
|
|
|
def test_routes_closed_input_auth_csrf_owner_and_append_only_records(route_client):
|
|
client = route_client
|
|
_, search, detail, _, _ = route_fixture(client)
|
|
before = len(verify_calls(client))
|
|
for changes in ({"keyword": " "}, {"keyword": None}, {"task_type": "contact"}, {"mode": "runtime"},
|
|
{"note_title": "客户端伪造标题"}, {"expected_outputs": {}}, {"search_digest": "bad"}, {"note_id": "bad"}):
|
|
assert client.post("/api/learning/routes/verify", json=replay_input(search, detail, **changes)).status_code == 422
|
|
assert client.post("/api/learning/routes/verify", json=replay_input(search, detail), headers={"X-CSRF-Token": "wrong"}).status_code == 403
|
|
assert len(verify_calls(client)) == before
|
|
result = replay(client, search, detail)
|
|
store = client.app_reference.state.store
|
|
service = client.app_reference.state.routes
|
|
owner = store.one("SELECT owner_id FROM learning_tasks WHERE id=$1", (search["task_id"],))["owner_id"]
|
|
assert service.catalogue(owner + "_other") == []
|
|
with pytest.raises(HTTPException) as error:
|
|
service.resolve(owner + "_other", resolve_input(search, detail))
|
|
assert error.value.status_code == 404
|
|
for sql in ("UPDATE validations SET data_json='{}' WHERE id=$1", "DELETE FROM validations WHERE id=$1"):
|
|
with pytest.raises((sqlite3.IntegrityError, StoreConflict)):
|
|
store.execute(sql, (result["id"],))
|
|
if store.backend == "sqlite":
|
|
assert Store(store.home).one("SELECT id FROM validations WHERE id=$1", (result["id"],))["id"] == result["id"]
|
|
with TestClient(client.app_reference, base_url=BASE) as anonymous:
|
|
assert anonymous.get("/api/learning/routes").status_code == 401
|
|
assert anonymous.post("/api/learning/routes/resolve", json=resolve_input()).status_code in {401, 403}
|
|
assert anonymous.post("/api/learning/routes/verify", json=replay_input(search, detail)).status_code in {401, 403}
|
|
|
|
|
|
def test_routes_exact_execution_profile_reaches_both_stages_and_busy_stops_actions(route_client):
|
|
client = route_client
|
|
install_profile_worker(client)
|
|
task, search, detail, _, _ = route_fixture(client)
|
|
response = client.post("/api/learning/routes/resolve", json=resolve_input(search, detail))
|
|
assert response.status_code == 200 and response.json()["execution_profile"] == EXECUTION_PROFILE
|
|
service = client.app_reference.state.routes
|
|
service.running.add(task["id"])
|
|
before = len(verify_calls(client))
|
|
blocked = client.post("/api/learning/routes/verify", json=replay_input(search, detail))
|
|
assert blocked.status_code == 409 and blocked.json()["detail"]["code"] == "ROUTE_BUSY"
|
|
assert len(verify_calls(client)) == before
|
|
service.running.discard(task["id"])
|
|
result = replay(client, search, detail)
|
|
assert result["route_status"] == "completed" and result["route"]["execution_profile"] == EXECUTION_PROFILE
|
|
for call in verify_calls(client):
|
|
assert all(call[key] == value for key, value in execution_metadata(EXECUTION_PROFILE).items())
|
|
assert service.running == set()
|
|
|
|
|
|
def test_route_pair_requires_one_task_platform_origin_and_exact_profile_contract():
|
|
base = {"task_id": "learn_fixture", "manifest": {"platform": "xiaohongshu", "entry_url": "https://www.xiaohongshu.com/explore", **execution_metadata(EXECUTION_PROFILE)}}
|
|
assert RouteService.compatible(base, base)
|
|
for change in ({"execution_profile": None}, {"execution_contract": {}},
|
|
{"entry_url": "https://other.test/explore"}, {"platform": "demo"}):
|
|
assert not RouteService.compatible(base, {**base, "manifest": {**base["manifest"], **change}})
|
|
assert not RouteService.compatible(base, {**base, "task_id": "learn_elsewhere"})
|
|
|
|
|
|
def test_routes_hold_shared_task_between_stages_and_release_on_failure(route_client):
|
|
client = route_client
|
|
task, search, detail, state, _ = route_fixture(client)
|
|
worker = client.worker_reference
|
|
previous_call = worker.call
|
|
entered, release = threading.Event(), threading.Event()
|
|
paused = False
|
|
|
|
async def call(method, params):
|
|
nonlocal paused
|
|
result = await previous_call(method, params)
|
|
if method == "verify" and not params.get("note_binding") and not paused:
|
|
paused = True
|
|
entered.set()
|
|
assert await asyncio.to_thread(release.wait, 30), "test-owned stage hold timed out"
|
|
return result
|
|
|
|
worker.call = call
|
|
prefix = "/api/learning/tasks/" + task["id"]
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
future = pool.submit(client.post, "/api/learning/routes/verify", json=replay_input(search, detail))
|
|
try:
|
|
assert entered.wait(30), "route did not reach the held search stage"
|
|
before = len(worker.calls)
|
|
# Each request has a separate ASGI context: none may inherit the
|
|
# route's internal permission to operate this task's browser.
|
|
requests = [(prefix + "/browser/open", {}),
|
|
(prefix + "/browser/ready", {"identity_label": "不应覆盖账号确认"}),
|
|
(prefix + "/browser/capture", {}),
|
|
(prefix + "/browser/action", {"kind": "read", "selector": '[data-testid="result"]', "output_name": "blocked"}),
|
|
(prefix + "/browser/close", {}),
|
|
("/api/learning/candidates/" + search["id"] + "/verify", {"expected_outputs": {"search_results": {"state": "nonempty"}}}),
|
|
("/api/learning/routes/verify", replay_input(search, detail))]
|
|
for endpoint, payload in requests:
|
|
blocked = client.post(endpoint, json=payload)
|
|
assert blocked.status_code == 409 and blocked.json()["detail"]["code"] == "ROUTE_BUSY", blocked.text
|
|
assert len(worker.calls) == before
|
|
finally:
|
|
release.set()
|
|
response = future.result(timeout=30)
|
|
assert response.status_code == 201 and response.json()["route_status"] == "completed", response.text
|
|
learning = client.app_reference.state.learning
|
|
assert learning.route_tasks == set() and learning.route_task.get() is None
|
|
assert client.app_reference.state.routes.running == set()
|
|
state["search_mode"] = "failed"
|
|
failed = replay(client, search, detail)
|
|
assert failed["route_status"] == "failed" and learning.route_tasks == set() and learning.route_task.get() is None
|
|
# The independent API is usable again after the route's finally block.
|
|
assert client.post(prefix + "/browser/capture", json={}).status_code == 200
|