329 lines
21 KiB
Python
329 lines
21 KiB
Python
"""Demand contracts with synthetic text, MockTransport and disposable SQLite/PG."""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import copy
|
||
import json
|
||
import sqlite3
|
||
import time
|
||
from dataclasses import replace
|
||
|
||
import httpx
|
||
import pytest
|
||
from fastapi import HTTPException
|
||
from fastapi.testclient import TestClient
|
||
|
||
from apps.server.analysis import checked_results, make_sources
|
||
from apps.server.db import Store, StoreConflict, digest, encoded, identifier, now
|
||
from apps.server.main import create_app
|
||
from apps.server.model_client import ModelCallError, ModelClient, ModelConfig, strict_json
|
||
from apps.server.tests.test_learning import FakeWorker, client, initialize
|
||
from apps.server.tests.test_postgres import pg_client
|
||
|
||
PRIMARY, FALLBACK = "fixture-primary-key-not-real", "fixture-fallback-key-not-real"
|
||
|
||
|
||
def config(**changes):
|
||
return replace(ModelConfig("http://127.0.0.1:9/v1", "fixture-text", "fixture", (("primary", PRIMARY), ("fallback", FALLBACK))), **changes)
|
||
|
||
|
||
def answer(payload):
|
||
value = {"results": [{"source_ref": source["source_ref"], "demand": "explicit" if "找装修公司" in source["text"] else "discussion",
|
||
"evidence": ["找装修公司"] if "找装修公司" in source["text"] else [source["text"][:1000]],
|
||
"role": "suspected_demand_side" if "找装修公司" in source["text"] else "participant",
|
||
"role_evidence": ["找装修公司"] if "找装修公司" in source["text"] else [source["text"][:1000]],
|
||
"expressions": [{"type": "self_need" if "找装修公司" in source["text"] else "case_share", "evidence": [source["text"][:1000]]}],
|
||
"business_relevance": {"value": "relevant", "evidence": [source["text"][:1000]]},
|
||
"stage": None, "region": None, "budget": None} for source in payload["sources"]]}
|
||
# Model-service fixtures follow the current contract. Direct parser and old
|
||
# immutable review fixtures intentionally retain the historical five axes.
|
||
if "business_context" in payload:
|
||
for item in value["results"]:
|
||
item["consultation"] = {"value": "no", "evidence": item["evidence"], "context_evidence": []}
|
||
return value
|
||
|
||
|
||
def wire(content, *, usage=True, finish="stop"):
|
||
value = {"model": "fixture-text", "choices": [{"finish_reason": finish, "message": {"role": "assistant", "content": json.dumps(content, ensure_ascii=False)}}]}
|
||
if usage:
|
||
value["usage"] = {"prompt_tokens": 100, "completion_tokens": 40, "total_tokens": 140, "prompt_cache_hit_tokens": 10, "prompt_cache_miss_tokens": 90}
|
||
return httpx.Response(200, json=value)
|
||
|
||
|
||
@pytest.fixture(params=["client", "pg_client"])
|
||
def analysis_client(request):
|
||
return request.getfixturevalue(request.param)
|
||
|
||
|
||
def seed(client, *, status="completed", bound=True, media=True, chars=None):
|
||
initialize(client)
|
||
store = client.app_reference.state.store
|
||
owner = store.one("SELECT id FROM operators")["id"]
|
||
stamp = now()
|
||
task, candidate, validation, discovery, reading = [identifier(prefix) for prefix in ("learn", "candidate", "validation", "discovery", "reading")]
|
||
detail = {"items": [{"title": "准备装修", "body": "我正在找装修公司，坐标杭州，预算20万。" if chars is None else "装" * chars, "author": "private-author-sentinel"}], "count": 1, "truncated": True, "source_url": "https://www.xiaohongshu.com/explore/" + "a" * 24}
|
||
comments = [{"comment_id": "private-comment-id-sentinel", "author": "private-comment-author-sentinel", "body": "好看，分享一下已完工的效果"}]
|
||
if media:
|
||
comments.append({"comment_id": "private-media-comment-id", "author": "private-media-author", "body": {"kind": "media", "text": None, "media": [{"kind": "image", "selector": "img.private-selector"}], "media_truncated": False}})
|
||
value = {"status": status, "model_connected": False, "discovery_run_id": discovery, "note_id": "a" * 24, "note_url": detail["source_url"], "validation_id": validation,
|
||
"detail": detail, "comments": {"items": comments, "count": len(comments), "truncated": True}, "evidence": []}
|
||
with store.connect(immediate=True) as conn:
|
||
conn.execute("INSERT INTO learning_tasks(id,owner_id,name,platform,entry_url,goal,account_key,parameters_json,status,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", (task, owner, "fixture", "demo", "http://127.0.0.1/fixture", "fixture", "fixture", "{}", "paused", stamp, stamp))
|
||
conn.execute("INSERT INTO candidates(id,task_id,name,version,digest,manifest_json,created_at) VALUES($1,$2,$3,$4,$5,$6,$7)", (candidate, task, "fixture", 1, "fixture", "{}", stamp))
|
||
conn.execute("INSERT INTO validations(id,candidate_id,task_id,kind,status,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6,$7)", (validation, candidate, task, "browser", "passed" if bound else "failed", encoded({"assertion_checks": [{"name": "note_source_binding", "passed": bound}]}), stamp))
|
||
conn.execute("INSERT INTO discovery_runs(id,owner_id,candidate_id,validation_id,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6)", (discovery, owner, candidate, validation, encoded({"conditions": {"service": "装修服务", "target_customer": "有装修需求的业主", "region": ""}}), stamp))
|
||
conn.execute("INSERT INTO reading_runs(id,owner_id,discovery_run_id,candidate_id,validation_id,data_json,created_at) VALUES($1,$2,$3,$4,$5,$6,$7)", (reading, owner, discovery, candidate, validation, encoded(value), stamp))
|
||
return reading, owner
|
||
|
||
|
||
def attach(client, handler, **changes):
|
||
service = client.app_reference.state.analysis
|
||
service.client = ModelClient(config(**changes), http_transport=httpx.MockTransport(handler))
|
||
return service
|
||
|
||
|
||
def test_analysis_frozen_sources_media_privacy_usage_reuse_and_ownership(analysis_client):
|
||
client = analysis_client
|
||
reading, owner = seed(client)
|
||
requests = []
|
||
def handler(request):
|
||
assert request.headers["authorization"] == "Bearer " + PRIMARY
|
||
payload = json.loads(request.content)
|
||
assert payload["thinking"] == {"type": "disabled"} and payload["response_format"] == {"type": "json_object"}
|
||
user = json.loads(payload["messages"][1]["content"])
|
||
assert set(user) == {"business_context", "sources"} and user["business_context"]["region"] is None
|
||
assert len(user["sources"]) == 2 and all(set(source) == {"source_ref", "text", "media_uninterpreted", "context_refs"} for source in user["sources"])
|
||
assert user["sources"][0]["context_refs"] == [] and user["sources"][1]["context_refs"] == ["note"]
|
||
for forbidden in ("private-", "xiaohongshu", "a" * 24, "img.private-selector"):
|
||
assert forbidden not in request.content.decode()
|
||
requests.append(user)
|
||
content = answer(user)
|
||
content["results"][0].update(region={"value": "杭州", "evidence": "坐标杭州"}, budget={"value": "20万", "evidence": "预算20万"})
|
||
return wire(content)
|
||
service = attach(client, handler)
|
||
store = service.store
|
||
original = store.one("SELECT data_json FROM reading_runs WHERE id=$1", (reading,))
|
||
endpoint = "/api/discovery/readings/" + reading + "/analyses"
|
||
first = client.post(endpoint, json={})
|
||
assert first.status_code == 201, first.text
|
||
run = first.json()
|
||
assert run["status"] == "completed" and len(run["sources"]) == len(run["results"]) == 3 and len(requests) == 1
|
||
assert run["results"][0]["demand"] == "explicit" and run["results"][0]["region"]["value"] == "杭州"
|
||
assert run["results"][-1]["demand"] == "insufficient" and run["results"][-1]["media_uninterpreted"]
|
||
assert "identity_unconfirmed" in run["results"][0]["missing_information"]
|
||
assert run["coverage"] == {"source_count": 3, "text_chars": sum(len(source["text"]) for source in run["sources"]), "reading_status": "completed", "reading_truncated": True, "comments_available": True, "max_sources": 21, "max_text_chars": 16000}
|
||
assert run["usage"]["total_tokens"] == 140 and run["usage"]["prompt_cache_hit_tokens"] == 10
|
||
assert run["cost"] == {"status": "unknown", "amount": None, "currency": None}
|
||
assert run["attempts"][0]["slot"] == "primary" and run["attempts"][0]["status"] == "succeeded"
|
||
assert client.post(endpoint, json={}).json() == run and len(requests) == 1
|
||
# Key rotation, including removing fallback, must not invalidate content reuse.
|
||
service.client.config = replace(service.client.config, keys=(("primary", "rotated-fixture-key"),))
|
||
assert client.post(endpoint, json={}).json() == run and len(requests) == 1
|
||
assert client.get(endpoint).json() == {"items": [run], "total": 1}
|
||
assert client.get("/api/discovery/analyses/" + run["id"]).json() == run
|
||
safe_config = client.get("/api/analysis/config").json()
|
||
assert safe_config["configured"] and safe_config["key_slots"] == ["primary"]
|
||
assert "127.0.0.1:9" not in encoded(safe_config) and PRIMARY not in encoded(run) and FALLBACK not in encoded(run)
|
||
assert store.one("SELECT data_json FROM reading_runs WHERE id=$1", (reading,)) == original
|
||
assert store.one("SELECT version FROM schema_versions WHERE version=7")["version"] == 7
|
||
expected_error = StoreConflict if store.backend == "postgresql" else sqlite3.IntegrityError
|
||
for query in ("UPDATE analysis_runs SET data_json='{}' WHERE id=$1", "DELETE FROM analysis_runs WHERE id=$1"):
|
||
with pytest.raises(expected_error):
|
||
store.execute(query, (run["id"],))
|
||
with pytest.raises(HTTPException) as error:
|
||
service.get(run["id"], "other-owner")
|
||
assert error.value.status_code == 404
|
||
if store.backend == "sqlite":
|
||
assert Store(store.home).one("SELECT data_json FROM analysis_runs WHERE id=$1", (run["id"],))["data_json"] == encoded({key: value for key, value in run.items() if key not in {"id", "created_at"}})
|
||
else:
|
||
restarted = create_app(store.home, FakeWorker(store.home), database_url=client.test_database_url, database_schema=client.test_schema)
|
||
assert restarted.state.analysis.get(run["id"], owner) == run
|
||
for operation in ("TRUNCATE analysis_runs CASCADE", "TRUNCATE reading_runs CASCADE"):
|
||
with pytest.raises(StoreConflict):
|
||
store.execute(operation)
|
||
assert service.get(run["id"], owner) == run
|
||
with TestClient(client.app_reference, base_url="http://127.0.0.1:8810") as anonymous:
|
||
assert anonymous.get(endpoint).status_code == anonymous.get("/api/analysis/config").status_code == 401
|
||
assert anonymous.post(endpoint, json={}).status_code in {401, 403}
|
||
|
||
|
||
def test_analysis_failed_model_output_retained_no_key_retry_and_partial_usable(analysis_client):
|
||
client = analysis_client
|
||
reading, _ = seed(client, status="partial")
|
||
calls = []
|
||
mode = {"invalid": True}
|
||
def handler(request):
|
||
calls.append(request)
|
||
user = json.loads(json.loads(request.content)["messages"][1]["content"])
|
||
content = answer(user)
|
||
if mode["invalid"]:
|
||
content["results"][0]["evidence"] = ["原文从未提及的伪造需求"]
|
||
return wire(content)
|
||
service = attach(client, handler)
|
||
endpoint = "/api/discovery/readings/" + reading + "/analyses"
|
||
failed = client.post(endpoint, json={}).json()
|
||
assert failed["status"] == "failed" and failed["error_codes"] == ["MODEL_OUTPUT_INVALID"] and failed["results"] == []
|
||
assert len(calls) == 1 and len(failed["attempts"]) == 1 and failed["usage"]["total_tokens"] == 140
|
||
assert "伪造需求" not in encoded(failed)
|
||
# Explicit new POST after failure is a new bounded attempt; no automatic retry.
|
||
mode["invalid"] = False
|
||
completed = client.post(endpoint, json={}).json()
|
||
assert completed["status"] == "completed" and completed["id"] != failed["id"] and len(calls) == 2
|
||
assert all("reading_partial" in result["missing_information"] for result in completed["results"])
|
||
assert service.list(reading, service.store.one("SELECT id FROM operators")["id"])[-1] == failed
|
||
|
||
|
||
def test_analysis_rejects_unusable_unbound_overlimit_missing_config_and_extra_input(analysis_client):
|
||
client = analysis_client
|
||
reading, _ = seed(client, bound=False)
|
||
calls = []
|
||
service = attach(client, lambda request: calls.append(request) or wire({"results": []}))
|
||
endpoint = "/api/discovery/readings/" + reading + "/analyses"
|
||
assert client.post(endpoint, json={}).json()["detail"]["code"] == "ANALYSIS_SOURCE_UNVERIFIED"
|
||
assert client.post(endpoint, json={"key": "must-not-accept"}).status_code == 422
|
||
service.client.config = replace(service.client.config, keys=())
|
||
assert not client.get("/api/analysis/config").json()["configured"]
|
||
# Source/budget failures are deterministic and do not reach the provider.
|
||
for status in ("failed", "partial"):
|
||
with pytest.raises(HTTPException):
|
||
make_sources({"status": status, "detail": None})
|
||
with pytest.raises(HTTPException) as error:
|
||
make_sources({"status": "completed", "detail": {"items": [{"title": "装修", "body": "装" * 16001}]}})
|
||
assert error.value.detail["code"] == "ANALYSIS_INPUT_LIMIT"
|
||
with pytest.raises(HTTPException):
|
||
make_sources({"status": "completed", "detail": {"items": [{"body": "装修"}]}, "comments": {"items": [{"body": "装修"}] * 21}})
|
||
assert not calls and service.store.one("SELECT count(*) AS n FROM analysis_runs")["n"] == 0
|
||
|
||
|
||
def test_analysis_busy_and_missing_config_no_provider_call(client):
|
||
reading, owner = seed(client)
|
||
calls = []
|
||
service = attach(client, lambda request: calls.append(request) or wire({"results": []}), keys=())
|
||
endpoint = "/api/discovery/readings/" + reading + "/analyses"
|
||
result = client.post(endpoint, json={})
|
||
assert result.status_code == 503 and result.json()["detail"]["code"] == "MODEL_NOT_CONFIGURED"
|
||
service.client.config = config()
|
||
async def run():
|
||
async with service.lock:
|
||
with pytest.raises(HTTPException) as error:
|
||
await service.create(reading, owner)
|
||
assert error.value.status_code == 409 and error.value.detail["code"] == "ANALYSIS_BUSY"
|
||
asyncio.run(run())
|
||
assert not calls
|
||
|
||
|
||
@pytest.mark.parametrize("failure", [401, 402, 429, 503, "network", "timeout"])
|
||
def test_model_distinct_fallback_safe_attempts_and_usage(failure):
|
||
calls, attempts = [], []
|
||
def handler(request):
|
||
calls.append(request.headers["authorization"])
|
||
if len(calls) == 1:
|
||
if failure == "network":
|
||
raise httpx.ConnectError("wire-private-secret-sentinel", request=request)
|
||
if failure == "timeout":
|
||
raise httpx.ReadTimeout("wire-private-secret-sentinel", request=request)
|
||
return httpx.Response(failure, text="wire-private-secret-sentinel " + PRIMARY)
|
||
return wire({"results": []}, usage=False)
|
||
result = asyncio.run(ModelClient(config(), http_transport=httpx.MockTransport(handler)).complete("system", {}, attempts))
|
||
assert strict_json(result) == {"results": []} and calls == ["Bearer " + PRIMARY, "Bearer " + FALLBACK]
|
||
assert attempts[0]["status"] == "failed" and attempts[1]["status"] == "succeeded" and attempts[1]["usage"] is None
|
||
assert PRIMARY not in encoded(attempts) and "sentinel" not in encoded(attempts)
|
||
|
||
|
||
@pytest.mark.parametrize("mode", ["bad_json", "too_large", "length", "redirect", "bad_request"])
|
||
def test_model_invalid_response_never_switches_key_or_stores_response(mode):
|
||
calls, attempts = [], []
|
||
def handler(request):
|
||
calls.append(request)
|
||
if mode == "bad_json":
|
||
return httpx.Response(200, text="wire-private-secret-sentinel")
|
||
if mode == "too_large":
|
||
return httpx.Response(200, content=b"x" * 262145)
|
||
if mode == "redirect":
|
||
return httpx.Response(302, headers={"Location": "https://external.test/secret"})
|
||
if mode == "bad_request":
|
||
return httpx.Response(400, text=PRIMARY)
|
||
return wire({"results": []}, finish="length")
|
||
with pytest.raises(ModelCallError):
|
||
asyncio.run(ModelClient(config(), http_transport=httpx.MockTransport(handler)).complete("system", {}, attempts))
|
||
assert len(calls) == len(attempts) == 1 and PRIMARY not in encoded(attempts) and "sentinel" not in encoded(attempts)
|
||
|
||
|
||
def test_model_env_key_dedupe_configuration_identity_and_safe_status(monkeypatch, client):
|
||
for name, value in {"ASSISTANT_MODEL_API_KEY": PRIMARY, "ASSISTANT_MODEL_FALLBACK_API_KEY": PRIMARY, "ASSISTANT_MODEL_BASE_URL": "https://api.example.test/v1", "ASSISTANT_MODEL_NAME": "fixture-text", "ASSISTANT_MODEL_PROVIDER": "fixture", "ASSISTANT_MODEL_THINKING": "disabled", "ASSISTANT_MODEL_TIMEOUT_SECONDS": "30", "ASSISTANT_MODEL_MAX_OUTPUT_TOKENS": "4096"}.items():
|
||
monkeypatch.setenv(name, value)
|
||
first = ModelConfig.from_env()
|
||
assert first.configured and first.public()["key_slots"] == ["primary"] and PRIMARY not in repr(first)
|
||
monkeypatch.setenv("ASSISTANT_MODEL_API_KEY", FALLBACK)
|
||
second = ModelConfig.from_env()
|
||
assert second.public()["key_slots"] == ["primary", "fallback"] and first.public()["config_digest"] == second.public()["config_digest"]
|
||
monkeypatch.setenv("ASSISTANT_MODEL_BASE_URL", "https://secret:password@example.test/v1")
|
||
assert not ModelConfig.from_env().configured and "password" not in encoded(ModelConfig.from_env().public())
|
||
monkeypatch.setenv("ASSISTANT_MODEL_BASE_URL", "http://remote.example.test/v1")
|
||
assert not ModelConfig.from_env().configured
|
||
reading, _ = seed(client)
|
||
calls = []
|
||
monkeypatch.setenv("ASSISTANT_MODEL_BASE_URL", "http://127.0.0.1:9/v1")
|
||
for invalid_key in ("中文凭据", "fixture-🔑-key"):
|
||
monkeypatch.setenv("ASSISTANT_MODEL_API_KEY", invalid_key)
|
||
invalid = ModelConfig.from_env()
|
||
assert not invalid.configured
|
||
client.app_reference.state.analysis.client = ModelClient(invalid, http_transport=httpx.MockTransport(lambda request: calls.append(request) or wire({"results": []})))
|
||
response = client.post("/api/discovery/readings/" + reading + "/analyses", json={})
|
||
assert response.status_code == 503 and response.json()["detail"]["code"] == "MODEL_NOT_CONFIGURED"
|
||
assert invalid_key not in encoded(client.get("/api/analysis/config").json())
|
||
assert not calls and client.app_reference.state.store.one("SELECT count(*) AS n FROM analysis_runs")["n"] == 0
|
||
|
||
|
||
def test_model_grounding_rejects_cross_source_duplicate_unknown_unquoted_facts_and_contact():
|
||
sources = [{"source_ref": "note", "text": "我找装修公司，杭州20万", "media_uninterpreted": False}, {"source_ref": "comment_1", "text": "已经完工很好看", "media_uninterpreted": False}]
|
||
valid = answer({"sources": sources})
|
||
assert checked_results(encoded(valid), sources, "completed")[0]["demand"] == "explicit"
|
||
mutations = [lambda v: v["results"].pop(), lambda v: v["results"][1].update(source_ref="note"), lambda v: v["results"][1].update(source_ref="unknown"),
|
||
lambda v: v["results"][1].update(demand="explicit", evidence=["找装修公司"]), lambda v: v["results"][0].update(region={"value": "上海", "evidence": "杭州20万"}),
|
||
lambda v: v["results"][0].update(phone="synthetic-contact"), lambda v: v["results"][0].update(demand="explicit", evidence=[])]
|
||
for mutate in mutations:
|
||
changed = copy.deepcopy(valid)
|
||
mutate(changed)
|
||
with pytest.raises(ModelCallError) as error:
|
||
checked_results(encoded(changed), sources, "completed")
|
||
assert error.value.code == "MODEL_OUTPUT_INVALID"
|
||
with pytest.raises(ModelCallError):
|
||
checked_results('{"results":[],"results":[]}', sources, "completed")
|
||
|
||
|
||
def test_model_deep_wire_and_content_json_stay_bounded_failed_records(client):
|
||
reading, _ = seed(client)
|
||
nested = "[" * 4000 + "0" + "]" * 4000
|
||
mode = {"wire": True}
|
||
def handler(request):
|
||
return httpx.Response(200, text=nested) if mode["wire"] else httpx.Response(200, json={"model": "fixture-text", "choices": [{"finish_reason": "stop", "message": {"content": nested}}], "usage": {"total_tokens": 140}})
|
||
service = attach(client, handler)
|
||
endpoint = "/api/discovery/readings/" + reading + "/analyses"
|
||
first = client.post(endpoint, json={}).json()
|
||
assert first["status"] == "failed" and first["error_codes"] == ["MODEL_RESPONSE_INVALID"] and len(first["attempts"]) == 1
|
||
mode["wire"] = False
|
||
second = client.post(endpoint, json={}).json()
|
||
assert second["status"] == "failed" and second["error_codes"] == ["MODEL_OUTPUT_INVALID"] and second["usage"]["total_tokens"] == 140
|
||
assert len(service.list(reading, service.store.one("SELECT id FROM operators")["id"])) == 2
|
||
|
||
|
||
def test_model_total_deadline_and_per_attempt_timeout_are_bounded(monkeypatch):
|
||
from apps.server import model_client
|
||
# Leave enough scheduling headroom on Windows to enter the fallback while
|
||
# retaining a total deadline shorter than two complete per-attempt windows.
|
||
monkeypatch.setattr(model_client, "TOTAL_TIMEOUT", 0.5)
|
||
attempts, calls = [], []
|
||
async def handler(request):
|
||
calls.append(request)
|
||
await asyncio.sleep(2)
|
||
return wire({"results": []})
|
||
started = time.monotonic()
|
||
with pytest.raises(ModelCallError) as error:
|
||
asyncio.run(ModelClient(config(timeout_seconds=0.3), http_transport=httpx.MockTransport(handler)).complete("system", {}, attempts))
|
||
assert error.value.code == "MODEL_TIMEOUT" and len(calls) == len(attempts) == 2
|
||
assert all(attempt["error_code"] == "MODEL_TIMEOUT" and attempt["status"] == "failed" for attempt in attempts)
|
||
assert [attempt["slot"] for attempt in attempts] == ["primary", "fallback"]
|
||
assert sum(attempt["duration_ms"] for attempt in attempts) < 900
|
||
assert time.monotonic() - started < 0.9
|