102 lines
5.8 KiB
Python
102 lines
5.8 KiB
Python
"""Fixed output-contract failure stages without model text or record backfill."""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from apps.server.analysis import MODEL_CONTRACT_CHECKS, checked_results, model_output_diagnostics
|
|
from apps.server.db import encoded, identifier, now
|
|
from apps.server.model_client import ModelCallError
|
|
from apps.server.tests.test_analysis import answer, attach, seed, wire
|
|
from apps.server.tests.test_learning import client
|
|
|
|
|
|
def test_base_diagnostics_cover_fixed_stages_without_changing_the_contract():
|
|
sources = [{"source_ref": "note", "text": "我正在找装修公司", "media_uninterpreted": False}]
|
|
original = answer({"sources": sources})
|
|
assert checked_results(encoded(original), sources, "completed")[0]["demand"] == "explicit"
|
|
cases = [
|
|
("base_payload", lambda value: value.update(raw_payload="untrusted-model-text")),
|
|
("base_coverage", lambda value: value["results"].clear()),
|
|
("base_fields", lambda value: value["results"][0].update(raw_field="untrusted-model-text")),
|
|
("base_source", lambda value: value["results"][0].update(source_ref="untrusted-model-text")),
|
|
("base_evidence", lambda value: value["results"][0].update(evidence=["untrusted-model-text"])),
|
|
("base_role", lambda value: value["results"][0].update(role="untrusted-model-text")),
|
|
("base_expressions", lambda value: value["results"][0].update(expressions=[])),
|
|
("base_need", lambda value: value["results"][0].update(demand="discussion")),
|
|
("base_relevance", lambda value: value["results"][0]["business_relevance"].update(value="untrusted-model-text")),
|
|
("base_facts", lambda value: value["results"][0].update(region={"value": "不在原文", "evidence": "我正在找装修公司"})),
|
|
]
|
|
for reason, mutate in cases:
|
|
value = copy.deepcopy(original)
|
|
mutate(value)
|
|
with pytest.raises(ModelCallError) as raised:
|
|
checked_results(encoded(value), sources, "completed")
|
|
error = raised.value
|
|
assert error.code == "MODEL_OUTPUT_INVALID" and error.contract_check == reason
|
|
assert str(error) == "MODEL_OUTPUT_INVALID" and "untrusted-model-text" not in repr(error)
|
|
assert model_output_diagnostics(error) == {"check": reason}
|
|
for value in ('{"results": [], "results": []}', "untrusted-model-text"):
|
|
with pytest.raises(ModelCallError) as raised:
|
|
checked_results(value, sources, "completed")
|
|
assert model_output_diagnostics(raised.value) == {"check": "base_payload"}
|
|
assert original == answer({"sources": sources})
|
|
|
|
|
|
def test_diagnostic_filter_only_serializes_allowlisted_stage_for_output_errors():
|
|
assert len(MODEL_CONTRACT_CHECKS) == 16
|
|
for check in MODEL_CONTRACT_CHECKS:
|
|
error = ModelCallError("MODEL_OUTPUT_INVALID")
|
|
error.contract_check = check
|
|
error.raw = "untrusted-model-text"
|
|
error.source_ref = "comment_private"
|
|
error.url = "https://private.invalid"
|
|
assert model_output_diagnostics(error) == {"check": check}
|
|
for check in (None, "", "unknown", "https://private.invalid", {"check": "base_fields"}, ["base_fields"], True, 1):
|
|
error = ModelCallError("MODEL_OUTPUT_INVALID")
|
|
error.contract_check = check
|
|
assert model_output_diagnostics(error) is None
|
|
error = ModelCallError("MODEL_NETWORK")
|
|
error.contract_check = "base_fields"
|
|
assert model_output_diagnostics(error) is None
|
|
assert model_output_diagnostics(ModelCallError("MODEL_OUTPUT_INVALID")) is None
|
|
|
|
|
|
def test_service_appends_safe_stage_without_raw_or_backfilling_old_failure(client):
|
|
reading_id, owner = seed(client)
|
|
calls = []
|
|
mode = {"invalid": True}
|
|
def handler(request):
|
|
payload = json.loads(json.loads(request.content)["messages"][1]["content"])
|
|
value = answer(payload)
|
|
if mode["invalid"]:
|
|
value["results"][0]["evidence"] = ["raw-model-only-secret-sentinel"]
|
|
calls.append(value)
|
|
return wire(value)
|
|
service = attach(client, handler)
|
|
previous_id = identifier("analysis")
|
|
previous = {"status": "failed", "reading_id": reading_id, "prompt_version": "renovation-demand-v4",
|
|
"result_version": "demand-result-v3", "error_codes": ["MODEL_OUTPUT_INVALID"], "results": []}
|
|
service.store.execute("INSERT INTO analysis_runs(id,owner_id,reading_run_id,data_json,created_at) VALUES($1,$2,$3,$4,$5)",
|
|
(previous_id, owner, reading_id, encoded(previous), now()))
|
|
before = service.store.one("SELECT data_json FROM analysis_runs WHERE id=$1", (previous_id,))
|
|
endpoint = f"/api/discovery/readings/{reading_id}/analyses"
|
|
failed_response = client.post(endpoint, json={})
|
|
assert failed_response.status_code == 201
|
|
failed = failed_response.json()
|
|
assert failed["status"] == "failed" and failed["error_codes"] == ["MODEL_OUTPUT_INVALID"]
|
|
assert failed["output_diagnostics"] == {"check": "base_evidence"} and failed["results"] == []
|
|
assert len(failed["attempts"]) == len(calls) == 1 and failed["attempts"][0]["status"] == "failed"
|
|
assert failed["attempts"][0]["error_code"] == "MODEL_OUTPUT_INVALID"
|
|
assert "raw-model-only-secret-sentinel" not in encoded(failed)
|
|
assert service.get(failed["id"], owner) == failed
|
|
mode["invalid"] = False
|
|
completed = client.post(endpoint, json={}).json()
|
|
assert completed["status"] == "completed" and "output_diagnostics" not in completed and len(calls) == 2
|
|
assert client.post(endpoint, json={}).json() == completed and len(calls) == 2
|
|
assert service.store.one("SELECT data_json FROM analysis_runs WHERE id=$1", (previous_id,)) == before
|
|
assert "output_diagnostics" not in service.get(previous_id, owner)
|
|
assert service.get(failed["id"], owner) == failed and client.worker_reference.calls == []
|