"""Fixed output-contract failure stages without model text or record backfill.""" from __future__ import annotations import copy import json import pytest from apps.server.analysis import MODEL_CONTRACT_CHECKS, checked_results, model_output_diagnostics from apps.server.db import encoded, identifier, now from apps.server.model_client import ModelCallError from apps.server.tests.test_analysis import answer, attach, seed, wire from apps.server.tests.test_learning import client def test_base_diagnostics_cover_fixed_stages_without_changing_the_contract(): sources = [{"source_ref": "note", "text": "我正在找装修公司", "media_uninterpreted": False}] original = answer({"sources": sources}) assert checked_results(encoded(original), sources, "completed")[0]["demand"] == "explicit" cases = [ ("base_payload", lambda value: value.update(raw_payload="untrusted-model-text")), ("base_coverage", lambda value: value["results"].clear()), ("base_fields", lambda value: value["results"][0].update(raw_field="untrusted-model-text")), ("base_source", lambda value: value["results"][0].update(source_ref="untrusted-model-text")), ("base_evidence", lambda value: value["results"][0].update(evidence=["untrusted-model-text"])), ("base_role", lambda value: value["results"][0].update(role="untrusted-model-text")), ("base_expressions", lambda value: value["results"][0].update(expressions=[])), ("base_need", lambda value: value["results"][0].update(demand="discussion")), ("base_relevance", lambda value: value["results"][0]["business_relevance"].update(value="untrusted-model-text")), ("base_facts", lambda value: value["results"][0].update(region={"value": "不在原文", "evidence": "我正在找装修公司"})), ] for reason, mutate in cases: value = copy.deepcopy(original) mutate(value) with pytest.raises(ModelCallError) as raised: checked_results(encoded(value), sources, "completed") error = raised.value assert error.code == "MODEL_OUTPUT_INVALID" and error.contract_check == reason assert str(error) == "MODEL_OUTPUT_INVALID" and "untrusted-model-text" not in repr(error) assert model_output_diagnostics(error) == {"check": reason} for value in ('{"results": [], "results": []}', "untrusted-model-text"): with pytest.raises(ModelCallError) as raised: checked_results(value, sources, "completed") assert model_output_diagnostics(raised.value) == {"check": "base_payload"} assert original == answer({"sources": sources}) def test_diagnostic_filter_only_serializes_allowlisted_stage_for_output_errors(): assert len(MODEL_CONTRACT_CHECKS) == 16 for check in MODEL_CONTRACT_CHECKS: error = ModelCallError("MODEL_OUTPUT_INVALID") error.contract_check = check error.raw = "untrusted-model-text" error.source_ref = "comment_private" error.url = "https://private.invalid" assert model_output_diagnostics(error) == {"check": check} for check in (None, "", "unknown", "https://private.invalid", {"check": "base_fields"}, ["base_fields"], True, 1): error = ModelCallError("MODEL_OUTPUT_INVALID") error.contract_check = check assert model_output_diagnostics(error) is None error = ModelCallError("MODEL_NETWORK") error.contract_check = "base_fields" assert model_output_diagnostics(error) is None assert model_output_diagnostics(ModelCallError("MODEL_OUTPUT_INVALID")) is None def test_service_appends_safe_stage_without_raw_or_backfilling_old_failure(client): reading_id, owner = seed(client) calls = [] mode = {"invalid": True} def handler(request): payload = json.loads(json.loads(request.content)["messages"][1]["content"]) value = answer(payload) if mode["invalid"]: value["results"][0]["evidence"] = ["raw-model-only-secret-sentinel"] calls.append(value) return wire(value) service = attach(client, handler) previous_id = identifier("analysis") previous = {"status": "failed", "reading_id": reading_id, "prompt_version": "renovation-demand-v4", "result_version": "demand-result-v3", "error_codes": ["MODEL_OUTPUT_INVALID"], "results": []} service.store.execute("INSERT INTO analysis_runs(id,owner_id,reading_run_id,data_json,created_at) VALUES($1,$2,$3,$4,$5)", (previous_id, owner, reading_id, encoded(previous), now())) before = service.store.one("SELECT data_json FROM analysis_runs WHERE id=$1", (previous_id,)) endpoint = f"/api/discovery/readings/{reading_id}/analyses" failed_response = client.post(endpoint, json={}) assert failed_response.status_code == 201 failed = failed_response.json() assert failed["status"] == "failed" and failed["error_codes"] == ["MODEL_OUTPUT_INVALID"] assert failed["output_diagnostics"] == {"check": "base_evidence"} and failed["results"] == [] assert len(failed["attempts"]) == len(calls) == 1 and failed["attempts"][0]["status"] == "failed" assert failed["attempts"][0]["error_code"] == "MODEL_OUTPUT_INVALID" assert "raw-model-only-secret-sentinel" not in encoded(failed) assert service.get(failed["id"], owner) == failed mode["invalid"] = False completed = client.post(endpoint, json={}).json() assert completed["status"] == "completed" and "output_diagnostics" not in completed and len(calls) == 2 assert client.post(endpoint, json={}).json() == completed and len(calls) == 2 assert service.store.one("SELECT data_json FROM analysis_runs WHERE id=$1", (previous_id,)) == before assert "output_diagnostics" not in service.get(previous_id, owner) assert service.get(failed["id"], owner) == failed and client.worker_reference.calls == []