219 lines
11 KiB
Python
219 lines
11 KiB
Python
"""试用制：草稿也注入（降一级），Agent 的回执当裁判，被证实的升级、被证伪的降级。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import pytest
|
||
|
||
from backend.orchestrator import orchestrator as orc
|
||
from backend.orchestrator.orchestrator import parse_trial_receipt
|
||
|
||
|
||
class TestLearnedBlock:
|
||
KNOWLEDGE = {"learned": {"门户结构": "有 8 个一级模块"}}
|
||
TRIALS = [{"id": 42, "key": "提交入口", "value": "提交按钮在表单底部，叫「提交填报」"}]
|
||
|
||
def test_verified_and_trial_are_separate_sections(self):
|
||
block = orc._learned_block(self.KNOWLEDGE, {}, self.TRIALS)
|
||
assert "已知信息（已确认" in block and "待验证线索" in block
|
||
|
||
def test_trial_entries_carry_their_id_for_the_receipt(self):
|
||
assert "[42]" in orc._learned_block(self.KNOWLEDGE, {}, self.TRIALS)
|
||
|
||
def test_page_always_wins_over_a_trial_hint(self):
|
||
# 错的线索会主动帮倒忙，所以必须写死「以页面为准」
|
||
block = orc._learned_block(self.KNOWLEDGE, {}, self.TRIALS)
|
||
assert "一律以页面为准" in block
|
||
|
||
def test_receipt_format_is_spelled_out(self):
|
||
block = orc._learned_block(self.KNOWLEDGE, {}, self.TRIALS)
|
||
assert "线索回执" in block and "相符" in block and "不符" in block and "没用上" in block
|
||
|
||
def test_no_trials_means_no_trial_section(self):
|
||
assert "待验证线索" not in orc._learned_block(self.KNOWLEDGE, {}, [])
|
||
|
||
def test_no_knowledge_at_all_produces_nothing(self):
|
||
assert orc._learned_block({}, {}, []) == ""
|
||
|
||
def test_trials_alone_still_render(self):
|
||
assert "待验证线索" in orc._learned_block({}, {}, self.TRIALS)
|
||
|
||
|
||
class TestReceiptParsing:
|
||
ALLOWED = {7, 42, 99}
|
||
|
||
def test_parses_the_documented_format(self):
|
||
text = "任务完成。\n\n线索回执：\n线索[42] 不符 提交按钮实际叫「保存并提交」\n线索[7] 相符 班级确实是下拉框"
|
||
assert parse_trial_receipt(text, self.ALLOWED) == [
|
||
{"id": 42, "outcome": "contradicted", "note": "提交按钮实际叫「保存并提交」"},
|
||
{"id": 7, "outcome": "confirmed", "note": "班级确实是下拉框"},
|
||
]
|
||
|
||
@pytest.mark.parametrize("line,outcome", [
|
||
("线索[7] 相符 对的", "confirmed"),
|
||
("线索[7]：相符，对的", "confirmed"),
|
||
("线索7 不符 错的", "contradicted"),
|
||
("线索[7] 没用上 没去到那页", "unused"),
|
||
("线索[7] 未用上 没去到那页", "unused"),
|
||
])
|
||
def test_tolerates_punctuation_and_wording_variants(self, line, outcome):
|
||
parsed = parse_trial_receipt(line, self.ALLOWED)
|
||
assert parsed and parsed[0]["outcome"] == outcome
|
||
|
||
def test_ids_that_were_never_injected_are_ignored(self):
|
||
# 防止模型凭空编一个编号出来，把没注入过的条目改掉
|
||
assert parse_trial_receipt("线索[500] 相符 编的", self.ALLOWED) == []
|
||
|
||
def test_later_lines_win_for_the_same_id(self):
|
||
text = "线索[7] 相符 先说对\n线索[7] 不符 后来发现不对"
|
||
assert parse_trial_receipt(text, self.ALLOWED)[0]["outcome"] == "contradicted"
|
||
|
||
@pytest.mark.parametrize("text", ["", "任务完成，没有回执", "线索很有用但没按格式写"])
|
||
def test_no_receipt_is_not_an_error(self, text):
|
||
assert parse_trial_receipt(text, self.ALLOWED) == []
|
||
|
||
def test_receipt_mixed_into_a_long_summary(self):
|
||
text = "【任务结果】已填未提交。\n\n一、操作过程\n1. 打开系统\n2. 填表\n\n【线索回执】\n线索[42] 相符 确实如此\n"
|
||
assert len(parse_trial_receipt(text, self.ALLOWED)) == 1
|
||
|
||
|
||
@pytest.mark.db
|
||
class TestTrialLifecycle:
|
||
KID, BID = "pytest_trial", "b1"
|
||
|
||
def _draft(self, db, key: str, value: str, category: str = "control_tip"):
|
||
db.add_learned_drafts(self.KID, self.BID, [{"key": key, "value": value, "category": category}], "t0")
|
||
return db.list_learned(knowledge_id=self.KID, business_id=self.BID)[0]
|
||
|
||
def test_drafts_are_offered_as_trials(self, test_school):
|
||
from backend.db import database as db
|
||
|
||
self._draft(db, "提交入口", "提交按钮在表单底部，叫「提交填报」，不要点保存草稿")
|
||
trials = db.trial_learned(self.KID, self.BID, 8)
|
||
assert len(trials) == 1 and trials[0]["status"] == "draft"
|
||
|
||
def test_verified_entries_are_not_offered_again(self, test_school):
|
||
from backend.db import database as db
|
||
|
||
entry = self._draft(db, "提交入口", "提交按钮在表单底部，叫「提交填报」，不要点保存草稿")
|
||
db.set_learned_status(entry["id"], "verified")
|
||
assert db.trial_learned(self.KID, self.BID, 8) == []
|
||
|
||
# 内容必须彼此差异够大，否则会被内容判重合并成一条（那是 add_learned_drafts 的正常行为）
|
||
DISTINCT = [
|
||
("班级控件", "班级是下拉框，点开后从选项里选择对应班级"),
|
||
("提交入口", "表单底部有「提交填报」按钮，旁边是保存草稿"),
|
||
("日期输入", "听课时间支持直接键入 yyyy-mm-dd 后回车确认"),
|
||
("菜单路径", "门户左侧导航进入辅导员管理再到工作日志"),
|
||
("附件上传", "相关附件一栏支持拖拽，也可点击选择文件"),
|
||
("审批状态", "列表最右列展示当前审批进度和处理人"),
|
||
]
|
||
|
||
def test_injection_is_capped(self, test_school):
|
||
from backend.db import database as db
|
||
|
||
for i, (key, value) in enumerate(self.DISTINCT):
|
||
db.add_learned_drafts(self.KID, self.BID, [{"key": key, "value": value, "category": "path"}], f"t{i}")
|
||
assert len(db.list_learned(knowledge_id=self.KID, business_id=self.BID)) == len(self.DISTINCT)
|
||
assert len(db.trial_learned(self.KID, self.BID, 3)) == 3
|
||
|
||
def test_contradicted_entries_sort_last(self, test_school):
|
||
from backend.db import database as db
|
||
|
||
good = self._draft(db, "班级控件", self.DISTINCT[0][1])
|
||
db.add_learned_drafts(self.KID, self.BID,
|
||
[{"key": "菜单路径", "value": self.DISTINCT[3][1], "category": "path"}], "t2")
|
||
bad = next(r for r in db.list_learned(knowledge_id=self.KID, business_id=self.BID) if r["key"] == "菜单路径")
|
||
db.record_trials("task_x", [bad["id"]])
|
||
db.apply_trial_outcomes("task_x", [{"id": bad["id"], "outcome": "contradicted", "note": "页面上不是这样"}], 2, 9)
|
||
order = [r["id"] for r in db.trial_learned(self.KID, self.BID, 8)]
|
||
assert order.index(good["id"]) < order.index(bad["id"])
|
||
|
||
def test_two_confirmations_promote(self, test_school):
|
||
from backend.db import database as db
|
||
|
||
entry = self._draft(db, "提交入口", "提交按钮在表单底部，叫「提交填报」，不要点保存草稿")
|
||
for task in ("task_1", "task_2"):
|
||
db.record_trials(task, [entry["id"]])
|
||
stats = db.apply_trial_outcomes(task, [{"id": entry["id"], "outcome": "confirmed", "note": "页面确实如此"}], 2, 1)
|
||
assert stats["promoted"] == 1
|
||
row = db.get_learned(entry["id"])
|
||
assert row["status"] == "verified" and row["verified_by"] == "trial" and row["confirmed"] == 2
|
||
assert "提交入口" in db.verified_learned(self.KID, self.BID)
|
||
|
||
def test_one_contradiction_demotes(self, test_school):
|
||
from backend.db import database as db
|
||
|
||
entry = self._draft(db, "错的经验", "这条说法与页面实际不符，应该被降级")
|
||
db.record_trials("task_1", [entry["id"]])
|
||
stats = db.apply_trial_outcomes("task_1", [{"id": entry["id"], "outcome": "contradicted", "note": "实际叫别的名字"}], 2, 1)
|
||
assert stats["demoted"] == 1
|
||
assert db.get_learned(entry["id"])["status"] == "rejected"
|
||
assert db.trial_learned(self.KID, self.BID, 8) == [] # 不再注入
|
||
|
||
def test_a_contradiction_blocks_promotion_even_with_confirmations(self, test_school):
|
||
from backend.db import database as db
|
||
|
||
entry = self._draft(db, "时灵时不灵", "这条有时对有时不对，不该升级")
|
||
db.apply_trial_outcomes("t1", [{"id": entry["id"], "outcome": "confirmed", "note": "对"}], 2, 9)
|
||
db.apply_trial_outcomes("t2", [{"id": entry["id"], "outcome": "contradicted", "note": "不对"}], 2, 9)
|
||
db.apply_trial_outcomes("t3", [{"id": entry["id"], "outcome": "confirmed", "note": "又对了"}], 2, 9)
|
||
row = db.get_learned(entry["id"])
|
||
assert row["status"] == "draft" and row["confirmed"] == 2 and row["contradicted"] == 1
|
||
|
||
def test_unused_changes_nothing(self, test_school):
|
||
from backend.db import database as db
|
||
|
||
entry = self._draft(db, "没用上的", "这次任务没走到相关页面，不算证据")
|
||
db.apply_trial_outcomes("t1", [{"id": entry["id"], "outcome": "unused", "note": "没去到那页"}], 2, 1)
|
||
row = db.get_learned(entry["id"])
|
||
assert row["status"] == "draft" and row["confirmed"] == 0 and row["contradicted"] == 0
|
||
|
||
def test_trial_history_is_traceable(self, test_school):
|
||
from backend.db import database as db
|
||
|
||
entry = self._draft(db, "可追溯", "要能查出这条是被哪次任务证实的")
|
||
db.apply_trial_outcomes("task_abc", [{"id": entry["id"], "outcome": "confirmed", "note": "页面确实如此"}], 2, 1)
|
||
history = db.trial_history(entry["id"])
|
||
assert history and history[0]["task_id"] == "task_abc" and history[0]["outcome"] == "confirmed"
|
||
|
||
def test_same_task_reporting_twice_does_not_double_count(self, test_school):
|
||
from backend.db import database as db
|
||
|
||
entry = self._draft(db, "重复回执", "同一个任务重复给回执不该算两次证据")
|
||
db.apply_trial_outcomes("t1", [{"id": entry["id"], "outcome": "confirmed", "note": "对"}], 9, 9)
|
||
db.apply_trial_outcomes("t1", [{"id": entry["id"], "outcome": "confirmed", "note": "对"}], 9, 9)
|
||
history = db.trial_history(entry["id"])
|
||
assert len(history) == 1
|
||
|
||
def test_unknown_entry_ids_are_ignored(self, test_school):
|
||
from backend.db import database as db
|
||
|
||
stats = db.apply_trial_outcomes("t1", [{"id": 99999999, "outcome": "confirmed"}], 2, 1)
|
||
assert stats["promoted"] == 0 and stats["confirmed"] == 0
|
||
|
||
def test_malformed_outcomes_are_ignored(self, test_school):
|
||
from backend.db import database as db
|
||
|
||
entry = self._draft(db, "格式乱的", "回执格式不对时不能改坏数据")
|
||
db.apply_trial_outcomes("t1", [{"id": entry["id"], "outcome": "编的状态"}, {"id": "abc"}, {}], 2, 1)
|
||
row = db.get_learned(entry["id"])
|
||
assert row["confirmed"] == 0 and row["contradicted"] == 0
|
||
|
||
|
||
class TestTrialSwitch:
|
||
def test_can_be_turned_off(self, monkeypatch, knowledge, business):
|
||
from backend.config import settings
|
||
from backend.orchestrator.orchestrator import Orchestrator
|
||
|
||
monkeypatch.setattr(settings, "trial_enabled", False)
|
||
assert Orchestrator._pick_trials(knowledge, business) == []
|
||
|
||
def test_database_failure_does_not_break_execution(self, monkeypatch, knowledge, business):
|
||
from backend.config import settings
|
||
from backend.db import database as db
|
||
from backend.orchestrator.orchestrator import Orchestrator
|
||
|
||
monkeypatch.setattr(settings, "trial_enabled", True)
|
||
monkeypatch.setattr(db, "trial_learned", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("库挂了")))
|
||
assert Orchestrator._pick_trials(knowledge, business) == []
|