266 lines
12 KiB
Python
266 lines
12 KiB
Python
"""智能采纳：什么条件下才敢自动动手，以及自动驳回不能像人工驳回那样是终局。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import uuid
|
||
|
||
import pytest
|
||
|
||
from server import autoreview, review
|
||
from server.autoreview import Policy
|
||
from server.db import database as hub
|
||
|
||
|
||
def entry(**over) -> dict:
|
||
return {
|
||
"id": 1,
|
||
"knowledge_id": "k",
|
||
"business_id": "b",
|
||
"key": "班级下拉框",
|
||
"value": "班级是下拉框，要先点开再从选项里选",
|
||
"category": "control_tip",
|
||
"school_count": 1,
|
||
"observations": 1,
|
||
**over,
|
||
}
|
||
|
||
|
||
ADOPT = ("adopt", "跨校成立")
|
||
REVIEW = ("review", "证据不足")
|
||
REJECT = ("reject", "内容太短")
|
||
|
||
|
||
@pytest.fixture
|
||
def no_llm(monkeypatch):
|
||
"""没配模型 Key：只有规则层说话。"""
|
||
from server.config import settings
|
||
|
||
monkeypatch.setattr(settings, "llm_api_key", "")
|
||
|
||
|
||
@pytest.fixture
|
||
def with_llm(monkeypatch):
|
||
from server.config import settings
|
||
|
||
monkeypatch.setattr(settings, "llm_api_key", "sk-test")
|
||
|
||
|
||
class TestCategoryGate:
|
||
"""系统规则和其他类只有一条路能过：Agent 在真实页面上验过。靠猜的一律不放行。"""
|
||
|
||
@pytest.mark.parametrize("category", ["rule", "note"])
|
||
def test_guessing_never_unlocks_a_high_risk_category(self, with_llm, category):
|
||
# 跨校观察 + 模型点头都只是「反复想到」，不是「去页面上对过」
|
||
plan = autoreview.decide(entry(category=category, school_count=5, observations=99),
|
||
ADOPT, ("adopt", "模型很确定"), Policy())
|
||
assert plan["action"] == "hold" and "必须人工确认" in plan["reason"]
|
||
|
||
@pytest.mark.parametrize("category", ["rule", "note"])
|
||
def test_real_page_verification_unlocks_it(self, no_llm, category):
|
||
plan = autoreview.decide(entry(category=category, confirmed=2), REVIEW, None, Policy())
|
||
assert plan["action"] == "verify"
|
||
assert plan["scope"] == "origin" # 实测只在本校做过，别急着推给别人
|
||
assert "真实页面上验证相符" in plan["reason"]
|
||
|
||
def test_one_verification_is_not_enough(self, no_llm):
|
||
plan = autoreview.decide(entry(category="rule", confirmed=1), REVIEW, None, Policy())
|
||
assert plan["action"] == "hold"
|
||
|
||
def test_a_single_contradiction_cancels_all_verifications(self, no_llm):
|
||
plan = autoreview.decide(entry(category="rule", confirmed=5, contradicted=1), REVIEW, None, Policy())
|
||
assert plan["action"] == "hold"
|
||
|
||
|
||
class TestTrialEvidence:
|
||
"""实测证据是最硬的，排在跨校观察之前。"""
|
||
|
||
def test_trial_beats_thin_observation_counts(self, no_llm):
|
||
plan = autoreview.decide(entry(school_count=1, observations=1, confirmed=2), REVIEW, None, Policy())
|
||
assert plan["action"] == "verify" and "真实页面上验证相符 2 次" in plan["reason"]
|
||
|
||
def test_trial_proven_helper(self):
|
||
assert autoreview.trial_proven({"confirmed": 2, "contradicted": 0}, Policy()) is True
|
||
assert autoreview.trial_proven({"confirmed": 2, "contradicted": 1}, Policy()) is False
|
||
assert autoreview.trial_proven({"confirmed": 1, "contradicted": 0}, Policy()) is False
|
||
|
||
def test_threshold_is_configurable(self):
|
||
assert autoreview.trial_proven({"confirmed": 1, "contradicted": 0}, Policy(min_trials=1)) is True
|
||
|
||
@pytest.mark.parametrize("category", ["control_tip", "path"])
|
||
def test_low_risk_categories_can_be_adopted(self, no_llm, category):
|
||
plan = autoreview.decide(entry(category=category, school_count=2), ADOPT, None, Policy())
|
||
assert plan["action"] == "verify"
|
||
|
||
|
||
class TestEvidencePath:
|
||
"""A 路：证据够硬。"""
|
||
|
||
def test_cross_school_adopts_and_publishes_to_everyone(self, no_llm):
|
||
plan = autoreview.decide(entry(school_count=2), ADOPT, None, Policy())
|
||
assert plan["action"] == "verify" and plan["scope"] == "all"
|
||
|
||
def test_single_school_with_enough_observations_stays_local(self, no_llm):
|
||
plan = autoreview.decide(entry(observations=3), ADOPT, None, Policy())
|
||
assert plan["action"] == "verify" and plan["scope"] == "origin"
|
||
|
||
def test_thin_evidence_without_a_model_is_held(self, no_llm):
|
||
plan = autoreview.decide(entry(), REVIEW, None, Policy())
|
||
assert plan["action"] == "hold" and "未配模型 Key" in plan["reason"]
|
||
|
||
def test_the_model_can_veto_strong_evidence(self, with_llm):
|
||
plan = autoreview.decide(entry(school_count=5), ADOPT, ("review", "表述含糊"), Policy(require_llm=True))
|
||
assert plan["action"] == "hold" and "模型不认可" in plan["reason"]
|
||
|
||
def test_veto_can_be_turned_off(self, with_llm):
|
||
plan = autoreview.decide(entry(school_count=5), ADOPT, ("review", "表述含糊"), Policy(require_llm=False))
|
||
assert plan["action"] == "verify"
|
||
|
||
|
||
class TestModelEndorsementPath:
|
||
"""B 路：证据薄但模型背书，只发回来源学校。"""
|
||
|
||
def test_model_endorsement_substitutes_for_observations(self, with_llm):
|
||
plan = autoreview.decide(entry(), REVIEW, ("adopt", "可复用的操作知识"), Policy())
|
||
assert plan["action"] == "verify"
|
||
assert plan["scope"] == "origin" # 没有跨校证据就不往别的学校推
|
||
assert "证据尚薄" in plan["reason"]
|
||
|
||
def test_model_endorsement_cannot_bypass_the_category_gate(self, with_llm):
|
||
plan = autoreview.decide(entry(category="rule"), REVIEW, ("adopt", "很有用"), Policy())
|
||
assert plan["action"] == "hold"
|
||
|
||
def test_a_rewrite_suggestion_holds_instead_of_adopting_the_original(self, with_llm):
|
||
reason = f"内容可复用\n{review.REWRITE_MARKER}改写后的更清晰表述"
|
||
plan = autoreview.decide(entry(), REVIEW, ("adopt", reason), Policy())
|
||
assert plan["action"] == "hold" and "建议改写" in plan["reason"]
|
||
|
||
def test_model_silence_holds(self, with_llm):
|
||
plan = autoreview.decide(entry(), REVIEW, ("review", "拿不准"), Policy())
|
||
assert plan["action"] == "hold"
|
||
|
||
|
||
class TestAutoReject:
|
||
def test_both_layers_agree_it_is_junk(self, no_llm):
|
||
plan = autoreview.decide(entry(value="太短"), REJECT, None, Policy())
|
||
assert plan["action"] == "reject"
|
||
|
||
def test_model_alone_can_flag_one_off_data(self, with_llm):
|
||
# 规则层漏掉的：长 URL 里带本次任务的 UUID，看起来像入口路径
|
||
plan = autoreview.decide(entry(category="path", value="https://example.edu/x/ff6be301-0fa3-4fc7"),
|
||
REVIEW, ("reject", "带 UUID 的一次性链接"), Policy())
|
||
assert plan["action"] == "reject"
|
||
|
||
def test_rule_adopt_blocks_a_model_rejection(self, with_llm):
|
||
# 两层意见相反时不动手
|
||
plan = autoreview.decide(entry(school_count=3), ADOPT, ("reject", "我觉得是重复"), Policy())
|
||
assert plan["action"] == "hold" and "模型不认可" in plan["reason"]
|
||
|
||
def test_auto_reject_can_be_turned_off(self, no_llm):
|
||
plan = autoreview.decide(entry(value="太短"), REJECT, None, Policy(auto_reject=False))
|
||
assert plan["action"] == "hold" and "自动驳回已关闭" in plan["reason"]
|
||
|
||
|
||
class TestPolicyThresholds:
|
||
def test_raising_the_cross_school_bar(self, no_llm):
|
||
assert autoreview.decide(entry(school_count=2), ADOPT, None, Policy(min_schools=3))["action"] == "hold"
|
||
|
||
def test_lowering_the_observation_bar(self, no_llm):
|
||
assert autoreview.decide(entry(observations=1), ADOPT, None, Policy(min_observations=1))["action"] == "verify"
|
||
|
||
def test_policy_reads_settings(self, monkeypatch):
|
||
from server.config import settings
|
||
|
||
monkeypatch.setattr(settings, "auto_min_schools", 4)
|
||
monkeypatch.setattr(settings, "auto_max_actions", 7)
|
||
policy = Policy.from_settings()
|
||
assert policy.min_schools == 4 and policy.max_actions == 7
|
||
|
||
|
||
@pytest.fixture
|
||
def hub_scope():
|
||
knowledge_id = f"pytest_{uuid.uuid4().hex[:8]}"
|
||
yield knowledge_id
|
||
with hub.get_conn() as conn:
|
||
hub._execute(conn, "DELETE FROM hub_entries WHERE knowledge_id = %s", (knowledge_id,))
|
||
hub._execute(conn, "DELETE FROM sync_log WHERE school_id LIKE 'pytest_%%'")
|
||
|
||
|
||
def draft(knowledge_id: str, key: str, value: str, category: str = "control_tip") -> dict:
|
||
return {"knowledge_id": knowledge_id, "business_id": "b1", "key": key, "value": value,
|
||
"category": category, "observations": 1, "source_task_id": "t1"}
|
||
|
||
|
||
@pytest.mark.db
|
||
class TestAutoRejectIsProvisional:
|
||
"""人工驳回是终局，自动驳回只是暂缓——机器误判不能永久吃掉一条知识。"""
|
||
|
||
VALUE = "这条内容会被驳回，用来验证自动驳回和人工驳回的区别"
|
||
|
||
def test_human_rejection_is_final(self, hub_scope):
|
||
hub.receive_drafts("pytest_a", [draft(hub_scope, "k", self.VALUE)])
|
||
entry_id = hub.list_entries(knowledge_id=hub_scope)[0]["id"]
|
||
hub.review_entry(entry_id, "reject", by="human")
|
||
stats = hub.receive_drafts("pytest_a", [draft(hub_scope, "k", self.VALUE)])
|
||
assert stats["skipped"] == 1 and stats["revived"] == 0
|
||
assert hub.get_entry(entry_id)["status"] == "rejected"
|
||
|
||
def test_auto_rejection_is_revived_when_observed_again(self, hub_scope):
|
||
hub.receive_drafts("pytest_a", [draft(hub_scope, "k", self.VALUE)])
|
||
entry_id = hub.list_entries(knowledge_id=hub_scope)[0]["id"]
|
||
hub.review_entry(entry_id, "reject", by="auto")
|
||
stats = hub.receive_drafts("pytest_b", [draft(hub_scope, "k", self.VALUE)])
|
||
assert stats["revived"] == 1 and stats["skipped"] == 0
|
||
row = hub.get_entry(entry_id)
|
||
assert row["status"] == "draft" and row["reviewed_by"] is None and row["version"] == 0
|
||
|
||
def test_a_revived_entry_keeps_its_cross_school_evidence(self, hub_scope):
|
||
hub.receive_drafts("pytest_a", [draft(hub_scope, "k", self.VALUE)])
|
||
entry_id = hub.list_entries(knowledge_id=hub_scope)[0]["id"]
|
||
hub.review_entry(entry_id, "reject", by="auto")
|
||
hub.receive_drafts("pytest_b", [draft(hub_scope, "k", self.VALUE)])
|
||
assert hub.get_entry(entry_id)["school_count"] == 2
|
||
|
||
|
||
@pytest.mark.db
|
||
class TestAutoReviewRun:
|
||
def test_dry_run_changes_nothing(self, hub_scope, monkeypatch):
|
||
from server.config import settings
|
||
|
||
monkeypatch.setattr(settings, "llm_api_key", "")
|
||
hub.receive_drafts("pytest_a", [draft(hub_scope, "菜单结构", "门户有 8 个一级模块，入口在左侧导航栏")])
|
||
hub.receive_drafts("pytest_b", [draft(hub_scope, "菜单结构", "门户有 8 个一级模块，入口在左侧导航栏")])
|
||
import asyncio
|
||
|
||
before = hub.list_entries(knowledge_id=hub_scope)[0]["status"]
|
||
result = asyncio.run(autoreview.run(dry_run=True, use_llm=False))
|
||
assert result["dry_run"] is True
|
||
assert hub.list_entries(knowledge_id=hub_scope)[0]["status"] == before == "draft"
|
||
|
||
def test_applying_marks_entries_as_auto(self, hub_scope, monkeypatch):
|
||
from server.config import settings
|
||
|
||
monkeypatch.setattr(settings, "llm_api_key", "")
|
||
value = "门户有 8 个一级模块，入口在左侧导航栏，点开后是二级菜单"
|
||
hub.receive_drafts("pytest_a", [draft(hub_scope, "菜单结构", value, category="path")])
|
||
hub.receive_drafts("pytest_b", [draft(hub_scope, "菜单结构", value, category="path")])
|
||
entry_id = hub.list_entries(knowledge_id=hub_scope)[0]["id"]
|
||
import asyncio
|
||
|
||
asyncio.run(autoreview.run(dry_run=False, ids=[entry_id], use_llm=False))
|
||
row = hub.get_entry(entry_id)
|
||
assert row["status"] == "verified" and row["reviewed_by"] == "auto" and row["scope"] == "all"
|
||
assert "自动采纳" in row["suggestion_note"]
|
||
|
||
def test_held_entries_get_a_reason_the_reviewer_can_read(self, hub_scope, monkeypatch):
|
||
from server.config import settings
|
||
|
||
monkeypatch.setattr(settings, "llm_api_key", "")
|
||
hub.receive_drafts("pytest_a", [draft(hub_scope, "系统规则", "同一天同一班级不能重复提交", category="rule")])
|
||
entry_id = hub.list_entries(knowledge_id=hub_scope)[0]["id"]
|
||
import asyncio
|
||
|
||
asyncio.run(autoreview.run(dry_run=False, ids=[entry_id], use_llm=False))
|
||
row = hub.get_entry(entry_id)
|
||
assert row["status"] == "draft"
|
||
assert "留给人工" in row["suggestion_note"] and "必须人工确认" in row["suggestion_note"]
|