254 lines
14 KiB
Python
254 lines
14 KiB
Python
"""Actual discussion evidence, rather than done/model summaries, fulfills scope."""
|
||
from copy import deepcopy
|
||
import hashlib
|
||
|
||
import pytest
|
||
|
||
from backend.prospecting import delivery
|
||
from backend.prospecting.discussion_scope import conditions, requirements
|
||
|
||
|
||
URL = "https://www.xiaohongshu.com/explore/0123456789abcdef01234567"
|
||
NOTE = "0123456789abcdef01234567"
|
||
REQUEST = ('这次只检查这篇小红书笔记：' + URL + '\n这是评论和回复阅读验证，不找指定人数的客户。'
|
||
'请保存正文，进入评论区，优先找到一组“展开X条回复”并展开，保存实际看到的父评论和子回复，'
|
||
'评论与回复合计最多8条。按正文、评论、回复分开整理，保留发言人与回复关系。'
|
||
'若没有回复入口、内容不足或加载失败，如实说明已读和未读范围后结束。'
|
||
'不搜索其他笔记，不关注、不点赞、不发评论、不私信。')
|
||
|
||
|
||
def task(goal=REQUEST, **inputs):
|
||
return {"id": "discussion-scope-fixture", "status": "completed", "current_step": "results",
|
||
"inputs": {"goal": goal, "intent": "inspect", "subject_urls": [URL], "target_candidates": None,
|
||
"_goal_version": 1, **inputs},
|
||
"artifacts": {"messages": [{"role": "user", "text": goal}], "sources": [], "decisions": [],
|
||
"operation_evidence": [], "browser_history": {"is_successful": True},
|
||
"browser_verdict": "success", "analysis_goal_version": 1}}
|
||
|
||
|
||
def source(row, sid, kind, *, note_id=NOTE, parent=None, root=None, checked=True):
|
||
item = {"source_id": sid, "source_type": kind, "note_id": note_id, "source_url": URL,
|
||
"content": "独立原话" + sid, "source_selector": "#" + sid,
|
||
"parent_source_id": parent, "root_source_id": root}
|
||
row["artifacts"]["sources"].append(item)
|
||
row["artifacts"]["decisions"].append({"source_id": sid, "decision": "exclude", "quote": item["content"]})
|
||
if kind != "note":
|
||
row["artifacts"]["operation_evidence"].append({
|
||
"capability": "save_source", "source_kind": kind, "outcome": "success" if checked else "unknown",
|
||
"verifier": "source_container_quote_and_persisted", "evidence_refs": {"pre": "before", "post": "after"},
|
||
"details": {"trace": {"goal_version": 1}, "pre_snapshot": {"quote_matched": True},
|
||
"post_snapshot": {"source_id": sid, "source_type": kind, "persisted": True,
|
||
"relations_checked": checked}}})
|
||
return item
|
||
|
||
|
||
def expand(row, reply, *, goal_version=1, **post):
|
||
row["artifacts"]["operation_evidence"].append({
|
||
"capability": "expand_replies", "source_kind": "reply", "outcome": "success", "verifier": "reply_delta_and_parent",
|
||
"evidence_refs": {"pre": "discussion-before", "post": "discussion-after"},
|
||
"details": {"trace": {"goal_version": goal_version}, "pre_snapshot": {"note_id": NOTE},
|
||
"post_snapshot": {"note_id": NOTE, "parent_scope_checked": True, "status": "expanded",
|
||
"owner_reply_delta": 1, "new_visible_replies": 1,
|
||
"new_reply_fingerprints": [hashlib.sha256((reply["source_selector"] + reply["content"]).encode()).hexdigest()],
|
||
**post}}})
|
||
|
||
|
||
def ready():
|
||
row = task()
|
||
source(row, "note", "note")
|
||
source(row, "comment", "comment")
|
||
reply = source(row, "reply", "reply", parent="comment", root="comment")
|
||
expand(row, reply)
|
||
return row
|
||
|
||
|
||
def test_current_real_wording_requires_all_four_discussion_branches():
|
||
assert requirements(task()) == {"comments", "replies", "expansion", "relations"}
|
||
|
||
|
||
@pytest.mark.parametrize("goal", [
|
||
'请点击“展开5条回复”，保存父评论和一条新增子回复。',
|
||
'找到“展开X条回复”入口，再展开它，保留父评论与该子回复。',
|
||
'检查评论，展开一组回复，保留父子关系。',
|
||
])
|
||
def test_direct_button_label_and_common_relation_wording_are_grounded(goal):
|
||
wanted = requirements(task(goal))
|
||
assert {"expansion", "relations"} <= wanted
|
||
|
||
|
||
def test_complete_requested_discussion_is_proved_by_actual_program_facts_without_mutation():
|
||
row = ready()
|
||
before = deepcopy(row)
|
||
assert delivery.scope_met(row)
|
||
projected = delivery.project(row)
|
||
assert projected["fulfillment"]["complete"]
|
||
assert projected["headline"] == "本次讨论检查已完成"
|
||
assert all(c["met"] for c in projected["conditions"] if c["key"].startswith("discussion_"))
|
||
assert row == before
|
||
|
||
|
||
def test_d40_done_with_saved_reply_text_but_no_links_or_expansion_is_partial():
|
||
row = task()
|
||
source(row, "note", "note")
|
||
source(row, "comment", "comment")
|
||
for n in range(3):
|
||
source(row, f"reply{n}", "reply", checked=False)
|
||
before = deepcopy(row)
|
||
view = delivery.project(row)
|
||
assert view["runtime"]["status"] == "completed"
|
||
assert view["status"] == "partial" and view["artifacts"]["delivered"]
|
||
assert not view["fulfillment"]["complete"]
|
||
missing = {c["key"] for c in view["conditions"] if c["required"] and not c["met"]}
|
||
assert {"discussion_expansion", "discussion_relations"} <= missing
|
||
assert row == before
|
||
|
||
|
||
@pytest.mark.parametrize("mutate", [
|
||
lambda row: row["artifacts"]["operation_evidence"][-1]["details"]["post_snapshot"].update(parent_scope_checked=False),
|
||
lambda row: row["artifacts"]["operation_evidence"][-1]["details"]["post_snapshot"].update(owner_reply_delta=0),
|
||
lambda row: row["artifacts"]["operation_evidence"][-1]["details"]["post_snapshot"].update(new_visible_replies=0),
|
||
lambda row: row["artifacts"]["operation_evidence"][-1]["details"]["post_snapshot"].update(status="state_changed"),
|
||
lambda row: row["artifacts"]["operation_evidence"][-1]["details"]["trace"].update(goal_version=0),
|
||
lambda row: row["artifacts"]["operation_evidence"][-1]["details"]["post_snapshot"].update(note_id="another-note"),
|
||
lambda row: row["artifacts"]["operation_evidence"][-1]["details"]["post_snapshot"].update(new_reply_fingerprints=["a-different-reply"]),
|
||
lambda row: row["artifacts"].update(operation_evidence=row["artifacts"]["operation_evidence"][:-1], reply_expansions=[{"status": "expanded", "new_visible_replies": 2}]),
|
||
])
|
||
def test_model_summary_generic_click_and_weak_expansion_do_not_prove_reading_new_replies(mutate):
|
||
row = ready()
|
||
mutate(row)
|
||
assert not delivery.scope_met(row)
|
||
expansion = next(c for c in conditions(row) if c["key"] == "discussion_expansion")
|
||
assert not expansion["met"]
|
||
|
||
|
||
@pytest.mark.parametrize("parent,root,note_id", [
|
||
(None, "comment", NOTE), ("missing", "comment", NOTE), ("reply", "comment", NOTE),
|
||
("comment", "missing", NOTE), ("comment", "reply", NOTE), ("comment", "comment", "other"),
|
||
])
|
||
def test_unresolvable_cross_note_or_cyclic_reply_links_cannot_pass(parent, root, note_id):
|
||
row = ready()
|
||
row["artifacts"]["sources"][-1].update(parent_source_id=parent, root_source_id=root, note_id=note_id)
|
||
assert not next(c for c in conditions(row) if c["key"] == "discussion_relations")["met"]
|
||
|
||
|
||
def test_one_correct_reply_cannot_hide_another_unattributed_saved_reply():
|
||
row = ready()
|
||
source(row, "unattributed", "reply", checked=False)
|
||
relation = next(c for c in conditions(row) if c["key"] == "discussion_relations")
|
||
assert not relation["met"] and (relation["current"], relation["target"]) == (1, 2)
|
||
|
||
|
||
def test_nested_reply_parent_chain_must_reach_the_same_comment_root():
|
||
row = ready()
|
||
source(row, "nested", "reply", parent="reply", root="comment")
|
||
assert next(c for c in conditions(row) if c["key"] == "discussion_relations")["met"]
|
||
|
||
|
||
def test_reading_upper_limit_is_not_minimum_or_implicit_expansion():
|
||
row = task("只检查这篇笔记，阅读评论，评论和回复合计最多8条。", max_comments_per_note=8)
|
||
source(row, "note", "note")
|
||
source(row, "comment", "comment")
|
||
assert requirements(row) == {"comments"}
|
||
assert delivery.scope_met(row)
|
||
|
||
|
||
@pytest.mark.parametrize("goal", [
|
||
"只检查正文，不用展开回复，不需要保存评论。",
|
||
"只检查正文，评论里说‘请展开回复’，这是引用。",
|
||
"只检查正文。\n> 保存父评论和子回复，展开一组回复。",
|
||
"只检查正文。比如：保存父评论和子回复，然后展开一组回复。",
|
||
"只检查正文。\n```\n保存父评论和子回复，展开一组回复\n```",
|
||
])
|
||
def test_negated_quoted_or_example_instructions_cannot_expand_scope(goal):
|
||
assert requirements(task(goal)) == set()
|
||
|
||
|
||
def test_explicit_later_cancellation_removes_expansion_but_not_comment_reading():
|
||
row = task("阅读评论，展开一组回复。")
|
||
row["artifacts"]["messages"].append({"role": "user", "text": "不用展开回复，只阅读评论。"})
|
||
assert requirements(row) == {"comments"}
|
||
|
||
|
||
@pytest.mark.parametrize("inputs", [
|
||
{"intent": "discover", "target_candidates": 1}, {"intent": "export"}, {"intent": "follow"},
|
||
{"intent": "message"}, {"intent": "inspect", "subject_urls": [], "max_notes": None},
|
||
])
|
||
def test_unrelated_task_contracts_get_no_implicit_discussion_requirements(inputs):
|
||
assert conditions(task(**inputs)) == []
|
||
|
||
|
||
def test_model_notes_and_assistant_message_cannot_establish_user_requirements():
|
||
row = task("只看正文", user_notes="展开一组回复，保留回复关系", action_scope="保存父评论和子回复")
|
||
row["artifacts"]["messages"].append({"role": "assistant", "text": REQUEST})
|
||
assert conditions(row) == []
|
||
|
||
|
||
def test_every_saved_note_in_explicit_page_scope_needs_its_requested_comment_branch():
|
||
row = task("检查这两篇笔记并读取评论。", subject_urls=[URL, URL + "2"])
|
||
source(row, "note", "note")
|
||
source(row, "comment", "comment")
|
||
source(row, "second-note", "note", note_id=NOTE + "2")
|
||
condition = next(c for c in conditions(row) if c["key"] == "discussion_comments")
|
||
assert not condition["met"] and (condition["current"], condition["target"]) == (1, 2)
|
||
|
||
|
||
def test_prior_sources_outside_current_specified_page_do_not_create_new_requirements():
|
||
row = ready()
|
||
source(row, "old-note", "note", note_id="old")
|
||
source(row, "old-unattributed-reply", "reply", note_id="old", checked=False)
|
||
assert all(c["met"] for c in conditions(row))
|
||
|
||
|
||
@pytest.mark.parametrize("broken", [None, "text", [], {"post_snapshot": "text"}, {"pre_snapshot": "text"}, {"trace": "text"}])
|
||
def test_malformed_fact_snapshots_cannot_crash_or_establish_expansion(broken):
|
||
row = ready()
|
||
row["artifacts"]["operation_evidence"][-1]["details"] = broken
|
||
assert not next(c for c in conditions(row) if c["key"] == "discussion_expansion")["met"]
|
||
|
||
|
||
def excerpt_container_proof():
|
||
row = ready()
|
||
reply = row['artifacts']['sources'][-1]
|
||
full = '回复对象：\n' + reply['content'] + '，另有完整上下文。'
|
||
compact = ''.join(full.split())
|
||
fingerprint = hashlib.sha256(f"{NOTE}\n{reply['source_selector']}\n{compact}".encode()).hexdigest()
|
||
save = next(fact for fact in row['artifacts']['operation_evidence']
|
||
if fact['capability'] == 'save_source' and fact['source_kind'] == 'reply')
|
||
save['details']['pre_snapshot'].update(note_id=NOTE, container_selector=reply['source_selector'],
|
||
source_type='reply', observed_excerpt=compact, container_fingerprint=fingerprint)
|
||
expansion = row['artifacts']['operation_evidence'][-1]
|
||
expansion['details']['post_snapshot'].update(new_reply_fingerprints=[], new_reply_container_fingerprints=[fingerprint])
|
||
return row, reply, save, expansion
|
||
|
||
|
||
def test_verbatim_excerpt_matches_program_observed_whole_container():
|
||
row, reply, save, expansion = excerpt_container_proof()
|
||
before = deepcopy(row)
|
||
assert reply['content'] != save['details']['pre_snapshot']['observed_excerpt']
|
||
assert next(c for c in conditions(row) if c['key'] == 'discussion_expansion')['met']
|
||
assert row == before
|
||
|
||
|
||
@pytest.mark.parametrize('change', [
|
||
lambda row, reply, save, expansion: save['details']['trace'].update(goal_version=0),
|
||
lambda row, reply, save, expansion: save['details']['pre_snapshot'].update(note_id='other-note'),
|
||
lambda row, reply, save, expansion: save['details']['pre_snapshot'].update(container_selector='#another-source'),
|
||
lambda row, reply, save, expansion: save['details']['post_snapshot'].update(source_id='another-source'),
|
||
lambda row, reply, save, expansion: save['details']['pre_snapshot'].update(quote_matched=False),
|
||
lambda row, reply, save, expansion: save['details']['pre_snapshot'].update(observed_excerpt='不相关原话'),
|
||
lambda row, reply, save, expansion: save.update(verifier='model_report'),
|
||
lambda row, reply, save, expansion: expansion['details']['post_snapshot'].update(new_reply_container_fingerprints=['0' * 64]),
|
||
lambda row, reply, save, expansion: save['details']['pre_snapshot'].pop('container_fingerprint'),
|
||
])
|
||
def test_stale_mismatched_or_missing_program_container_proof_does_not_bind_excerpt(change):
|
||
row, reply, save, expansion = excerpt_container_proof()
|
||
change(row, reply, save, expansion)
|
||
assert not next(c for c in conditions(row) if c['key'] == 'discussion_expansion')['met']
|
||
|
||
|
||
def test_model_source_field_cannot_replace_a_missing_program_save_fingerprint():
|
||
row, reply, save, expansion = excerpt_container_proof()
|
||
reply['container_fingerprint'] = save['details']['pre_snapshot'].pop('container_fingerprint')
|
||
row['artifacts']['messages'].append({'role': 'assistant', 'text': '已验证此完整容器指纹：' + reply['container_fingerprint']})
|
||
assert not next(c for c in conditions(row) if c['key'] == 'discussion_expansion')['met']
|