115 lines
6.0 KiB
Python
115 lines
6.0 KiB
Python
"""Browser UTF-16 truncation cannot break evidence storage or API UTF-8."""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from apps.server.db import digest, encoded
|
|
from apps.server.learning import normalize_worker_unicode
|
|
from apps.server.tests.test_learning import client, initialize, working_candidate
|
|
from apps.server.tests.test_postgres import pg_client
|
|
from apps.server.worker import WorkerError
|
|
|
|
|
|
def utf16_prefix(value: str, units: int) -> str:
|
|
"""Reproduce JavaScript slice() cutting through an emoji surrogate pair."""
|
|
return value.encode("utf-16-le")[: units * 2].decode("utf-16-le", "surrogatepass")
|
|
|
|
|
|
def test_worker_unicode_preserves_valid_scalars_and_canonical_json():
|
|
valid = {"中文🦁": ["装修🦁🏡", "e\u0301", "é", "\x00\ufffd\uffff", "𠀀"], "count": 2, "flag": True, "missing": None}
|
|
old_json, old_digest = encoded(valid), digest(valid)
|
|
assert normalize_worker_unicode(valid) == valid
|
|
assert encoded(normalize_worker_unicode(valid)) == old_json
|
|
assert digest(normalize_worker_unicode(valid)) == old_digest
|
|
assert normalize_worker_unicode("a\ud83eb\udd81c") == "a\ufffdb\ufffdc"
|
|
assert normalize_worker_unicode("\ud83e\udd81") == "🦁"
|
|
assert utf16_prefix("装修🦁", 3) == "装修\ud83e"
|
|
|
|
|
|
@pytest.fixture(params=["client", "pg_client"])
|
|
def unicode_client(request):
|
|
return request.getfixturevalue(request.param)
|
|
|
|
|
|
def test_truncated_emoji_capture_verify_and_failure_roundtrip(unicode_client):
|
|
client = unicode_client
|
|
initialize(client)
|
|
task, candidate = working_candidate(client)
|
|
prefix = "/api/learning/tasks/" + task["id"]
|
|
endpoint = "/api/learning/candidates/" + candidate["id"]
|
|
assert client.post(endpoint + "/check").json()["status"] == "passed"
|
|
store = client.app_reference.state.store
|
|
old_observations = store.rows("SELECT id,data_json FROM observations WHERE task_id=$1 ORDER BY " + store.order_column(), (task["id"],))
|
|
old_candidate = store.one("SELECT digest,manifest_json FROM candidates WHERE id=$1", (candidate["id"],))
|
|
old_validation = store.one("SELECT id,data_json FROM validations WHERE candidate_id=$1", (candidate["id"],))
|
|
worker = client.worker_reference
|
|
original_capture, original_call = worker.capture, worker.call
|
|
broken = utf16_prefix("文" * 29999 + "🦁", 30000)
|
|
good = "文" * 29999 + "🦁后续"
|
|
raw_captures = []
|
|
|
|
def malformed_capture(task_id):
|
|
value = original_capture(task_id)
|
|
value.update(text=broken, viewport_text="可见\udc00文本", title="有效标题🦁")
|
|
value["elements"][0]["label"] = "搜索\ud83e"
|
|
value["elements"][0]["read_hints"] = [{"selector": ".title", "label": "标题\ud83e"}]
|
|
raw_captures.append(copy.deepcopy(value))
|
|
return value
|
|
|
|
worker.capture = malformed_capture
|
|
|
|
async def unicode_call(method, params):
|
|
value = await original_call(method, params)
|
|
if method == "verify":
|
|
value["outputs"]["results"][0]["text"] += "🦁" + utf16_prefix("装修🦁", 3)
|
|
intact = original_capture(params["task_id"])
|
|
intact["text"] = good
|
|
value["evidence"].append(intact)
|
|
return value
|
|
|
|
worker.call = unicode_call
|
|
capture = client.post(prefix + "/browser/capture", json={})
|
|
assert capture.status_code == 200, capture.text
|
|
saved = capture.json()
|
|
assert saved["text"] == "文" * 29999 + "\ufffd"
|
|
assert saved["viewport_text"] == "可见\ufffd文本" and saved["title"] == "有效标题🦁"
|
|
assert saved["elements"][0]["label"] == "搜索\ufffd"
|
|
assert saved["elements"][0]["read_hints"][0]["label"] == "标题\ufffd"
|
|
assert raw_captures[0]["text"] == broken # Worker evidence was not mutated.
|
|
verification = client.post(endpoint + "/verify", json={"expected_outputs": {"results": "建材"}})
|
|
assert verification.status_code == 200, verification.text
|
|
record = verification.json()
|
|
assert record["status"] == "passed" and len(record["evidence"]) == 2
|
|
assert record["actual_outputs"]["results"][0]["text"].endswith("🦁装修\ufffd")
|
|
assert all(client.get(item["screenshot_url"]).status_code == 200 for item in record["evidence"])
|
|
|
|
# Failed worker calls bypass the normal response path; their actual evidence
|
|
# still needs to append safely without manufacturing a successful step.
|
|
async def failed_action(method, params):
|
|
if method == "action":
|
|
raise WorkerError("fixture failure", "WAIT_TIMEOUT", before=worker.capture(params["task_id"]), after=worker.capture(params["task_id"]))
|
|
return await unicode_call(method, params)
|
|
|
|
worker.call = failed_action
|
|
failure = client.post(prefix + "/browser/action", json={"kind": "read", "selector": '[data-testid="result"]', "output_name": "failure_output"})
|
|
assert failure.status_code == 409 and len(failure.json()["detail"]["failure_observation_ids"]) == 2
|
|
current = client.get(prefix)
|
|
assert current.status_code == 200
|
|
detail = current.json()
|
|
assert len(detail["steps"]) == 3
|
|
assert detail["observations"][-1]["text"].endswith("\ufffd")
|
|
intact_observation = next(item for item in detail["observations"] if item["id"] == record["evidence"][-1]["observation_id"])
|
|
assert intact_observation["text"] == "文" * 29999 + "🦁"
|
|
assert client.get("/api/learning/validations").status_code == 200
|
|
reread = client.get(endpoint).json()
|
|
assert reread["digest"] == candidate["digest"] and reread["manifest"] == candidate["manifest"]
|
|
assert store.one("SELECT digest,manifest_json FROM candidates WHERE id=$1", (candidate["id"],)) == old_candidate
|
|
assert store.one("SELECT id,data_json FROM validations WHERE id=$1", (old_validation["id"],)) == old_validation
|
|
assert store.rows("SELECT id,data_json FROM observations WHERE task_id=$1 ORDER BY " + store.order_column(), (task["id"],))[:len(old_observations)] == old_observations
|
|
for row in store.rows("SELECT data_json FROM observations WHERE task_id=$1", (task["id"],)):
|
|
row["data_json"].encode("utf-8")
|
|
json.loads(row["data_json"])
|