"""Browser UTF-16 truncation cannot break evidence storage or API UTF-8.""" from __future__ import annotations import copy import json import pytest from apps.server.db import digest, encoded from apps.server.learning import normalize_worker_unicode from apps.server.tests.test_learning import client, initialize, working_candidate from apps.server.tests.test_postgres import pg_client from apps.server.worker import WorkerError def utf16_prefix(value: str, units: int) -> str: """Reproduce JavaScript slice() cutting through an emoji surrogate pair.""" return value.encode("utf-16-le")[: units * 2].decode("utf-16-le", "surrogatepass") def test_worker_unicode_preserves_valid_scalars_and_canonical_json(): valid = {"中文🦁": ["装修🦁🏡", "e\u0301", "é", "\x00\ufffd\uffff", "𠀀"], "count": 2, "flag": True, "missing": None} old_json, old_digest = encoded(valid), digest(valid) assert normalize_worker_unicode(valid) == valid assert encoded(normalize_worker_unicode(valid)) == old_json assert digest(normalize_worker_unicode(valid)) == old_digest assert normalize_worker_unicode("a\ud83eb\udd81c") == "a\ufffdb\ufffdc" assert normalize_worker_unicode("\ud83e\udd81") == "🦁" assert utf16_prefix("装修🦁", 3) == "装修\ud83e" @pytest.fixture(params=["client", "pg_client"]) def unicode_client(request): return request.getfixturevalue(request.param) def test_truncated_emoji_capture_verify_and_failure_roundtrip(unicode_client): client = unicode_client initialize(client) task, candidate = working_candidate(client) prefix = "/api/learning/tasks/" + task["id"] endpoint = "/api/learning/candidates/" + candidate["id"] assert client.post(endpoint + "/check").json()["status"] == "passed" store = client.app_reference.state.store old_observations = store.rows("SELECT id,data_json FROM observations WHERE task_id=$1 ORDER BY " + store.order_column(), (task["id"],)) old_candidate = store.one("SELECT digest,manifest_json FROM candidates WHERE id=$1", (candidate["id"],)) old_validation = store.one("SELECT id,data_json FROM validations WHERE candidate_id=$1", (candidate["id"],)) worker = client.worker_reference original_capture, original_call = worker.capture, worker.call broken = utf16_prefix("文" * 29999 + "🦁", 30000) good = "文" * 29999 + "🦁后续" raw_captures = [] def malformed_capture(task_id): value = original_capture(task_id) value.update(text=broken, viewport_text="可见\udc00文本", title="有效标题🦁") value["elements"][0]["label"] = "搜索\ud83e" value["elements"][0]["read_hints"] = [{"selector": ".title", "label": "标题\ud83e"}] raw_captures.append(copy.deepcopy(value)) return value worker.capture = malformed_capture async def unicode_call(method, params): value = await original_call(method, params) if method == "verify": value["outputs"]["results"][0]["text"] += "🦁" + utf16_prefix("装修🦁", 3) intact = original_capture(params["task_id"]) intact["text"] = good value["evidence"].append(intact) return value worker.call = unicode_call capture = client.post(prefix + "/browser/capture", json={}) assert capture.status_code == 200, capture.text saved = capture.json() assert saved["text"] == "文" * 29999 + "\ufffd" assert saved["viewport_text"] == "可见\ufffd文本" and saved["title"] == "有效标题🦁" assert saved["elements"][0]["label"] == "搜索\ufffd" assert saved["elements"][0]["read_hints"][0]["label"] == "标题\ufffd" assert raw_captures[0]["text"] == broken # Worker evidence was not mutated. verification = client.post(endpoint + "/verify", json={"expected_outputs": {"results": "建材"}}) assert verification.status_code == 200, verification.text record = verification.json() assert record["status"] == "passed" and len(record["evidence"]) == 2 assert record["actual_outputs"]["results"][0]["text"].endswith("🦁装修\ufffd") assert all(client.get(item["screenshot_url"]).status_code == 200 for item in record["evidence"]) # Failed worker calls bypass the normal response path; their actual evidence # still needs to append safely without manufacturing a successful step. async def failed_action(method, params): if method == "action": raise WorkerError("fixture failure", "WAIT_TIMEOUT", before=worker.capture(params["task_id"]), after=worker.capture(params["task_id"])) return await unicode_call(method, params) worker.call = failed_action failure = client.post(prefix + "/browser/action", json={"kind": "read", "selector": '[data-testid="result"]', "output_name": "failure_output"}) assert failure.status_code == 409 and len(failure.json()["detail"]["failure_observation_ids"]) == 2 current = client.get(prefix) assert current.status_code == 200 detail = current.json() assert len(detail["steps"]) == 3 assert detail["observations"][-1]["text"].endswith("\ufffd") intact_observation = next(item for item in detail["observations"] if item["id"] == record["evidence"][-1]["observation_id"]) assert intact_observation["text"] == "文" * 29999 + "🦁" assert client.get("/api/learning/validations").status_code == 200 reread = client.get(endpoint).json() assert reread["digest"] == candidate["digest"] and reread["manifest"] == candidate["manifest"] assert store.one("SELECT digest,manifest_json FROM candidates WHERE id=$1", (candidate["id"],)) == old_candidate assert store.one("SELECT id,data_json FROM validations WHERE id=$1", (old_validation["id"],)) == old_validation assert store.rows("SELECT id,data_json FROM observations WHERE task_id=$1 ORDER BY " + store.order_column(), (task["id"],))[:len(old_observations)] == old_observations for row in store.rows("SELECT data_json FROM observations WHERE task_id=$1", (task["id"],)): row["data_json"].encode("utf-8") json.loads(row["data_json"])