"""Whitelisted action-schema diagnostics do not copy model data or retry.""" import json from types import SimpleNamespace import pytest from pydantic import BaseModel, ConfigDict, ValidationError from pydantic_core import PydanticCustomError from backend.browser import diagnostics, llm_factory from backend.prospecting import usage class Click(BaseModel): model_config = ConfigDict(extra="forbid") index: int class Action(BaseModel): click: Click class Output(BaseModel): model_config = ConfigDict(extra="forbid") action: list[Action] def failure(): try: Output.model_validate({"action": [{"click": {"index": "PRIVATE-VALUE", "PRIVATE-KEY": "PRIVATE-VALUE"}}], "https://private.invalid/key": "PRIVATE"}) except ValidationError as error: return error def test_only_schema_declared_path_and_builtin_error_type_survive(): entry = {"errors": diagnostics.validation_fields(failure(), Output)} serialized = json.dumps(entry) assert {"loc": ["action", 0, "click", "index"], "type": "int_parsing"} in entry["errors"] assert {"loc": ["action", 0, "click", ""], "type": "extra_forbidden"} in entry["errors"] assert all(value not in serialized for value in ["PRIVATE", "https", "input", "ctx", "msg", "url"]) custom = SimpleNamespace(errors=lambda **_kwargs: [{"loc": ("https://private.invalid/key",), "type": "PRIVATE-CUSTOM-TYPE", "input": "PRIVATE", "msg": "PRIVATE"}]) assert diagnostics.validation_fields(custom, Output) == [{"loc": [""], "type": "validation_error"}] @pytest.fixture def local(tmp_path, monkeypatch): def directory(tid): path = tmp_path / tid path.mkdir(exist_ok=True) return path monkeypatch.setattr(diagnostics, "task_doc_dir", directory) monkeypatch.setattr(usage, "task_doc_dir", directory) return tmp_path def test_runs_and_tasks_keep_independent_schema_sequences_and_private_log(local): with diagnostics.run_scope("one", "run_one") as first: diagnostics.schema_error(None, failure(), Output) with diagnostics.run_scope("two", "run_two") as second: assert diagnostics.consecutive_schema_errors() == 0 diagnostics.schema_error(None, failure(), Output) assert diagnostics.consecutive_schema_errors() == 1 assert diagnostics.summary(second)["events"][0]["run_id"] == "run_two" with diagnostics.run_scope("one", "run_three") as third: assert diagnostics.consecutive_schema_errors() == 0 diagnostics.schema_error(None, failure(), Output) log = (local / "one" / "model-diagnostics.jsonl").read_text() assert "PRIVATE" not in log and "https" not in log assert len(log.splitlines()) == 2 and first.events[0]["run_id"] == "run_one" assert third.events[0]["run_id"] == "run_three" async def test_real_adapter_records_safe_diagnostic_and_usage_once_without_network_retry(local, monkeypatch): import openai from browser_use.llm.exceptions import ModelProviderError from browser_use.llm.messages import UserMessage sent = [] class Client: def __init__(self, **kwargs): assert kwargs["max_retries"] == 0 self.chat = SimpleNamespace(completions=SimpleNamespace(create=self.create)) async def __aenter__(self): return self async def __aexit__(self, *args): return False async def create(self, **kwargs): sent.append(kwargs) return SimpleNamespace(model="deepseek-flash", choices=[SimpleNamespace( message=SimpleNamespace(content=None, tool_calls=[SimpleNamespace(type="function", function=SimpleNamespace( name="return_structured_output", arguments='{"action":[{"click":{"index":"PRIVATE-RESPONSE"}}]}'))]), finish_reason="tool_calls")], usage=SimpleNamespace(prompt_tokens=20, completion_tokens=4, total_tokens=24)) monkeypatch.setattr(openai, "AsyncOpenAI", Client) with usage.usage_scope("adapter", "browser"), diagnostics.run_scope("adapter", "run_fixture") as diag: with pytest.raises(ModelProviderError) as caught: await llm_factory.DeepSeekBrowserChat("deepseek-flash", "PRIVATE-KEY").ainvoke([UserMessage(content="PRIVATE-PROMPT")], Output) assert diagnostics.consecutive_schema_errors() == 1 assert len(sent) == 1 assert "action.0.click.index (int_parsing)" in str(caught.value) assert caught.value.status_code == 422 assert "PRIVATE" not in str(caught.value) and caught.value.__suppress_context__ assert usage.snapshot("adapter")["total_tokens"] == 24 assert usage.snapshot("adapter")["parse_failed_requests"] == 1 assert "PRIVATE" not in (local / "adapter" / "model-diagnostics.jsonl").read_text() assert diagnostics.summary(diag)["events"][0]["errors"] == [{"loc": ["action", 0, "click", "index"], "type": "int_parsing"}] def test_diagnostic_storage_failure_is_explicit_without_retry_or_exception(local, monkeypatch): def broken(_tid): raise OSError("PRIVATE failure details") monkeypatch.setattr(diagnostics, "task_doc_dir", broken) with diagnostics.run_scope("storage", "run_fixture") as diag: diagnostics.schema_error(None, failure(), Output) assert diagnostics.summary(diag)["storage_error"] is True assert len(diag.events) == 1 and "PRIVATE" not in json.dumps(diagnostics.summary(diag))