114 lines
5.3 KiB
Python
114 lines
5.3 KiB
Python
"""Whitelisted action-schema diagnostics do not copy model data or retry."""
|
|
import json
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
from pydantic import BaseModel, ConfigDict, ValidationError
|
|
from pydantic_core import PydanticCustomError
|
|
|
|
from backend.browser import diagnostics, llm_factory
|
|
from backend.prospecting import usage
|
|
|
|
|
|
class Click(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
index: int
|
|
|
|
|
|
class Action(BaseModel):
|
|
click: Click
|
|
|
|
|
|
class Output(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
action: list[Action]
|
|
|
|
|
|
def failure():
|
|
try:
|
|
Output.model_validate({"action": [{"click": {"index": "PRIVATE-VALUE", "PRIVATE-KEY": "PRIVATE-VALUE"}}], "https://private.invalid/key": "PRIVATE"})
|
|
except ValidationError as error:
|
|
return error
|
|
|
|
|
|
def test_only_schema_declared_path_and_builtin_error_type_survive():
|
|
entry = {"errors": diagnostics.validation_fields(failure(), Output)}
|
|
serialized = json.dumps(entry)
|
|
assert {"loc": ["action", 0, "click", "index"], "type": "int_parsing"} in entry["errors"]
|
|
assert {"loc": ["action", 0, "click", "<field>"], "type": "extra_forbidden"} in entry["errors"]
|
|
assert all(value not in serialized for value in ["PRIVATE", "https", "input", "ctx", "msg", "url"])
|
|
custom = SimpleNamespace(errors=lambda **_kwargs: [{"loc": ("https://private.invalid/key",), "type": "PRIVATE-CUSTOM-TYPE", "input": "PRIVATE", "msg": "PRIVATE"}])
|
|
assert diagnostics.validation_fields(custom, Output) == [{"loc": ["<field>"], "type": "validation_error"}]
|
|
|
|
|
|
@pytest.fixture
|
|
def local(tmp_path, monkeypatch):
|
|
def directory(tid):
|
|
path = tmp_path / tid
|
|
path.mkdir(exist_ok=True)
|
|
return path
|
|
monkeypatch.setattr(diagnostics, "task_doc_dir", directory)
|
|
monkeypatch.setattr(usage, "task_doc_dir", directory)
|
|
return tmp_path
|
|
|
|
|
|
def test_runs_and_tasks_keep_independent_schema_sequences_and_private_log(local):
|
|
with diagnostics.run_scope("one", "run_one") as first:
|
|
diagnostics.schema_error(None, failure(), Output)
|
|
with diagnostics.run_scope("two", "run_two") as second:
|
|
assert diagnostics.consecutive_schema_errors() == 0
|
|
diagnostics.schema_error(None, failure(), Output)
|
|
assert diagnostics.consecutive_schema_errors() == 1
|
|
assert diagnostics.summary(second)["events"][0]["run_id"] == "run_two"
|
|
with diagnostics.run_scope("one", "run_three") as third:
|
|
assert diagnostics.consecutive_schema_errors() == 0
|
|
diagnostics.schema_error(None, failure(), Output)
|
|
log = (local / "one" / "model-diagnostics.jsonl").read_text()
|
|
assert "PRIVATE" not in log and "https" not in log
|
|
assert len(log.splitlines()) == 2 and first.events[0]["run_id"] == "run_one"
|
|
assert third.events[0]["run_id"] == "run_three"
|
|
|
|
|
|
async def test_real_adapter_records_safe_diagnostic_and_usage_once_without_network_retry(local, monkeypatch):
|
|
import openai
|
|
from browser_use.llm.exceptions import ModelProviderError
|
|
from browser_use.llm.messages import UserMessage
|
|
sent = []
|
|
class Client:
|
|
def __init__(self, **kwargs):
|
|
assert kwargs["max_retries"] == 0
|
|
self.chat = SimpleNamespace(completions=SimpleNamespace(create=self.create))
|
|
async def __aenter__(self):
|
|
return self
|
|
async def __aexit__(self, *args):
|
|
return False
|
|
async def create(self, **kwargs):
|
|
sent.append(kwargs)
|
|
return SimpleNamespace(model="deepseek-flash", choices=[SimpleNamespace(
|
|
message=SimpleNamespace(content=None, tool_calls=[SimpleNamespace(type="function", function=SimpleNamespace(
|
|
name="return_structured_output", arguments='{"action":[{"click":{"index":"PRIVATE-RESPONSE"}}]}'))]), finish_reason="tool_calls")],
|
|
usage=SimpleNamespace(prompt_tokens=20, completion_tokens=4, total_tokens=24))
|
|
monkeypatch.setattr(openai, "AsyncOpenAI", Client)
|
|
with usage.usage_scope("adapter", "browser"), diagnostics.run_scope("adapter", "run_fixture") as diag:
|
|
with pytest.raises(ModelProviderError) as caught:
|
|
await llm_factory.DeepSeekBrowserChat("deepseek-flash", "PRIVATE-KEY").ainvoke([UserMessage(content="PRIVATE-PROMPT")], Output)
|
|
assert diagnostics.consecutive_schema_errors() == 1
|
|
assert len(sent) == 1
|
|
assert "action.0.click.index (int_parsing)" in str(caught.value)
|
|
assert caught.value.status_code == 422
|
|
assert "PRIVATE" not in str(caught.value) and caught.value.__suppress_context__
|
|
assert usage.snapshot("adapter")["total_tokens"] == 24
|
|
assert usage.snapshot("adapter")["parse_failed_requests"] == 1
|
|
assert "PRIVATE" not in (local / "adapter" / "model-diagnostics.jsonl").read_text()
|
|
assert diagnostics.summary(diag)["events"][0]["errors"] == [{"loc": ["action", 0, "click", "index"], "type": "int_parsing"}]
|
|
|
|
|
|
def test_diagnostic_storage_failure_is_explicit_without_retry_or_exception(local, monkeypatch):
|
|
def broken(_tid):
|
|
raise OSError("PRIVATE failure details")
|
|
monkeypatch.setattr(diagnostics, "task_doc_dir", broken)
|
|
with diagnostics.run_scope("storage", "run_fixture") as diag:
|
|
diagnostics.schema_error(None, failure(), Output)
|
|
assert diagnostics.summary(diag)["storage_error"] is True
|
|
assert len(diag.events) == 1 and "PRIVATE" not in json.dumps(diagnostics.summary(diag))
|