111 lines
6.5 KiB
Python
111 lines
6.5 KiB
Python
"""Request and action observation keeps metadata separate from private data."""
|
|
import hashlib
|
|
import json
|
|
from types import SimpleNamespace as Obj
|
|
|
|
import pytest
|
|
|
|
from backend.browser import diagnostics, llm_factory, structured_output, tools
|
|
from backend.prospecting import usage
|
|
|
|
|
|
@pytest.fixture
|
|
def local(tmp_path, monkeypatch):
|
|
def directory(tid):
|
|
path = tmp_path / tid
|
|
path.mkdir(exist_ok=True)
|
|
return path
|
|
monkeypatch.setattr(usage, 'task_doc_dir', directory)
|
|
monkeypatch.setattr(diagnostics, 'task_doc_dir', directory)
|
|
return tmp_path
|
|
|
|
|
|
def test_request_metadata_follows_only_action_union_not_parameter_names(local):
|
|
function = {'type': 'function', 'function': {'name': 'return_structured_output', 'parameters': {
|
|
'description': 'PRIVATE-PROMPT', 'properties': {'action': {'items': {'$ref': '#/$defs/Action'}}},
|
|
'$defs': {'Action': {'anyOf': [{'$ref': '#/$defs/Save'}, {'$ref': '#/$defs/Scroll'}]},
|
|
'Save': {'properties': {'save_source': {'$ref': '#/$defs/Params'}}},
|
|
'Scroll': {'properties': {'scroll': {'type': 'object'}}},
|
|
'Params': {'properties': {'content': {'default': 'PRIVATE-QUOTE'},
|
|
'source_url': {'default': 'https://private.invalid/'}}}}}}}
|
|
call = usage.UsageCall('fixture', 'a'*32, 'browser', 'deepseek-flash')
|
|
diagnostics.record_request_contract(call, function)
|
|
text = (local/'fixture'/'model-request-contracts.jsonl').read_text()
|
|
row = json.loads(text)
|
|
assert row['action_names'] == ['save_source', 'scroll']
|
|
assert row['kind'] == 'request_prepared' and row['call_id'] == call.call_id
|
|
assert row['schema_sha256'] == hashlib.sha256(json.dumps(function, ensure_ascii=False, sort_keys=True).encode()).hexdigest()
|
|
assert all(value not in text for value in ('PRIVATE', 'https://', 'source_url', 'content', 'parameters'))
|
|
|
|
|
|
def test_action_boundary_counts_only_actual_invocations_and_keeps_scope(local):
|
|
assert diagnostics.record_browser_action('save_source', 'invoked') is None
|
|
with diagnostics.run_scope('fixture', 'run_one'):
|
|
blocked = diagnostics.record_browser_action('click', 'blocked')
|
|
action_id = diagnostics.record_browser_action('save_source', 'invoked')
|
|
diagnostics.record_browser_action('save_source', 'returned', action_id=action_id)
|
|
assert diagnostics.record_browser_action('https://private.invalid/', 'invoked') is None
|
|
rows = [json.loads(line) for line in (local/'fixture'/'browser-actions.jsonl').read_text().splitlines()]
|
|
assert [row['status'] for row in rows] == ['blocked', 'invoked', 'returned']
|
|
assert rows[1]['action_id'] == rows[2]['action_id'] == action_id != blocked
|
|
assert all(row['task_id'] == 'fixture' and row['run_id'] == 'run_one' for row in rows)
|
|
assert 'https://' not in json.dumps(rows)
|
|
|
|
|
|
async def test_real_adapter_outbound_off_schema_is_audited_and_joinable_to_response(local, monkeypatch):
|
|
import openai
|
|
from browser_use.agent.views import AgentOutput
|
|
from browser_use.llm.messages import UserMessage
|
|
monkeypatch.setattr(tools.settings, 'reply_relation_preflight_enabled', False)
|
|
output = AgentOutput.type_with_custom_actions(tools.build_prospecting_tools('fixture').registry.create_action_model())
|
|
sent = []
|
|
class Client:
|
|
def __init__(self, **options):
|
|
assert options['max_retries'] == 0
|
|
self.chat = Obj(completions=Obj(create=self.create))
|
|
async def __aenter__(self): return self
|
|
async def __aexit__(self, *_args): return False
|
|
async def create(self, **params):
|
|
sent.append(params)
|
|
return Obj(model='deepseek-flash', choices=[Obj(finish_reason='tool_calls', message=Obj(content=None,
|
|
tool_calls=[Obj(type='function', function=Obj(name=structured_output.FUNCTION_NAME,
|
|
arguments=json.dumps({'memory':'PRIVATE-RESPONSE', 'action':[{'inspect_source_candidates': {'max_comments': 1}}]})))]))],
|
|
usage=Obj(prompt_tokens=40, completion_tokens=8, total_tokens=48))
|
|
monkeypatch.setattr(openai, 'AsyncOpenAI', Client)
|
|
with usage.usage_scope('fixture', 'browser'):
|
|
await llm_factory.DeepSeekBrowserChat('deepseek-flash', 'PRIVATE-KEY').ainvoke([UserMessage(content='PRIVATE-PROMPT')], output)
|
|
row = json.loads((local/'fixture'/'model-request-contracts.jsonl').read_text())
|
|
assert len(sent) == 1
|
|
assert not {'inspect_reply_relations', 'resolve_reply_target'} & set(row['action_names'])
|
|
assert {'save_source', 'advance_comments', 'expand_replies'} <= set(row['action_names'])
|
|
assert row['schema_sha256'] == hashlib.sha256(json.dumps(sent[0]['tools'][0], ensure_ascii=False, sort_keys=True).encode()).hexdigest()
|
|
events = [json.loads(line) for line in (local/'fixture'/'model-usage.jsonl').read_text().splitlines()]
|
|
assert any(event['kind'] == 'response' and event['call_id'] == row['call_id'] for event in events)
|
|
assert any(event['kind'] == 'finished' and event['call_id'] == row['call_id'] and event['status'] == 'succeeded' for event in events)
|
|
assert 'PRIVATE' not in (local/'fixture'/'model-request-contracts.jsonl').read_text()
|
|
|
|
|
|
async def test_observation_storage_failure_does_not_retry_or_change_model_result(local, monkeypatch):
|
|
import openai
|
|
from pydantic import BaseModel
|
|
from browser_use.llm.messages import UserMessage
|
|
class Output(BaseModel):
|
|
action: str
|
|
calls = []
|
|
class Client:
|
|
def __init__(self, **_options): self.chat = Obj(completions=Obj(create=self.create))
|
|
async def __aenter__(self): return self
|
|
async def __aexit__(self, *_args): return False
|
|
async def create(self, **params):
|
|
calls.append(params)
|
|
return Obj(model='deepseek-flash', choices=[Obj(finish_reason='tool_calls', message=Obj(content=None,
|
|
tool_calls=[Obj(type='function', function=Obj(name=structured_output.FUNCTION_NAME, arguments='{"action":"done"}'))]))], usage=None)
|
|
monkeypatch.setattr(openai, 'AsyncOpenAI', Client)
|
|
def broken(_tid): raise OSError('PRIVATE-STORAGE')
|
|
monkeypatch.setattr(diagnostics, 'task_doc_dir', broken)
|
|
with usage.usage_scope('fixture', 'browser'), diagnostics.run_scope('fixture', 'run_one') as state:
|
|
result = await llm_factory.DeepSeekBrowserChat('deepseek-flash', 'PRIVATE-KEY').ainvoke([UserMessage(content='PRIVATE-PROMPT')], Output)
|
|
diagnostics.record_browser_action('save_source', 'invoked')
|
|
assert state.storage_error is True
|
|
assert result.completion.action == 'done' and len(calls) == 1
|