"""Optional public-web research via DeepSeek Search/Fetch. These calls do not use the browser's cookies or establish a Xiaohongshu reading. The caller must opt in. API credentials are only sent to the fixed DeepSeek host. """ from __future__ import annotations import asyncio import ipaddress import json import socket from datetime import datetime, timezone from typing import Any from urllib.parse import urlsplit, urlunsplit import httpx from backend.browser import llm_keys from backend.browser.llm_factory import get_llm_config from backend.config import settings from backend.db import database as db API_ROOT = "https://api.deepseek.com/v1/tools" MAX_RESPONSE_BYTES = 1_048_576 MAX_CONTENT_CHARS = 24_000 class ResearchError(RuntimeError): def __init__(self, message: str, status_code: int = 502): super().__init__(message) self.status_code = status_code def _public_host(host: str) -> bool: """Reject local names and literal non-public addresses before DNS lookup.""" host = host.lower().rstrip(".") if not host or host == "localhost" or host.endswith((".localhost", ".local", ".internal", ".lan")): return False try: return ipaddress.ip_address(host).is_global except ValueError: return "." in host def _normalise_url(url: str) -> str: if not isinstance(url, str) or not url.strip() or len(url) > 8192: raise ValueError("网页地址不能为空且不能超过 8192 个字符") url = url.strip() if any(ord(char) < 33 or char == "\\" for char in url): raise ValueError("网页地址包含不允许的字符") try: parsed = urlsplit(url) host = (parsed.hostname or "").encode("idna").decode("ascii").lower().rstrip(".") port = parsed.port except (ValueError, UnicodeError) as error: raise ValueError("网页地址格式不正确") from error if parsed.scheme.lower() not in {"http", "https"} or parsed.username is not None or parsed.password is not None: raise ValueError("只接受不携带账号密码的公开 HTTP/HTTPS 网页") if not _public_host(host): raise ValueError("不支持本机、内网或保留地址") expected_port = 443 if parsed.scheme.lower() == "https" else 80 if port is not None and port != expected_port: raise ValueError("公开网页只支持标准 HTTP/HTTPS 端口") authority = f"[{host}]" if ":" in host else host if port is not None: authority += f":{port}" return urlunsplit((parsed.scheme.lower(), authority, parsed.path or "/", parsed.query, "")) async def validate_public_url(url: str) -> str: """Validate the supplied host; the remote Fetch service owns redirects.""" normalised = _normalise_url(url) parsed = urlsplit(normalised) host = parsed.hostname or "" try: literal = ipaddress.ip_address(host) except ValueError: literal = None if literal is None: try: addresses = await asyncio.wait_for( asyncio.to_thread(socket.getaddrinfo, host, parsed.port or (443 if parsed.scheme == "https" else 80), type=socket.SOCK_STREAM), timeout=5, ) except (OSError, asyncio.TimeoutError) as error: raise ValueError("无法确认网页域名的公开地址") from error if not addresses or any(not ipaddress.ip_address(address[4][0]).is_global for address in addresses): raise ValueError("网页域名解析到了内网或保留地址") return normalised def _api_key() -> str: cfg = get_llm_config() if cfg["provider"] == "deepseek": key = cfg.get("api_key", "") else: # A selected Qwen provider must never send its credential to DeepSeek. primary = db.get_setting("llm_api_key") or settings.deepseek_api_key key, _using_fallback = llm_keys.choose("deepseek", primary) if not key or str(key).strip().lower() in {"sk-your-key-here", "sk-你的key"}: raise ResearchError("请先在设置中配置 DeepSeek API Key,再使用公开资料检索。", 503) return str(key) def _error_for_status(status: int) -> ResearchError: message = { 400: "DeepSeek 拒绝了检索参数。", 401: "DeepSeek API Key 无效,请检查设置。", 402: "DeepSeek 余额不足,请充值或配置备用 Key。", 403: "DeepSeek Fetch 拒绝访问该网页。", 404: "该网页没有可读取的公开正文。", 408: "DeepSeek 读取网页超时。", 429: "DeepSeek 检索限流,请稍后再试。", }.get(status, "DeepSeek 公开资料服务暂时不可用。") # Preserve actionable provider status without returning its response body. return ResearchError(message, status if status in {400, 401, 402, 403, 404, 408, 429} else 502) async def _request(operation: str, payload: dict[str, Any]) -> dict: key = _api_key() chunks: list[bytes] = [] total = 0 try: async with httpx.AsyncClient(timeout=httpx.Timeout(30.0, connect=10.0), follow_redirects=False, trust_env=False) as client: async with client.stream( "POST", f"{API_ROOT}/{operation}", headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, json=payload, ) as response: if response.status_code != 200: raise _error_for_status(response.status_code) async for chunk in response.aiter_bytes(): total += len(chunk) if total > MAX_RESPONSE_BYTES: raise ResearchError("DeepSeek 返回内容过大,本次未保存。") chunks.append(chunk) except httpx.TimeoutException as error: raise ResearchError("DeepSeek 公开资料请求超时,本次未自动重试。", 504) from error except httpx.HTTPError as error: raise ResearchError("无法连接 DeepSeek 公开资料服务。") from error try: data = json.loads(b"".join(chunks)) except (ValueError, UnicodeError) as error: raise ResearchError("DeepSeek 返回了无法解析的资料。") from error if not isinstance(data, dict): raise ResearchError("DeepSeek 返回格式不正确。") return data def _timestamp() -> str: return datetime.now(timezone.utc).isoformat() def _text(value: Any, limit: int) -> str: return value[:limit] if isinstance(value, str) else "" async def search_public(query: str, count: int = 3) -> dict: query = query.strip() if isinstance(query, str) else "" if not query or len(query) > 128: raise ValueError("搜索词须为 1 至 128 个字符") if isinstance(count, bool) or not isinstance(count, int) or not 1 <= count <= 10: raise ValueError("每次公开检索条数须为 1 至 10") raw = await _request("search", {"query": query, "count": count, "need_content": False}) entries = raw.get("results") if not isinstance(entries, list): raise ResearchError("DeepSeek 搜索结果缺少 results 列表。") results: list[dict] = [] dropped = 0 for entry in entries[:count]: if not isinstance(entry, dict): raise ResearchError("DeepSeek 搜索结果条目格式不正确。") try: url = _normalise_url(entry.get("url", "")) except ValueError: dropped += 1 continue results.append({ "title": _text(entry.get("title"), 300), "url": url, "snippet": _text(entry.get("snippet"), 1000), "summary": _text(entry.get("summary"), 4000), "site_name": _text(entry.get("site_name"), 200), "date": _text(entry.get("date"), 64) or None, }) return { "provider": "deepseek", "source_type": "public_web_search", "query": query, "results": results, "retrieved_at": _timestamp(), "dropped_results": dropped, "notice": "公开搜索摘要,不代表已读取小红书登录后正文或评论。", } async def fetch_public(url: str) -> dict: url = await validate_public_url(url) raw = await _request("fetch", {"url": url}) content = raw.get("content") if not isinstance(content, str) or not content.strip(): raise ResearchError("该网页没有可提取的公开正文。", 404) return { "provider": "deepseek", "source_type": "public_web_fetch", "url": url, "title": _text(raw.get("title"), 300), "content": content[:MAX_CONTENT_CHARS], "content_truncated": len(content) > MAX_CONTENT_CHARS, "retrieved_at": _timestamp(), "notice": "第三方抓取的公开正文,不含浏览器登录态;网页访问及重定向由 DeepSeek 服务处理。", }