206 lines
8.6 KiB
Python
206 lines
8.6 KiB
Python
"""Optional public-web research via DeepSeek Search/Fetch.
|
||
|
||
These calls do not use the browser's cookies or establish a Xiaohongshu reading.
|
||
The caller must opt in. API credentials are only sent to the fixed DeepSeek host.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import ipaddress
|
||
import json
|
||
import socket
|
||
from datetime import datetime, timezone
|
||
from typing import Any
|
||
from urllib.parse import urlsplit, urlunsplit
|
||
|
||
import httpx
|
||
|
||
from backend.browser import llm_keys
|
||
from backend.browser.llm_factory import get_llm_config
|
||
from backend.config import settings
|
||
from backend.db import database as db
|
||
|
||
API_ROOT = "https://api.deepseek.com/v1/tools"
|
||
MAX_RESPONSE_BYTES = 1_048_576
|
||
MAX_CONTENT_CHARS = 24_000
|
||
|
||
|
||
class ResearchError(RuntimeError):
|
||
def __init__(self, message: str, status_code: int = 502):
|
||
super().__init__(message)
|
||
self.status_code = status_code
|
||
|
||
|
||
def _public_host(host: str) -> bool:
|
||
"""Reject local names and literal non-public addresses before DNS lookup."""
|
||
host = host.lower().rstrip(".")
|
||
if not host or host == "localhost" or host.endswith((".localhost", ".local", ".internal", ".lan")):
|
||
return False
|
||
try:
|
||
return ipaddress.ip_address(host).is_global
|
||
except ValueError:
|
||
return "." in host
|
||
|
||
|
||
def _normalise_url(url: str) -> str:
|
||
if not isinstance(url, str) or not url.strip() or len(url) > 8192:
|
||
raise ValueError("网页地址不能为空且不能超过 8192 个字符")
|
||
url = url.strip()
|
||
if any(ord(char) < 33 or char == "\\" for char in url):
|
||
raise ValueError("网页地址包含不允许的字符")
|
||
try:
|
||
parsed = urlsplit(url)
|
||
host = (parsed.hostname or "").encode("idna").decode("ascii").lower().rstrip(".")
|
||
port = parsed.port
|
||
except (ValueError, UnicodeError) as error:
|
||
raise ValueError("网页地址格式不正确") from error
|
||
if parsed.scheme.lower() not in {"http", "https"} or parsed.username is not None or parsed.password is not None:
|
||
raise ValueError("只接受不携带账号密码的公开 HTTP/HTTPS 网页")
|
||
if not _public_host(host):
|
||
raise ValueError("不支持本机、内网或保留地址")
|
||
expected_port = 443 if parsed.scheme.lower() == "https" else 80
|
||
if port is not None and port != expected_port:
|
||
raise ValueError("公开网页只支持标准 HTTP/HTTPS 端口")
|
||
authority = f"[{host}]" if ":" in host else host
|
||
if port is not None:
|
||
authority += f":{port}"
|
||
return urlunsplit((parsed.scheme.lower(), authority, parsed.path or "/", parsed.query, ""))
|
||
|
||
|
||
async def validate_public_url(url: str) -> str:
|
||
"""Validate the supplied host; the remote Fetch service owns redirects."""
|
||
normalised = _normalise_url(url)
|
||
parsed = urlsplit(normalised)
|
||
host = parsed.hostname or ""
|
||
try:
|
||
literal = ipaddress.ip_address(host)
|
||
except ValueError:
|
||
literal = None
|
||
if literal is None:
|
||
try:
|
||
addresses = await asyncio.wait_for(
|
||
asyncio.to_thread(socket.getaddrinfo, host, parsed.port or (443 if parsed.scheme == "https" else 80), type=socket.SOCK_STREAM),
|
||
timeout=5,
|
||
)
|
||
except (OSError, asyncio.TimeoutError) as error:
|
||
raise ValueError("无法确认网页域名的公开地址") from error
|
||
if not addresses or any(not ipaddress.ip_address(address[4][0]).is_global for address in addresses):
|
||
raise ValueError("网页域名解析到了内网或保留地址")
|
||
return normalised
|
||
|
||
|
||
def _api_key() -> str:
|
||
cfg = get_llm_config()
|
||
if cfg["provider"] == "deepseek":
|
||
key = cfg.get("api_key", "")
|
||
else:
|
||
# A selected Qwen provider must never send its credential to DeepSeek.
|
||
primary = db.get_setting("llm_api_key") or settings.deepseek_api_key
|
||
key, _using_fallback = llm_keys.choose("deepseek", primary)
|
||
if not key or str(key).strip().lower() in {"sk-your-key-here", "sk-你的key"}:
|
||
raise ResearchError("请先在设置中配置 DeepSeek API Key，再使用公开资料检索。", 503)
|
||
return str(key)
|
||
|
||
|
||
def _error_for_status(status: int) -> ResearchError:
|
||
message = {
|
||
400: "DeepSeek 拒绝了检索参数。",
|
||
401: "DeepSeek API Key 无效，请检查设置。",
|
||
402: "DeepSeek 余额不足，请充值或配置备用 Key。",
|
||
403: "DeepSeek Fetch 拒绝访问该网页。",
|
||
404: "该网页没有可读取的公开正文。",
|
||
408: "DeepSeek 读取网页超时。",
|
||
429: "DeepSeek 检索限流，请稍后再试。",
|
||
}.get(status, "DeepSeek 公开资料服务暂时不可用。")
|
||
# Preserve actionable provider status without returning its response body.
|
||
return ResearchError(message, status if status in {400, 401, 402, 403, 404, 408, 429} else 502)
|
||
|
||
|
||
async def _request(operation: str, payload: dict[str, Any]) -> dict:
|
||
key = _api_key()
|
||
chunks: list[bytes] = []
|
||
total = 0
|
||
try:
|
||
async with httpx.AsyncClient(timeout=httpx.Timeout(30.0, connect=10.0), follow_redirects=False, trust_env=False) as client:
|
||
async with client.stream(
|
||
"POST", f"{API_ROOT}/{operation}",
|
||
headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
|
||
json=payload,
|
||
) as response:
|
||
if response.status_code != 200:
|
||
raise _error_for_status(response.status_code)
|
||
async for chunk in response.aiter_bytes():
|
||
total += len(chunk)
|
||
if total > MAX_RESPONSE_BYTES:
|
||
raise ResearchError("DeepSeek 返回内容过大，本次未保存。")
|
||
chunks.append(chunk)
|
||
except httpx.TimeoutException as error:
|
||
raise ResearchError("DeepSeek 公开资料请求超时，本次未自动重试。", 504) from error
|
||
except httpx.HTTPError as error:
|
||
raise ResearchError("无法连接 DeepSeek 公开资料服务。") from error
|
||
try:
|
||
data = json.loads(b"".join(chunks))
|
||
except (ValueError, UnicodeError) as error:
|
||
raise ResearchError("DeepSeek 返回了无法解析的资料。") from error
|
||
if not isinstance(data, dict):
|
||
raise ResearchError("DeepSeek 返回格式不正确。")
|
||
return data
|
||
|
||
|
||
def _timestamp() -> str:
|
||
return datetime.now(timezone.utc).isoformat()
|
||
|
||
|
||
def _text(value: Any, limit: int) -> str:
|
||
return value[:limit] if isinstance(value, str) else ""
|
||
|
||
|
||
async def search_public(query: str, count: int = 3) -> dict:
|
||
query = query.strip() if isinstance(query, str) else ""
|
||
if not query or len(query) > 128:
|
||
raise ValueError("搜索词须为 1 至 128 个字符")
|
||
if isinstance(count, bool) or not isinstance(count, int) or not 1 <= count <= 10:
|
||
raise ValueError("每次公开检索条数须为 1 至 10")
|
||
raw = await _request("search", {"query": query, "count": count, "need_content": False})
|
||
entries = raw.get("results")
|
||
if not isinstance(entries, list):
|
||
raise ResearchError("DeepSeek 搜索结果缺少 results 列表。")
|
||
results: list[dict] = []
|
||
dropped = 0
|
||
for entry in entries[:count]:
|
||
if not isinstance(entry, dict):
|
||
raise ResearchError("DeepSeek 搜索结果条目格式不正确。")
|
||
try:
|
||
url = _normalise_url(entry.get("url", ""))
|
||
except ValueError:
|
||
dropped += 1
|
||
continue
|
||
results.append({
|
||
"title": _text(entry.get("title"), 300),
|
||
"url": url,
|
||
"snippet": _text(entry.get("snippet"), 1000),
|
||
"summary": _text(entry.get("summary"), 4000),
|
||
"site_name": _text(entry.get("site_name"), 200),
|
||
"date": _text(entry.get("date"), 64) or None,
|
||
})
|
||
return {
|
||
"provider": "deepseek", "source_type": "public_web_search", "query": query,
|
||
"results": results, "retrieved_at": _timestamp(), "dropped_results": dropped,
|
||
"notice": "公开搜索摘要，不代表已读取小红书登录后正文或评论。",
|
||
}
|
||
|
||
|
||
async def fetch_public(url: str) -> dict:
|
||
url = await validate_public_url(url)
|
||
raw = await _request("fetch", {"url": url})
|
||
content = raw.get("content")
|
||
if not isinstance(content, str) or not content.strip():
|
||
raise ResearchError("该网页没有可提取的公开正文。", 404)
|
||
return {
|
||
"provider": "deepseek", "source_type": "public_web_fetch", "url": url,
|
||
"title": _text(raw.get("title"), 300), "content": content[:MAX_CONTENT_CHARS],
|
||
"content_truncated": len(content) > MAX_CONTENT_CHARS, "retrieved_at": _timestamp(),
|
||
"notice": "第三方抓取的公开正文，不含浏览器登录态；网页访问及重定向由 DeepSeek 服务处理。",
|
||
}
|