108 lines
6.4 KiB
Python
108 lines
6.4 KiB
Python
"""逐来源文字分析；分批请求不会限制任务总阅读数量。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import asyncio
|
||
|
||
import httpx
|
||
|
||
from backend.browser.llm_factory import get_llm_config
|
||
from backend.prospecting.contracts import AnalysisOutput, ProspectingBrief
|
||
from backend.prospecting.prompts import analysis_prompt
|
||
from backend.prospecting import usage as model_usage
|
||
from backend.prospecting.evidence_context import related_context as _related_context
|
||
from backend.prospecting.people import own_region_quote, region_quote_supports
|
||
|
||
ANALYSIS_BATCH_SIZE = 12
|
||
|
||
|
||
def validate_analysis(payload: dict, sources: list[dict], brief: ProspectingBrief | None = None) -> tuple[list[dict], list[dict]]:
|
||
parsed = AnalysisOutput.model_validate(payload)
|
||
by_id = {source["source_id"]: source for source in sources}
|
||
returned = [row.source_id for row in parsed.results]
|
||
if len(returned) != len(set(returned)) or set(returned) != set(by_id):
|
||
raise ValueError("模型结果与本轮来源不一一对应，未保存为待核线索")
|
||
decisions, leads = [], []
|
||
for row in parsed.results:
|
||
source = by_id[row.source_id]
|
||
quote = row.quote.strip()
|
||
if (row.decision != "unclear" and not quote) or (quote and quote not in source["content"]):
|
||
raise ValueError("模型引用不在对应来源原文中，未保存为待核线索")
|
||
region_quote = row.region_quote.strip()
|
||
if region_quote and not own_region_quote(source["content"], region_quote):
|
||
raise ValueError("地区依据必须来自对应发言人原话，不能继承上下文或使用IP属地")
|
||
if row.region_match != "unknown" and not region_quote:
|
||
raise ValueError("地区判断缺少对应来源原话，未保存为已匹配地区")
|
||
if brief and brief.region and row.region_match == "yes":
|
||
if not region_quote_supports(brief.region, region_quote):
|
||
raise ValueError("地区匹配缺少对应来源本人地点原话，未保存为已匹配地区")
|
||
metadata = {key: source.get(key) for key in (
|
||
"source_type", "source_url", "author", "author_profile_url", "page_time", "ip_location",
|
||
"parent_source_id", "root_source_id", "reply_to", "observed_at",
|
||
) if source.get(key) is not None}
|
||
decision = {**metadata, **row.model_dump(), "region_quote": region_quote,
|
||
"region_scope": brief.region if brief else None}
|
||
decisions.append(decision)
|
||
if row.decision == "keep":
|
||
leads.append({**decision, "id": source["source_id"], "status": "pending"})
|
||
return decisions, leads
|
||
|
||
|
||
async def analyze_sources(brief: ProspectingBrief, sources: list[dict], context_sources: list[dict] | None = None) -> dict:
|
||
if not sources:
|
||
raise ValueError("没有取得可核对的页面文字，无法判断是否存在需求")
|
||
cfg = get_llm_config()
|
||
if not cfg["api_key"]:
|
||
raise ValueError("请先在设置中填写 DeepSeek API Key")
|
||
decisions, leads, summaries, models = [], [], [], []
|
||
usage: dict[str, int] = {}
|
||
async with httpx.AsyncClient(timeout=60) as client:
|
||
for offset in range(0, len(sources), ANALYSIS_BATCH_SIZE):
|
||
batch = sources[offset:offset + ANALYSIS_BATCH_SIZE]
|
||
payload = {
|
||
"model": cfg["model"], "messages": [{"role": "user", "content": analysis_prompt(brief, batch, _related_context(batch, context_sources or sources))}],
|
||
"temperature": 0, "max_tokens": 8192, "response_format": {"type": "json_object"},
|
||
}
|
||
if cfg["provider"] == "deepseek":
|
||
payload["thinking"] = {"type": "disabled"}
|
||
phase = model_usage.current_scope()[1]
|
||
call = model_usage.start_call(cfg["model"], phase=phase if phase in {"analysis", "analysis_incremental", "review"} else "analysis")
|
||
call_status, status_code = "connection_error", None
|
||
try:
|
||
response = await client.post(cfg["base_url"].rstrip("/") + "/chat/completions", json=payload,
|
||
headers={"Authorization": "Bearer " + cfg["api_key"]})
|
||
if response.status_code >= 400:
|
||
call_status, status_code = "http_error", response.status_code
|
||
raise ValueError({401: "DeepSeek Key 无效，请在设置中更换", 402: "模型服务余额不足", 429: "模型请求受限，请稍后继续分析"}.get(response.status_code, f"文字分析接口返回 HTTP {response.status_code}"))
|
||
call_status = "invalid_response"
|
||
body = response.json()
|
||
if not isinstance(body, dict):
|
||
raise TypeError("invalid analysis response")
|
||
model_usage.record_response(call, body.get("usage"), model=body.get("model") or cfg["model"])
|
||
call_status = "schema_error"
|
||
parsed = json.loads(body["choices"][0]["message"].get("content") or "")
|
||
batch_decisions, batch_leads = validate_analysis(parsed, batch, brief)
|
||
call_status = "succeeded"
|
||
except httpx.HTTPError as error:
|
||
raise ValueError("文字分析服务暂时无法连接；页面资料已保留，可以继续分析") from error
|
||
except (ValueError, TypeError, KeyError, IndexError) as error:
|
||
if isinstance(error, ValueError) and str(error).startswith(("DeepSeek", "模型", "文字分析接口")):
|
||
raise
|
||
raise ValueError("文字分析结果未通过来源与原话核验；已保留页面资料，可重试分析") from error
|
||
except asyncio.CancelledError:
|
||
call_status = "cancelled"
|
||
raise
|
||
finally:
|
||
model_usage.finish_call(call, call_status, http_status=status_code)
|
||
decisions.extend(batch_decisions)
|
||
leads.extend(batch_leads)
|
||
summaries.append(parsed.get("summary", ""))
|
||
models.append(body.get("model", cfg["model"]))
|
||
for key, value in (body.get("usage") or {}).items():
|
||
if isinstance(value, int):
|
||
usage[key] = usage.get(key, 0) + value
|
||
return {"decisions": decisions, "leads": leads, "analysis_model": models[-1], "analysis_models": list(dict.fromkeys(models)),
|
||
"analysis_usage": usage, "analysis_summary": "\n".join(summaries),
|
||
"analysis_batches": len(models)}
|