"""逐来源文字分析;分批请求不会限制任务总阅读数量。""" from __future__ import annotations import json import asyncio import httpx from backend.browser.llm_factory import get_llm_config from backend.prospecting.contracts import AnalysisOutput, ProspectingBrief from backend.prospecting.prompts import analysis_prompt from backend.prospecting import usage as model_usage from backend.prospecting.evidence_context import related_context as _related_context from backend.prospecting.people import own_region_quote, region_quote_supports ANALYSIS_BATCH_SIZE = 12 def validate_analysis(payload: dict, sources: list[dict], brief: ProspectingBrief | None = None) -> tuple[list[dict], list[dict]]: parsed = AnalysisOutput.model_validate(payload) by_id = {source["source_id"]: source for source in sources} returned = [row.source_id for row in parsed.results] if len(returned) != len(set(returned)) or set(returned) != set(by_id): raise ValueError("模型结果与本轮来源不一一对应,未保存为待核线索") decisions, leads = [], [] for row in parsed.results: source = by_id[row.source_id] quote = row.quote.strip() if (row.decision != "unclear" and not quote) or (quote and quote not in source["content"]): raise ValueError("模型引用不在对应来源原文中,未保存为待核线索") region_quote = row.region_quote.strip() if region_quote and not own_region_quote(source["content"], region_quote): raise ValueError("地区依据必须来自对应发言人原话,不能继承上下文或使用IP属地") if row.region_match != "unknown" and not region_quote: raise ValueError("地区判断缺少对应来源原话,未保存为已匹配地区") if brief and brief.region and row.region_match == "yes": if not region_quote_supports(brief.region, region_quote): raise ValueError("地区匹配缺少对应来源本人地点原话,未保存为已匹配地区") metadata = {key: source.get(key) for key in ( "source_type", "source_url", "author", "author_profile_url", "page_time", "ip_location", "parent_source_id", "root_source_id", "reply_to", "observed_at", ) if source.get(key) is not None} decision = {**metadata, **row.model_dump(), "region_quote": region_quote, "region_scope": brief.region if brief else None} decisions.append(decision) if row.decision == "keep": leads.append({**decision, "id": source["source_id"], "status": "pending"}) return decisions, leads async def analyze_sources(brief: ProspectingBrief, sources: list[dict], context_sources: list[dict] | None = None) -> dict: if not sources: raise ValueError("没有取得可核对的页面文字,无法判断是否存在需求") cfg = get_llm_config() if not cfg["api_key"]: raise ValueError("请先在设置中填写 DeepSeek API Key") decisions, leads, summaries, models = [], [], [], [] usage: dict[str, int] = {} async with httpx.AsyncClient(timeout=60) as client: for offset in range(0, len(sources), ANALYSIS_BATCH_SIZE): batch = sources[offset:offset + ANALYSIS_BATCH_SIZE] payload = { "model": cfg["model"], "messages": [{"role": "user", "content": analysis_prompt(brief, batch, _related_context(batch, context_sources or sources))}], "temperature": 0, "max_tokens": 8192, "response_format": {"type": "json_object"}, } if cfg["provider"] == "deepseek": payload["thinking"] = {"type": "disabled"} phase = model_usage.current_scope()[1] call = model_usage.start_call(cfg["model"], phase=phase if phase in {"analysis", "analysis_incremental", "review"} else "analysis") call_status, status_code = "connection_error", None try: response = await client.post(cfg["base_url"].rstrip("/") + "/chat/completions", json=payload, headers={"Authorization": "Bearer " + cfg["api_key"]}) if response.status_code >= 400: call_status, status_code = "http_error", response.status_code raise ValueError({401: "DeepSeek Key 无效,请在设置中更换", 402: "模型服务余额不足", 429: "模型请求受限,请稍后继续分析"}.get(response.status_code, f"文字分析接口返回 HTTP {response.status_code}")) call_status = "invalid_response" body = response.json() if not isinstance(body, dict): raise TypeError("invalid analysis response") model_usage.record_response(call, body.get("usage"), model=body.get("model") or cfg["model"]) call_status = "schema_error" parsed = json.loads(body["choices"][0]["message"].get("content") or "") batch_decisions, batch_leads = validate_analysis(parsed, batch, brief) call_status = "succeeded" except httpx.HTTPError as error: raise ValueError("文字分析服务暂时无法连接;页面资料已保留,可以继续分析") from error except (ValueError, TypeError, KeyError, IndexError) as error: if isinstance(error, ValueError) and str(error).startswith(("DeepSeek", "模型", "文字分析接口")): raise raise ValueError("文字分析结果未通过来源与原话核验;已保留页面资料,可重试分析") from error except asyncio.CancelledError: call_status = "cancelled" raise finally: model_usage.finish_call(call, call_status, http_status=status_code) decisions.extend(batch_decisions) leads.extend(batch_leads) summaries.append(parsed.get("summary", "")) models.append(body.get("model", cfg["model"])) for key, value in (body.get("usage") or {}).items(): if isinstance(value, int): usage[key] = usage.get(key, 0) + value return {"decisions": decisions, "leads": leads, "analysis_model": models[-1], "analysis_models": list(dict.fromkeys(models)), "analysis_usage": usage, "analysis_summary": "\n".join(summaries), "analysis_batches": len(models)}