feat: 大模型使用场景化配置与去重多源记录

- 新增 configs/llm_models.yaml: 4 个场景(event_extraction/daily_report/stock_report/embedding)
  可独立配置 provider/model/api_key_env/base_url_env/temperature 等,含用途与模型要求说明
- 新增 configs/loader.py: YAML 场景加载器(优先级: CLI 参数 > YAML > .env > 内置默认)
- llm/client.py: load_llm_config 支持 scene 参数,LLMConfig 增加 max_attempts
- embedding/factory+remote+local: provider/model/batch_limit 支持场景覆盖
- scheduler/reporter+stock_reporter: 日报/个股摘要接入场景配置
- dedup: Fingerprint.source_ids 多源记录 + 旧库自动迁移 + DedupResult 多源字段
- scripts/run_dedup: uniques JSON 的 sources 字段 + data/deduped/{day}/sources.json 汇总
- scripts/run_event_extraction: 接入 event_extraction 场景
- 补充测试: 场景优先级/零值、多源合并、旧库迁移、embedding 场景覆盖
This commit is contained in:
2026-08-12 07:57:10 +08:00
parent 0c032196d2
commit 3c65701449
21 changed files with 886 additions and 80 deletions
+17 -9
View File
@@ -17,7 +17,10 @@ import time
from collections import Counter
from datetime import date, datetime, timedelta
from pathlib import Path
from typing import Any
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from llm.client import LLMConfig
from dotenv import load_dotenv
from loguru import logger
@@ -519,10 +522,10 @@ def _generate_ai_summary(news: dict, cninfo: dict, day_str: str,
return ""
try:
from llm.client import load_llm_config, make_sync_client
config = load_llm_config()
from llm.client import SCENE_DAILY_REPORT, load_llm_config, make_sync_client
config = load_llm_config(scene=SCENE_DAILY_REPORT)
client = make_sync_client(config)
return _llm_summarize(client, config.model, lines, day_str)
return _llm_summarize(client, config, lines, day_str)
except Exception as e:
logger.warning("AI 摘要生成失败: {}", e)
return ""
@@ -548,8 +551,12 @@ def _split_lines_into_chunks(lines: list[str], max_chars: int = 3000) -> list[li
return chunks
def _llm_summarize(client, model: str, lines: list[str], day_str: str) -> str:
"""LLM 摘要:单块直接总结,多块先分段总结再合并。"""
def _llm_summarize(client, config: LLMConfig, lines: list[str], day_str: str) -> str:
"""LLM 摘要:单块直接总结,多块先分段总结再合并。
config 为 llm.client.LLMConfig(daily_report 场景),提供 model / temperature。
"""
model = config.model
chunks = _split_lines_into_chunks(lines)
if len(chunks) == 1:
@@ -609,9 +616,10 @@ def _build_prompt(lines: list[str], day_str: str) -> str:
直接输出要点列表:"""
def _llm_call(client, model: str, prompt: str, max_tokens: int = 1500) -> str:
def _llm_call(client, config: LLMConfig, prompt: str, max_tokens: int = 1500) -> str:
"""单次 LLM 调用(带重试),返回 strip 后的文本。
config 为 llm.client.LLMConfig(daily_report 场景),提供 model / temperature。
失败按指数退避重试 `_LLM_RETRY_TIMES` 次(默认 3),全部失败则抛出最后一次异常。
若 finish_reason 为 'length' 则说明达到 max_tokens 上限被截断。
"""
@@ -619,12 +627,12 @@ def _llm_call(client, model: str, prompt: str, max_tokens: int = 1500) -> str:
for attempt in range(_LLM_RETRY_TIMES):
try:
resp = client.chat.completions.create(
model=model,
model=config.model,
messages=[
{"role": "system", "content": "你是 A 股日报撰写助手,输出简洁、有洞察的新闻摘要。"},
{"role": "user", "content": prompt},
],
temperature=0.3,
temperature=config.temperature,
max_tokens=max_tokens,
)
content = (resp.choices[0].message.content or "").strip()
+3 -3
View File
@@ -232,7 +232,7 @@ def _generate_ai_summary(company_name: str, announcements: list[dict],
news: list[dict], research: list[dict],
irm: list[dict]) -> str:
"""LLM 生成个股要点分析。"""
from llm.client import load_llm_config, make_sync_client
from llm.client import SCENE_STOCK_REPORT, load_llm_config, make_sync_client
lines = []
@@ -277,12 +277,12 @@ def _generate_ai_summary(company_name: str, announcements: list[dict],
直接输出要点列表:"""
try:
config = load_llm_config()
config = load_llm_config(scene=SCENE_STOCK_REPORT)
client = make_sync_client(config)
resp = client.chat.completions.create(
model=config.model,
messages=[{"role": "user", "content": prompt}],
temperature=0.3, max_tokens=500,
temperature=config.temperature, max_tokens=500,
)
return (resp.choices[0].message.content or "").strip()
except Exception as e: