feat: 大模型使用场景化配置与去重多源记录
- 新增 configs/llm_models.yaml: 4 个场景(event_extraction/daily_report/stock_report/embedding)
可独立配置 provider/model/api_key_env/base_url_env/temperature 等,含用途与模型要求说明
- 新增 configs/loader.py: YAML 场景加载器(优先级: CLI 参数 > YAML > .env > 内置默认)
- llm/client.py: load_llm_config 支持 scene 参数,LLMConfig 增加 max_attempts
- embedding/factory+remote+local: provider/model/batch_limit 支持场景覆盖
- scheduler/reporter+stock_reporter: 日报/个股摘要接入场景配置
- dedup: Fingerprint.source_ids 多源记录 + 旧库自动迁移 + DedupResult 多源字段
- scripts/run_dedup: uniques JSON 的 sources 字段 + data/deduped/{day}/sources.json 汇总
- scripts/run_event_extraction: 接入 event_extraction 场景
- 补充测试: 场景优先级/零值、多源合并、旧库迁移、embedding 场景覆盖
This commit is contained in:
+17
-9
@@ -17,7 +17,10 @@ import time
|
||||
from collections import Counter
|
||||
from datetime import date, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from llm.client import LLMConfig
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from loguru import logger
|
||||
@@ -519,10 +522,10 @@ def _generate_ai_summary(news: dict, cninfo: dict, day_str: str,
|
||||
return ""
|
||||
|
||||
try:
|
||||
from llm.client import load_llm_config, make_sync_client
|
||||
config = load_llm_config()
|
||||
from llm.client import SCENE_DAILY_REPORT, load_llm_config, make_sync_client
|
||||
config = load_llm_config(scene=SCENE_DAILY_REPORT)
|
||||
client = make_sync_client(config)
|
||||
return _llm_summarize(client, config.model, lines, day_str)
|
||||
return _llm_summarize(client, config, lines, day_str)
|
||||
except Exception as e:
|
||||
logger.warning("AI 摘要生成失败: {}", e)
|
||||
return ""
|
||||
@@ -548,8 +551,12 @@ def _split_lines_into_chunks(lines: list[str], max_chars: int = 3000) -> list[li
|
||||
return chunks
|
||||
|
||||
|
||||
def _llm_summarize(client, model: str, lines: list[str], day_str: str) -> str:
|
||||
"""LLM 摘要:单块直接总结,多块先分段总结再合并。"""
|
||||
def _llm_summarize(client, config: LLMConfig, lines: list[str], day_str: str) -> str:
|
||||
"""LLM 摘要:单块直接总结,多块先分段总结再合并。
|
||||
|
||||
config 为 llm.client.LLMConfig(daily_report 场景),提供 model / temperature。
|
||||
"""
|
||||
model = config.model
|
||||
chunks = _split_lines_into_chunks(lines)
|
||||
|
||||
if len(chunks) == 1:
|
||||
@@ -609,9 +616,10 @@ def _build_prompt(lines: list[str], day_str: str) -> str:
|
||||
直接输出要点列表:"""
|
||||
|
||||
|
||||
def _llm_call(client, model: str, prompt: str, max_tokens: int = 1500) -> str:
|
||||
def _llm_call(client, config: LLMConfig, prompt: str, max_tokens: int = 1500) -> str:
|
||||
"""单次 LLM 调用(带重试),返回 strip 后的文本。
|
||||
|
||||
config 为 llm.client.LLMConfig(daily_report 场景),提供 model / temperature。
|
||||
失败按指数退避重试 `_LLM_RETRY_TIMES` 次(默认 3),全部失败则抛出最后一次异常。
|
||||
若 finish_reason 为 'length' 则说明达到 max_tokens 上限被截断。
|
||||
"""
|
||||
@@ -619,12 +627,12 @@ def _llm_call(client, model: str, prompt: str, max_tokens: int = 1500) -> str:
|
||||
for attempt in range(_LLM_RETRY_TIMES):
|
||||
try:
|
||||
resp = client.chat.completions.create(
|
||||
model=model,
|
||||
model=config.model,
|
||||
messages=[
|
||||
{"role": "system", "content": "你是 A 股日报撰写助手,输出简洁、有洞察的新闻摘要。"},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
temperature=0.3,
|
||||
temperature=config.temperature,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
content = (resp.choices[0].message.content or "").strip()
|
||||
|
||||
@@ -232,7 +232,7 @@ def _generate_ai_summary(company_name: str, announcements: list[dict],
|
||||
news: list[dict], research: list[dict],
|
||||
irm: list[dict]) -> str:
|
||||
"""LLM 生成个股要点分析。"""
|
||||
from llm.client import load_llm_config, make_sync_client
|
||||
from llm.client import SCENE_STOCK_REPORT, load_llm_config, make_sync_client
|
||||
|
||||
lines = []
|
||||
|
||||
@@ -277,12 +277,12 @@ def _generate_ai_summary(company_name: str, announcements: list[dict],
|
||||
直接输出要点列表:"""
|
||||
|
||||
try:
|
||||
config = load_llm_config()
|
||||
config = load_llm_config(scene=SCENE_STOCK_REPORT)
|
||||
client = make_sync_client(config)
|
||||
resp = client.chat.completions.create(
|
||||
model=config.model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.3, max_tokens=500,
|
||||
temperature=config.temperature, max_tokens=500,
|
||||
)
|
||||
return (resp.choices[0].message.content or "").strip()
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user