feat: 大模型使用场景化配置与去重多源记录

- 新增 configs/llm_models.yaml: 4 个场景(event_extraction/daily_report/stock_report/embedding)
  可独立配置 provider/model/api_key_env/base_url_env/temperature 等,含用途与模型要求说明
- 新增 configs/loader.py: YAML 场景加载器(优先级: CLI 参数 > YAML > .env > 内置默认)
- llm/client.py: load_llm_config 支持 scene 参数,LLMConfig 增加 max_attempts
- embedding/factory+remote+local: provider/model/batch_limit 支持场景覆盖
- scheduler/reporter+stock_reporter: 日报/个股摘要接入场景配置
- dedup: Fingerprint.source_ids 多源记录 + 旧库自动迁移 + DedupResult 多源字段
- scripts/run_dedup: uniques JSON 的 sources 字段 + data/deduped/{day}/sources.json 汇总
- scripts/run_event_extraction: 接入 event_extraction 场景
- 补充测试: 场景优先级/零值、多源合并、旧库迁移、embedding 场景覆盖
This commit is contained in:
2026-08-12 07:57:10 +08:00
parent 0c032196d2
commit 3c65701449
21 changed files with 886 additions and 80 deletions
+111
View File
@@ -362,3 +362,114 @@ def test_load_llm_config_missing_model_raises(monkeypatch: pytest.MonkeyPatch) -
monkeypatch.delenv("LLM_MODEL", raising=False)
with pytest.raises(ValueError, match="模型"):
load_llm_config(provider="deepseek")
# --------------------------------------------------------------------------- #
# load_llm_config —— configs/llm_models.yaml 场景配置
# --------------------------------------------------------------------------- #
def _patch_scene(monkeypatch: pytest.MonkeyPatch, cfg: dict) -> None:
"""替换场景加载,模拟 configs/llm_models.yaml 中的某场景配置。"""
monkeypatch.setattr(
"llm.client.load_scene_config",
lambda scene: cfg if scene == "daily_report" else {},
)
def test_load_llm_config_scene_overrides_env(monkeypatch: pytest.MonkeyPatch) -> None:
"""YAML 场景配置优先于 .env:provider / model / temperature / timeout / max_attempts。"""
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-env")
monkeypatch.setenv("DEEPSEEK_MODEL", "deepseek-env-model")
monkeypatch.setenv("QWEN_API_KEY", "sk-qwen")
_patch_scene(monkeypatch, {
"provider": "qwen",
"model": "qwen-max",
"temperature": 0.5,
"timeout_sec": 99,
"max_attempts": 5,
})
cfg = load_llm_config(scene="daily_report")
assert cfg.provider == "qwen"
assert cfg.model == "qwen-max"
assert cfg.api_key == "sk-qwen"
assert cfg.temperature == 0.5
assert cfg.timeout_sec == 99
assert cfg.max_attempts == 5
def test_load_llm_config_scene_blank_fields_fall_back_to_env(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""YAML 场景未配置的字段(如 model 留空)回退 .env,保持向后兼容。"""
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-env")
monkeypatch.setenv("DEEPSEEK_MODEL", "deepseek-env-model")
_patch_scene(monkeypatch, {"provider": "deepseek", "model": "", "temperature": 0.7})
cfg = load_llm_config(scene="daily_report")
assert cfg.provider == "deepseek"
assert cfg.model == "deepseek-env-model"
assert cfg.api_key == "sk-env"
assert cfg.temperature == 0.7
def test_load_llm_config_scene_api_key_env_name(monkeypatch: pytest.MonkeyPatch) -> None:
"""api_key_env 指向自定义环境变量时,优先使用该变量。"""
monkeypatch.setenv("MY_CUSTOM_KEY", "sk-custom")
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-default")
monkeypatch.setenv("DEEPSEEK_MODEL", "deepseek-m")
_patch_scene(monkeypatch, {
"provider": "deepseek",
"model": "deepseek-scene-m",
"api_key_env": "MY_CUSTOM_KEY",
})
cfg = load_llm_config(scene="daily_report")
assert cfg.api_key == "sk-custom"
assert cfg.model == "deepseek-scene-m"
def test_load_llm_config_scene_explicit_args_win(monkeypatch: pytest.MonkeyPatch) -> None:
"""CLI/显式参数优先级最高,覆盖 YAML 场景。"""
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-env")
_patch_scene(monkeypatch, {"provider": "qwen", "model": "qwen-max"})
monkeypatch.setenv("QWEN_API_KEY", "sk-qwen")
cfg = load_llm_config(provider="deepseek", model="deepseek-chat", scene="daily_report")
assert cfg.provider == "deepseek"
assert cfg.model == "deepseek-chat"
def test_load_llm_config_scene_missing_model_raises(monkeypatch: pytest.MonkeyPatch) -> None:
"""场景与 .env 都未配置模型时必须报错(无内置兜底)。"""
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-test")
monkeypatch.delenv("DEEPSEEK_MODEL", raising=False)
monkeypatch.delenv("LLM_MODEL", raising=False)
_patch_scene(monkeypatch, {"provider": "deepseek", "model": ""})
with pytest.raises(ValueError, match="模型"):
load_llm_config(scene="daily_report")
def test_load_llm_config_real_yaml_parseable() -> None:
"""真实 configs/llm_models.yaml 必须可解析且包含全部场景(回归保护)。"""
from configs.loader import load_defaults, load_scene_config
for scene in ("event_extraction", "daily_report", "stock_report", "embedding"):
assert isinstance(load_scene_config(scene), dict)
assert isinstance(load_defaults(), dict)
def test_load_llm_config_temperature_zero_is_respected(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""temperature=0 是合法配置,不应被 or 链回退默认。"""
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-env")
monkeypatch.setenv("DEEPSEEK_MODEL", "deepseek-m")
_patch_scene(monkeypatch, {"provider": "deepseek", "model": "deepseek-m", "temperature": 0})
cfg = load_llm_config(scene="daily_report")
assert cfg.temperature == 0.0
def test_load_llm_config_scene_max_attempts_zero() -> None:
"""max_attempts=0 由 _pick_int 显式处理。"""
from llm.client import _pick_int
assert _pick_int({"max_attempts": 0}, "max_attempts", 3) == 0
assert _pick_int({"max_attempts": ""}, "max_attempts", 3) == 3
assert _pick_int({}, "max_attempts", 3) == 3