feat: 大模型使用场景化配置与去重多源记录
- 新增 configs/llm_models.yaml: 4 个场景(event_extraction/daily_report/stock_report/embedding)
可独立配置 provider/model/api_key_env/base_url_env/temperature 等,含用途与模型要求说明
- 新增 configs/loader.py: YAML 场景加载器(优先级: CLI 参数 > YAML > .env > 内置默认)
- llm/client.py: load_llm_config 支持 scene 参数,LLMConfig 增加 max_attempts
- embedding/factory+remote+local: provider/model/batch_limit 支持场景覆盖
- scheduler/reporter+stock_reporter: 日报/个股摘要接入场景配置
- dedup: Fingerprint.source_ids 多源记录 + 旧库自动迁移 + DedupResult 多源字段
- scripts/run_dedup: uniques JSON 的 sources 字段 + data/deduped/{day}/sources.json 汇总
- scripts/run_event_extraction: 接入 event_extraction 场景
- 补充测试: 场景优先级/零值、多源合并、旧库迁移、embedding 场景覆盖
This commit is contained in:
@@ -362,3 +362,114 @@ def test_load_llm_config_missing_model_raises(monkeypatch: pytest.MonkeyPatch) -
|
||||
monkeypatch.delenv("LLM_MODEL", raising=False)
|
||||
with pytest.raises(ValueError, match="模型"):
|
||||
load_llm_config(provider="deepseek")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# load_llm_config —— configs/llm_models.yaml 场景配置
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _patch_scene(monkeypatch: pytest.MonkeyPatch, cfg: dict) -> None:
|
||||
"""替换场景加载,模拟 configs/llm_models.yaml 中的某场景配置。"""
|
||||
monkeypatch.setattr(
|
||||
"llm.client.load_scene_config",
|
||||
lambda scene: cfg if scene == "daily_report" else {},
|
||||
)
|
||||
|
||||
|
||||
def test_load_llm_config_scene_overrides_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""YAML 场景配置优先于 .env:provider / model / temperature / timeout / max_attempts。"""
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-env")
|
||||
monkeypatch.setenv("DEEPSEEK_MODEL", "deepseek-env-model")
|
||||
monkeypatch.setenv("QWEN_API_KEY", "sk-qwen")
|
||||
_patch_scene(monkeypatch, {
|
||||
"provider": "qwen",
|
||||
"model": "qwen-max",
|
||||
"temperature": 0.5,
|
||||
"timeout_sec": 99,
|
||||
"max_attempts": 5,
|
||||
})
|
||||
cfg = load_llm_config(scene="daily_report")
|
||||
assert cfg.provider == "qwen"
|
||||
assert cfg.model == "qwen-max"
|
||||
assert cfg.api_key == "sk-qwen"
|
||||
assert cfg.temperature == 0.5
|
||||
assert cfg.timeout_sec == 99
|
||||
assert cfg.max_attempts == 5
|
||||
|
||||
|
||||
def test_load_llm_config_scene_blank_fields_fall_back_to_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""YAML 场景未配置的字段(如 model 留空)回退 .env,保持向后兼容。"""
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-env")
|
||||
monkeypatch.setenv("DEEPSEEK_MODEL", "deepseek-env-model")
|
||||
_patch_scene(monkeypatch, {"provider": "deepseek", "model": "", "temperature": 0.7})
|
||||
cfg = load_llm_config(scene="daily_report")
|
||||
assert cfg.provider == "deepseek"
|
||||
assert cfg.model == "deepseek-env-model"
|
||||
assert cfg.api_key == "sk-env"
|
||||
assert cfg.temperature == 0.7
|
||||
|
||||
|
||||
def test_load_llm_config_scene_api_key_env_name(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""api_key_env 指向自定义环境变量时,优先使用该变量。"""
|
||||
monkeypatch.setenv("MY_CUSTOM_KEY", "sk-custom")
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-default")
|
||||
monkeypatch.setenv("DEEPSEEK_MODEL", "deepseek-m")
|
||||
_patch_scene(monkeypatch, {
|
||||
"provider": "deepseek",
|
||||
"model": "deepseek-scene-m",
|
||||
"api_key_env": "MY_CUSTOM_KEY",
|
||||
})
|
||||
cfg = load_llm_config(scene="daily_report")
|
||||
assert cfg.api_key == "sk-custom"
|
||||
assert cfg.model == "deepseek-scene-m"
|
||||
|
||||
|
||||
def test_load_llm_config_scene_explicit_args_win(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""CLI/显式参数优先级最高,覆盖 YAML 场景。"""
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-env")
|
||||
_patch_scene(monkeypatch, {"provider": "qwen", "model": "qwen-max"})
|
||||
monkeypatch.setenv("QWEN_API_KEY", "sk-qwen")
|
||||
cfg = load_llm_config(provider="deepseek", model="deepseek-chat", scene="daily_report")
|
||||
assert cfg.provider == "deepseek"
|
||||
assert cfg.model == "deepseek-chat"
|
||||
|
||||
|
||||
def test_load_llm_config_scene_missing_model_raises(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""场景与 .env 都未配置模型时必须报错(无内置兜底)。"""
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-test")
|
||||
monkeypatch.delenv("DEEPSEEK_MODEL", raising=False)
|
||||
monkeypatch.delenv("LLM_MODEL", raising=False)
|
||||
_patch_scene(monkeypatch, {"provider": "deepseek", "model": ""})
|
||||
with pytest.raises(ValueError, match="模型"):
|
||||
load_llm_config(scene="daily_report")
|
||||
|
||||
|
||||
def test_load_llm_config_real_yaml_parseable() -> None:
|
||||
"""真实 configs/llm_models.yaml 必须可解析且包含全部场景(回归保护)。"""
|
||||
from configs.loader import load_defaults, load_scene_config
|
||||
|
||||
for scene in ("event_extraction", "daily_report", "stock_report", "embedding"):
|
||||
assert isinstance(load_scene_config(scene), dict)
|
||||
assert isinstance(load_defaults(), dict)
|
||||
|
||||
|
||||
def test_load_llm_config_temperature_zero_is_respected(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""temperature=0 是合法配置,不应被 or 链回退默认。"""
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-env")
|
||||
monkeypatch.setenv("DEEPSEEK_MODEL", "deepseek-m")
|
||||
_patch_scene(monkeypatch, {"provider": "deepseek", "model": "deepseek-m", "temperature": 0})
|
||||
cfg = load_llm_config(scene="daily_report")
|
||||
assert cfg.temperature == 0.0
|
||||
|
||||
|
||||
def test_load_llm_config_scene_max_attempts_zero() -> None:
|
||||
"""max_attempts=0 由 _pick_int 显式处理。"""
|
||||
from llm.client import _pick_int
|
||||
|
||||
assert _pick_int({"max_attempts": 0}, "max_attempts", 3) == 0
|
||||
assert _pick_int({"max_attempts": ""}, "max_attempts", 3) == 3
|
||||
assert _pick_int({}, "max_attempts", 3) == 3
|
||||
|
||||
Reference in New Issue
Block a user