feat: 大模型使用场景化配置与去重多源记录

- 新增 configs/llm_models.yaml: 4 个场景(event_extraction/daily_report/stock_report/embedding)
  可独立配置 provider/model/api_key_env/base_url_env/temperature 等,含用途与模型要求说明
- 新增 configs/loader.py: YAML 场景加载器(优先级: CLI 参数 > YAML > .env > 内置默认)
- llm/client.py: load_llm_config 支持 scene 参数,LLMConfig 增加 max_attempts
- embedding/factory+remote+local: provider/model/batch_limit 支持场景覆盖
- scheduler/reporter+stock_reporter: 日报/个股摘要接入场景配置
- dedup: Fingerprint.source_ids 多源记录 + 旧库自动迁移 + DedupResult 多源字段
- scripts/run_dedup: uniques JSON 的 sources 字段 + data/deduped/{day}/sources.json 汇总
- scripts/run_event_extraction: 接入 event_extraction 场景
- 补充测试: 场景优先级/零值、多源合并、旧库迁移、embedding 场景覆盖

tests/test_dedup.py

@@ -380,3 +380,123 @@ def test_stats_aggregates_by_source(tmp_db: Path) -> None:
         assert stats.total == 3
         assert stats.by_source == {"cls": 2, "sina": 1}
         assert stats.earliest is not None
+
+
+# --------------------------------------------------------------------------- #
+# 多源记录(source_ids)
+# --------------------------------------------------------------------------- #
+
+def test_fingerprint_source_ids_default_to_source() -> None:
+    """source_ids 未显式给定时,自动包含主源 source_id。"""
+    fp = Fingerprint(
+        url_hash="h", content_hash="c", simhash=0,
+        source_id="cls", url="u", title="t",
+    )
+    assert fp.source_ids == ["cls"]
+
+
+def test_fingerprint_source_ids_keeps_main_source_first() -> None:
+    """source_ids 无论怎么传,主源 source_id 始终居首且去重。"""
+    fp = Fingerprint(
+        url_hash="h", content_hash="c", simhash=0,
+        source_id="cls", url="u", title="t",
+        source_ids=["sina", "cls", "eastmoney", "sina"],
+    )
+    assert fp.source_ids[0] == "cls"
+    assert len(fp.source_ids) == len(set(fp.source_ids))  # 无重复
+
+
+def test_store_persists_source_ids(tmp_db: Path) -> None:
+    fp = Fingerprint(
+        url_hash="h", content_hash="c", simhash=0,
+        source_id="cls", url="u", title="t",
+        source_ids=["cls", "sina", "eastmoney"],
+    )
+    with FingerprintStore(tmp_db) as store:
+        store.upsert(fp)
+        got = store.get_by_url_hash("h")
+        assert got is not None
+        assert got.source_ids == ["cls", "sina", "eastmoney"]
+
+
+def test_store_migrates_old_schema_without_source_ids(tmp_db: Path) -> None:
+    """旧库(无 source_ids 列)打开时应自动迁移,旧数据回退为 [source_id]。"""
+    import sqlite3
+
+    conn = sqlite3.connect(tmp_db)
+    conn.executescript(
+        "CREATE TABLE fingerprints ("
+        " url_hash TEXT PRIMARY KEY, content_hash TEXT NOT NULL, simhash_hex TEXT NOT NULL,"
+        " source_id TEXT NOT NULL, url TEXT NOT NULL, title TEXT NOT NULL,"
+        " publish_date TEXT, ingested_at TEXT NOT NULL);"
+    )
+    conn.execute(
+        "INSERT INTO fingerprints VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+        ("old1", "ch1", "0000000000000000", "cls", "u1", "t1", "2026-06-01", "2026-06-01T00:00:00"),
+    )
+    conn.commit()
+    conn.close()
+
+    with FingerprintStore(tmp_db) as store:
+        got = store.get_by_url_hash("old1")
+        assert got is not None
+        assert got.source_ids == ["cls"]  # 迁移后回退主源
+        # 迁移后可正常写入多源
+        store.upsert(Fingerprint(
+            url_hash="new1", content_hash="c2", simhash=1,
+            source_id="sina", url="u2", title="t2",
+            source_ids=["sina", "cls"],
+        ))
+        assert store.get_by_url_hash("new1") is not None  # type: ignore[union-attr]
+
+
+def test_ingest_merges_sources_on_duplicate(tmp_db: Path) -> None:
+    """同一内容被多个源发布时,重复文章的来源并入唯一新闻指纹。"""
+    body = "宁德时代今日发布新一代麒麟电池,能量密度 255Wh/kg。" * 4
+    a1 = _article(source_id="cls", url="https://cls/a", url_hash="aaaa111111111111",
+                  content=body)
+    a2 = _article(source_id="sina", url="https://sina/b", url_hash="bbbb222222222222",
+                  content=body)
+    a3 = _article(source_id="eastmoney", url="https://em/c", url_hash="cccc333333333333",
+                  content=body)
+
+    with Deduper(db_path=tmp_db) as d:
+        r1 = d.ingest(a1)
+        assert not r1.is_duplicate
+        r2 = d.ingest(a2)
+        assert r2.is_duplicate
+        assert r2.matched_layer == DedupLayer.CONTENT
+        assert r2.matched_source_id == "cls"
+        # 命中后 all_source_ids 立即包含两个源
+        assert r2.all_source_ids == ["cls", "sina"]
+
+        r3 = d.ingest(a3)
+        assert r3.is_duplicate
+        assert r3.all_source_ids == ["cls", "sina", "eastmoney"]
+
+        # 指纹库持久化多源
+        matched = d.store.get_by_url_hash("aaaa111111111111")
+        assert matched is not None
+        assert matched.source_ids == ["cls", "sina", "eastmoney"]
+        assert d.stats().total == 1  # 内容组只算 1 条唯一
+
+
+def test_check_reports_all_sources_without_writing(tmp_db: Path) -> None:
+    """check(只读)命中重复时也能看到全部来源,且不写库。"""
+    body = "宁德时代发布新一代麒麟电池产品。" * 6
+    a1 = _article(source_id="cls", url="https://cls/a", url_hash="aaaa111111111111",
+                  content=body)
+    a2 = _article(source_id="sina", url="https://sina/b", url_hash="bbbb222222222222",
+                  content=body)
+
+    with Deduper(db_path=tmp_db) as d:
+        d.ingest(a1)
+        d.ingest(a2)
+        # 第三次来一篇同样内容的文章,仅 check
+        a3 = _article(source_id="eastmoney", url="https://em/c", url_hash="cccc333333333333",
+                      content=body)
+        result = d.check(a3)
+        assert result.is_duplicate
+        assert result.matched_source_id == "cls"
+        assert result.all_source_ids == ["cls", "sina"]
+        assert d.stats().total == 1  # check 不写库

tests/test_embedding.py

@@ -167,6 +167,27 @@ def test_resolve_provider_type_env_override(monkeypatch: pytest.MonkeyPatch) ->
     assert resolve_provider_type() == EmbeddingProviderType.LOCAL_BGE
 
 
+def test_resolve_provider_type_scene_override(monkeypatch: pytest.MonkeyPatch) -> None:
+    """configs/llm_models.yaml 的 scenes.embedding.provider 优先于 .env。"""
+    monkeypatch.delenv("EMBEDDING_PROVIDER", raising=False)
+    monkeypatch.setattr(
+        "embedding.factory.load_scene_config",
+        lambda scene: {"provider": "local-bge"} if scene == "embedding" else {},
+    )
+    assert resolve_provider_type() == EmbeddingProviderType.LOCAL_BGE
+
+
+def test_resolve_provider_type_explicit_arg_beats_scene(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """显式参数优先级最高,覆盖 YAML 场景。"""
+    monkeypatch.setattr(
+        "embedding.factory.load_scene_config",
+        lambda scene: {"provider": "local-bge"} if scene == "embedding" else {},
+    )
+    assert resolve_provider_type("dashscope") == EmbeddingProviderType.DASHSCOPE
+
+
 def test_resolve_provider_type_unknown_raises() -> None:
     with pytest.raises(EmbeddingError):
         resolve_provider_type("anthropic-emb")

tests/test_llm.py

@@ -362,3 +362,114 @@ def test_load_llm_config_missing_model_raises(monkeypatch: pytest.MonkeyPatch) -
     monkeypatch.delenv("LLM_MODEL", raising=False)
     with pytest.raises(ValueError, match="模型"):
         load_llm_config(provider="deepseek")
+
+
+# --------------------------------------------------------------------------- #
+# load_llm_config —— configs/llm_models.yaml 场景配置
+# --------------------------------------------------------------------------- #
+
+def _patch_scene(monkeypatch: pytest.MonkeyPatch, cfg: dict) -> None:
+    """替换场景加载,模拟 configs/llm_models.yaml 中的某场景配置。"""
+    monkeypatch.setattr(
+        "llm.client.load_scene_config",
+        lambda scene: cfg if scene == "daily_report" else {},
+    )
+
+
+def test_load_llm_config_scene_overrides_env(monkeypatch: pytest.MonkeyPatch) -> None:
+    """YAML 场景配置优先于 .env:provider / model / temperature / timeout / max_attempts。"""
+    monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-env")
+    monkeypatch.setenv("DEEPSEEK_MODEL", "deepseek-env-model")
+    monkeypatch.setenv("QWEN_API_KEY", "sk-qwen")
+    _patch_scene(monkeypatch, {
+        "provider": "qwen",
+        "model": "qwen-max",
+        "temperature": 0.5,
+        "timeout_sec": 99,
+        "max_attempts": 5,
+    })
+    cfg = load_llm_config(scene="daily_report")
+    assert cfg.provider == "qwen"
+    assert cfg.model == "qwen-max"
+    assert cfg.api_key == "sk-qwen"
+    assert cfg.temperature == 0.5
+    assert cfg.timeout_sec == 99
+    assert cfg.max_attempts == 5
+
+
+def test_load_llm_config_scene_blank_fields_fall_back_to_env(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """YAML 场景未配置的字段(如 model 留空)回退 .env,保持向后兼容。"""
+    monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-env")
+    monkeypatch.setenv("DEEPSEEK_MODEL", "deepseek-env-model")
+    _patch_scene(monkeypatch, {"provider": "deepseek", "model": "", "temperature": 0.7})
+    cfg = load_llm_config(scene="daily_report")
+    assert cfg.provider == "deepseek"
+    assert cfg.model == "deepseek-env-model"
+    assert cfg.api_key == "sk-env"
+    assert cfg.temperature == 0.7
+
+
+def test_load_llm_config_scene_api_key_env_name(monkeypatch: pytest.MonkeyPatch) -> None:
+    """api_key_env 指向自定义环境变量时,优先使用该变量。"""
+    monkeypatch.setenv("MY_CUSTOM_KEY", "sk-custom")
+    monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-default")
+    monkeypatch.setenv("DEEPSEEK_MODEL", "deepseek-m")
+    _patch_scene(monkeypatch, {
+        "provider": "deepseek",
+        "model": "deepseek-scene-m",
+        "api_key_env": "MY_CUSTOM_KEY",
+    })
+    cfg = load_llm_config(scene="daily_report")
+    assert cfg.api_key == "sk-custom"
+    assert cfg.model == "deepseek-scene-m"
+
+
+def test_load_llm_config_scene_explicit_args_win(monkeypatch: pytest.MonkeyPatch) -> None:
+    """CLI/显式参数优先级最高,覆盖 YAML 场景。"""
+    monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-env")
+    _patch_scene(monkeypatch, {"provider": "qwen", "model": "qwen-max"})
+    monkeypatch.setenv("QWEN_API_KEY", "sk-qwen")
+    cfg = load_llm_config(provider="deepseek", model="deepseek-chat", scene="daily_report")
+    assert cfg.provider == "deepseek"
+    assert cfg.model == "deepseek-chat"
+
+
+def test_load_llm_config_scene_missing_model_raises(monkeypatch: pytest.MonkeyPatch) -> None:
+    """场景与 .env 都未配置模型时必须报错(无内置兜底)。"""
+    monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-test")
+    monkeypatch.delenv("DEEPSEEK_MODEL", raising=False)
+    monkeypatch.delenv("LLM_MODEL", raising=False)
+    _patch_scene(monkeypatch, {"provider": "deepseek", "model": ""})
+    with pytest.raises(ValueError, match="模型"):
+        load_llm_config(scene="daily_report")
+
+
+def test_load_llm_config_real_yaml_parseable() -> None:
+    """真实 configs/llm_models.yaml 必须可解析且包含全部场景(回归保护)。"""
+    from configs.loader import load_defaults, load_scene_config
+
+    for scene in ("event_extraction", "daily_report", "stock_report", "embedding"):
+        assert isinstance(load_scene_config(scene), dict)
+    assert isinstance(load_defaults(), dict)
+
+
+def test_load_llm_config_temperature_zero_is_respected(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """temperature=0 是合法配置,不应被 or 链回退默认。"""
+    monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-env")
+    monkeypatch.setenv("DEEPSEEK_MODEL", "deepseek-m")
+    _patch_scene(monkeypatch, {"provider": "deepseek", "model": "deepseek-m", "temperature": 0})
+    cfg = load_llm_config(scene="daily_report")
+    assert cfg.temperature == 0.0
+
+
+def test_load_llm_config_scene_max_attempts_zero() -> None:
+    """max_attempts=0 由 _pick_int 显式处理。"""
+    from llm.client import _pick_int
+
+    assert _pick_int({"max_attempts": 0}, "max_attempts", 3) == 0
+    assert _pick_int({"max_attempts": ""}, "max_attempts", 3) == 3
+    assert _pick_int({}, "max_attempts", 3) == 3

tests/test_report_builder.py

@@ -113,10 +113,20 @@ class TestLlmCallRetry:
 
         return SimpleNamespace(chat=SimpleNamespace(completions=Completions())), n
 
+    @staticmethod
+    def _cfg():
+        from llm.client import LLMConfig
+
+        return LLMConfig(
+            provider="deepseek", model="deepseek-v4-flash",
+            api_key="sk-test", base_url="https://api.deepseek.com",
+            temperature=0.3,
+        )
+
     def test_success_first_try(self) -> None:
         from scheduler.reporter import _llm_call
         client, n = self._fake_client(0)
-        out = _llm_call(client, "deepseek-v4-flash", "p")
+        out = _llm_call(client, self._cfg(), "p")
         assert out == "今日要点摘要"
         assert n["count"] == 1
 
@@ -125,7 +135,7 @@ class TestLlmCallRetry:
         monkeypatch.setattr(rep, "_LLM_RETRY_TIMES", 3)
         monkeypatch.setattr(rep, "_LLM_RETRY_BACKOFF_SEC", 0.01)
         client, n = self._fake_client(2)  # 前 2 次失败，第 3 次成功
-        out = rep._llm_call(client, "deepseek-v4-flash", "p")
+        out = rep._llm_call(client, self._cfg(), "p")
         assert out == "今日要点摘要"
         assert n["count"] == 3
 
@@ -135,7 +145,7 @@ class TestLlmCallRetry:
         monkeypatch.setattr(rep, "_LLM_RETRY_BACKOFF_SEC", 0.01)
         client, n = self._fake_client(99)  # 一直失败
         with pytest.raises(ConnectionError):
-            rep._llm_call(client, "deepseek-v4-flash", "p")
+            rep._llm_call(client, self._cfg(), "p")
         assert n["count"] == 2  # 重试 2 次后放弃