feat: 大模型使用场景化配置与去重多源记录

- 新增 configs/llm_models.yaml: 4 个场景(event_extraction/daily_report/stock_report/embedding) 可独立配置 provider/model/api_key_env/base_url_env/temperature 等,含用途与模型要求说明 - 新增 configs/loader.py: YAML 场景加载器(优先级: CLI 参数 > YAML > .env > 内置默认) - llm/client.py: load_llm_config 支持 scene 参数,LLMConfig 增加 max_attempts - embedding/factory+remote+local: provider/model/batch_limit 支持场景覆盖 - scheduler/reporter+stock_reporter: 日报/个股摘要接入场景配置 - dedup: Fingerprint.source_ids 多源记录 + 旧库自动迁移 + DedupResult 多源字段 - scripts/run_dedup: uniques JSON 的 sources 字段 + data/deduped/{day}/sources.json 汇总 - scripts/run_event_extraction: 接入 event_extraction 场景 - 补充测试: 场景优先级/零值、多源合并、旧库迁移、embedding 场景覆盖
2026-08-12 07:57:10 +08:00
parent 0c032196d2
commit 3c65701449
21 changed files with 886 additions and 80 deletions
@@ -380,3 +380,123 @@ def test_stats_aggregates_by_source(tmp_db: Path) -> None:
        assert stats.total == 3
        assert stats.by_source == {"cls": 2, "sina": 1}
        assert stats.earliest is not None
+
+
+# --------------------------------------------------------------------------- #
+# 多源记录(source_ids)
+# --------------------------------------------------------------------------- #
+
+def test_fingerprint_source_ids_default_to_source() -> None:
+    """source_ids 未显式给定时,自动包含主源 source_id。"""
+    fp = Fingerprint(
+        url_hash="h", content_hash="c", simhash=0,
+        source_id="cls", url="u", title="t",
+    )
+    assert fp.source_ids == ["cls"]
+
+
+def test_fingerprint_source_ids_keeps_main_source_first() -> None:
+    """source_ids 无论怎么传,主源 source_id 始终居首且去重。"""
+    fp = Fingerprint(
+        url_hash="h", content_hash="c", simhash=0,
+        source_id="cls", url="u", title="t",
+        source_ids=["sina", "cls", "eastmoney", "sina"],
+    )
+    assert fp.source_ids[0] == "cls"
+    assert len(fp.source_ids) == len(set(fp.source_ids))  # 无重复
+
+
+def test_store_persists_source_ids(tmp_db: Path) -> None:
+    fp = Fingerprint(
+        url_hash="h", content_hash="c", simhash=0,
+        source_id="cls", url="u", title="t",
+        source_ids=["cls", "sina", "eastmoney"],
+    )
+    with FingerprintStore(tmp_db) as store:
+        store.upsert(fp)
+        got = store.get_by_url_hash("h")
+        assert got is not None
+        assert got.source_ids == ["cls", "sina", "eastmoney"]
+
+
+def test_store_migrates_old_schema_without_source_ids(tmp_db: Path) -> None:
+    """旧库(无 source_ids 列)打开时应自动迁移,旧数据回退为 [source_id]。"""
+    import sqlite3
+
+    conn = sqlite3.connect(tmp_db)
+    conn.executescript(
+        "CREATE TABLE fingerprints ("
+        " url_hash TEXT PRIMARY KEY, content_hash TEXT NOT NULL, simhash_hex TEXT NOT NULL,"
+        " source_id TEXT NOT NULL, url TEXT NOT NULL, title TEXT NOT NULL,"
+        " publish_date TEXT, ingested_at TEXT NOT NULL);"
+    )
+    conn.execute(
+        "INSERT INTO fingerprints VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+        ("old1", "ch1", "0000000000000000", "cls", "u1", "t1", "2026-06-01", "2026-06-01T00:00:00"),
+    )
+    conn.commit()
+    conn.close()
+
+    with FingerprintStore(tmp_db) as store:
+        got = store.get_by_url_hash("old1")
+        assert got is not None
+        assert got.source_ids == ["cls"]  # 迁移后回退主源
+        # 迁移后可正常写入多源
+        store.upsert(Fingerprint(
+            url_hash="new1", content_hash="c2", simhash=1,
+            source_id="sina", url="u2", title="t2",
+            source_ids=["sina", "cls"],
+        ))
+        assert store.get_by_url_hash("new1") is not None  # type: ignore[union-attr]
+
+
+def test_ingest_merges_sources_on_duplicate(tmp_db: Path) -> None:
+    """同一内容被多个源发布时,重复文章的来源并入唯一新闻指纹。"""
+    body = "宁德时代今日发布新一代麒麟电池,能量密度 255Wh/kg。" * 4
+    a1 = _article(source_id="cls", url="https://cls/a", url_hash="aaaa111111111111",
+                  content=body)
+    a2 = _article(source_id="sina", url="https://sina/b", url_hash="bbbb222222222222",
+                  content=body)
+    a3 = _article(source_id="eastmoney", url="https://em/c", url_hash="cccc333333333333",
+                  content=body)
+
+    with Deduper(db_path=tmp_db) as d:
+        r1 = d.ingest(a1)
+        assert not r1.is_duplicate
+        r2 = d.ingest(a2)
+        assert r2.is_duplicate
+        assert r2.matched_layer == DedupLayer.CONTENT
+        assert r2.matched_source_id == "cls"
+        # 命中后 all_source_ids 立即包含两个源
+        assert r2.all_source_ids == ["cls", "sina"]
+
+        r3 = d.ingest(a3)
+        assert r3.is_duplicate
+        assert r3.all_source_ids == ["cls", "sina", "eastmoney"]
+
+        # 指纹库持久化多源
+        matched = d.store.get_by_url_hash("aaaa111111111111")
+        assert matched is not None
+        assert matched.source_ids == ["cls", "sina", "eastmoney"]
+        assert d.stats().total == 1  # 内容组只算 1 条唯一
+
+
+def test_check_reports_all_sources_without_writing(tmp_db: Path) -> None:
+    """check(只读)命中重复时也能看到全部来源,且不写库。"""
+    body = "宁德时代发布新一代麒麟电池产品。" * 6
+    a1 = _article(source_id="cls", url="https://cls/a", url_hash="aaaa111111111111",
+                  content=body)
+    a2 = _article(source_id="sina", url="https://sina/b", url_hash="bbbb222222222222",
+                  content=body)
+
+    with Deduper(db_path=tmp_db) as d:
+        d.ingest(a1)
+        d.ingest(a2)
+        # 第三次来一篇同样内容的文章,仅 check
+        a3 = _article(source_id="eastmoney", url="https://em/c", url_hash="cccc333333333333",
+                      content=body)
+        result = d.check(a3)
+        assert result.is_duplicate
+        assert result.matched_source_id == "cls"
+        assert result.all_source_ids == ["cls", "sina"]
+        assert d.stats().total == 1  # check 不写库