feat: 打通多源新闻记录链路,检索/日报/知识库可见多源 (方案A+B)

- 模型层: EmbeddingResult/SearchResult 新增 sources 字段(主源居首,旧产物兜底)
- M5 run_embedding: events/deduped 产物透传 sources 进 EmbeddingResult
- M6 run_qdrant_ingest: payload 写入 sources(M5 → 回查 M4 → 兜底 [主源])
- vectorstore: 检索读取 payload.sources
- 日报 HTML / CLI search / MCP: 多源显示「财联社 / 新浪 [多源]」
- 新增 scripts/backfill_qdrant_sources.py: 指纹库为权威源,scroll+upsert 回填存量
  (本地模式 set_payload 逐点 0.65s 不可行,改走 ingest 同款快速路径)
- 新增 tests/test_multisource.py 10 个;全量 276 passed
This commit is contained in:
2026-08-22 22:26:10 +08:00
parent 8fa27ad65b
commit 80828310d6
11 changed files with 490 additions and 17 deletions
+25 -2
View File
@@ -52,18 +52,26 @@ def _result_to_point(
) -> dict[str, Any] | None:
"""把 EmbeddingResult JSON dict 转换为 Qdrant Point 格式。
Payload 包含 title/url/source_id/publish_time/event/计数 等。
Payload 包含 title/url/source_id/sources/publish_time/event/计数 等。
事件字段优先从 M4 ExtractedEvent 补充(EmbeddingResult 本身不含 event)。
sources 多源字段优先取 M5 产物,缺失时回查 M4,再兜底 [主源]。
"""
vector = obj.get("vector")
if not vector:
return None
url_hash = obj["url_hash"]
source_id = obj.get("source_id") or ""
sources = obj.get("sources") or None
if not sources and events_dir is not None:
sources = _load_sources_from_m4(events_dir, url_hash)
if not sources:
sources = [source_id] if source_id else []
payload = {
"url_hash": url_hash,
"title": obj.get("title") or "",
"url": obj.get("url") or "",
"source_id": obj.get("source_id") or "",
"source_id": source_id,
"sources": sources,
"publish_time": obj.get("publish_time"),
"char_count": obj.get("char_count"),
"word_count": obj.get("word_count"),
@@ -93,6 +101,21 @@ def _result_to_point(
return {"id": url_hash, "vector": vector, "payload": payload}
def _load_sources_from_m4(events_dir: Path, url_hash: str) -> list[str] | None:
"""从 M4 ExtractedEvent JSON 读取 sources 多源字段(旧 M5 产物兜底用)。"""
event_file = events_dir / f"{url_hash}.json"
if not event_file.is_file():
return None
try:
obj = json.loads(event_file.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return None
sources = obj.get("sources")
if isinstance(sources, list) and sources:
return [s for s in sources if s]
return None
def _load_event_from_m4(events_dir: Path, url_hash: str) -> dict[str, Any] | None:
"""从 M4 ExtractedEvent JSON 中提取事件 payload 子集。"""
event_file = events_dir / f"{url_hash}.json"