feat: 打通多源新闻记录链路,检索/日报/知识库可见多源 (方案A+B)

- 模型层: EmbeddingResult/SearchResult 新增 sources 字段(主源居首,旧产物兜底)
- M5 run_embedding: events/deduped 产物透传 sources 进 EmbeddingResult
- M6 run_qdrant_ingest: payload 写入 sources(M5 → 回查 M4 → 兜底 [主源])
- vectorstore: 检索读取 payload.sources
- 日报 HTML / CLI search / MCP: 多源显示「财联社 / 新浪 [多源]」
- 新增 scripts/backfill_qdrant_sources.py: 指纹库为权威源,scroll+upsert 回填存量
  (本地模式 set_payload 逐点 0.65s 不可行,改走 ingest 同款快速路径)
- 新增 tests/test_multisource.py 10 个;全量 276 passed
This commit is contained in:
2026-08-22 22:26:10 +08:00
parent 8fa27ad65b
commit 80828310d6
11 changed files with 490 additions and 17 deletions
+17 -2
View File
@@ -4,8 +4,9 @@ from __future__ import annotations
from datetime import datetime
from enum import StrEnum
from typing import Self
from pydantic import BaseModel, Field
from pydantic import BaseModel, Field, model_validator
class EmbeddingProviderType(StrEnum):
@@ -19,7 +20,11 @@ class EmbeddingResult(BaseModel):
"""单篇文章的嵌入结果(落盘格式)。"""
url_hash: str = Field(..., description="主键,与 Article.url_hash 一致")
source_id: str = Field(..., description="来源源 id")
source_id: str = Field(..., description="主来源源 id")
sources: list[str] = Field(
default_factory=list,
description="该唯一新闻的全部来源(主源 source_id 居首),来自 M3 去重多源记录",
)
title: str = Field(..., description="原文标题(便于人工检索)")
text: str = Field(
..., description="实际送入 embedder 的文本(已截断/拼接)"
@@ -33,6 +38,16 @@ class EmbeddingResult(BaseModel):
char_count: int = Field(default=0, ge=0, description="text 字符数,便于排查")
publish_time: datetime | None = None
@model_validator(mode="after")
def _ensure_sources(self) -> Self:
"""保证 sources 非空、去重且以主源 source_id 开头(旧产物无字段时兜底)。"""
seen: list[str] = []
for s in [self.source_id, *self.sources]:
if s and s not in seen:
seen.append(s)
self.sources = seen
return self
def short_summary(self) -> str:
return (
f"[{self.source_id}] {self.title[:30]} "