feat: 打通多源新闻记录链路,检索/日报/知识库可见多源 (方案A+B)

- 模型层: EmbeddingResult/SearchResult 新增 sources 字段(主源居首,旧产物兜底)
- M5 run_embedding: events/deduped 产物透传 sources 进 EmbeddingResult
- M6 run_qdrant_ingest: payload 写入 sources(M5 → 回查 M4 → 兜底 [主源])
- vectorstore: 检索读取 payload.sources
- 日报 HTML / CLI search / MCP: 多源显示「财联社 / 新浪 [多源]」
- 新增 scripts/backfill_qdrant_sources.py: 指纹库为权威源,scroll+upsert 回填存量
  (本地模式 set_payload 逐点 0.65s 不可行,改走 ingest 同款快速路径)
- 新增 tests/test_multisource.py 10 个;全量 276 passed
This commit is contained in:
2026-08-22 22:26:10 +08:00
parent 8fa27ad65b
commit 80828310d6
11 changed files with 490 additions and 17 deletions
+19 -10
View File
@@ -89,8 +89,11 @@ def _build_text_from_event(
event_path: Path,
processed_root: Path,
day: str,
) -> tuple[str, Article, str | None] | None:
"""从 ExtractedEvent JSON 构造嵌入文本与文章元数据。"""
) -> tuple[str, Article, str | None, list[str] | None] | None:
"""从 ExtractedEvent JSON 构造嵌入文本与文章元数据。
返回 (text, article, summary, sources);sources 为 M3 多源记录。
"""
try:
obj: dict[str, Any] = json.loads(event_path.read_text(encoding="utf-8"))
except json.JSONDecodeError as e:
@@ -98,6 +101,7 @@ def _build_text_from_event(
return None
article, head, summary = _from_event_dict(obj)
sources = obj.get("sources") or None
# 真正的正文要去 processed/ 找
real_article = _load_article_by_hash(processed_root, day, article.url_hash)
if real_article is None:
@@ -110,17 +114,21 @@ def _build_text_from_event(
update={"publish_time": article.publish_time or real_article.publish_time}
)
text = compose_text(real_article, head=head, summary=summary)
return text, real_article, summary
return text, real_article, summary, sources
def _build_text_from_article(article_path: Path) -> tuple[str, Article, str | None] | None:
def _build_text_from_article(
article_path: Path,
) -> tuple[str, Article, str | None, list[str] | None] | None:
try:
obj = json.loads(article_path.read_text(encoding="utf-8"))
article = Article.model_validate(obj)
except (json.JSONDecodeError, ValidationError) as e:
logger.warning("跳过损坏 article 文件 {}: {}", article_path, e)
return None
return compose_text(article), article, None
# deduped uniques JSON 含 sources 多源字段;processed 产物无此字段 → None
sources = obj.get("sources") or None
return compose_text(article), article, None, sources
def _collect_inputs(args: argparse.Namespace) -> list[tuple[Path, str]]:
@@ -192,8 +200,8 @@ async def _run(args: argparse.Namespace) -> int:
return 0
logger.info("待嵌入文章数: {} (跳过已处理 {}; input={})", len(files), skipped, args.input)
# 准备每篇文本
prepared: list[tuple[str, Article, str | None]] = []
# 准备每篇文本: (文本, 文章, 摘要, 多源列表)
prepared: list[tuple[str, Article, str | None, list[str] | None]] = []
for fp, kind in files:
if kind == "event":
built = _build_text_from_event(fp, Path(args.processed_root), args.date)
@@ -234,12 +242,12 @@ async def _run(args: argparse.Namespace) -> int:
batch_size = args.batch_size
for i in range(0, len(prepared), batch_size):
batch = prepared[i : i + batch_size]
texts = [t for t, _, _ in batch]
texts = [t for t, _, _, _ in batch]
try:
vectors = await provider.embed_batch(texts)
except EmbeddingError as e:
logger.warning("批 {} 嵌入失败: {}", i // batch_size, e)
for _, art, _ in batch:
for _, art, _, _ in batch:
fail_cnt += 1
with failed_path.open("a", encoding="utf-8") as f:
f.write(
@@ -252,7 +260,7 @@ async def _run(args: argparse.Namespace) -> int:
)
continue
for (text, article, _summary), vec in zip(batch, vectors, strict=True):
for (text, article, _summary, sources), vec in zip(batch, vectors, strict=True):
if len(vec) != provider.dim:
logger.warning(
"维度不一致 url_hash={} 实际={} 预期={}",
@@ -261,6 +269,7 @@ async def _run(args: argparse.Namespace) -> int:
result = EmbeddingResult(
url_hash=article.url_hash,
source_id=article.source_id,
sources=sources or [],
title=article.title,
text=text,
vector=vec,