feat: 打通多源新闻记录链路,检索/日报/知识库可见多源 (方案A+B)
- 模型层: EmbeddingResult/SearchResult 新增 sources 字段(主源居首,旧产物兜底) - M5 run_embedding: events/deduped 产物透传 sources 进 EmbeddingResult - M6 run_qdrant_ingest: payload 写入 sources(M5 → 回查 M4 → 兜底 [主源]) - vectorstore: 检索读取 payload.sources - 日报 HTML / CLI search / MCP: 多源显示「财联社 / 新浪 [多源]」 - 新增 scripts/backfill_qdrant_sources.py: 指纹库为权威源,scroll+upsert 回填存量 (本地模式 set_payload 逐点 0.65s 不可行,改走 ingest 同款快速路径) - 新增 tests/test_multisource.py 10 个;全量 276 passed
This commit is contained in:
+19
-10
@@ -89,8 +89,11 @@ def _build_text_from_event(
|
||||
event_path: Path,
|
||||
processed_root: Path,
|
||||
day: str,
|
||||
) -> tuple[str, Article, str | None] | None:
|
||||
"""从 ExtractedEvent JSON 构造嵌入文本与文章元数据。"""
|
||||
) -> tuple[str, Article, str | None, list[str] | None] | None:
|
||||
"""从 ExtractedEvent JSON 构造嵌入文本与文章元数据。
|
||||
|
||||
返回 (text, article, summary, sources);sources 为 M3 多源记录。
|
||||
"""
|
||||
try:
|
||||
obj: dict[str, Any] = json.loads(event_path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError as e:
|
||||
@@ -98,6 +101,7 @@ def _build_text_from_event(
|
||||
return None
|
||||
|
||||
article, head, summary = _from_event_dict(obj)
|
||||
sources = obj.get("sources") or None
|
||||
# 真正的正文要去 processed/ 找
|
||||
real_article = _load_article_by_hash(processed_root, day, article.url_hash)
|
||||
if real_article is None:
|
||||
@@ -110,17 +114,21 @@ def _build_text_from_event(
|
||||
update={"publish_time": article.publish_time or real_article.publish_time}
|
||||
)
|
||||
text = compose_text(real_article, head=head, summary=summary)
|
||||
return text, real_article, summary
|
||||
return text, real_article, summary, sources
|
||||
|
||||
|
||||
def _build_text_from_article(article_path: Path) -> tuple[str, Article, str | None] | None:
|
||||
def _build_text_from_article(
|
||||
article_path: Path,
|
||||
) -> tuple[str, Article, str | None, list[str] | None] | None:
|
||||
try:
|
||||
obj = json.loads(article_path.read_text(encoding="utf-8"))
|
||||
article = Article.model_validate(obj)
|
||||
except (json.JSONDecodeError, ValidationError) as e:
|
||||
logger.warning("跳过损坏 article 文件 {}: {}", article_path, e)
|
||||
return None
|
||||
return compose_text(article), article, None
|
||||
# deduped uniques JSON 含 sources 多源字段;processed 产物无此字段 → None
|
||||
sources = obj.get("sources") or None
|
||||
return compose_text(article), article, None, sources
|
||||
|
||||
|
||||
def _collect_inputs(args: argparse.Namespace) -> list[tuple[Path, str]]:
|
||||
@@ -192,8 +200,8 @@ async def _run(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
logger.info("待嵌入文章数: {} (跳过已处理 {}; input={})", len(files), skipped, args.input)
|
||||
|
||||
# 准备每篇文本
|
||||
prepared: list[tuple[str, Article, str | None]] = []
|
||||
# 准备每篇文本: (文本, 文章, 摘要, 多源列表)
|
||||
prepared: list[tuple[str, Article, str | None, list[str] | None]] = []
|
||||
for fp, kind in files:
|
||||
if kind == "event":
|
||||
built = _build_text_from_event(fp, Path(args.processed_root), args.date)
|
||||
@@ -234,12 +242,12 @@ async def _run(args: argparse.Namespace) -> int:
|
||||
batch_size = args.batch_size
|
||||
for i in range(0, len(prepared), batch_size):
|
||||
batch = prepared[i : i + batch_size]
|
||||
texts = [t for t, _, _ in batch]
|
||||
texts = [t for t, _, _, _ in batch]
|
||||
try:
|
||||
vectors = await provider.embed_batch(texts)
|
||||
except EmbeddingError as e:
|
||||
logger.warning("批 {} 嵌入失败: {}", i // batch_size, e)
|
||||
for _, art, _ in batch:
|
||||
for _, art, _, _ in batch:
|
||||
fail_cnt += 1
|
||||
with failed_path.open("a", encoding="utf-8") as f:
|
||||
f.write(
|
||||
@@ -252,7 +260,7 @@ async def _run(args: argparse.Namespace) -> int:
|
||||
)
|
||||
continue
|
||||
|
||||
for (text, article, _summary), vec in zip(batch, vectors, strict=True):
|
||||
for (text, article, _summary, sources), vec in zip(batch, vectors, strict=True):
|
||||
if len(vec) != provider.dim:
|
||||
logger.warning(
|
||||
"维度不一致 url_hash={} 实际={} 预期={}",
|
||||
@@ -261,6 +269,7 @@ async def _run(args: argparse.Namespace) -> int:
|
||||
result = EmbeddingResult(
|
||||
url_hash=article.url_hash,
|
||||
source_id=article.source_id,
|
||||
sources=sources or [],
|
||||
title=article.title,
|
||||
text=text,
|
||||
vector=vec,
|
||||
|
||||
Reference in New Issue
Block a user