feat: 打通多源新闻记录链路,检索/日报/知识库可见多源 (方案A+B)
- 模型层: EmbeddingResult/SearchResult 新增 sources 字段(主源居首,旧产物兜底) - M5 run_embedding: events/deduped 产物透传 sources 进 EmbeddingResult - M6 run_qdrant_ingest: payload 写入 sources(M5 → 回查 M4 → 兜底 [主源]) - vectorstore: 检索读取 payload.sources - 日报 HTML / CLI search / MCP: 多源显示「财联社 / 新浪 [多源]」 - 新增 scripts/backfill_qdrant_sources.py: 指纹库为权威源,scroll+upsert 回填存量 (本地模式 set_payload 逐点 0.65s 不可行,改走 ingest 同款快速路径) - 新增 tests/test_multisource.py 10 个;全量 276 passed
This commit is contained in:
+17
-2
@@ -4,8 +4,9 @@ from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Self
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
|
||||
class EmbeddingProviderType(StrEnum):
|
||||
@@ -19,7 +20,11 @@ class EmbeddingResult(BaseModel):
|
||||
"""单篇文章的嵌入结果(落盘格式)。"""
|
||||
|
||||
url_hash: str = Field(..., description="主键,与 Article.url_hash 一致")
|
||||
source_id: str = Field(..., description="来源源 id")
|
||||
source_id: str = Field(..., description="主来源源 id")
|
||||
sources: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="该唯一新闻的全部来源(主源 source_id 居首),来自 M3 去重多源记录",
|
||||
)
|
||||
title: str = Field(..., description="原文标题(便于人工检索)")
|
||||
text: str = Field(
|
||||
..., description="实际送入 embedder 的文本(已截断/拼接)"
|
||||
@@ -33,6 +38,16 @@ class EmbeddingResult(BaseModel):
|
||||
char_count: int = Field(default=0, ge=0, description="text 字符数,便于排查")
|
||||
publish_time: datetime | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _ensure_sources(self) -> Self:
|
||||
"""保证 sources 非空、去重且以主源 source_id 开头(旧产物无字段时兜底)。"""
|
||||
seen: list[str] = []
|
||||
for s in [self.source_id, *self.sources]:
|
||||
if s and s not in seen:
|
||||
seen.append(s)
|
||||
self.sources = seen
|
||||
return self
|
||||
|
||||
def short_summary(self) -> str:
|
||||
return (
|
||||
f"[{self.source_id}] {self.title[:30]} "
|
||||
|
||||
Reference in New Issue
Block a user