feat: 打通多源新闻记录链路,检索/日报/知识库可见多源 (方案A+B)
- 模型层: EmbeddingResult/SearchResult 新增 sources 字段(主源居首,旧产物兜底) - M5 run_embedding: events/deduped 产物透传 sources 进 EmbeddingResult - M6 run_qdrant_ingest: payload 写入 sources(M5 → 回查 M4 → 兜底 [主源]) - vectorstore: 检索读取 payload.sources - 日报 HTML / CLI search / MCP: 多源显示「财联社 / 新浪 [多源]」 - 新增 scripts/backfill_qdrant_sources.py: 指纹库为权威源,scroll+upsert 回填存量 (本地模式 set_payload 逐点 0.65s 不可行,改走 ingest 同款快速路径) - 新增 tests/test_multisource.py 10 个;全量 276 passed
This commit is contained in:
@@ -52,18 +52,26 @@ def _result_to_point(
|
||||
) -> dict[str, Any] | None:
|
||||
"""把 EmbeddingResult JSON dict 转换为 Qdrant Point 格式。
|
||||
|
||||
Payload 包含 title/url/source_id/publish_time/event/计数 等。
|
||||
Payload 包含 title/url/source_id/sources/publish_time/event/计数 等。
|
||||
事件字段优先从 M4 ExtractedEvent 补充(EmbeddingResult 本身不含 event)。
|
||||
sources 多源字段优先取 M5 产物,缺失时回查 M4,再兜底 [主源]。
|
||||
"""
|
||||
vector = obj.get("vector")
|
||||
if not vector:
|
||||
return None
|
||||
url_hash = obj["url_hash"]
|
||||
source_id = obj.get("source_id") or ""
|
||||
sources = obj.get("sources") or None
|
||||
if not sources and events_dir is not None:
|
||||
sources = _load_sources_from_m4(events_dir, url_hash)
|
||||
if not sources:
|
||||
sources = [source_id] if source_id else []
|
||||
payload = {
|
||||
"url_hash": url_hash,
|
||||
"title": obj.get("title") or "",
|
||||
"url": obj.get("url") or "",
|
||||
"source_id": obj.get("source_id") or "",
|
||||
"source_id": source_id,
|
||||
"sources": sources,
|
||||
"publish_time": obj.get("publish_time"),
|
||||
"char_count": obj.get("char_count"),
|
||||
"word_count": obj.get("word_count"),
|
||||
@@ -93,6 +101,21 @@ def _result_to_point(
|
||||
return {"id": url_hash, "vector": vector, "payload": payload}
|
||||
|
||||
|
||||
def _load_sources_from_m4(events_dir: Path, url_hash: str) -> list[str] | None:
|
||||
"""从 M4 ExtractedEvent JSON 读取 sources 多源字段(旧 M5 产物兜底用)。"""
|
||||
event_file = events_dir / f"{url_hash}.json"
|
||||
if not event_file.is_file():
|
||||
return None
|
||||
try:
|
||||
obj = json.loads(event_file.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return None
|
||||
sources = obj.get("sources")
|
||||
if isinstance(sources, list) and sources:
|
||||
return [s for s in sources if s]
|
||||
return None
|
||||
|
||||
|
||||
def _load_event_from_m4(events_dir: Path, url_hash: str) -> dict[str, Any] | None:
|
||||
"""从 M4 ExtractedEvent JSON 中提取事件 payload 子集。"""
|
||||
event_file = events_dir / f"{url_hash}.json"
|
||||
|
||||
Reference in New Issue
Block a user