simon
80828310d6
- 模型层: EmbeddingResult/SearchResult 新增 sources 字段(主源居首,旧产物兜底) - M5 run_embedding: events/deduped 产物透传 sources 进 EmbeddingResult - M6 run_qdrant_ingest: payload 写入 sources(M5 → 回查 M4 → 兜底 [主源]) - vectorstore: 检索读取 payload.sources - 日报 HTML / CLI search / MCP: 多源显示「财联社 / 新浪 [多源]」 - 新增 scripts/backfill_qdrant_sources.py: 指纹库为权威源,scroll+upsert 回填存量 (本地模式 set_payload 逐点 0.65s 不可行,改走 ingest 同款快速路径) - 新增 tests/test_multisource.py 10 个;全量 276 passed
65 lines
2.2 KiB
Python
65 lines
2.2 KiB
Python
"""Embedding 模块的数据模型 (M5)。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime
|
|
from enum import StrEnum
|
|
from typing import Self
|
|
|
|
from pydantic import BaseModel, Field, model_validator
|
|
|
|
|
|
class EmbeddingProviderType(StrEnum):
|
|
"""支持的 embedding provider 标识。"""
|
|
|
|
DASHSCOPE = "dashscope" # 远程 Qwen / 百炼
|
|
LOCAL_BGE = "local-bge" # 本地 BGE-M3
|
|
|
|
|
|
class EmbeddingResult(BaseModel):
|
|
"""单篇文章的嵌入结果(落盘格式)。"""
|
|
|
|
url_hash: str = Field(..., description="主键,与 Article.url_hash 一致")
|
|
source_id: str = Field(..., description="主来源源 id")
|
|
sources: list[str] = Field(
|
|
default_factory=list,
|
|
description="该唯一新闻的全部来源(主源 source_id 居首),来自 M3 去重多源记录",
|
|
)
|
|
title: str = Field(..., description="原文标题(便于人工检索)")
|
|
text: str = Field(
|
|
..., description="实际送入 embedder 的文本(已截断/拼接)"
|
|
)
|
|
vector: list[float] = Field(..., description="嵌入向量")
|
|
dim: int = Field(..., gt=0, description="向量维度")
|
|
|
|
provider: str = Field(..., description="dashscope / local-bge")
|
|
model: str = Field(..., description="嵌入模型名")
|
|
embedded_at: datetime = Field(default_factory=datetime.now)
|
|
char_count: int = Field(default=0, ge=0, description="text 字符数,便于排查")
|
|
publish_time: datetime | None = None
|
|
|
|
@model_validator(mode="after")
|
|
def _ensure_sources(self) -> Self:
|
|
"""保证 sources 非空、去重且以主源 source_id 开头(旧产物无字段时兜底)。"""
|
|
seen: list[str] = []
|
|
for s in [self.source_id, *self.sources]:
|
|
if s and s not in seen:
|
|
seen.append(s)
|
|
self.sources = seen
|
|
return self
|
|
|
|
def short_summary(self) -> str:
|
|
return (
|
|
f"[{self.source_id}] {self.title[:30]} "
|
|
f"dim={self.dim} provider={self.provider}"
|
|
)
|
|
|
|
|
|
class EmbeddingError(Exception):
|
|
"""嵌入调用失败。"""
|
|
|
|
def __init__(self, reason: str, *, attempts: int = 0) -> None:
|
|
super().__init__(reason)
|
|
self.reason = reason
|
|
self.attempts = attempts
|