fix: 回填脚本改为定向回填多源 point(本地模式全量不可行)

- 本地模式 Qdrant 写入逐点 0.4~0.65s,全量 3.3 万条需 4~6h 且占单进程锁
- 改为仅回填指纹库 227 条多源记录(单源点展示层兜底 source_id):1m57s 完成
- 测试改为 _load_multi_source 语义(NULL/损坏/单源排除)
This commit is contained in:
2026-08-22 22:39:10 +08:00
parent 80828310d6
commit 639d857ffb
3 changed files with 90 additions and 111 deletions
+16 -9
View File
@@ -166,7 +166,7 @@ def test_build_text_from_article_reads_sources(tmp_path: Path) -> None:
# --------------------------------------------------------------------------- #
# 回填脚本 _load_fingerprint_sources
# 回填脚本 _load_multi_source
# --------------------------------------------------------------------------- #
def _make_fp_db(tmp_path: Path) -> Path:
@@ -183,23 +183,30 @@ def _make_fp_db(tmp_path: Path) -> Path:
conn.execute(
"INSERT INTO fingerprints VALUES (?, ?, ?)", ("h2", "zqrb", None)
)
conn.execute(
"INSERT INTO fingerprints VALUES (?, ?, ?)", ("h3", "cls", "not-json")
)
conn.execute(
"INSERT INTO fingerprints VALUES (?, ?, ?)",
("h4", "cls", json.dumps(["cls"])),
)
conn.commit()
conn.close()
return db
def test_load_fingerprint_sources_parses_and_falls_back(tmp_path: Path) -> None:
from scripts.backfill_qdrant_sources import _load_fingerprint_sources
def test_load_multi_source_only_multi(tmp_path: Path) -> None:
"""仅返回多源记录;NULL/损坏/单源均排除。"""
from scripts.backfill_qdrant_sources import _load_multi_source
db = _make_fp_db(tmp_path)
res = _load_fingerprint_sources(db)
assert res["h1"] == ["cls", "sina"]
assert res["h2"] == ["zqrb"] # source_ids NULL → 回退 [主源]
res = _load_multi_source(db)
assert res == {"h1": ["cls", "sina"]}
def test_load_fingerprint_sources_main_first(tmp_path: Path) -> None:
def test_load_multi_source_main_first(tmp_path: Path) -> None:
"""source_ids 顺序异常时保证主源居首。"""
from scripts.backfill_qdrant_sources import _load_fingerprint_sources
from scripts.backfill_qdrant_sources import _load_multi_source
db = tmp_path / "fp.sqlite3"
conn = sqlite3.connect(db)
@@ -213,5 +220,5 @@ def test_load_fingerprint_sources_main_first(tmp_path: Path) -> None:
)
conn.commit()
conn.close()
res = _load_fingerprint_sources(db)
res = _load_multi_source(db)
assert res["h1"] == ["cls", "sina"]