fix: 回填脚本改为定向回填多源 point(本地模式全量不可行)
- 本地模式 Qdrant 写入逐点 0.4~0.65s,全量 3.3 万条需 4~6h 且占单进程锁 - 改为仅回填指纹库 227 条多源记录(单源点展示层兜底 source_id):1m57s 完成 - 测试改为 _load_multi_source 语义(NULL/损坏/单源排除)
This commit is contained in:
@@ -166,7 +166,7 @@ def test_build_text_from_article_reads_sources(tmp_path: Path) -> None:
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 回填脚本 _load_fingerprint_sources
|
||||
# 回填脚本 _load_multi_source
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _make_fp_db(tmp_path: Path) -> Path:
|
||||
@@ -183,23 +183,30 @@ def _make_fp_db(tmp_path: Path) -> Path:
|
||||
conn.execute(
|
||||
"INSERT INTO fingerprints VALUES (?, ?, ?)", ("h2", "zqrb", None)
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO fingerprints VALUES (?, ?, ?)", ("h3", "cls", "not-json")
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO fingerprints VALUES (?, ?, ?)",
|
||||
("h4", "cls", json.dumps(["cls"])),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return db
|
||||
|
||||
|
||||
def test_load_fingerprint_sources_parses_and_falls_back(tmp_path: Path) -> None:
|
||||
from scripts.backfill_qdrant_sources import _load_fingerprint_sources
|
||||
def test_load_multi_source_only_multi(tmp_path: Path) -> None:
|
||||
"""仅返回多源记录;NULL/损坏/单源均排除。"""
|
||||
from scripts.backfill_qdrant_sources import _load_multi_source
|
||||
|
||||
db = _make_fp_db(tmp_path)
|
||||
res = _load_fingerprint_sources(db)
|
||||
assert res["h1"] == ["cls", "sina"]
|
||||
assert res["h2"] == ["zqrb"] # source_ids NULL → 回退 [主源]
|
||||
res = _load_multi_source(db)
|
||||
assert res == {"h1": ["cls", "sina"]}
|
||||
|
||||
|
||||
def test_load_fingerprint_sources_main_first(tmp_path: Path) -> None:
|
||||
def test_load_multi_source_main_first(tmp_path: Path) -> None:
|
||||
"""source_ids 顺序异常时保证主源居首。"""
|
||||
from scripts.backfill_qdrant_sources import _load_fingerprint_sources
|
||||
from scripts.backfill_qdrant_sources import _load_multi_source
|
||||
|
||||
db = tmp_path / "fp.sqlite3"
|
||||
conn = sqlite3.connect(db)
|
||||
@@ -213,5 +220,5 @@ def test_load_fingerprint_sources_main_first(tmp_path: Path) -> None:
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
res = _load_fingerprint_sources(db)
|
||||
res = _load_multi_source(db)
|
||||
assert res["h1"] == ["cls", "sina"]
|
||||
|
||||
Reference in New Issue
Block a user