fix: 修复 xwlb 日期错位与重复行问题(方案 A)

- run_xwlb: 新增 --date(处理日)参数,落盘改为处理日目录,下游 M2-M6 打通
- run_xwlb: 幂等落盘(按 url_hash 去重,重写 index 替代裸追加)
- pipeline: xwlb 步骤传 --date {date}
- run_extractor: xwlb publish_time 从 url 兜底解析真实播出日
- 新增 tests/test_run_xwlb.py 9 个测试
- 清理 60 天历史死数据(用户决策 A1)
- docs: 文档重构收尾(删除 deployment.md,已在服务器直接修改)
This commit is contained in:
2026-08-22 19:11:32 +08:00
parent 65ead54b4f
commit 7ea8925209
5 changed files with 306 additions and 19 deletions
+21
View File
@@ -115,9 +115,30 @@ def _process_one(rec: dict, raw_dir: Path, out_dir: Path,
logger.warning("提取失败 {} {}: {}", rec["source_id"], rec["url"], e.reason)
return None
# xwlb 的假 HTML 无时间节点,从 url(xwlb://YYYY-MM-DD/sid)兜底解析真实播出日
if src_id == "xwlb" and article.publish_time is None:
article = _fill_xwlb_publish_time(article)
return _save_article(article, out_dir)
def _fill_xwlb_publish_time(article: Article) -> Article:
"""从 xwlb url 解析播出日期填充 publish_time,失败时原样返回。"""
import re as _re
m = _re.match(r"xwlb://(\d{4}-\d{2}-\d{2})", article.url)
if not m:
return article
try:
pt = datetime.strptime(m.group(1), "%Y-%m-%d")
except ValueError:
return article
return article.model_copy(update={
"publish_time": pt,
"publish_time_raw": m.group(1),
})
def _process_cninfo(rec: dict, html_path: Path, out_dir: Path) -> Article | None:
"""处理 cninfo 公告记录:从 meta JSON 解析结构化数据。"""
import re