fix: 修复 xwlb 日期错位与重复行问题(方案 A)
- run_xwlb: 新增 --date(处理日)参数,落盘改为处理日目录,下游 M2-M6 打通
- run_xwlb: 幂等落盘(按 url_hash 去重,重写 index 替代裸追加)
- pipeline: xwlb 步骤传 --date {date}
- run_extractor: xwlb publish_time 从 url 兜底解析真实播出日
- 新增 tests/test_run_xwlb.py 9 个测试
- 清理 60 天历史死数据(用户决策 A1)
- docs: 文档重构收尾(删除 deployment.md,已在服务器直接修改)
This commit is contained in:
@@ -115,9 +115,30 @@ def _process_one(rec: dict, raw_dir: Path, out_dir: Path,
|
||||
logger.warning("提取失败 {} {}: {}", rec["source_id"], rec["url"], e.reason)
|
||||
return None
|
||||
|
||||
# xwlb 的假 HTML 无时间节点,从 url(xwlb://YYYY-MM-DD/sid)兜底解析真实播出日
|
||||
if src_id == "xwlb" and article.publish_time is None:
|
||||
article = _fill_xwlb_publish_time(article)
|
||||
|
||||
return _save_article(article, out_dir)
|
||||
|
||||
|
||||
def _fill_xwlb_publish_time(article: Article) -> Article:
|
||||
"""从 xwlb url 解析播出日期填充 publish_time,失败时原样返回。"""
|
||||
import re as _re
|
||||
|
||||
m = _re.match(r"xwlb://(\d{4}-\d{2}-\d{2})", article.url)
|
||||
if not m:
|
||||
return article
|
||||
try:
|
||||
pt = datetime.strptime(m.group(1), "%Y-%m-%d")
|
||||
except ValueError:
|
||||
return article
|
||||
return article.model_copy(update={
|
||||
"publish_time": pt,
|
||||
"publish_time_raw": m.group(1),
|
||||
})
|
||||
|
||||
|
||||
def _process_cninfo(rec: dict, html_path: Path, out_dir: Path) -> Article | None:
|
||||
"""处理 cninfo 公告记录:从 meta JSON 解析结构化数据。"""
|
||||
import re
|
||||
|
||||
Reference in New Issue
Block a user