fix: 修复补跑误含 cninfo 三步与时区不一致 (P1-2/P1-3)

- P1-2: 提取 DEFAULT_NEWS_STEPS(全链路去 report+cninfo),定时/补跑/默认三处统一;
  启动补跑不再误执行 cninfo 公告管道
- P1-3: 新增 scheduler/timeutil.py(调度时区统一入口,SCHEDULE_TZ 可覆盖,
  默认 Asia/Shanghai);run_scheduler 定时/补跑/--once、crawler 补跑保护、
  reporter 兜底日期全部改用调度时区,避免系统时区非上海时日期错位
- 新增 6 个测试;全量 282 passed
This commit is contained in:
2026-08-23 05:40:52 +08:00
parent 639d857ffb
commit 7b33182f67
7 changed files with 197 additions and 20 deletions
+105 -2
View File
@@ -8,6 +8,7 @@
from __future__ import annotations
import json
from datetime import UTC
from pathlib import Path
from types import SimpleNamespace
@@ -306,8 +307,7 @@ def test_crawler_today_runs_normally(monkeypatch: pytest.MonkeyPatch) -> None:
def test_report_step_uses_date_str(monkeypatch: pytest.MonkeyPatch) -> None:
"""report 步骤必须使用传入的 date_str,而非 date.today()(P0-4)。"""
from scheduler import pipeline
from scheduler import reporter
from scheduler import pipeline, reporter
received: list[str] = []
@@ -346,3 +346,106 @@ def test_pipeline_backfill_skips_crawler_keeps_rest(
assert all(s.success for s in result.steps)
state = pipeline._load_pipeline_state(state_path)
assert state["20260101"]["crawler"]["status"] == "ok"
# --------------------------------------------------------------------------- #
# P1-2: 默认步骤不含 cninfo 独立管道 / P1-3: 调度时区统一
# --------------------------------------------------------------------------- #
def test_default_news_steps_excludes_cninfo_and_report() -> None:
"""新闻链路默认步骤不含 report 与 cninfo 三步(P1-2)。"""
from scheduler import DEFAULT_NEWS_STEPS
assert "report" not in DEFAULT_NEWS_STEPS
assert "cninfo_crawl" not in DEFAULT_NEWS_STEPS
assert "cninfo_extract" not in DEFAULT_NEWS_STEPS
assert "cninfo_pdf" not in DEFAULT_NEWS_STEPS
# 新闻链路核心步骤齐全
assert {"crawler", "xwlb", "extractor", "dedup", "llm",
"embedding", "qdrant"} <= set(DEFAULT_NEWS_STEPS)
def test_run_pipeline_default_steps_excludes_cninfo(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
) -> None:
"""run_pipeline 默认 steps 不含 cninfo(只跑新闻链路)。"""
from scheduler import pipeline
from scheduler.timeutil import today_str
calls: list[str] = []
def _fake_run(cmd, timeout=None): # noqa: ARG001
name = next(c.split(".")[-1] for c in cmd if "scripts.run_" in c)
calls.append(name)
return SimpleNamespace(returncode=0)
monkeypatch.setattr(pipeline.subprocess, "run", _fake_run)
state_path = tmp_path / "state.json"
today = today_str() # 今天:避免 crawler 补跑保护跳过
result = pipeline.run_pipeline(
today, state_path=state_path,
)
assert result.all_success is True
assert calls == ["run_crawler", "run_xwlb", "run_extractor", "run_dedup",
"run_event_extraction", "run_embedding", "run_qdrant_ingest"]
assert "cninfo" not in " ".join(calls)
def test_scheduler_daemon_backfill_uses_default_news_steps(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""守护进程补跑 steps 使用 DEFAULT_NEWS_STEPS(不含 cninfo 三步)。"""
import inspect
import scripts.run_scheduler as rs
from scheduler import DEFAULT_NEWS_STEPS
# 验证补跑逻辑引用的常量:通过源码断言 + 常量内容双重保证
src = inspect.getsource(rs._daemon)
assert "list(DEFAULT_NEWS_STEPS)" in src
assert "cninfo_crawl" not in DEFAULT_NEWS_STEPS
def test_today_str_matches_schedule_timezone(monkeypatch: pytest.MonkeyPatch) -> None:
"""today_str 按调度时区返回 YYYYMMDD(P1-3)。"""
from datetime import datetime
from zoneinfo import ZoneInfo
from scheduler.timeutil import today_str
s = today_str()
assert len(s) == 8 and s.isdigit()
# 与调度时区(默认 Asia/Shanghai)当前日期一致
expect = datetime.now(ZoneInfo("Asia/Shanghai")).strftime("%Y%m%d")
assert s == expect
def test_today_str_respects_env_override(monkeypatch: pytest.MonkeyPatch) -> None:
"""SCHEDULE_TZ 环境变量可覆盖调度时区(P1-3)。"""
from datetime import datetime
from scheduler.timeutil import today_str
# 覆盖为 UTC 后,today_str 应返回 UTC 日期(而非默认 Asia/Shanghai)
monkeypatch.setenv("SCHEDULE_TZ", "UTC")
s = today_str()
expect = datetime.now(UTC).strftime("%Y%m%d")
assert s == expect
def test_once_uses_today_str_when_date_missing(monkeypatch: pytest.MonkeyPatch) -> None:
"""--once 不带 --date 时按调度时区取今天(P1-3)。"""
import scripts.run_scheduler as rs
captured: dict[str, str] = {}
def _fake_pipeline(date_str, steps=None, resume=False): # noqa: ARG001
captured["date"] = date_str
monkeypatch.setattr(rs, "run_pipeline", _fake_pipeline)
from types import SimpleNamespace
args = SimpleNamespace(steps=None, resume=False, date=None)
rc = rs._once(args)
assert rc == 0
assert captured["date"] == rs.today_str()