fix: 修复补跑误含 cninfo 三步与时区不一致 (P1-2/P1-3)
- P1-2: 提取 DEFAULT_NEWS_STEPS(全链路去 report+cninfo),定时/补跑/默认三处统一; 启动补跑不再误执行 cninfo 公告管道 - P1-3: 新增 scheduler/timeutil.py(调度时区统一入口,SCHEDULE_TZ 可覆盖, 默认 Asia/Shanghai);run_scheduler 定时/补跑/--once、crawler 补跑保护、 reporter 兜底日期全部改用调度时区,避免系统时区非上海时日期错位 - 新增 6 个测试;全量 282 passed
This commit is contained in:
+105
-2
@@ -8,6 +8,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -306,8 +307,7 @@ def test_crawler_today_runs_normally(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
||||
def test_report_step_uses_date_str(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""report 步骤必须使用传入的 date_str,而非 date.today()(P0-4)。"""
|
||||
from scheduler import pipeline
|
||||
from scheduler import reporter
|
||||
from scheduler import pipeline, reporter
|
||||
|
||||
received: list[str] = []
|
||||
|
||||
@@ -346,3 +346,106 @@ def test_pipeline_backfill_skips_crawler_keeps_rest(
|
||||
assert all(s.success for s in result.steps)
|
||||
state = pipeline._load_pipeline_state(state_path)
|
||||
assert state["20260101"]["crawler"]["status"] == "ok"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# P1-2: 默认步骤不含 cninfo 独立管道 / P1-3: 调度时区统一
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def test_default_news_steps_excludes_cninfo_and_report() -> None:
|
||||
"""新闻链路默认步骤不含 report 与 cninfo 三步(P1-2)。"""
|
||||
from scheduler import DEFAULT_NEWS_STEPS
|
||||
|
||||
assert "report" not in DEFAULT_NEWS_STEPS
|
||||
assert "cninfo_crawl" not in DEFAULT_NEWS_STEPS
|
||||
assert "cninfo_extract" not in DEFAULT_NEWS_STEPS
|
||||
assert "cninfo_pdf" not in DEFAULT_NEWS_STEPS
|
||||
# 新闻链路核心步骤齐全
|
||||
assert {"crawler", "xwlb", "extractor", "dedup", "llm",
|
||||
"embedding", "qdrant"} <= set(DEFAULT_NEWS_STEPS)
|
||||
|
||||
|
||||
def test_run_pipeline_default_steps_excludes_cninfo(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""run_pipeline 默认 steps 不含 cninfo(只跑新闻链路)。"""
|
||||
from scheduler import pipeline
|
||||
from scheduler.timeutil import today_str
|
||||
|
||||
calls: list[str] = []
|
||||
|
||||
def _fake_run(cmd, timeout=None): # noqa: ARG001
|
||||
name = next(c.split(".")[-1] for c in cmd if "scripts.run_" in c)
|
||||
calls.append(name)
|
||||
return SimpleNamespace(returncode=0)
|
||||
|
||||
monkeypatch.setattr(pipeline.subprocess, "run", _fake_run)
|
||||
state_path = tmp_path / "state.json"
|
||||
today = today_str() # 今天:避免 crawler 补跑保护跳过
|
||||
result = pipeline.run_pipeline(
|
||||
today, state_path=state_path,
|
||||
)
|
||||
assert result.all_success is True
|
||||
assert calls == ["run_crawler", "run_xwlb", "run_extractor", "run_dedup",
|
||||
"run_event_extraction", "run_embedding", "run_qdrant_ingest"]
|
||||
assert "cninfo" not in " ".join(calls)
|
||||
|
||||
|
||||
def test_scheduler_daemon_backfill_uses_default_news_steps(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""守护进程补跑 steps 使用 DEFAULT_NEWS_STEPS(不含 cninfo 三步)。"""
|
||||
import inspect
|
||||
|
||||
import scripts.run_scheduler as rs
|
||||
from scheduler import DEFAULT_NEWS_STEPS
|
||||
|
||||
# 验证补跑逻辑引用的常量:通过源码断言 + 常量内容双重保证
|
||||
src = inspect.getsource(rs._daemon)
|
||||
assert "list(DEFAULT_NEWS_STEPS)" in src
|
||||
assert "cninfo_crawl" not in DEFAULT_NEWS_STEPS
|
||||
|
||||
|
||||
def test_today_str_matches_schedule_timezone(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""today_str 按调度时区返回 YYYYMMDD(P1-3)。"""
|
||||
from datetime import datetime
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from scheduler.timeutil import today_str
|
||||
|
||||
s = today_str()
|
||||
assert len(s) == 8 and s.isdigit()
|
||||
# 与调度时区(默认 Asia/Shanghai)当前日期一致
|
||||
expect = datetime.now(ZoneInfo("Asia/Shanghai")).strftime("%Y%m%d")
|
||||
assert s == expect
|
||||
|
||||
|
||||
def test_today_str_respects_env_override(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""SCHEDULE_TZ 环境变量可覆盖调度时区(P1-3)。"""
|
||||
from datetime import datetime
|
||||
|
||||
from scheduler.timeutil import today_str
|
||||
|
||||
# 覆盖为 UTC 后,today_str 应返回 UTC 日期(而非默认 Asia/Shanghai)
|
||||
monkeypatch.setenv("SCHEDULE_TZ", "UTC")
|
||||
s = today_str()
|
||||
expect = datetime.now(UTC).strftime("%Y%m%d")
|
||||
assert s == expect
|
||||
|
||||
|
||||
def test_once_uses_today_str_when_date_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""--once 不带 --date 时按调度时区取今天(P1-3)。"""
|
||||
import scripts.run_scheduler as rs
|
||||
|
||||
captured: dict[str, str] = {}
|
||||
|
||||
def _fake_pipeline(date_str, steps=None, resume=False): # noqa: ARG001
|
||||
captured["date"] = date_str
|
||||
|
||||
monkeypatch.setattr(rs, "run_pipeline", _fake_pipeline)
|
||||
from types import SimpleNamespace
|
||||
|
||||
args = SimpleNamespace(steps=None, resume=False, date=None)
|
||||
rc = rs._once(args)
|
||||
assert rc == 0
|
||||
assert captured["date"] == rs.today_str()
|
||||
|
||||
Reference in New Issue
Block a user