fix: 修复补跑历史日期的静默空转与日报日期错位 (P0-3/P0-4)
- run_step: 补跑保护——历史日期跳过 crawler(首页只含当天内容)并告警,复用已有 raw 数据 - run_step: report 步骤改用传入的 date_str,不再硬编码 date.today() - 新增 4 个补跑相关测试,修正 3 个既有测试适配新行为 - 全量 259 passed(3 个 crawler 基线失败与本次无关)
This commit is contained in:
@@ -266,3 +266,83 @@ def test_once_rejects_resume_with_steps(monkeypatch: pytest.MonkeyPatch) -> None
|
||||
date="20260616")
|
||||
rc = rs._once(args)
|
||||
assert rc == 2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# P0-3: 补跑模式跳过抓取 / P0-4: report 使用传入日期
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def test_crawler_backfill_skips_past_date(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""补跑历史日期时跳过 crawler(首页只含当天内容),不执行子进程。"""
|
||||
from scheduler import pipeline
|
||||
|
||||
def _fail_run(cmd, timeout=None): # noqa: ARG001
|
||||
raise AssertionError("补跑模式不应执行子进程")
|
||||
|
||||
monkeypatch.setattr(pipeline.subprocess, "run", _fail_run)
|
||||
sr = pipeline.run_step("crawler", "20260101")
|
||||
assert sr.success is True
|
||||
assert "跳过" in sr.tail_msg
|
||||
assert sr.elapsed_sec == 0.0
|
||||
|
||||
|
||||
def test_crawler_today_runs_normally(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""当天日期正常执行 crawler(不受补跑保护影响)。"""
|
||||
from datetime import date
|
||||
|
||||
from scheduler import pipeline
|
||||
|
||||
called: list[str] = []
|
||||
|
||||
def _fake_run(cmd, timeout=None): # noqa: ARG001
|
||||
called.append(" ".join(cmd))
|
||||
return SimpleNamespace(returncode=0)
|
||||
|
||||
monkeypatch.setattr(pipeline.subprocess, "run", _fake_run)
|
||||
sr = pipeline.run_step("crawler", date.today().strftime("%Y%m%d"))
|
||||
assert sr.success is True
|
||||
assert called and "run_crawler" in called[0]
|
||||
|
||||
|
||||
def test_report_step_uses_date_str(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""report 步骤必须使用传入的 date_str,而非 date.today()(P0-4)。"""
|
||||
from scheduler import pipeline
|
||||
from scheduler import reporter
|
||||
|
||||
received: list[str] = []
|
||||
|
||||
def _fake_generate(day_str, upload=True): # noqa: ARG001
|
||||
received.append(day_str)
|
||||
return 12345
|
||||
|
||||
monkeypatch.setattr(reporter, "generate_report", _fake_generate)
|
||||
sr = pipeline.run_step("report", "20260101")
|
||||
assert sr.success is True
|
||||
assert received == ["20260101"]
|
||||
|
||||
|
||||
def test_pipeline_backfill_skips_crawler_keeps_rest(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""补跑历史日期:全链路中仅 crawler 跳过,其余步骤照常执行。"""
|
||||
from scheduler import pipeline
|
||||
|
||||
calls: list[str] = []
|
||||
|
||||
def _fake_run(cmd, timeout=None): # noqa: ARG001
|
||||
name = next(c.split(".")[-1] for c in cmd if "scripts.run_" in c)
|
||||
calls.append(name)
|
||||
return SimpleNamespace(returncode=0)
|
||||
|
||||
monkeypatch.setattr(pipeline.subprocess, "run", _fake_run)
|
||||
state_path = tmp_path / "state.json"
|
||||
result = pipeline.run_pipeline(
|
||||
"20260101",
|
||||
steps=["crawler", "extractor", "dedup"],
|
||||
state_path=state_path,
|
||||
)
|
||||
# crawler 跳过(未执行子进程),extractor/dedup 正常
|
||||
assert calls == ["run_extractor", "run_dedup"]
|
||||
assert all(s.success for s in result.steps)
|
||||
state = pipeline._load_pipeline_state(state_path)
|
||||
assert state["20260101"]["crawler"]["status"] == "ok"
|
||||
|
||||
Reference in New Issue
Block a user