simon/news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修复补跑误含 cninfo 三步与时区不一致 (P1-2/P1-3)

Browse Source 
- P1-2: 提取 DEFAULT_NEWS_STEPS(全链路去 report+cninfo),定时/补跑/默认三处统一;
  启动补跑不再误执行 cninfo 公告管道
- P1-3: 新增 scheduler/timeutil.py(调度时区统一入口,SCHEDULE_TZ 可覆盖,
  默认 Asia/Shanghai);run_scheduler 定时/补跑/--once、crawler 补跑保护、
  reporter 兜底日期全部改用调度时区,避免系统时区非上海时日期错位
- 新增 6 个测试;全量 282 passed
This commit is contained in:
simon
2026-08-23 05:40:52 +08:00
parent 639d857ffb
commit 7b33182f67
7 changed files with 197 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/run_scheduler.py
+18 -14
View File
@@ -16,15 +16,16 @@ from __future__ import annotations
import argparse
import signal
import sys
from datetime import date, datetime
from pathlib import Path
from typing import Any
from dotenv import load_dotenv
from loguru import logger
from scheduler import STEP_COMMANDS, run_pipeline
from scheduler import DEFAULT_NEWS_STEPS, run_pipeline
from scheduler.stock_reporter import generate_all_stock_reports
from scheduler.timeutil import now as tz_now
from scheduler.timeutil import schedule_tz, today_str
def _setup_logger(level: str) -> None:
@@ -69,7 +70,7 @@ def _once(args: argparse.Namespace) -> int:
if args.resume and args.steps:
logger.error("--resume 与 --steps 不能同时使用(断点续跑针对全链路)")
return 2
run_pipeline(args.date, steps=steps, resume=args.resume)
run_pipeline(args.date or today_str(), steps=steps, resume=args.resume)
return 0
@@ -92,17 +93,18 @@ def _daemon(args: argparse.Namespace) -> int:
scheduler = BackgroundScheduler()
# 包装函数:每次触发时重新计算日期,避免 date.today() 在注册时冻结。
# 包装函数:每次触发时按调度时区重新计算日期(P1-3),
# 避免 date.today() 在注册时冻结或与 cron 时区不一致。
def _scheduled_pipeline(steps: list[str] | None = None) -> None:
run_pipeline(date.today().strftime("%Y%m%d"), steps=steps)
run_pipeline(today_str(), steps=steps)
for hour, minute in times:
trigger = CronTrigger(hour=hour, minute=minute, timezone="Asia/Shanghai")
trigger = CronTrigger(hour=hour, minute=minute, timezone=str(schedule_tz()))
is_first = (hour == first_hour and minute == first_minute)
job_kwargs: dict | None = None
if is_first:
job_kwargs = {
"steps": [k for k in STEP_COMMANDS if k not in ("report", "cninfo_crawl", "cninfo_extract", "cninfo_pdf")] + ["report"]
"steps": list(DEFAULT_NEWS_STEPS) + ["report"]
}
scheduler.add_job(
_scheduled_pipeline,
@@ -118,7 +120,7 @@ def _daemon(args: argparse.Namespace) -> int:
cninfo_raw = os.environ.get("CNINFO_SCHEDULE_TIME", "06:30")
cninfo_parts = cninfo_raw.split(":")
cninfo_h, cninfo_m = int(cninfo_parts[0]), int(cninfo_parts[1]) if len(cninfo_parts) > 1 else 0
cninfo_trigger = CronTrigger(hour=cninfo_h, minute=cninfo_m, timezone="Asia/Shanghai")
cninfo_trigger = CronTrigger(hour=cninfo_h, minute=cninfo_m, timezone=str(schedule_tz()))
cninfo_steps = ["cninfo_crawl", "cninfo_extract", "cninfo_pdf",
"dedup", "llm", "embedding", "qdrant"]
scheduler.add_job(
@@ -135,7 +137,7 @@ def _daemon(args: argparse.Namespace) -> int:
if stock_raw:
stock_parts = stock_raw.split(":")
stock_h, stock_m = int(stock_parts[0]), int(stock_parts[1]) if len(stock_parts) > 1 else 0
stock_trigger = CronTrigger(hour=stock_h, minute=stock_m, timezone="Asia/Shanghai")
stock_trigger = CronTrigger(hour=stock_h, minute=stock_m, timezone=str(schedule_tz()))
scheduler.add_job(
generate_all_stock_reports,
trigger=stock_trigger,
@@ -159,7 +161,7 @@ def _daemon(args: argparse.Namespace) -> int:
logger.info("调度器已启动,等待触发... (按 Ctrl+C 退出)")
# 启动时检查是否有因重启/宕机错过的定时任务,30 分钟内补跑
now = datetime.now()
now = tz_now()
for hour, minute in times:
scheduled = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
missed_minutes = (now - scheduled).total_seconds() / 60
@@ -168,10 +170,12 @@ def _daemon(args: argparse.Namespace) -> int:
"检测到错过的定时任务 {:02d}:{:02d} ({} 分钟前),立即补跑一次",
hour, minute, int(missed_minutes),
)
steps = [k for k in STEP_COMMANDS if k != "report"]
# P1-2:补跑只跑新闻链路(DEFAULT_NEWS_STEPS 不含 report 与 cninfo 三步),
# cninfo 公告管道由其自身定时任务负责,避免重复执行整套公告管道。
steps = list(DEFAULT_NEWS_STEPS)
if (hour, minute) == sorted_times[0]:
steps.append("report")
run_pipeline(date.today().strftime("%Y%m%d"), steps=steps)
run_pipeline(today_str(), steps=steps)
import contextlib
@@ -186,8 +190,8 @@ def main() -> int:
parser = argparse.ArgumentParser(description="A 股新闻定时任务 (M7)")
parser.add_argument("--once", action="store_true", help="立即执行一次全链路")
parser.add_argument(
"--date", default=date.today().strftime("%Y%m%d"),
help="日期 YYYYMMDD (仅 --once 模式)",
"--date", default=None,
help="日期 YYYYMMDD (仅 --once 模式,默认按调度时区取今天)",
)
parser.add_argument("--steps", default=None,
help="仅执行指定步骤,逗号分隔 (如 crawler,extractor)")