simon
7b33182f67
- P1-2: 提取 DEFAULT_NEWS_STEPS(全链路去 report+cninfo),定时/补跑/默认三处统一; 启动补跑不再误执行 cninfo 公告管道 - P1-3: 新增 scheduler/timeutil.py(调度时区统一入口,SCHEDULE_TZ 可覆盖, 默认 Asia/Shanghai);run_scheduler 定时/补跑/--once、crawler 补跑保护、 reporter 兜底日期全部改用调度时区,避免系统时区非上海时日期错位 - 新增 6 个测试;全量 282 passed
215 lines
7.9 KiB
Python
215 lines
7.9 KiB
Python
"""M7 定时任务调度入口。
|
|
|
|
支持两种模式:
|
|
--once 立刻执行一次全链路
|
|
(默认) 启动 APScheduler,按 .env 中 SCHEDULE_TIMES 定时执行
|
|
|
|
用法:
|
|
uv run python -m scripts.run_scheduler # 启动定时服务
|
|
uv run python -m scripts.run_scheduler --once # 立即执行一次
|
|
uv run python -m scripts.run_scheduler --once --date 20260616
|
|
uv run python -m scripts.run_scheduler --once --steps crawler,extractor
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import signal
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from dotenv import load_dotenv
|
|
from loguru import logger
|
|
|
|
from scheduler import DEFAULT_NEWS_STEPS, run_pipeline
|
|
from scheduler.stock_reporter import generate_all_stock_reports
|
|
from scheduler.timeutil import now as tz_now
|
|
from scheduler.timeutil import schedule_tz, today_str
|
|
|
|
|
|
def _setup_logger(level: str) -> None:
|
|
logger.remove()
|
|
logger.add(
|
|
sys.stderr,
|
|
level=level,
|
|
format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}",
|
|
)
|
|
log_path = Path("logs") / "scheduler.log"
|
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
logger.add(log_path, level="DEBUG", rotation="10 MB", retention=5, encoding="utf-8")
|
|
|
|
|
|
def _parse_schedule_times(raw: str) -> list[tuple[int, int]]:
|
|
"""解析 SCHEDULE_TIMES 环境变量。
|
|
|
|
格式: "07:00,12:00,18:00,22:00"
|
|
返回: [(7,0), (12,0), (18,0), (22,0)]
|
|
"""
|
|
out: list[tuple[int, int]] = []
|
|
for part in raw.split(","):
|
|
part = part.strip()
|
|
if not part:
|
|
continue
|
|
try:
|
|
h, m = part.split(":")
|
|
out.append((int(h), int(m)))
|
|
except ValueError:
|
|
logger.warning("SCHEDULE_TIMES 格式错误: {!r},跳过", part)
|
|
return out
|
|
|
|
|
|
def _once(args: argparse.Namespace) -> int:
|
|
"""单次执行模式。
|
|
|
|
默认全量执行;--resume 时断点续跑(跳过连续成功步骤,从失败/未执行步骤继续)。
|
|
"""
|
|
steps = None
|
|
if args.steps:
|
|
steps = [s.strip() for s in args.steps.split(",")]
|
|
if args.resume and args.steps:
|
|
logger.error("--resume 与 --steps 不能同时使用(断点续跑针对全链路)")
|
|
return 2
|
|
run_pipeline(args.date or today_str(), steps=steps, resume=args.resume)
|
|
return 0
|
|
|
|
|
|
def _daemon(args: argparse.Namespace) -> int:
|
|
"""守护进程模式(APScheduler)。"""
|
|
import os
|
|
|
|
from apscheduler.schedulers.background import BackgroundScheduler # noqa: E402
|
|
from apscheduler.triggers.cron import CronTrigger # noqa: E402
|
|
|
|
times_raw = os.environ.get("SCHEDULE_TIMES", "07:00,12:00,18:00,22:00")
|
|
times = _parse_schedule_times(times_raw)
|
|
if not times:
|
|
logger.error("SCHEDULE_TIMES 为空或全部非法,无法启动定时任务")
|
|
return 2
|
|
|
|
# 找出最早的时间(当天首次运行),仅该次追加日报步骤
|
|
sorted_times = sorted(times)
|
|
first_hour, first_minute = sorted_times[0] if sorted_times else (0, 0)
|
|
|
|
scheduler = BackgroundScheduler()
|
|
|
|
# 包装函数:每次触发时按调度时区重新计算日期(P1-3),
|
|
# 避免 date.today() 在注册时冻结或与 cron 时区不一致。
|
|
def _scheduled_pipeline(steps: list[str] | None = None) -> None:
|
|
run_pipeline(today_str(), steps=steps)
|
|
|
|
for hour, minute in times:
|
|
trigger = CronTrigger(hour=hour, minute=minute, timezone=str(schedule_tz()))
|
|
is_first = (hour == first_hour and minute == first_minute)
|
|
job_kwargs: dict | None = None
|
|
if is_first:
|
|
job_kwargs = {
|
|
"steps": list(DEFAULT_NEWS_STEPS) + ["report"]
|
|
}
|
|
scheduler.add_job(
|
|
_scheduled_pipeline,
|
|
trigger=trigger,
|
|
kwargs=job_kwargs,
|
|
id=f"pipeline_{hour:02d}{minute:02d}",
|
|
name=f"全链路 {'+日报' if is_first else ''} {hour:02d}:{minute:02d}",
|
|
)
|
|
logger.info("已注册定时任务: {}每天 {:02d}:{:02d}{}", trigger, hour, minute,
|
|
" (含日报)" if is_first else "")
|
|
|
|
# cninfo 公告管道(可配置,默认 06:30)
|
|
cninfo_raw = os.environ.get("CNINFO_SCHEDULE_TIME", "06:30")
|
|
cninfo_parts = cninfo_raw.split(":")
|
|
cninfo_h, cninfo_m = int(cninfo_parts[0]), int(cninfo_parts[1]) if len(cninfo_parts) > 1 else 0
|
|
cninfo_trigger = CronTrigger(hour=cninfo_h, minute=cninfo_m, timezone=str(schedule_tz()))
|
|
cninfo_steps = ["cninfo_crawl", "cninfo_extract", "cninfo_pdf",
|
|
"dedup", "llm", "embedding", "qdrant"]
|
|
scheduler.add_job(
|
|
_scheduled_pipeline,
|
|
trigger=cninfo_trigger,
|
|
kwargs={"steps": cninfo_steps},
|
|
id="pipeline_cninfo",
|
|
name=f"cninfo 公告管道 {cninfo_h:02d}:{cninfo_m:02d}",
|
|
)
|
|
logger.info("已注册定时任务: cninfo 公告管道 每天 {:02d}:{:02d}", cninfo_h, cninfo_m)
|
|
|
|
# 个股日报(可配置,默认 07:30, 设为空可禁用)
|
|
stock_raw = os.environ.get("STOCK_REPORT_TIME", "07:30")
|
|
if stock_raw:
|
|
stock_parts = stock_raw.split(":")
|
|
stock_h, stock_m = int(stock_parts[0]), int(stock_parts[1]) if len(stock_parts) > 1 else 0
|
|
stock_trigger = CronTrigger(hour=stock_h, minute=stock_m, timezone=str(schedule_tz()))
|
|
scheduler.add_job(
|
|
generate_all_stock_reports,
|
|
trigger=stock_trigger,
|
|
id="stock_report",
|
|
name=f"个股日报 {stock_h:02d}:{stock_m:02d}",
|
|
)
|
|
logger.info("已注册定时任务: 个股日报 每天 {:02d}:{:02d}", stock_h, stock_m)
|
|
else:
|
|
logger.info("STOCK_REPORT_TIME 为空, 已禁用个股日报")
|
|
|
|
# 优雅退出
|
|
def _shutdown(signum: int, frame: Any) -> None:
|
|
logger.info("收到信号 {}, 关闭调度器...", signum)
|
|
scheduler.shutdown(wait=False)
|
|
raise SystemExit(0)
|
|
|
|
signal.signal(signal.SIGINT, _shutdown)
|
|
signal.signal(signal.SIGTERM, _shutdown)
|
|
|
|
scheduler.start()
|
|
logger.info("调度器已启动,等待触发... (按 Ctrl+C 退出)")
|
|
|
|
# 启动时检查是否有因重启/宕机错过的定时任务,30 分钟内补跑
|
|
now = tz_now()
|
|
for hour, minute in times:
|
|
scheduled = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
|
|
missed_minutes = (now - scheduled).total_seconds() / 60
|
|
if 0 < missed_minutes < 30:
|
|
logger.warning(
|
|
"检测到错过的定时任务 {:02d}:{:02d} ({} 分钟前),立即补跑一次",
|
|
hour, minute, int(missed_minutes),
|
|
)
|
|
# P1-2:补跑只跑新闻链路(DEFAULT_NEWS_STEPS 不含 report 与 cninfo 三步),
|
|
# cninfo 公告管道由其自身定时任务负责,避免重复执行整套公告管道。
|
|
steps = list(DEFAULT_NEWS_STEPS)
|
|
if (hour, minute) == sorted_times[0]:
|
|
steps.append("report")
|
|
run_pipeline(today_str(), steps=steps)
|
|
|
|
import contextlib
|
|
|
|
with contextlib.suppress(SystemExit, KeyboardInterrupt):
|
|
# 保持主线程存活,直到收到退出信号
|
|
signal.pause()
|
|
|
|
return 0
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="A 股新闻定时任务 (M7)")
|
|
parser.add_argument("--once", action="store_true", help="立即执行一次全链路")
|
|
parser.add_argument(
|
|
"--date", default=None,
|
|
help="日期 YYYYMMDD (仅 --once 模式,默认按调度时区取今天)",
|
|
)
|
|
parser.add_argument("--steps", default=None,
|
|
help="仅执行指定步骤,逗号分隔 (如 crawler,extractor)")
|
|
parser.add_argument(
|
|
"--resume", action="store_true",
|
|
help="断点续跑(仅 --once):跳过连续成功步骤,从上次失败/未执行步骤继续",
|
|
)
|
|
parser.add_argument("--log-level", default="INFO")
|
|
args = parser.parse_args()
|
|
|
|
_setup_logger(args.log_level)
|
|
load_dotenv()
|
|
|
|
if args.once:
|
|
return _once(args)
|
|
return _daemon(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|