simon/news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Token Plan 迁移与 .env 热加载,并修复日报 AI 摘要为空

Browse Source 
Token Plan 迁移 / 配置热加载:
- configs/llm_models.yaml: 各场景切到 Token Plan(deepseek-v4.1-flash / qwen3.6-flash)
- 新增 configs/runtime_env.py: .env 按 (mtime_ns, size) 热加载并同步 os.environ,
  统一 env_get 取值;llm / embedding / vectorstore / mcp / pipeline 改用 env_get
- configs/loader.py / scripts/run_scheduler.py 等配套调整
- 新增 tests/test_hot_reload.py

日报 AI 摘要为空修复(2026-09-25):
- 根因: 推理模型的 reasoning token 与正文共用 max_tokens, 预算 1500 被"思考"
  占满 -> text_tokens=0 / finish_reason=length, 摘要静默为空且不重试
- daily_report 场景新增 max_tokens(默认 4000, YAML 保存即热生效);
  LLMConfig 支持可选 max_tokens; 分块预算 800 -> 2000
- _llm_call 拆出 _call_once, 正文为空时自动加倍预算重试(上限 16000),
  用尽才降级返回空串; 网络异常重试语义不变
- docs/user-guide.md 新增 FAQ; continuation.md 记录本次排查
- 已重跑 2026-09-25 日报(report_id=357)补回 466 字摘要

测试: 相关用例 56 passed(test_hot_reload 12 passed);
      ruff 无新增问题; 3 个 crawler 既有失败与本改动无关
This commit is contained in:
simon
2026-09-25 11:13:37 +08:00
parent 2eaea2ee81
commit ff911cf6f7
19 changed files with 1024 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/run_scheduler.py
+147 -65
View File
@@ -14,14 +14,15 @@
from __future__ import annotations
import argparse
import json
import signal
import sys
from pathlib import Path
from typing import Any
from dotenv import load_dotenv
from loguru import logger
from configs.runtime_env import ensure_env_loaded, env_raw, start_env_watcher
from scheduler import DEFAULT_NEWS_STEPS, run_pipeline
from scheduler.stock_reporter import generate_all_stock_reports
from scheduler.timeutil import now as tz_now
@@ -59,6 +60,131 @@ def _parse_schedule_times(raw: str) -> list[tuple[int, int]]:
return out
def _parse_hhmm(raw: str, default: tuple[int, int]) -> tuple[int, int]:
"""解析 "HH:MM"；非法或越界时记警告并返回 default。"""
parts = raw.split(":")
try:
hour = int(parts[0])
minute = int(parts[1]) if len(parts) > 1 else 0
except (ValueError, IndexError):
logger.warning("时间格式错误: {!r},改用默认 {:02d}:{:02d}", raw, *default)
return default
if not (0 <= hour <= 23 and 0 <= minute <= 59):
logger.warning("时间越界: {!r},改用默认 {:02d}:{:02d}", raw, *default)
return default
return hour, minute
def _scheduled_pipeline(steps: list[str] | None = None) -> None:
"""定时触发的全链路：每次触发时按调度时区重新计算日期。"""
run_pipeline(today_str(), steps=steps)
def _desired_jobs() -> dict[str, dict[str, Any]]:
"""按当前环境变量算出「期望的」定时任务集合。
每次调用都通过 env_get 读取，所以改 .env 后无需重启即可反映到 _sync_jobs。
"""
jobs: dict[str, dict[str, Any]] = {}
times = _parse_schedule_times(
env_raw("SCHEDULE_TIMES", "07:00,12:00,18:00,22:00") or ""
)
if times:
first = min(times)
for hour, minute in times:
with_report = (hour, minute) == first
jobs[f"pipeline_{hour:02d}{minute:02d}"] = {
"kind": "pipeline",
"hour": hour,
"minute": minute,
"steps": list(DEFAULT_NEWS_STEPS) + (["report"] if with_report else []),
"name": f"全链路 {'+日报' if with_report else ''} {hour:02d}:{minute:02d}",
}
cninfo_h, cninfo_m = _parse_hhmm(
env_raw("CNINFO_SCHEDULE_TIME", "06:30") or "06:30", (6, 30)
)
jobs["pipeline_cninfo"] = {
"kind": "pipeline",
"hour": cninfo_h,
"minute": cninfo_m,
"steps": ["cninfo_crawl", "cninfo_extract", "cninfo_pdf",
"dedup", "llm", "embedding", "qdrant"],
"name": f"cninfo 公告管道 {cninfo_h:02d}:{cninfo_m:02d}",
}
# STOCK_REPORT_TIME 缺省 07:30；显式留空表示禁用
stock_raw = env_raw("STOCK_REPORT_TIME")
if stock_raw is None:
stock_raw = "07:30"
if stock_raw:
stock_h, stock_m = _parse_hhmm(stock_raw, (7, 30))
jobs["stock_report"] = {
"kind": "stock",
"hour": stock_h,
"minute": stock_m,
"name": f"个股日报 {stock_h:02d}:{stock_m:02d}",
}
return jobs
_jobs_sig: str | None = None
def _sync_jobs(scheduler: Any) -> bool:
"""把「期望任务」同步到 APScheduler；只在配置变化时增删。返回是否变更。
以 ``_`` 开头的内部任务（如配置热同步自身）不参与增删。
"""
global _jobs_sig
from apscheduler.triggers.cron import CronTrigger # noqa: E402
desired = _desired_jobs()
if not any(jid.startswith("pipeline_") for jid in desired):
logger.error("SCHEDULE_TIMES 为空或全部非法,保留现有定时任务不改动")
return False
sig = json.dumps(desired, sort_keys=True, ensure_ascii=False)
if sig == _jobs_sig:
return False
for jid, spec in desired.items():
trigger = CronTrigger(
hour=spec["hour"], minute=spec["minute"], timezone=str(schedule_tz())
)
if spec["kind"] == "pipeline":
scheduler.add_job(
_scheduled_pipeline,
trigger=trigger,
kwargs={"steps": spec["steps"]},
id=jid,
name=spec["name"],
replace_existing=True,
)
else:
scheduler.add_job(
generate_all_stock_reports,
trigger=trigger,
id=jid,
name=spec["name"],
replace_existing=True,
)
for job in scheduler.get_jobs():
if job.id.startswith("_") or job.id in desired:
continue
scheduler.remove_job(job.id)
_jobs_sig = sig
logger.info(
"定时任务已同步: {}",
", ".join(f"{s['name']}" for s in desired.values()),
)
return True
def _once(args: argparse.Namespace) -> int:
"""单次执行模式。
@@ -76,77 +202,20 @@ def _once(args: argparse.Namespace) -> int:
def _daemon(args: argparse.Namespace) -> int:
"""守护进程模式(APScheduler)。"""
import os
from apscheduler.schedulers.background import BackgroundScheduler # noqa: E402
from apscheduler.triggers.cron import CronTrigger # noqa: E402
times_raw = os.environ.get("SCHEDULE_TIMES", "07:00,12:00,18:00,22:00")
times = _parse_schedule_times(times_raw)
times = _parse_schedule_times(
env_raw("SCHEDULE_TIMES", "07:00,12:00,18:00,22:00") or ""
)
if not times:
logger.error("SCHEDULE_TIMES 为空或全部非法,无法启动定时任务")
return 2
# 找出最早的时间(当天首次运行),仅该次追加日报步骤
sorted_times = sorted(times)
first_hour, first_minute = sorted_times[0] if sorted_times else (0, 0)
scheduler = BackgroundScheduler()
# 包装函数:每次触发时按调度时区重新计算日期(P1-3),
# 避免 date.today() 在注册时冻结或与 cron 时区不一致。
def _scheduled_pipeline(steps: list[str] | None = None) -> None:
run_pipeline(today_str(), steps=steps)
for hour, minute in times:
trigger = CronTrigger(hour=hour, minute=minute, timezone=str(schedule_tz()))
is_first = (hour == first_hour and minute == first_minute)
job_kwargs: dict | None = None
if is_first:
job_kwargs = {
"steps": list(DEFAULT_NEWS_STEPS) + ["report"]
}
scheduler.add_job(
_scheduled_pipeline,
trigger=trigger,
kwargs=job_kwargs,
id=f"pipeline_{hour:02d}{minute:02d}",
name=f"全链路 {'+日报' if is_first else ''} {hour:02d}:{minute:02d}",
)
logger.info("已注册定时任务: {}每天 {:02d}:{:02d}{}", trigger, hour, minute,
" (含日报)" if is_first else "")
# cninfo 公告管道(可配置,默认 06:30)
cninfo_raw = os.environ.get("CNINFO_SCHEDULE_TIME", "06:30")
cninfo_parts = cninfo_raw.split(":")
cninfo_h, cninfo_m = int(cninfo_parts[0]), int(cninfo_parts[1]) if len(cninfo_parts) > 1 else 0
cninfo_trigger = CronTrigger(hour=cninfo_h, minute=cninfo_m, timezone=str(schedule_tz()))
cninfo_steps = ["cninfo_crawl", "cninfo_extract", "cninfo_pdf",
"dedup", "llm", "embedding", "qdrant"]
scheduler.add_job(
_scheduled_pipeline,
trigger=cninfo_trigger,
kwargs={"steps": cninfo_steps},
id="pipeline_cninfo",
name=f"cninfo 公告管道 {cninfo_h:02d}:{cninfo_m:02d}",
)
logger.info("已注册定时任务: cninfo 公告管道 每天 {:02d}:{:02d}", cninfo_h, cninfo_m)
# 个股日报(可配置,默认 07:30, 设为空可禁用)
stock_raw = os.environ.get("STOCK_REPORT_TIME", "07:30")
if stock_raw:
stock_parts = stock_raw.split(":")
stock_h, stock_m = int(stock_parts[0]), int(stock_parts[1]) if len(stock_parts) > 1 else 0
stock_trigger = CronTrigger(hour=stock_h, minute=stock_m, timezone=str(schedule_tz()))
scheduler.add_job(
generate_all_stock_reports,
trigger=stock_trigger,
id="stock_report",
name=f"个股日报 {stock_h:02d}:{stock_m:02d}",
)
logger.info("已注册定时任务: 个股日报 每天 {:02d}:{:02d}", stock_h, stock_m)
else:
logger.info("STOCK_REPORT_TIME 为空, 已禁用个股日报")
# 首次注册；此后由 _config_watch 每 30s 热同步，改 .env 无需重启
_sync_jobs(scheduler)
# 优雅退出
def _shutdown(signum: int, frame: Any) -> None:
@@ -158,7 +227,19 @@ def _daemon(args: argparse.Namespace) -> int:
signal.signal(signal.SIGTERM, _shutdown)
scheduler.start()
logger.info("调度器已启动,等待触发... (按 Ctrl+C 退出)")
# 配置热同步：每 30s 重新计算 SCHEDULE_TIMES / CNINFO_SCHEDULE_TIME / STOCK_REPORT_TIME
scheduler.add_job(
_sync_jobs,
"interval",
seconds=30,
args=[scheduler],
id="_config_watch",
name="配置热同步",
replace_existing=True,
)
# .env 热加载监听：provider / model / key / base_url 等改动无需重启
start_env_watcher()
logger.info("调度器已启动,等待触发... (按 Ctrl+C 退出；改 .env 无需重启)")
# 启动时检查是否有因重启/宕机错过的定时任务,30 分钟内补跑
now = tz_now()
@@ -203,7 +284,8 @@ def main() -> int:
args = parser.parse_args()
_setup_logger(args.log_level)
load_dotenv()
# .env 热加载（改文件后常驻进程无需重启；--once 也会即时读取最新配置）
ensure_env_loaded()
if args.once:
return _once(args)