Initial commit

2026-07-18 15:51:01 +08:00
commit f2c80c5a9c
799 changed files with 133475 additions and 0 deletions
@@ -0,0 +1,217 @@
+"""定时任务主流程(M7)。
+
+编排 M1→M6 全链路,每一步调用已有脚本。
+单步失败记录日志但不阻断后续(后续步骤可能使用旧缓存数据,降级继续)。
+"""
+
+from __future__ import annotations
+
+import subprocess
+import time
+from dataclasses import dataclass, field
+from datetime import date, datetime
+
+from loguru import logger
+
+# 步骤超时(秒)
+STEP_TIMEOUTS: dict[str, int] = {
+    "crawler": 900,     # M1 抓取(含 Playwright 浏览器,13 源约 8-12 min)
+    "xwlb": 60,         # M1 新闻联播 API(纯 HTTP,秒级)
+    "extractor": 300,   # M2 正文提取
+    "dedup": 120,       # M3 去重
+    "llm": 900,         # M4 LLM 事件抽取(API 调用,100 篇约 30s 但加限流余量)
+    "embedding": 300,   # M5 向量化
+    "qdrant": 300,      # M6 入库(数据量大时需较长时间)
+    "report":  30,      # 日报生成+上传
+    "cninfo_crawl":     900,  # cninfo watchlist URL 驱动(SPA 渲染,每只约 25s)
+}
+
+# 步骤对应的 uv run 命令(参数中 {date} 会被替换为实际日期)
+# crawler 不支持 --date,固定写当天目录; dedup 不加 --reset 以保持增量
+STEP_COMMANDS: dict[str, list[str]] = {
+    "crawler":   ["uv", "run", "python", "-m", "scripts.run_crawler"],        # 无 --date
+    "xwlb":      ["uv", "run", "python", "-m", "scripts.run_xwlb"],
+    "extractor": ["uv", "run", "python", "-m", "scripts.run_extractor", "--date", "{date}"],
+    "dedup":     ["uv", "run", "python", "-m", "scripts.run_dedup", "--date", "{date}"],
+    "llm":       ["uv", "run", "python", "-m", "scripts.run_event_extraction", "--date", "{date}"],
+    "embedding": ["uv", "run", "python", "-m", "scripts.run_embedding", "--date", "{date}"],
+    "qdrant":    ["uv", "run", "python", "-m", "scripts.run_qdrant_ingest", "--date", "{date}"],
+    "report":        [],  # 特殊步骤:仅在每天首次定时任务时追加
+    "cninfo_crawl":   ["uv", "run", "a-share", "cninfo"],
+    "cninfo_extract": ["uv", "run", "a-share", "extract", "--source", "cninfo", "--date", "{date}"],
+    "cninfo_pdf":     ["uv", "run", "a-share", "cninfo", "--enrich-pdf", "--pdf-limit", "100"],
+}
+
+
+@dataclass
+class StepResult:
+    name: str
+    success: bool
+    elapsed_sec: float
+    exit_code: int | None = None
+    tail_msg: str = ""
+    started_at: datetime | None = None
+
+
+@dataclass
+class PipelineResult:
+    steps: list[StepResult] = field(default_factory=list)
+    started_at: datetime | None = None
+    finished_at: datetime | None = None
+
+    @property
+    def all_success(self) -> bool:
+        return all(s.success for s in self.steps)
+
+
+def run_step(name: str, date_str: str) -> StepResult:
+    """执行单个 pipeline 步骤。
+
+    参数:
+        name:     步骤名(crawler/extractor/.../report)
+        date_str: YYYYMMDD 日期字符串
+
+    返回: StepResult。
+    """
+    # report 步骤:内部函数,不走子进程
+    # 日报默认统计"昨天"的数据（因为今天的数据由当天的定时任务处理）。
+    # 如果昨天没有事件数据，向前回溯最多 3 天，取最近有数据的日期。
+    if name == "report":
+        started = datetime.now()
+        try:
+            from datetime import timedelta  # noqa: E402
+            from pathlib import Path  # noqa: E402
+
+            from .reporter import generate_report  # noqa: E402
+
+            # 向前回溯找最近有事件数据的日期（最多回溯 3 天）
+            report_date: str | None = None
+            for offset in range(1, 4):
+                candidate = (date.today() - timedelta(days=offset)).strftime("%Y%m%d")
+                ev_dir = Path(f"data/events/{candidate}")
+                if ev_dir.is_dir() and list(ev_dir.glob("*.json")):
+                    report_date = candidate
+                    break
+
+            if report_date is None:
+                # 没有任何事件数据，仍然尝试生成昨天日报（至少展示管道统计）
+                report_date = (date.today() - timedelta(days=1)).strftime("%Y%m%d")
+                logger.warning(
+                    "日报: 近 3 日均无事件数据 ({} ~ {}), 日报将只含管道统计",
+                    (date.today() - timedelta(days=3)).strftime("%Y%m%d"),
+                    (date.today() - timedelta(days=1)).strftime("%Y%m%d"),
+                )
+            elif report_date != (date.today() - timedelta(days=1)).strftime("%Y%m%d"):
+                logger.warning(
+                    "日报: 昨天 ({}) 无事件数据, 回退使用 {}",
+                    (date.today() - timedelta(days=1)).strftime("%Y%m%d"),
+                    report_date,
+                )
+
+            path = generate_report(report_date, upload=True)
+            elapsed = (datetime.now() - started).total_seconds()
+            ok = path is not None
+            return StepResult(
+                name=name, success=ok, elapsed_sec=elapsed,
+                tail_msg=str(path) if path else f"无数据 (report_date={report_date})",
+                started_at=started,
+            )
+        except Exception as e:  # noqa: BLE001
+            elapsed = (datetime.now() - started).total_seconds()
+            logger.exception("日报生成异常: {}", e)
+            return StepResult(name=name, success=False, elapsed_sec=elapsed,
+                              tail_msg=str(e)[:200], started_at=started)
+
+    cmd = STEP_COMMANDS.get(name)
+    if cmd is None:
+        return StepResult(name=name, success=False, elapsed_sec=0,
+                          tail_msg=f"未知步骤: {name}")
+
+    full_cmd = [arg.replace("{date}", date_str) for arg in cmd]
+    # 超时优先级:
+    #   1. TIMEOUT_{NAME} 环境变量 (单步精确控制)
+    #   2. PIPELINE_STEP_TIMEOUT 环境变量 (全局兜底, 覆盖硬编码)
+    #   3. STEP_TIMEOUTS 硬编码字典 (代码内默认值)
+    #   4. 1800s (最终兜底)
+    import os
+    specific_key = f"TIMEOUT_{name.upper()}"
+    if specific_key in os.environ:
+        timeout = int(os.environ[specific_key])
+    elif "PIPELINE_STEP_TIMEOUT" in os.environ:
+        timeout = int(os.environ["PIPELINE_STEP_TIMEOUT"])
+    else:
+        timeout = STEP_TIMEOUTS.get(name, 1800)
+    started = datetime.now()
+    logger.info("步骤 {} 开始: {}", name, " ".join(full_cmd))
+
+    try:
+        # 不捕获输出,子进程日志直接流到终端(用户能看到每个源的抓取进度)
+        proc = subprocess.run(
+            full_cmd,
+            timeout=timeout,
+        )
+        elapsed = (datetime.now() - started).total_seconds()
+        ok = proc.returncode == 0
+
+        # dedup 返回 1 是"重复率过高"(无新文章的正常场景)
+        if name == "dedup" and proc.returncode == 1:
+            ok = True
+            logger.info("dedup 重复率超过阈值(无新数据场景,视为成功)")
+
+        tail_msg = f"rc={proc.returncode}" if not ok else ""
+
+        if ok:
+            logger.info("步骤 {} 完成 ({}s) ✅", name, elapsed)
+        else:
+            logger.error("步骤 {} 失败 rc={} ({}s)", name, proc.returncode, elapsed)
+
+        return StepResult(
+            name=name, success=ok, elapsed_sec=elapsed,
+            exit_code=proc.returncode, tail_msg=tail_msg,
+            started_at=started,
+        )
+    except subprocess.TimeoutExpired:
+        elapsed = (datetime.now() - started).total_seconds()
+        logger.error("步骤 {} 超时 (>{:.0f}s)", name, elapsed)
+        return StepResult(name=name, success=False, elapsed_sec=elapsed,
+                          tail_msg="超时", started_at=started)
+    except Exception as e:  # noqa: BLE001
+        elapsed = (datetime.now() - started).total_seconds()
+        logger.exception("步骤 {} 异常: {}", name, e)
+        return StepResult(name=name, success=False, elapsed_sec=elapsed,
+                          tail_msg=str(e)[:200], started_at=started)
+
+
+def run_pipeline(date_str: str, *, steps: list[str] | None = None) -> PipelineResult:
+    """串联执行全链路(M1→M6)。
+
+    参数:
+        date_str: YYYYMMDD。
+        steps:    可选步骤列表,默认全部 6 步。
+    """
+    names = steps or [k for k in STEP_COMMANDS if k not in ("report", "cninfo_crawl", "cninfo_extract", "cninfo_pdf")]
+    result = PipelineResult(started_at=datetime.now())
+
+    for name in names:
+        sr = run_step(name, date_str)
+        result.steps.append(sr)
+        if not sr.success:
+            logger.warning("步骤 {} 失败,后续步骤继续(可能降级)", name)
+        # 步间留一点缓冲
+        time.sleep(0.5)
+
+    result.finished_at = datetime.now()
+    total = (result.finished_at - result.started_at).total_seconds() if result.started_at else 0
+    succ = sum(1 for s in result.steps if s.success)
+    rate = succ / max(len(result.steps), 1)
+    logger.info(
+        "Pipeline 完成: {}/{} 步骤成功 ({:.0%}) 总耗时 {:.0f}s",
+        succ, len(result.steps), rate, total,
+    )
+
+    # 输出摘要
+    for s in result.steps:
+        flag = "✅" if s.success else "❌"
+        logger.info("  {} {}  {}s", flag, s.name, s.elapsed_sec)
+
+    return result