feat: 日报结构化入库（M10 前后端分离数据层）

- 新增 report_db 包: MySQL 连接/建表/幂等写入 (news_report/news_event, myquant 库) - 新增 report_import 包: 历史 178 份日报 HTML 解析入库, 表头驱动列映射 - reporter.py 完全切换: generate_report 结构化入库, 不再生成/上传 HTML - CLI: 新增 report-import 子命令 - 依赖: uv add pymysql; 配置: NEWS_DB_* / REPORT_HISTORY_DIR - 文档: docs/report_db_design.md(实现逻辑), docs/db_schema.md(表结构供 API/前端) - 测试: 24 个单测通过 (parser/builder/models/importer)
2026-08-03 21:32:07 +08:00
parent f2c80c5a9c
commit 366e60e8a9
25 changed files with 1934 additions and 33 deletions
@@ -21,6 +21,8 @@ from typing import Any
 from dotenv import load_dotenv
 from loguru import logger

+from report_db.models import EventRow, ReportData  # noqa: F401 - 供 _build_report_data 注解使用
+
 # 确保 .env 已加载（模块级常量依赖环境变量）
 load_dotenv()

@@ -843,7 +845,7 @@ def _render_xwlb_section(xwlb: dict | None) -> str:
 def _render_html(news: dict, cninfo: dict, pipeline: dict,
                 ai_summary: str, day_str: str,
                 xwlb: dict | None = None) -> str:
-    """组装完整 HTML。"""
+    """组装完整 HTML（M10 起废弃：日报已改为结构化入库，此函数不再被调用，保留以便回退）。"""

    # AI 摘要 → HTML
    summary_html = _re.sub(r"\*\*(.+?)\*\*", r"<strong>\1</strong>", ai_summary)
@@ -941,8 +943,76 @@ def _render_html(news: dict, cninfo: dict, pipeline: dict,
 # 生成 + 上传
 # --------------------------------------------------------------------------- #

-def generate_report(day_str: str | None = None, *, upload: bool = True) -> Path | None:
-    """生成每日摘要 HTML 报告,可选上传到 Web 服务器。"""
+def _build_report_data(news: dict, cninfo: dict, pipeline: dict,
+                       ai_summary: str, day_str: str,
+                       xwlb: dict | None = None) -> ReportData:
+    """组装结构化日报数据（M10：写入 MySQL 的前置步骤）。
+
+    事件板块映射：news["high"]→news / cninfo["high"]→cninfo / xwlb["items"]→xwlb。
+    数据总览统计以 JSON 快照存入 stats（前端自行解析）。
+    """
+    events: list[EventRow] = []
+
+    def _rows(items: list[dict], section: str) -> None:
+        for i, e in enumerate(items, 1):
+            ev = e.get("event", {})
+            events.append(
+                EventRow(
+                    section=section,
+                    rank=i,
+                    importance=ev.get("importance"),
+                    event_type=ev.get("event_type"),
+                    title=str(e.get("title", ""))[:512],
+                    summary=(ev.get("summary") or None),
+                    sentiment=ev.get("sentiment") or None,
+                    source=e.get("source_id") or None,
+                    url=e.get("url") or None,
+                )
+            )
+
+    _rows(news.get("high", []), "news")
+    _rows(cninfo.get("high", []), "cninfo")
+    if xwlb:
+        _rows(xwlb.get("items", []), "xwlb")
+
+    stats: dict[str, Any] = {
+        "pipeline": pipeline,
+        "news": {
+            "total": news.get("total", 0),
+            "hi_threshold": news.get("hi_threshold"),
+            "sentiments": news.get("sentiments", {}),
+            "importances": news.get("importances", {}),
+            "event_types": news.get("event_types", {}),
+        },
+        "cninfo": {
+            "total": cninfo.get("total", 0),
+            "hi_threshold": cninfo.get("hi_threshold"),
+            "by_day": cninfo.get("by_day", {}),
+            "announcement": cninfo.get("announcement", 0),
+            "research": cninfo.get("research", 0),
+            "irm": cninfo.get("irm", 0),
+        },
+    }
+    if xwlb:
+        stats["xwlb"] = {"total": len(xwlb.get("items", [])), "date": xwlb.get("date", "")}
+
+    return ReportData(
+        report_date=datetime.strptime(day_str, "%Y%m%d").date(),
+        report_type="finance",
+        file_name="",  # 新生成日报唯一键退化为 (report_date, finance, "")
+        generated_at=datetime.now(),
+        ai_summary=ai_summary or None,
+        stats=stats,
+        events=events,
+    )
+
+
+def generate_report(day_str: str | None = None, *, upload: bool = True) -> int | None:
+    """生成每日日报并结构化入库（M10 完全切换，不再生成 HTML）。
+
+    `upload` 参数保留以兼容 scheduler/pipeline.py 调用，已无实际作用。
+    返回 report_id（成功）或 None（无数据/失败）。
+    """
    day_str = day_str or date.today().strftime("%Y%m%d")
    logger.info("生成日报: {}", day_str)

@@ -963,25 +1033,26 @@ def generate_report(day_str: str | None = None, *, upload: bool = True) -> Path
    # AI 摘要(新闻联播 + 新闻 + cninfo)
    ai_summary = _generate_ai_summary(news, cninfo, day_str, xwlb=xwlb)

-    # 渲染
-    html = _render_html(news, cninfo, pipeline, ai_summary, day_str, xwlb=xwlb)
+    # 结构化入库（替代原 HTML 渲染 + 上传）
+    report = _build_report_data(news, cninfo, pipeline, ai_summary, day_str, xwlb=xwlb)
+    try:
+        from report_db import connect, save_report

-    # 保存（文件名含时间，支持一天多次生成）
-    out_dir = Path("data/reports")
-    out_dir.mkdir(parents=True, exist_ok=True)
-    file_tag = f"{day_str}_{datetime.now():%H%M}"
-    html_path = out_dir / f"finance_news_daily_{file_tag}.html"
-    html_path.write_text(html, encoding="utf-8")
-    logger.info("日报已保存: {} ({} KB)", html_path, len(html) // 1024)
+        conn = connect()
+        try:
+            report_id = save_report(conn, report)
+        finally:
+            conn.close()
+    except Exception as e:
+        logger.exception("日报入库失败: {}", e)
+        return None

-    if upload:
-        _upload(html_path, file_tag)
-
-    return html_path
+    logger.info("日报已入库: report_id={}", report_id)
+    return report_id


 def _upload(html_path: Path, file_tag: str) -> bool:
-    """上传 HTML 报告到 Web 服务器。"""
+    """上传 HTML 报告到 Web 服务器（M10 起废弃：不再被调用，保留以便回退）。"""
    today_str = date.today().strftime("%Y%m%d")
    remote_dir = f"{UPLOAD_BASE}/{today_str}/"
    logger.info("上传日报到 {}:{}", UPLOAD_HOST, remote_dir)