Files
simon d4a55bcaaa feat: M9 日报结构化入库(HTML 改为写入 MySQL news_report/news_event,report_type=intl)
- 新增 report_db/ 包(models/schema/db,复用 news 项目实现,幂等 upsert)
- reporter.py: _build_report_data + generate_report 写库返回 report_id
- pipeline/cli 适配 report_id 返回值;HTML 渲染/上传保留 deprecated
- 新增 tests/test_report_db.py;.env.example 增加 NEWS_DB_* 配置
- 已部署 pi5 并验证:真实生成 report_id=184(12 事件)+ 幂等覆盖
2026-08-04 11:09:17 +08:00

162 lines
5.3 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""日报结构化入库:DB 连接、建表、写入。
与 news 项目 report_db/db.py 保持一致(日志改用 logging,遵守本项目规范)。
"""
from __future__ import annotations
import json
import logging
import os
from dataclasses import dataclass
from typing import Any
import pymysql
from .models import ReportData
from .schema import DDL_STATEMENTS
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class DbConfig:
"""MySQL 连接配置(来自环境变量 NEWS_DB_*)。"""
host: str
port: int
user: str
password: str
name: str
def load_db_config() -> DbConfig:
"""从环境变量读取 NEWS_DB_*,缺失密码时抛异常(禁止默认密码)。"""
host = os.environ.get("NEWS_DB_HOST", "192.168.1.10")
port = int(os.environ.get("NEWS_DB_PORT", "13306"))
user = os.environ.get("NEWS_DB_USER", "myquant")
password = os.environ.get("NEWS_DB_PASSWORD", "")
name = os.environ.get("NEWS_DB_NAME", "myquant")
if not password:
logger.error("NEWS_DB_PASSWORD 未配置,请在 .env 中设置")
raise ValueError("NEWS_DB_PASSWORD 未配置")
return DbConfig(host=host, port=port, user=user, password=password, name=name)
def connect(cfg: DbConfig | None = None) -> pymysql.Connection:
"""建立短连接(autocommit=False)。失败时记录日志并抛出。"""
cfg = cfg or load_db_config()
try:
conn = pymysql.connect(
host=cfg.host,
port=cfg.port,
user=cfg.user,
password=cfg.password,
database=cfg.name,
charset="utf8mb4",
autocommit=False,
cursorclass=pymysql.cursors.DictCursor,
)
except Exception:
logger.exception("连接 MySQL 失败: host=%s port=%s user=%s", cfg.host, cfg.port, cfg.user)
raise
logger.debug("MySQL 已连接: %s/%s", cfg.host, cfg.name)
return conn
def init_schema(conn: pymysql.Connection) -> None:
"""建表(CREATE TABLE IF NOT EXISTS ×2),幂等。"""
with conn.cursor() as cur:
for ddl in DDL_STATEMENTS:
cur.execute(ddl)
conn.commit()
logger.info("news_report / news_event 建表完成")
def save_report(conn: pymysql.Connection, report: ReportData) -> int:
"""事务内写入一份日报。
幂等策略:
- 主表按 (report_date, report_type, file_name) 唯一键 upsert
- 事件表 DELETE 该 report 旧行后全量 INSERT(整份覆盖一致)。
返回 report_id。
"""
with conn.cursor() as cur:
stats_json = json.dumps(report.stats, ensure_ascii=False) if report.stats else None
cur.execute(
"""
INSERT INTO news_report
(report_date, report_type, file_name, generated_at, ai_summary, stats)
VALUES (%s, %s, %s, %s, %s, %s)
ON DUPLICATE KEY UPDATE
generated_at = VALUES(generated_at),
ai_summary = VALUES(ai_summary),
stats = VALUES(stats)
""",
(
report.report_date,
report.report_type,
report.file_name,
report.generated_at,
report.ai_summary,
stats_json,
),
)
cur.execute(
"SELECT id FROM news_report WHERE report_date=%s AND report_type=%s AND file_name=%s",
(report.report_date, report.report_type, report.file_name),
)
row = cur.fetchone()
if row is None: # pragma: no cover - 理论不可达
raise RuntimeError("写入 news_report 后查询不到 report_id")
report_id: int = row["id"]
cur.execute("DELETE FROM news_event WHERE report_id=%s", (report_id,))
for ev in report.events:
cur.execute(
"""
INSERT INTO news_event
(report_id, section, rank, importance, event_type, title,
summary, sentiment, source, url)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
""",
(
report_id,
ev.section,
ev.rank,
ev.importance,
ev.event_type,
ev.title,
ev.summary,
ev.sentiment,
ev.source,
ev.url,
),
)
conn.commit()
logger.info("日报已入库: report_id=%s date=%s type=%s events=%s",
report_id, report.report_date, report.report_type, len(report.events))
return report_id
def fetch_report(conn: pymysql.Connection, report_id: int) -> dict[str, Any] | None:
"""读侧辅助(联调/测试用),返回主表行。"""
with conn.cursor() as cur:
cur.execute("SELECT * FROM news_report WHERE id=%s", (report_id,))
return cur.fetchone()
def exists_report(
conn: pymysql.Connection,
report_date: Any,
report_type: str,
file_name: str,
) -> bool:
"""判断主表是否已存在该唯一键记录(历史导入幂等用)。"""
with conn.cursor() as cur:
cur.execute(
"SELECT 1 FROM news_report WHERE report_date=%s AND report_type=%s AND file_name=%s",
(report_date, report_type, file_name),
)
return cur.fetchone() is not None