feat: M9 日报结构化入库（HTML 改为写入 MySQL news_report/news_event，report_type=intl）
- 新增 report_db/ 包（models/schema/db，复用 news 项目实现，幂等 upsert） - reporter.py: _build_report_data + generate_report 写库返回 report_id - pipeline/cli 适配 report_id 返回值；HTML 渲染/上传保留 deprecated - 新增 tests/test_report_db.py；.env.example 增加 NEWS_DB_* 配置 - 已部署 pi5 并验证：真实生成 report_id=184（12 事件）+ 幂等覆盖
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
"""日报结构化入库。
|
||||
|
||||
职责：日报内容（intl）结构化后写入 MySQL（news_report / news_event），
|
||||
与 news 项目共用同一表结构与数据契约（见 docs/db_schema.md、report_db_design.md）。
|
||||
本包不提供 API 与前端。
|
||||
"""
|
||||
|
||||
from .db import connect, exists_report, fetch_report, init_schema, load_db_config, save_report
|
||||
from .models import EventRow, ReportData
|
||||
|
||||
__all__ = [
|
||||
"EventRow",
|
||||
"ReportData",
|
||||
"connect",
|
||||
"exists_report",
|
||||
"fetch_report",
|
||||
"init_schema",
|
||||
"load_db_config",
|
||||
"save_report",
|
||||
]
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
"""日报结构化入库：DB 连接、建表、写入。
|
||||
|
||||
与 news 项目 report_db/db.py 保持一致（日志改用 logging，遵守本项目规范）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import pymysql
|
||||
|
||||
from .models import ReportData
|
||||
from .schema import DDL_STATEMENTS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DbConfig:
|
||||
"""MySQL 连接配置（来自环境变量 NEWS_DB_*）。"""
|
||||
|
||||
host: str
|
||||
port: int
|
||||
user: str
|
||||
password: str
|
||||
name: str
|
||||
|
||||
|
||||
def load_db_config() -> DbConfig:
|
||||
"""从环境变量读取 NEWS_DB_*，缺失密码时抛异常（禁止默认密码）。"""
|
||||
host = os.environ.get("NEWS_DB_HOST", "192.168.1.10")
|
||||
port = int(os.environ.get("NEWS_DB_PORT", "13306"))
|
||||
user = os.environ.get("NEWS_DB_USER", "myquant")
|
||||
password = os.environ.get("NEWS_DB_PASSWORD", "")
|
||||
name = os.environ.get("NEWS_DB_NAME", "myquant")
|
||||
if not password:
|
||||
logger.error("NEWS_DB_PASSWORD 未配置，请在 .env 中设置")
|
||||
raise ValueError("NEWS_DB_PASSWORD 未配置")
|
||||
return DbConfig(host=host, port=port, user=user, password=password, name=name)
|
||||
|
||||
|
||||
def connect(cfg: DbConfig | None = None) -> pymysql.Connection:
|
||||
"""建立短连接（autocommit=False）。失败时记录日志并抛出。"""
|
||||
cfg = cfg or load_db_config()
|
||||
try:
|
||||
conn = pymysql.connect(
|
||||
host=cfg.host,
|
||||
port=cfg.port,
|
||||
user=cfg.user,
|
||||
password=cfg.password,
|
||||
database=cfg.name,
|
||||
charset="utf8mb4",
|
||||
autocommit=False,
|
||||
cursorclass=pymysql.cursors.DictCursor,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("连接 MySQL 失败: host=%s port=%s user=%s", cfg.host, cfg.port, cfg.user)
|
||||
raise
|
||||
logger.debug("MySQL 已连接: %s/%s", cfg.host, cfg.name)
|
||||
return conn
|
||||
|
||||
|
||||
def init_schema(conn: pymysql.Connection) -> None:
|
||||
"""建表（CREATE TABLE IF NOT EXISTS ×2），幂等。"""
|
||||
with conn.cursor() as cur:
|
||||
for ddl in DDL_STATEMENTS:
|
||||
cur.execute(ddl)
|
||||
conn.commit()
|
||||
logger.info("news_report / news_event 建表完成")
|
||||
|
||||
|
||||
def save_report(conn: pymysql.Connection, report: ReportData) -> int:
|
||||
"""事务内写入一份日报。
|
||||
|
||||
幂等策略：
|
||||
- 主表按 (report_date, report_type, file_name) 唯一键 upsert；
|
||||
- 事件表 DELETE 该 report 旧行后全量 INSERT（整份覆盖一致）。
|
||||
返回 report_id。
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
stats_json = json.dumps(report.stats, ensure_ascii=False) if report.stats else None
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO news_report
|
||||
(report_date, report_type, file_name, generated_at, ai_summary, stats)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
generated_at = VALUES(generated_at),
|
||||
ai_summary = VALUES(ai_summary),
|
||||
stats = VALUES(stats)
|
||||
""",
|
||||
(
|
||||
report.report_date,
|
||||
report.report_type,
|
||||
report.file_name,
|
||||
report.generated_at,
|
||||
report.ai_summary,
|
||||
stats_json,
|
||||
),
|
||||
)
|
||||
cur.execute(
|
||||
"SELECT id FROM news_report WHERE report_date=%s AND report_type=%s AND file_name=%s",
|
||||
(report.report_date, report.report_type, report.file_name),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None: # pragma: no cover - 理论不可达
|
||||
raise RuntimeError("写入 news_report 后查询不到 report_id")
|
||||
report_id: int = row["id"]
|
||||
|
||||
cur.execute("DELETE FROM news_event WHERE report_id=%s", (report_id,))
|
||||
for ev in report.events:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO news_event
|
||||
(report_id, section, rank, importance, event_type, title,
|
||||
summary, sentiment, source, url)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""",
|
||||
(
|
||||
report_id,
|
||||
ev.section,
|
||||
ev.rank,
|
||||
ev.importance,
|
||||
ev.event_type,
|
||||
ev.title,
|
||||
ev.summary,
|
||||
ev.sentiment,
|
||||
ev.source,
|
||||
ev.url,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
logger.info("日报已入库: report_id=%s date=%s type=%s events=%s",
|
||||
report_id, report.report_date, report.report_type, len(report.events))
|
||||
return report_id
|
||||
|
||||
|
||||
def fetch_report(conn: pymysql.Connection, report_id: int) -> dict[str, Any] | None:
|
||||
"""读侧辅助（联调/测试用），返回主表行。"""
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT * FROM news_report WHERE id=%s", (report_id,))
|
||||
return cur.fetchone()
|
||||
|
||||
|
||||
def exists_report(
|
||||
conn: pymysql.Connection,
|
||||
report_date: Any,
|
||||
report_type: str,
|
||||
file_name: str,
|
||||
) -> bool:
|
||||
"""判断主表是否已存在该唯一键记录（历史导入幂等用）。"""
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT 1 FROM news_report WHERE report_date=%s AND report_type=%s AND file_name=%s",
|
||||
(report_date, report_type, file_name),
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
@@ -0,0 +1,37 @@
|
||||
"""日报结构化入库：数据模型。
|
||||
|
||||
与 news 项目 report_db/models.py 保持一致（intl 板块复用同一表结构）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class EventRow(BaseModel):
|
||||
"""一条事件记录，对应 news_event 一行。"""
|
||||
|
||||
section: str # xwlb | news | cninfo | intl
|
||||
rank: int # 板块内序号（从 1 开始）
|
||||
importance: int | None = None
|
||||
event_type: str | None = None
|
||||
title: str
|
||||
summary: str | None = None
|
||||
sentiment: str | None = None # positive | negative | neutral | ''
|
||||
source: str | None = None
|
||||
url: str | None = None
|
||||
|
||||
|
||||
class ReportData(BaseModel):
|
||||
"""一份完整日报：news_report 一行 + news_event 多行。"""
|
||||
|
||||
report_date: date
|
||||
report_type: str # finance | intl
|
||||
file_name: str = "" # 源文件名（新生成日报可为空）
|
||||
generated_at: datetime
|
||||
ai_summary: str | None = None
|
||||
stats: dict[str, Any] = Field(default_factory=dict)
|
||||
events: list[EventRow] = Field(default_factory=list)
|
||||
@@ -0,0 +1,44 @@
|
||||
"""MySQL DDL：日报结构化入库（表前缀 news_，目标 MariaDB 10.11）。
|
||||
|
||||
与 news 项目 docs/db_schema.md 保持一致（news_report / news_event）。
|
||||
历史日报已由 news 项目 report-import 导入，本包只负责新日报写入与幂等建表。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
DDL_STATEMENTS: list[str] = [
|
||||
# 日报主表
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS news_report (
|
||||
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||
report_date DATE NOT NULL COMMENT '日报日期',
|
||||
report_type VARCHAR(16) NOT NULL COMMENT 'finance=A股日报 / intl=国际财经日报',
|
||||
file_name VARCHAR(160) NOT NULL DEFAULT '' COMMENT '源文件名（历史解析）；新生成可为空',
|
||||
generated_at DATETIME NOT NULL COMMENT '生成时间',
|
||||
ai_summary TEXT NULL COMMENT 'AI 摘要全文',
|
||||
stats JSON NULL COMMENT '数据总览统计快照（管道/情绪/重要度/事件类型/来源分布）',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_report_file (report_date, report_type, file_name),
|
||||
KEY idx_report_date (report_date)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='每日日报主表'
|
||||
""",
|
||||
# 日报事件明细
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS news_event (
|
||||
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||
report_id BIGINT UNSIGNED NOT NULL COMMENT 'FK → news_report.id',
|
||||
section VARCHAR(16) NOT NULL COMMENT '板块: xwlb=新闻联播 / news=财经新闻 / cninfo=公告调研 / intl=国际重要事件',
|
||||
rank INT NOT NULL DEFAULT 0 COMMENT '板块内序号',
|
||||
importance INT NULL COMMENT '重要度 1-5',
|
||||
event_type VARCHAR(64) NULL COMMENT '事件类型',
|
||||
title VARCHAR(512) NOT NULL COMMENT '标题',
|
||||
summary TEXT NULL COMMENT '摘要/正文',
|
||||
sentiment VARCHAR(8) NULL COMMENT 'positive/negative/neutral',
|
||||
source VARCHAR(64) NULL COMMENT '来源',
|
||||
url VARCHAR(512) NULL COMMENT '原文链接',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
KEY idx_report_section (report_id, section),
|
||||
KEY idx_title (title(255))
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='日报事件明细'
|
||||
""",
|
||||
]
|
||||
Reference in New Issue
Block a user