feat: 日报结构化入库（M10 前后端分离数据层）
- 新增 report_db 包: MySQL 连接/建表/幂等写入 (news_report/news_event, myquant 库) - 新增 report_import 包: 历史 178 份日报 HTML 解析入库, 表头驱动列映射 - reporter.py 完全切换: generate_report 结构化入库, 不再生成/上传 HTML - CLI: 新增 report-import 子命令 - 依赖: uv add pymysql; 配置: NEWS_DB_* / REPORT_HISTORY_DIR - 文档: docs/report_db_design.md(实现逻辑), docs/db_schema.md(表结构供 API/前端) - 测试: 24 个单测通过 (parser/builder/models/importer)
This commit is contained in:
@@ -0,0 +1,19 @@
|
||||
"""日报结构化入库（Milestone 10）。
|
||||
|
||||
职责：日报内容（finance/intl）结构化后写入 MySQL（news_report / news_event），
|
||||
供用户另行实现的 API/前端读取。本项目不提供 API 与前端。
|
||||
"""
|
||||
|
||||
from .db import connect, exists_report, fetch_report, init_schema, load_db_config, save_report
|
||||
from .models import EventRow, ReportData
|
||||
|
||||
__all__ = [
|
||||
"EventRow",
|
||||
"ReportData",
|
||||
"connect",
|
||||
"exists_report",
|
||||
"fetch_report",
|
||||
"init_schema",
|
||||
"load_db_config",
|
||||
"save_report",
|
||||
]
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
"""日报结构化入库：DB 连接、建表、写入。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import pymysql
|
||||
from loguru import logger
|
||||
|
||||
from .models import ReportData
|
||||
from .schema import DDL_STATEMENTS
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DbConfig:
|
||||
"""MySQL 连接配置（来自环境变量 NEWS_DB_*）。"""
|
||||
|
||||
host: str
|
||||
port: int
|
||||
user: str
|
||||
password: str
|
||||
name: str
|
||||
|
||||
|
||||
def load_db_config() -> DbConfig:
|
||||
"""从环境变量读取 NEWS_DB_*，缺失密码时抛异常（禁止默认密码）。"""
|
||||
host = os.environ.get("NEWS_DB_HOST", "127.0.0.1")
|
||||
port = int(os.environ.get("NEWS_DB_PORT", "13306"))
|
||||
user = os.environ.get("NEWS_DB_USER", "myquant")
|
||||
password = os.environ.get("NEWS_DB_PASSWORD", "")
|
||||
name = os.environ.get("NEWS_DB_NAME", "myquant")
|
||||
if not password:
|
||||
logger.error("NEWS_DB_PASSWORD 未配置，请在 .env 中设置")
|
||||
raise ValueError("NEWS_DB_PASSWORD 未配置")
|
||||
return DbConfig(host=host, port=port, user=user, password=password, name=name)
|
||||
|
||||
|
||||
def connect(cfg: DbConfig | None = None) -> pymysql.Connection:
|
||||
"""建立短连接（autocommit=False）。失败时记录日志并抛出。"""
|
||||
cfg = cfg or load_db_config()
|
||||
try:
|
||||
conn = pymysql.connect(
|
||||
host=cfg.host,
|
||||
port=cfg.port,
|
||||
user=cfg.user,
|
||||
password=cfg.password,
|
||||
database=cfg.name,
|
||||
charset="utf8mb4",
|
||||
autocommit=False,
|
||||
cursorclass=pymysql.cursors.DictCursor,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("连接 MySQL 失败: host={} port={} user={}", cfg.host, cfg.port, cfg.user)
|
||||
raise
|
||||
logger.debug("MySQL 已连接: {}/{}", cfg.host, cfg.name)
|
||||
return conn
|
||||
|
||||
|
||||
def init_schema(conn: pymysql.Connection) -> None:
|
||||
"""建表（CREATE TABLE IF NOT EXISTS ×2），幂等。"""
|
||||
with conn.cursor() as cur:
|
||||
for ddl in DDL_STATEMENTS:
|
||||
cur.execute(ddl)
|
||||
conn.commit()
|
||||
logger.info("news_report / news_event 建表完成")
|
||||
|
||||
|
||||
def save_report(conn: pymysql.Connection, report: ReportData) -> int:
|
||||
"""事务内写入一份日报。
|
||||
|
||||
幂等策略：
|
||||
- 主表按 (report_date, report_type, file_name) 唯一键 upsert；
|
||||
- 事件表 DELETE 该 report 旧行后全量 INSERT（整份覆盖一致）。
|
||||
返回 report_id。
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
stats_json = json.dumps(report.stats, ensure_ascii=False) if report.stats else None
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO news_report
|
||||
(report_date, report_type, file_name, generated_at, ai_summary, stats)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
generated_at = VALUES(generated_at),
|
||||
ai_summary = VALUES(ai_summary),
|
||||
stats = VALUES(stats)
|
||||
""",
|
||||
(
|
||||
report.report_date,
|
||||
report.report_type,
|
||||
report.file_name,
|
||||
report.generated_at,
|
||||
report.ai_summary,
|
||||
stats_json,
|
||||
),
|
||||
)
|
||||
cur.execute(
|
||||
"SELECT id FROM news_report WHERE report_date=%s AND report_type=%s AND file_name=%s",
|
||||
(report.report_date, report.report_type, report.file_name),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None: # pragma: no cover - 理论不可达
|
||||
raise RuntimeError("写入 news_report 后查询不到 report_id")
|
||||
report_id: int = row["id"]
|
||||
|
||||
cur.execute("DELETE FROM news_event WHERE report_id=%s", (report_id,))
|
||||
for ev in report.events:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO news_event
|
||||
(report_id, section, rank, importance, event_type, title,
|
||||
summary, sentiment, source, url)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""",
|
||||
(
|
||||
report_id,
|
||||
ev.section,
|
||||
ev.rank,
|
||||
ev.importance,
|
||||
ev.event_type,
|
||||
ev.title,
|
||||
ev.summary,
|
||||
ev.sentiment,
|
||||
ev.source,
|
||||
ev.url,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
logger.info("日报已入库: report_id={} date={} type={} events={}",
|
||||
report_id, report.report_date, report.report_type, len(report.events))
|
||||
return report_id
|
||||
|
||||
|
||||
def fetch_report(conn: pymysql.Connection, report_id: int) -> dict[str, Any] | None:
|
||||
"""读侧辅助（联调/测试用），返回主表行。"""
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT * FROM news_report WHERE id=%s", (report_id,))
|
||||
return cur.fetchone()
|
||||
|
||||
|
||||
def exists_report(
|
||||
conn: pymysql.Connection,
|
||||
report_date: Any,
|
||||
report_type: str,
|
||||
file_name: str,
|
||||
) -> bool:
|
||||
"""判断主表是否已存在该唯一键记录（历史导入幂等用）。"""
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT 1 FROM news_report WHERE report_date=%s AND report_type=%s AND file_name=%s",
|
||||
(report_date, report_type, file_name),
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
@@ -0,0 +1,34 @@
|
||||
"""日报结构化入库：数据模型。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class EventRow(BaseModel):
|
||||
"""一条事件记录，对应 news_event 一行。"""
|
||||
|
||||
section: str # xwlb | news | cninfo | intl
|
||||
rank: int # 板块内序号（从 1 开始）
|
||||
importance: int | None = None
|
||||
event_type: str | None = None
|
||||
title: str
|
||||
summary: str | None = None
|
||||
sentiment: str | None = None # positive | negative | neutral | ''
|
||||
source: str | None = None
|
||||
url: str | None = None
|
||||
|
||||
|
||||
class ReportData(BaseModel):
|
||||
"""一份完整日报：news_report 一行 + news_event 多行。"""
|
||||
|
||||
report_date: date
|
||||
report_type: str # finance | intl
|
||||
file_name: str = "" # 源文件名（新生成日报可为空）
|
||||
generated_at: datetime
|
||||
ai_summary: str | None = None
|
||||
stats: dict[str, Any] = Field(default_factory=dict)
|
||||
events: list[EventRow] = Field(default_factory=list)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""MySQL DDL：日报结构化入库（表前缀 news_，目标 MariaDB 10.11）。
|
||||
|
||||
与 project_plan.md「十八、Milestone 10」及 docs/report_db_design.md 保持一致。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
DDL_STATEMENTS: list[str] = [
|
||||
# 日报主表
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS news_report (
|
||||
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||
report_date DATE NOT NULL COMMENT '日报日期',
|
||||
report_type VARCHAR(16) NOT NULL COMMENT 'finance=A股日报 / intl=国际财经日报',
|
||||
file_name VARCHAR(160) NOT NULL DEFAULT '' COMMENT '源文件名（历史解析）；新生成可为空',
|
||||
generated_at DATETIME NOT NULL COMMENT '生成时间',
|
||||
ai_summary TEXT NULL COMMENT 'AI 摘要全文',
|
||||
stats JSON NULL COMMENT '数据总览统计快照（管道/情绪/重要度/事件类型/来源分布）',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE KEY uk_report_file (report_date, report_type, file_name),
|
||||
KEY idx_report_date (report_date)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='每日日报主表'
|
||||
""",
|
||||
# 日报事件明细
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS news_event (
|
||||
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||
report_id BIGINT UNSIGNED NOT NULL COMMENT 'FK → news_report.id',
|
||||
section VARCHAR(16) NOT NULL COMMENT '板块: xwlb=新闻联播 / news=财经新闻 / cninfo=公告调研 / intl=国际重要事件',
|
||||
rank INT NOT NULL DEFAULT 0 COMMENT '板块内序号',
|
||||
importance INT NULL COMMENT '重要度 1-5',
|
||||
event_type VARCHAR(64) NULL COMMENT '事件类型',
|
||||
title VARCHAR(512) NOT NULL COMMENT '标题',
|
||||
summary TEXT NULL COMMENT '摘要/正文',
|
||||
sentiment VARCHAR(8) NULL COMMENT 'positive/negative/neutral',
|
||||
source VARCHAR(64) NULL COMMENT '来源',
|
||||
url VARCHAR(512) NULL COMMENT '原文链接',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
KEY idx_report_section (report_id, section),
|
||||
KEY idx_title (title(255))
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='日报事件明细'
|
||||
""",
|
||||
]
|
||||
Reference in New Issue
Block a user