feat: M9 日报结构化入库(HTML 改为写入 MySQL news_report/news_event,report_type=intl)
- 新增 report_db/ 包(models/schema/db,复用 news 项目实现,幂等 upsert) - reporter.py: _build_report_data + generate_report 写库返回 report_id - pipeline/cli 适配 report_id 返回值;HTML 渲染/上传保留 deprecated - 新增 tests/test_report_db.py;.env.example 增加 NEWS_DB_* 配置 - 已部署 pi5 并验证:真实生成 report_id=184(12 事件)+ 幂等覆盖
This commit is contained in:
+113
-28
@@ -9,12 +9,15 @@
|
||||
import json
|
||||
import logging
|
||||
from collections import Counter
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import markdown
|
||||
import yaml
|
||||
|
||||
from report_db.models import EventRow, ReportData # noqa: F401 - 供 _build_report_data 注解使用
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MAX_HIGH_EVENTS = 30
|
||||
@@ -130,8 +133,8 @@ def _try_parse_time(time_str: str) -> datetime | None:
|
||||
# - 有时区 → astimezone 转为 UTC
|
||||
# - 无时区 → 假设为 UTC(多数财经新闻 API 使用 UTC)
|
||||
if dt.tzinfo is None:
|
||||
return dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
return dt.replace(tzinfo=UTC)
|
||||
return dt.astimezone(UTC)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
@@ -168,7 +171,7 @@ def _load_events_window(now: datetime) -> list[dict]:
|
||||
文章 dict 列表,含 title/title_zh/url/source_id/events 等
|
||||
"""
|
||||
# cutoff 使用 UTC-aware,与 _try_parse_time 返回的 UTC datetime 对齐
|
||||
cutoff = now.astimezone(timezone.utc) - timedelta(hours=_REPORT_WINDOW_HOURS)
|
||||
cutoff = now.astimezone(UTC) - timedelta(hours=_REPORT_WINDOW_HOURS)
|
||||
articles: list[dict] = []
|
||||
skipped_empty_pt = 0
|
||||
for day_str in _dates_in_window(now):
|
||||
@@ -551,17 +554,98 @@ def _generate_ai_summary(articles: list[dict], day_str: str) -> str:
|
||||
return "⚠️ AI 摘要合并失败,以下为各批次原始摘要:\n\n" + "\n\n".join(partial_summaries)
|
||||
|
||||
|
||||
def generate_report() -> Path | None:
|
||||
"""生成 HTML 日报(覆盖过去 25 小时数据)。
|
||||
def _build_report_data(
|
||||
now: datetime,
|
||||
stats: dict,
|
||||
high_events: list[dict],
|
||||
sentiments: Counter,
|
||||
importances: Counter,
|
||||
event_types: Counter,
|
||||
sources: Counter,
|
||||
ai_summary: str,
|
||||
) -> ReportData:
|
||||
"""组装结构化日报数据(M9:写入 MySQL 的前置步骤)。
|
||||
|
||||
命名规则: intl_news_daily_{YYYYMMDD_HHMMSS}.html — 支持一天多份日报。
|
||||
与 news 项目 report_db_design.md 数据契约一致:
|
||||
- 事件全部归入 `intl` 板块(intl 日报无 xwlb/news/cninfo 板块);
|
||||
- report_type="intl"、file_name="" → 唯一键退化为 (report_date, intl, ""),
|
||||
同一天重复生成 = UPDATE 覆盖(幂等);
|
||||
- 数据总览统计以 JSON 快照存入 stats(前端自行解析)。
|
||||
"""
|
||||
events: list[EventRow] = []
|
||||
for i, ev in enumerate(high_events, 1):
|
||||
article = ev.get("article", {})
|
||||
title = (article.get("title_zh") or article.get("title") or "").strip()[:512]
|
||||
url = article.get("url") or None
|
||||
src = _url_source_label(url, article.get("source_id", ""))
|
||||
# 归一化:"" / "?" 不入库,留 None(DB 仅存 positive/negative/neutral)
|
||||
sentiment = ev.get("sentiment") or None
|
||||
if sentiment in ("", "?"):
|
||||
sentiment = None
|
||||
event_type = ev.get("event_type") or None
|
||||
if event_type in ("", "?"):
|
||||
event_type = None
|
||||
events.append(
|
||||
EventRow(
|
||||
section="intl",
|
||||
rank=i,
|
||||
importance=ev.get("importance") or None,
|
||||
event_type=event_type,
|
||||
title=title or "(无标题)",
|
||||
summary=ev.get("summary_zh") or None,
|
||||
sentiment=sentiment,
|
||||
source=src if src not in (None, "", "?") else None,
|
||||
url=url,
|
||||
)
|
||||
)
|
||||
|
||||
stats_snapshot: dict[str, Any] = {
|
||||
"pipeline": {
|
||||
"raw_total": stats.get("raw_total", 0),
|
||||
"processed": stats.get("proc", 0),
|
||||
"deduped": stats.get("deduped", 0),
|
||||
"embedded": stats.get("emb_count", 0),
|
||||
"qdrant": stats.get("qdrant_count", 0),
|
||||
},
|
||||
"sentiment": {k: v for k, v in sentiments.items() if k not in ("", "?")},
|
||||
"importance": [
|
||||
{"importance": k, "count": v} for k, v in sorted(importances.items())
|
||||
],
|
||||
"event_types": [
|
||||
{"event_type": k, "count": v}
|
||||
for k, v in event_types.most_common(10)
|
||||
if k not in ("", "?")
|
||||
],
|
||||
"source_dist": [
|
||||
{"source": _source_name(k), "count": v}
|
||||
for k, v in sources.most_common(15)
|
||||
if k not in ("", "?")
|
||||
],
|
||||
}
|
||||
|
||||
return ReportData(
|
||||
report_date=now.date(),
|
||||
report_type="intl",
|
||||
file_name="", # 新生成日报唯一键退化为 (report_date, intl, "")
|
||||
generated_at=now,
|
||||
ai_summary=ai_summary or None,
|
||||
stats=stats_snapshot,
|
||||
events=events,
|
||||
)
|
||||
|
||||
|
||||
def generate_report() -> int | None:
|
||||
"""生成日报并结构化入库(覆盖过去 25 小时数据)。
|
||||
|
||||
M9 起完全切换:不再产出 HTML,日报内容写入 MySQL
|
||||
(news_report / news_event,report_type="intl",file_name="",
|
||||
同一天重复生成 → 幂等覆盖,不产生多行)。
|
||||
|
||||
Returns:
|
||||
HTML 文件路径,无数据时返回 None
|
||||
report_id(成功)或 None(无数据/失败)
|
||||
"""
|
||||
now = datetime.now()
|
||||
ts = now.strftime("%Y%m%d_%H%M%S")
|
||||
date_str = now.strftime("%Y%m%d") # 用于上传目录
|
||||
logger.info("生成日报: %s(窗口: 过去 %d 小时)", ts, _REPORT_WINDOW_HOURS)
|
||||
|
||||
# 加载过去 25 小时数据
|
||||
@@ -595,13 +679,10 @@ def generate_report() -> Path | None:
|
||||
)
|
||||
|
||||
high = _get_high(all_events, 4)
|
||||
hi_threshold = 4
|
||||
if len(high) < 3:
|
||||
high = _get_high(all_events, 3)
|
||||
hi_threshold = 3
|
||||
if len(high) < 3:
|
||||
high = sorted(all_events, key=lambda e: -e.get("importance", 0))
|
||||
hi_threshold = 0
|
||||
# 事件级去重:同一 URL + 同一标题 → 合并
|
||||
high = _dedup_events(high)
|
||||
high = high[:_MAX_HIGH_EVENTS]
|
||||
@@ -609,20 +690,24 @@ def generate_report() -> Path | None:
|
||||
# AI 摘要
|
||||
ai_summary = _generate_ai_summary(articles, ts)
|
||||
|
||||
# 渲染 HTML
|
||||
html = _render_html(ts, stats, articles, high, hi_threshold,
|
||||
sentiments, importances, event_types, sources, ai_summary)
|
||||
# 结构化入库(替代原 HTML 渲染 + 上传)
|
||||
report = _build_report_data(
|
||||
now, stats, high, sentiments, importances, event_types, sources, ai_summary
|
||||
)
|
||||
try:
|
||||
from report_db import connect, save_report
|
||||
|
||||
# 本地保存(文件名含时间戳)
|
||||
_REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
html_path = _REPORT_DIR / f"intl_news_daily_{ts}.html"
|
||||
html_path.write_text(html, encoding="utf-8")
|
||||
logger.info("日报已保存: %s (%d KB)", html_path, len(html) // 1024)
|
||||
conn = connect()
|
||||
try:
|
||||
report_id = save_report(conn, report)
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.exception("日报入库失败: %s", e)
|
||||
return None
|
||||
|
||||
# 自动上传到日期子目录
|
||||
_upload_report(html_path, date_str)
|
||||
|
||||
return html_path
|
||||
logger.info("日报已入库: report_id=%s", report_id)
|
||||
return report_id
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -644,7 +729,7 @@ def _load_report_config() -> dict:
|
||||
|
||||
|
||||
def _upload_report(html_path: Path, day_str: str) -> bool:
|
||||
"""上传日报到 Web 服务器(配置来自 system.yaml)。"""
|
||||
"""上传日报到 Web 服务器(M9 起弃用:日报已改为写库,保留以便回退)。"""
|
||||
import subprocess
|
||||
|
||||
config = _load_report_config()
|
||||
@@ -678,7 +763,7 @@ def _upload_report(html_path: Path, day_str: str) -> bool:
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# HTML 渲染
|
||||
# HTML 渲染(M9 起弃用:日报已改为写库,以下渲染函数保留以便回退)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
_HTML_TEMPLATE = """<!DOCTYPE html>
|
||||
@@ -798,7 +883,7 @@ def _render_html(
|
||||
sources: Counter,
|
||||
ai_summary: str,
|
||||
) -> str:
|
||||
"""组装完整 HTML。"""
|
||||
"""组装完整 HTML(M9 起弃用,保留以便回退)。"""
|
||||
|
||||
# AI 摘要 Markdown → HTML
|
||||
summary_html = _md_to_html(ai_summary) if ai_summary.strip() else "<p>暂无 AI 摘要</p>"
|
||||
@@ -915,7 +1000,7 @@ def _dedup_events(events: list[dict]) -> list[dict]:
|
||||
|
||||
|
||||
def _render_event_table(events: list[dict]) -> str:
|
||||
"""渲染事件表格。"""
|
||||
"""渲染事件表格(M9 起弃用,保留以便回退)。"""
|
||||
if not events:
|
||||
return "<p>暂无符合条件的数据</p>"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user