feat: 日报结构化入库（M10 前后端分离数据层）
- 新增 report_db 包: MySQL 连接/建表/幂等写入 (news_report/news_event, myquant 库) - 新增 report_import 包: 历史 178 份日报 HTML 解析入库, 表头驱动列映射 - reporter.py 完全切换: generate_report 结构化入库, 不再生成/上传 HTML - CLI: 新增 report-import 子命令 - 依赖: uv add pymysql; 配置: NEWS_DB_* / REPORT_HISTORY_DIR - 文档: docs/report_db_design.md(实现逻辑), docs/db_schema.md(表结构供 API/前端) - 测试: 24 个单测通过 (parser/builder/models/importer)
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
"""历史日报解析与批量导入（Milestone 10）。
|
||||
|
||||
将 doorcome 历史日报 HTML（finance/intl）解析为结构化数据并写入 MySQL。
|
||||
"""
|
||||
|
||||
from .importer import ImportStats, import_history
|
||||
from .parser import (
|
||||
ReportParseError,
|
||||
parse_finance_report,
|
||||
parse_intl_report,
|
||||
parse_report,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ImportStats",
|
||||
"ReportParseError",
|
||||
"import_history",
|
||||
"parse_finance_report",
|
||||
"parse_intl_report",
|
||||
"parse_report",
|
||||
]
|
||||
@@ -0,0 +1,89 @@
|
||||
"""历史日报批量导入（解析 → 入库，幂等）。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from report_db import connect, exists_report, save_report
|
||||
from report_import.parser import ReportParseError, parse_report
|
||||
|
||||
_REPORT_FILE_RE = re.compile(r"([a-z]+)_news_daily_\d{8}(?:_\d{4,6})?\.html$")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportStats:
|
||||
"""一次导入的统计结果。"""
|
||||
|
||||
scanned: int = 0 # 扫描到的日报文件数
|
||||
imported: int = 0 # 新入库
|
||||
skipped: int = 0 # 已存在（幂等跳过）
|
||||
failed: int = 0 # 解析失败
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def _match_file(path: Path, date_str: str | None, report_type: str | None) -> bool:
|
||||
"""按文件名判断是否属于本次导入范围。"""
|
||||
m = _REPORT_FILE_RE.search(path.name)
|
||||
if not m:
|
||||
return False
|
||||
if report_type is not None and m.group(1) != report_type:
|
||||
return False
|
||||
return date_str is None or date_str in path.name
|
||||
|
||||
|
||||
def import_history(
|
||||
report_dir: str | Path,
|
||||
date_str: str | None = None,
|
||||
report_type: str | None = None,
|
||||
*,
|
||||
force: bool = False,
|
||||
) -> ImportStats:
|
||||
"""扫描 {report_dir}/{YYYYMMDD}/ 下全部 `*_news_daily_*.html` 并入库。
|
||||
|
||||
- 幂等：主表唯一键 (report_date, report_type, file_name) 已存在则跳过；
|
||||
- `force=True` 时跳过存在性检查，直接覆盖重导；
|
||||
- 单文件解析失败不影响其他文件。
|
||||
"""
|
||||
stats = ImportStats()
|
||||
root = Path(report_dir)
|
||||
if not root.is_dir():
|
||||
logger.error("日报目录不存在: {}", root)
|
||||
raise FileNotFoundError(f"日报目录不存在: {root}")
|
||||
|
||||
files = sorted(p for p in root.glob("*/[a-z]*_news_daily_*.html") if _match_file(p, date_str, report_type))
|
||||
stats.scanned = len(files)
|
||||
logger.info("扫描到日报文件 {} 份: {}", stats.scanned, root)
|
||||
|
||||
conn = connect()
|
||||
try:
|
||||
for path in files:
|
||||
try:
|
||||
html = path.read_text(encoding="utf-8")
|
||||
report = parse_report(html, path.name)
|
||||
except ReportParseError as e:
|
||||
stats.failed += 1
|
||||
stats.errors.append(f"{path.name}: {e}")
|
||||
logger.warning("解析失败: {} ({})", path.name, e)
|
||||
continue
|
||||
except Exception as e: # 防御未知异常，不中断批量
|
||||
stats.failed += 1
|
||||
stats.errors.append(f"{path.name}: {type(e).__name__}: {e}")
|
||||
logger.exception("读取/解析异常: {}", path.name)
|
||||
continue
|
||||
|
||||
if not force and exists_report(conn, report.report_date, report.report_type, report.file_name):
|
||||
stats.skipped += 1
|
||||
logger.debug("已存在, 跳过: {}", path.name)
|
||||
continue
|
||||
save_report(conn, report)
|
||||
stats.imported += 1
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
logger.info("导入完成: scanned={} imported={} skipped={} failed={}",
|
||||
stats.scanned, stats.imported, stats.skipped, stats.failed)
|
||||
return stats
|
||||
@@ -0,0 +1,298 @@
|
||||
"""历史日报 HTML 解析器（finance / intl）。
|
||||
|
||||
策略：表头驱动列映射，不依赖列位置；板块按 h2 标题识别；
|
||||
数据总览按 h3 标题归类为 stats JSON 快照（前端自行解析）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date, datetime
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
from report_db.models import EventRow, ReportData
|
||||
|
||||
# 情绪图标 → sentiment 值
|
||||
_SENTIMENT_ICON: dict[str, str] = {"⚪": "neutral", "🔴": "negative", "🟢": "positive"}
|
||||
|
||||
# 数据总览 h3 标题关键词 → stats key
|
||||
_STATS_SECTION_KEYS: list[tuple[str, str]] = [
|
||||
("管道", "pipeline"),
|
||||
("各源", "sources"),
|
||||
("情绪", "sentiment"),
|
||||
("重要度", "importance"),
|
||||
("事件类型", "event_types"),
|
||||
("来源", "source_dist"),
|
||||
]
|
||||
|
||||
|
||||
class ReportParseError(Exception):
|
||||
"""整份文件解析失败。"""
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 文件名 / 时间解析
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _parse_datetime_from_filename(file_name: str) -> tuple[date, datetime] | None:
|
||||
"""从文件名解析日报日期与生成时间。
|
||||
|
||||
支持 `{type}_news_daily_{YYYYMMDD}.html` 与带时间戳的
|
||||
`{type}_news_daily_{YYYYMMDD}_{HHMMSS}.html` / `..._{HHMM}.html`。
|
||||
"""
|
||||
m = re.search(r"_daily_(\d{8})(?:_(\d{4})(\d{2})?)?", file_name)
|
||||
if not m:
|
||||
return None
|
||||
day = date(int(m.group(1)[:4]), int(m.group(1)[4:6]), int(m.group(1)[6:8]))
|
||||
hh = mm = ss = 0
|
||||
if m.group(2):
|
||||
hh, mm = int(m.group(2)[:2]), int(m.group(2)[2:4])
|
||||
ss = int(m.group(3) or 0)
|
||||
return day, datetime(day.year, day.month, day.day, hh, mm, ss)
|
||||
|
||||
|
||||
def _parse_header_generated_at(soup: BeautifulSoup, fallback: datetime) -> datetime:
|
||||
"""从 <header> 中"生成于 YYYY-MM-DD HH:MM:SS"解析生成时间。"""
|
||||
p = soup.select_one("header p")
|
||||
if p:
|
||||
m = re.search(r"生成于 (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})", p.get_text())
|
||||
if m:
|
||||
return datetime.strptime(m.group(1), "%Y-%m-%d %H:%M:%S")
|
||||
return fallback
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# AI 摘要
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _parse_ai_summary(soup: BeautifulSoup) -> str | None:
|
||||
"""AI 摘要：<div class="ai-summary">，li 逐行输出。"""
|
||||
div = soup.select_one("div.ai-summary")
|
||||
if div is None:
|
||||
return None
|
||||
items = [li.get_text(strip=True) for li in div.find_all("li") if li.get_text(strip=True)]
|
||||
if items:
|
||||
return "\n".join(items)
|
||||
text = re.sub(r"\s+", " ", div.get_text(strip=True))
|
||||
return text or None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 事件表解析
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _to_int(text: str) -> int | None:
|
||||
m = re.search(r"\d+", text or "")
|
||||
return int(m.group(0)) if m else None
|
||||
|
||||
|
||||
def _parse_summary(cell: Tag) -> tuple[str | None, str | None]:
|
||||
"""摘要列：剥除 <small>[来源]</small>，返回 (摘要, 来源)。"""
|
||||
source = None
|
||||
small = cell.find("small")
|
||||
if small:
|
||||
m = re.search(r"\[([^\]]+)\]", small.get_text())
|
||||
if m:
|
||||
source = m.group(1)
|
||||
small.decompose()
|
||||
text = re.sub(r"\s+", " ", cell.get_text(strip=True))
|
||||
return (text or None, source)
|
||||
|
||||
|
||||
def _clean_title(cell: Tag) -> str:
|
||||
"""标题列：剥除 <small> 股票代码标注等，返回纯标题。"""
|
||||
for small in cell.find_all("small"):
|
||||
small.decompose()
|
||||
return re.sub(r"\s+", " ", cell.get_text(strip=True))
|
||||
|
||||
|
||||
def _parse_event_table(table: Tag, section: str) -> list[EventRow]:
|
||||
"""事件表 → EventRow 列表。表头驱动列映射，兼容 xwlb/news/cninfo/intl 四类表。"""
|
||||
rows = table.find_all("tr")
|
||||
if len(rows) < 2:
|
||||
return []
|
||||
header_cells = rows[0].find_all(["th", "td"])
|
||||
col_index: dict[str, int] = {}
|
||||
icon_col: int | None = None
|
||||
for i, cell in enumerate(header_cells):
|
||||
text = cell.get_text(strip=True)
|
||||
if text:
|
||||
col_index[text] = i
|
||||
elif icon_col is None:
|
||||
icon_col = i
|
||||
|
||||
out: list[EventRow] = []
|
||||
for row in rows[1:]:
|
||||
tds = row.find_all("td")
|
||||
if not tds:
|
||||
continue
|
||||
|
||||
def col(name: str, tds: list[Tag] = tds) -> Tag | None: # noqa: B008 - 绑定循环变量
|
||||
idx = col_index.get(name)
|
||||
return tds[idx] if idx is not None and idx < len(tds) else None
|
||||
|
||||
title_cell = col("标题")
|
||||
if title_cell is None:
|
||||
continue
|
||||
a = title_cell.find("a")
|
||||
url = a.get("href") if a else None
|
||||
|
||||
summary_cell = col("摘要")
|
||||
summary: str | None = None
|
||||
source: str | None = None
|
||||
if summary_cell is not None:
|
||||
summary, source = _parse_summary(summary_cell)
|
||||
if source is None:
|
||||
src_cell = col("源")
|
||||
if src_cell is not None and src_cell.get_text(strip=True):
|
||||
source = src_cell.get_text(strip=True)
|
||||
|
||||
sentiment = None
|
||||
if icon_col is not None and icon_col < len(tds):
|
||||
sentiment = _SENTIMENT_ICON.get(tds[icon_col].get_text(strip=True))
|
||||
|
||||
imp_cell = col("重要度")
|
||||
type_cell = col("事件类型")
|
||||
out.append(
|
||||
EventRow(
|
||||
section=section,
|
||||
rank=_to_int(tds[0].get_text(strip=True)) or len(out) + 1,
|
||||
importance=_to_int(imp_cell.get_text(strip=True)) if imp_cell else None,
|
||||
event_type=type_cell.get_text(strip=True) if type_cell else None,
|
||||
title=_clean_title(title_cell),
|
||||
summary=summary,
|
||||
sentiment=sentiment,
|
||||
source=source,
|
||||
url=url,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _collect_events(soup: BeautifulSoup, report_type: str) -> list[EventRow]:
|
||||
"""按 h2 板块标题收集各事件表。"""
|
||||
events: list[EventRow] = []
|
||||
for h2 in soup.find_all("h2"):
|
||||
title = h2.get_text()
|
||||
table = h2.find_next_sibling("table")
|
||||
if table is None:
|
||||
continue
|
||||
section: str | None = None
|
||||
if "新闻联播" in title:
|
||||
section = "xwlb"
|
||||
elif "公告" in title or "调研" in title:
|
||||
section = "cninfo"
|
||||
elif "重要事件" in title:
|
||||
section = "intl" if report_type == "intl" else "news"
|
||||
if section is not None:
|
||||
events.extend(_parse_event_table(table, section))
|
||||
return events
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 数据总览 stats
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _table_to_rows(table: Tag) -> list[dict[str, str]]:
|
||||
"""表格 → [{表头: 值, ...}, ...]（首行为表头）。"""
|
||||
rows: list[list[str]] = []
|
||||
for tr in table.find_all("tr"):
|
||||
cells = [re.sub(r"\s+", " ", c.get_text(strip=True)) for c in tr.find_all(["th", "td"])]
|
||||
if cells:
|
||||
rows.append(cells)
|
||||
if not rows:
|
||||
return []
|
||||
header = rows[0]
|
||||
return [dict(zip(header, r, strict=False)) for r in rows[1:]]
|
||||
|
||||
|
||||
def _parse_stats_block(h3: Tag) -> tuple[str, object] | None:
|
||||
"""h3 数据总览区块 → (stats_key, value)。缺失/未知板块返回 None。"""
|
||||
title = h3.get_text()
|
||||
key = next((k for kw, k in _STATS_SECTION_KEYS if kw in title), None)
|
||||
if key is None:
|
||||
return None
|
||||
block = h3.find_next_sibling()
|
||||
if block is None:
|
||||
return key, {}
|
||||
|
||||
if block.name == "table":
|
||||
return key, _table_to_rows(block)
|
||||
|
||||
classes = block.get("class", []) if isinstance(block.get("class"), list) else []
|
||||
if "stats-grid" in classes:
|
||||
cards: dict[str, str | int] = {}
|
||||
for card in block.find_all("div", class_="stat-card"):
|
||||
label = card.select_one(".label")
|
||||
num = card.select_one(".num")
|
||||
if label is not None:
|
||||
num_text = num.get_text(strip=True) if num else ""
|
||||
cards[label.get_text(strip=True)] = _to_int(num_text) if _to_int(num_text) is not None else num_text
|
||||
return key, cards
|
||||
if "source-grid" in classes:
|
||||
items: dict[str, str | int] = {}
|
||||
for item in block.find_all("div", class_="source-item"):
|
||||
name = item.select_one(".s-name")
|
||||
count = item.select_one(".s-count")
|
||||
if name is not None:
|
||||
count_text = count.get_text(strip=True) if count else ""
|
||||
items[name.get_text(strip=True)] = _to_int(count_text) or count_text
|
||||
return key, items
|
||||
if "sentiment-bar" in classes:
|
||||
legend = block.find_next_sibling("div", class_="sentiment-legend")
|
||||
spans = legend.find_all("span") if legend else []
|
||||
return key, [re.sub(r"\s+", " ", s.get_text(strip=True)) for s in spans]
|
||||
|
||||
text = re.sub(r"\s+", " ", block.get_text(strip=True))
|
||||
return key, text[:500]
|
||||
|
||||
|
||||
def _collect_stats(soup: BeautifulSoup) -> dict[str, object]:
|
||||
stats: dict[str, object] = {}
|
||||
for h3 in soup.find_all("h3"):
|
||||
parsed = _parse_stats_block(h3)
|
||||
if parsed is not None:
|
||||
stats[parsed[0]] = parsed[1]
|
||||
return stats
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 主入口
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _parse(html: str, file_name: str, report_type: str) -> ReportData:
|
||||
parsed = _parse_datetime_from_filename(file_name)
|
||||
if parsed is None:
|
||||
raise ReportParseError(f"无法从文件名解析日报日期: {file_name}")
|
||||
day, gen_from_file = parsed
|
||||
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
generated_at = _parse_header_generated_at(soup, gen_from_file)
|
||||
|
||||
return ReportData(
|
||||
report_date=day,
|
||||
report_type=report_type,
|
||||
file_name=file_name,
|
||||
generated_at=generated_at,
|
||||
ai_summary=_parse_ai_summary(soup),
|
||||
stats=_collect_stats(soup),
|
||||
events=_collect_events(soup, report_type),
|
||||
)
|
||||
|
||||
|
||||
def parse_finance_report(html: str, file_name: str) -> ReportData:
|
||||
"""解析 A 股日报 finance_news_daily_*.html。"""
|
||||
return _parse(html, file_name, "finance")
|
||||
|
||||
|
||||
def parse_intl_report(html: str, file_name: str) -> ReportData:
|
||||
"""解析国际财经日报 intl_news_daily_*.html。"""
|
||||
return _parse(html, file_name, "intl")
|
||||
|
||||
|
||||
def parse_report(html: str, file_name: str) -> ReportData:
|
||||
"""按文件名前缀自动分流 finance / intl。"""
|
||||
if "intl_news_daily" in file_name:
|
||||
return parse_intl_report(html, file_name)
|
||||
return parse_finance_report(html, file_name)
|
||||
Reference in New Issue
Block a user