feat: 日报结构化入库(M10 前后端分离数据层)
- 新增 report_db 包: MySQL 连接/建表/幂等写入 (news_report/news_event, myquant 库) - 新增 report_import 包: 历史 178 份日报 HTML 解析入库, 表头驱动列映射 - reporter.py 完全切换: generate_report 结构化入库, 不再生成/上传 HTML - CLI: 新增 report-import 子命令 - 依赖: uv add pymysql; 配置: NEWS_DB_* / REPORT_HISTORY_DIR - 文档: docs/report_db_design.md(实现逻辑), docs/db_schema.md(表结构供 API/前端) - 测试: 24 个单测通过 (parser/builder/models/importer)
This commit is contained in:
@@ -0,0 +1,98 @@
|
||||
"""历史日报解析器单元测试(基于真实样例 HTML)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from report_import.parser import (
|
||||
ReportParseError,
|
||||
parse_finance_report,
|
||||
parse_intl_report,
|
||||
parse_report,
|
||||
)
|
||||
|
||||
FIXTURES = Path(__file__).parent / "fixtures"
|
||||
FINANCE_HTML = (FIXTURES / "finance_news_daily_20260710_0720.html").read_text(encoding="utf-8")
|
||||
INTL_HTML = (FIXTURES / "intl_news_daily_20260711_070304.html").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
class TestFinanceParse:
|
||||
def test_metadata(self) -> None:
|
||||
r = parse_finance_report(FINANCE_HTML, "finance_news_daily_20260710_0720.html")
|
||||
assert r.report_date == date(2026, 7, 10)
|
||||
assert r.report_type == "finance"
|
||||
assert r.file_name == "finance_news_daily_20260710_0720.html"
|
||||
assert r.generated_at == datetime(2026, 7, 11, 7, 20, 27) # header"生成于"优先
|
||||
|
||||
def test_ai_summary_lines(self) -> None:
|
||||
r = parse_finance_report(FINANCE_HTML, "finance_news_daily_20260710_0720.html")
|
||||
assert r.ai_summary is not None
|
||||
assert len(r.ai_summary.splitlines()) >= 5
|
||||
assert "碳达峰" in r.ai_summary
|
||||
|
||||
def test_sections_and_ranks(self) -> None:
|
||||
r = parse_finance_report(FINANCE_HTML, "finance_news_daily_20260710_0720.html")
|
||||
sections = {e.section for e in r.events}
|
||||
assert sections == {"xwlb", "news", "cninfo"}
|
||||
assert sum(1 for e in r.events if e.section == "xwlb") == 16
|
||||
assert sum(1 for e in r.events if e.section == "news") == 20
|
||||
assert sum(1 for e in r.events if e.section == "cninfo") == 20
|
||||
|
||||
def test_event_fields(self) -> None:
|
||||
r = parse_finance_report(FINANCE_HTML, "finance_news_daily_20260710_0720.html")
|
||||
xwlb = next(e for e in r.events if e.section == "xwlb")
|
||||
assert xwlb.importance == 4
|
||||
assert xwlb.event_type == "新闻联播"
|
||||
assert xwlb.sentiment == "neutral"
|
||||
assert "张国清" in xwlb.title
|
||||
news = next(e for e in r.events if e.section == "news" and e.source)
|
||||
assert news.source # 新闻板块带来源
|
||||
|
||||
def test_stats_keys(self) -> None:
|
||||
r = parse_finance_report(FINANCE_HTML, "finance_news_daily_20260710_0720.html")
|
||||
for key in ("pipeline", "sources", "sentiment", "importance", "event_types"):
|
||||
assert key in r.stats, f"缺少 stats.{key}"
|
||||
assert r.stats["pipeline"]["M1 原始文章"] == 758
|
||||
|
||||
|
||||
class TestIntlParse:
|
||||
def test_metadata(self) -> None:
|
||||
r = parse_intl_report(INTL_HTML, "intl_news_daily_20260711_070304.html")
|
||||
assert r.report_date == date(2026, 7, 11)
|
||||
assert r.report_type == "intl"
|
||||
assert r.generated_at == datetime(2026, 7, 11, 7, 3, 30)
|
||||
|
||||
def test_events_and_source_from_small(self) -> None:
|
||||
r = parse_intl_report(INTL_HTML, "intl_news_daily_20260711_070304.html")
|
||||
assert len(r.events) == 19
|
||||
first = r.events[0]
|
||||
assert first.section == "intl"
|
||||
assert first.source == "investinglive.com" # 从摘要 <small>[来源]</small> 提取
|
||||
assert first.url.startswith("https://investinglive.com/")
|
||||
assert first.importance == 4
|
||||
assert first.event_type == "地缘政治"
|
||||
# 摘要中不应残留 [来源] 标记
|
||||
assert first.summary is not None and "[investinglive.com]" not in first.summary
|
||||
|
||||
def test_stats_keys(self) -> None:
|
||||
r = parse_intl_report(INTL_HTML, "intl_news_daily_20260711_070304.html")
|
||||
for key in ("pipeline", "sentiment", "importance", "event_types", "source_dist"):
|
||||
assert key in r.stats
|
||||
assert r.stats["source_dist"][0]["来源"] == "ForexLive"
|
||||
|
||||
|
||||
class TestParseReportDispatch:
|
||||
def test_dispatch_finance(self) -> None:
|
||||
r = parse_report(FINANCE_HTML, "finance_news_daily_20260710_0720.html")
|
||||
assert r.report_type == "finance"
|
||||
|
||||
def test_dispatch_intl(self) -> None:
|
||||
r = parse_report(INTL_HTML, "intl_news_daily_20260711_070304.html")
|
||||
assert r.report_type == "intl"
|
||||
|
||||
def test_bad_filename_raises(self) -> None:
|
||||
with pytest.raises(ReportParseError):
|
||||
parse_report("<html></html>", "not_a_report.html")
|
||||
Reference in New Issue
Block a user