482 lines
17 KiB
Python
482 lines
17 KiB
Python
"""M2 正文提取模块单元测试。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from extractor import Article, ExtractError, extract_article
|
|
from extractor.parser import (
|
|
_clean_content,
|
|
_count_chinese,
|
|
_fallback_time_from_html,
|
|
_is_boilerplate,
|
|
_is_reasonable_time,
|
|
_normalize_time,
|
|
_refine_title,
|
|
_resolve_publish_time,
|
|
_url_hash,
|
|
)
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 构造测试 HTML
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
SAMPLE_HTML = """
|
|
<html>
|
|
<head><title>宁德时代发布麒麟电池 - 财联社</title></head>
|
|
<body>
|
|
<header><nav>首页 财经 股票</nav></header>
|
|
<div class="ad">广告:抢购618</div>
|
|
|
|
<article>
|
|
<h1>宁德时代发布新一代麒麟电池 能量密度达 255Wh/kg</h1>
|
|
<div class="meta">
|
|
<span class="time">2026-06-15 14:30:00</span>
|
|
<span class="author">记者 张三</span>
|
|
</div>
|
|
<p>财联社6月15日电,宁德时代今日正式发布了新一代麒麟电池产品,能量密度达到 255 Wh/kg,
|
|
显著优于上一代产品的 213 Wh/kg。该产品定位高端电动车市场。</p>
|
|
<p>据公司公告,该电池将于2026年第三季度量产,首批应用于多款新能源汽车。
|
|
公司股价应声上涨5.2%,创年内新高。</p>
|
|
<p>分析师认为,这将进一步巩固宁德时代在动力电池领域的全球领先地位,
|
|
预计2026年公司动力电池出货量将同比增长30%以上。</p>
|
|
<p>同时,公司还披露了海外建厂计划,德国与匈牙利工厂将于2027年投产。</p>
|
|
</article>
|
|
|
|
<aside class="related">
|
|
<h3>相关阅读</h3>
|
|
<ul><li><a href="/a/1.html">比亚迪发布刀片电池升级版</a></li>
|
|
<li><a href="/a/2.html">国轩高科上半年扭亏为盈</a></li></ul>
|
|
</aside>
|
|
|
|
<section class="comments">
|
|
<h3>评论 (88)</h3>
|
|
<p>用户A: 太牛了!</p>
|
|
<p>用户B: 行业要变天</p>
|
|
</section>
|
|
|
|
<footer>版权所有 财联社</footer>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
SHORT_HTML = """
|
|
<html><body><h1>很短</h1><p>太短</p></body></html>
|
|
"""
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 主提取流程
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def test_extract_basic() -> None:
|
|
article = extract_article(
|
|
html=SAMPLE_HTML,
|
|
source_id="cls",
|
|
url="https://www.cls.cn/detail/123456",
|
|
)
|
|
assert isinstance(article, Article)
|
|
assert article.source_id == "cls"
|
|
assert article.source_name == "财联社"
|
|
assert "宁德时代" in article.title
|
|
assert "麒麟电池" in article.title
|
|
assert article.url_hash == _url_hash("https://www.cls.cn/detail/123456")
|
|
|
|
# 关键正文短语必须保留
|
|
assert "255 Wh/kg" in article.content or "255Wh/kg" in article.content
|
|
assert "海外建厂" in article.content
|
|
|
|
# 时间被解析
|
|
assert article.publish_time is not None
|
|
assert article.publish_time.year == 2026
|
|
assert article.publish_time.month == 6
|
|
assert article.publish_time.day == 15
|
|
|
|
# 字数
|
|
assert article.word_count > 30
|
|
|
|
|
|
def test_extract_h1_preferred_over_short_title() -> None:
|
|
"""<title> 仅有站名时应优先 H1。"""
|
|
html = """
|
|
<html><head><title>财联社</title></head><body>
|
|
<h1>这是一个明显更长更详细的真实文章标题</h1>
|
|
<article>
|
|
<p>正文段落一,正文段落一,正文段落一,正文段落一,正文段落一。</p>
|
|
<p>正文段落二,正文段落二,正文段落二,正文段落二,正文段落二。</p>
|
|
<p>正文段落三,正文段落三,正文段落三,正文段落三,正文段落三。</p>
|
|
</article></body></html>
|
|
"""
|
|
article = extract_article(html, "cls", "https://www.cls.cn/detail/1")
|
|
assert article.title == "这是一个明显更长更详细的真实文章标题"
|
|
|
|
|
|
def test_extract_too_short_raises() -> None:
|
|
with pytest.raises(ExtractError):
|
|
extract_article(SHORT_HTML, "cls", "https://www.cls.cn/detail/2")
|
|
|
|
|
|
def test_extract_empty_html_raises() -> None:
|
|
with pytest.raises(ExtractError):
|
|
extract_article("", "cls", "https://www.cls.cn/detail/3")
|
|
with pytest.raises(ExtractError):
|
|
extract_article(" \n\n ", "cls", "https://www.cls.cn/detail/4")
|
|
|
|
|
|
def test_extract_unknown_source_has_no_source_name() -> None:
|
|
article = extract_article(SAMPLE_HTML, "unknown_src", "https://example.com/a/1")
|
|
assert article.source_name is None
|
|
assert article.source_id == "unknown_src"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 模板兜底检测(M2.2)
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def test_is_boilerplate_eastmoney_legal_disclaimer() -> None:
|
|
text = (
|
|
"郑重声明: 1.根据《证券法》规定,禁止编造、传播虚假信息或者误导性信息,"
|
|
"扰乱证券市场;2.用户在本社区发表的所有资料、言论等仅代表个人观点。"
|
|
)
|
|
is_bp, reason = _is_boilerplate(text)
|
|
assert is_bp
|
|
assert reason is not None and "郑重声明" in reason
|
|
|
|
|
|
def test_is_boilerplate_yicai_ad_copyright() -> None:
|
|
text = (
|
|
"第一财经广告合作,请点击这里。此内容为第一财经原创,著作权归第一财经所有。"
|
|
"未经第一财经书面授权,不得以任何方式加以使用。"
|
|
)
|
|
is_bp, reason = _is_boilerplate(text)
|
|
assert is_bp
|
|
|
|
|
|
def test_is_boilerplate_sina_embedded_post() -> None:
|
|
text = (
|
|
"北京红竹 今天 16:02:49 本质上就是一句话:资金开始从核心抱团,进入扩散阶段。"
|
|
"银行这边今天已经明显走弱。银行和科技之间还是跷跷板关系。" * 2
|
|
)
|
|
is_bp, _ = _is_boilerplate(text)
|
|
assert is_bp
|
|
|
|
|
|
def test_is_boilerplate_long_real_article_passes() -> None:
|
|
"""真实长篇文章中即使提到"郑重声明"等词,长度 > 800 不应误判。"""
|
|
legitimate = (
|
|
"公司发布郑重声明,回应近期市场关注的多项议题。根据《证券法》及相关法律法规要求,"
|
|
"公司将严格履行信息披露义务。" * 30 # ~1200 字
|
|
)
|
|
assert len(legitimate) > 800
|
|
is_bp, _ = _is_boilerplate(legitimate)
|
|
assert not is_bp
|
|
|
|
|
|
def test_is_boilerplate_empty_returns_false() -> None:
|
|
assert _is_boilerplate("") == (False, None)
|
|
assert _is_boilerplate("普通正文,没有任何模板特征,长度也合理。" * 5) == (False, None)
|
|
|
|
|
|
def test_extract_article_raises_on_boilerplate() -> None:
|
|
"""模拟一篇 GNE 兜底失败的页面,extract_article 应抛 ExtractError。"""
|
|
html = """
|
|
<html><head><title>东方财富</title></head><body>
|
|
<h1>是否有中国船只通过霍尔木兹海峡？外交部回应</h1>
|
|
<article>
|
|
郑重声明: 1.根据《证券法》规定,禁止编造、传播虚假信息或者误导性信息,
|
|
扰乱证券市场;2.用户在本社区发表的所有资料、言论等仅代表个人观点,与本网站立场无关。
|
|
《东方财富社区管理规定》
|
|
</article>
|
|
</body></html>
|
|
"""
|
|
with pytest.raises(ExtractError) as exc:
|
|
extract_article(
|
|
html,
|
|
source_id="eastmoney",
|
|
url="https://finance.eastmoney.com/a/123.html",
|
|
)
|
|
assert "模板" in str(exc.value)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# title 校正
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def test_refine_title_uses_h1_when_gne_is_substring() -> None:
|
|
html = "<html><body><h1>完整真实标题</h1></body></html>"
|
|
assert _refine_title(html, "完整真实") == "完整真实标题"
|
|
|
|
|
|
def test_refine_title_uses_h1_when_gne_has_site_suffix() -> None:
|
|
html = "<html><body><h1>真实标题</h1></body></html>"
|
|
assert _refine_title(html, "真实标题 - 财联社") == "真实标题"
|
|
|
|
|
|
def test_refine_title_falls_back_to_gne_when_no_h1() -> None:
|
|
html = "<html><body><p>x</p></body></html>"
|
|
assert _refine_title(html, "GNE 标题") == "GNE 标题"
|
|
|
|
|
|
def test_refine_title_returns_empty_when_both_missing() -> None:
|
|
assert _refine_title("<html></html>", "") == ""
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# content 清理
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def test_clean_content_strips_repeated_header() -> None:
|
|
raw = "宁德时代发布新一代麒麟电池\n2026-06-15 14:30:00\n记者 张三\n\n正文第一段。\n\n\n\n正文第二段。"
|
|
cleaned = _clean_content(
|
|
raw,
|
|
title="宁德时代发布新一代麒麟电池",
|
|
author="记者 张三",
|
|
time_raw="2026-06-15 14:30:00",
|
|
)
|
|
assert cleaned.startswith("正文第一段")
|
|
assert "正文第二段" in cleaned
|
|
# 连续 4 个 \n 应被压缩为 \n\n
|
|
assert "\n\n\n" not in cleaned
|
|
|
|
|
|
def test_clean_content_keeps_body_when_no_header_match() -> None:
|
|
raw = "段落一\n段落二\n段落三"
|
|
cleaned = _clean_content(raw, title="完全不同的标题", author=None, time_raw=None)
|
|
assert cleaned == "段落一\n段落二\n段落三"
|
|
|
|
|
|
def test_clean_content_handles_empty() -> None:
|
|
assert _clean_content("", "", None, None) == ""
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 时间标准化
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def test_normalize_time_iso() -> None:
|
|
dt = _normalize_time("2026-06-15 14:30:00")
|
|
assert dt is not None and dt == datetime(2026, 6, 15, 14, 30, 0)
|
|
|
|
|
|
def test_normalize_time_chinese_format() -> None:
|
|
dt = _normalize_time("2026年6月15日 14时30分")
|
|
assert dt is not None
|
|
assert dt.year == 2026 and dt.month == 6 and dt.day == 15
|
|
assert dt.hour == 14 and dt.minute == 30
|
|
|
|
|
|
def test_normalize_time_chinese_seconds() -> None:
|
|
dt = _normalize_time("2026年1月1日 9时5分3秒")
|
|
assert dt is not None and dt.second == 3
|
|
|
|
|
|
def test_normalize_time_invalid_returns_none() -> None:
|
|
assert _normalize_time("不是时间") is None
|
|
assert _normalize_time("") is None
|
|
assert _normalize_time(None) is None
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 时间合理性 + HTML 兜底
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def test_is_reasonable_time_within_range() -> None:
|
|
ref = datetime(2026, 6, 16, 12, 0, 0)
|
|
assert _is_reasonable_time(datetime(2026, 6, 15, 8, 0), ref)
|
|
assert _is_reasonable_time(datetime(2025, 12, 1, 0, 0), ref)
|
|
# 同一天即将到来的时间
|
|
assert _is_reasonable_time(datetime(2026, 6, 16, 23, 0), ref)
|
|
|
|
|
|
def test_is_reasonable_time_rejects_far_future() -> None:
|
|
ref = datetime(2026, 6, 16, 12, 0)
|
|
assert not _is_reasonable_time(datetime(2026, 6, 18, 0, 0), ref)
|
|
|
|
|
|
def test_is_reasonable_time_rejects_far_past() -> None:
|
|
ref = datetime(2026, 6, 16, 12, 0)
|
|
# 距 ref 超过 365 天 -> 不合理(模拟 GNE 抓到的 2019 年页脚时间)
|
|
assert not _is_reasonable_time(datetime(2019, 1, 16, 10, 40), ref)
|
|
|
|
|
|
def test_is_reasonable_time_strips_tzinfo() -> None:
|
|
"""带时区的时间也能与 naive ref 比较。"""
|
|
from datetime import UTC
|
|
|
|
aware = datetime(2026, 6, 16, 12, 0, tzinfo=UTC)
|
|
assert _is_reasonable_time(aware, datetime(2026, 6, 16, 12, 0))
|
|
|
|
|
|
def test_is_reasonable_time_handles_none() -> None:
|
|
assert _is_reasonable_time(None) is False
|
|
|
|
|
|
def test_fallback_time_from_html_finds_eastmoney_pattern() -> None:
|
|
"""模拟 eastmoney 真实结构,兜底应能命中 .infos 内的中文日期。"""
|
|
html = """
|
|
<div class="infos">
|
|
<div class="item">2026年06月16日 17:54</div>
|
|
<div class="item">来源:发改委网站</div>
|
|
</div>
|
|
"""
|
|
dt, raw = _fallback_time_from_html(html)
|
|
assert dt is not None
|
|
assert dt.year == 2026 and dt.month == 6 and dt.day == 16
|
|
assert dt.hour == 17 and dt.minute == 54
|
|
assert raw is not None and "2026" in raw
|
|
|
|
|
|
def test_fallback_time_skips_unreasonable_dates() -> None:
|
|
"""页脚备案/版权时间(2019-01-16)出现在前,真实发布时间在后,应跳过前者。"""
|
|
html = """
|
|
<html><body>
|
|
<header>
|
|
<div class="biaobei">备案号 京ICP-XXX 备案日期: 2019-01-16</div>
|
|
</header>
|
|
<div class="infos">
|
|
<div class="item">2026年06月16日 17:54</div>
|
|
</div>
|
|
</body></html>
|
|
"""
|
|
dt, raw = _fallback_time_from_html(html)
|
|
assert dt is not None
|
|
assert dt.year == 2026
|
|
assert "2026" in (raw or "")
|
|
|
|
|
|
def test_fallback_time_returns_none_when_no_match() -> None:
|
|
dt, raw = _fallback_time_from_html("<html><body>没有日期</body></html>")
|
|
assert dt is None and raw is None
|
|
|
|
|
|
def test_resolve_publish_time_uses_gne_when_reasonable() -> None:
|
|
today = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
dt, raw = _resolve_publish_time("<html></html>", today)
|
|
assert dt is not None
|
|
assert raw == today
|
|
|
|
|
|
def test_resolve_publish_time_falls_back_when_gne_unreasonable() -> None:
|
|
"""模拟 eastmoney 场景:GNE 给出 2019-01-16,HTML 中含真实时间。"""
|
|
html = '<div class="infos"><div class="item">2026年06月16日 17:54</div></div>'
|
|
dt, raw = _resolve_publish_time(html, "2019-01-16 10:40:21")
|
|
assert dt is not None and dt.year == 2026 and dt.month == 6 and dt.day == 16
|
|
# raw 应反映兜底来源,而非原始 GNE 字符串
|
|
assert "2026" in (raw or "")
|
|
|
|
|
|
def test_resolve_publish_time_keeps_gne_raw_when_all_fail() -> None:
|
|
"""GNE 不合理且 HTML 也无可用时间 -> publish_time None,raw 保留 GNE。"""
|
|
dt, raw = _resolve_publish_time("<html><body>无</body></html>", "2010-01-01 00:00:00")
|
|
assert dt is None
|
|
assert raw == "2010-01-01 00:00:00"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 端到端:真实 eastmoney 结构应解析出 2026 时间
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def test_extract_article_uses_html_fallback_for_eastmoney_like() -> None:
|
|
"""模拟真实 eastmoney 文章页:GNE 拿到页脚错误时间,extractor 应兜底。"""
|
|
html = """
|
|
<html><head><title>事关六张网建设</title></head><body>
|
|
<footer>备案信息发布时间 2019-01-16 10:40:21</footer>
|
|
<div id="topbox" class="topbox">
|
|
<div class="title">事关“六张网”建设 国家发展改革委召开重要座谈会</div>
|
|
<div class="tipbox">
|
|
<div class="infos">
|
|
<div class="item">2026年06月16日 17:54</div>
|
|
<div class="item">来源:发改委网站</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<article>
|
|
<h1>事关“六张网”建设 国家发展改革委召开重要座谈会</h1>
|
|
<p>近日,国家发展改革委召开座谈会,围绕加快推进“六张网”建设的具体举措进行专题研讨。</p>
|
|
<p>会议指出,“六张网”建设关系国家长远发展,要从体制机制、关键技术、重点项目三方面协同推进。</p>
|
|
<p>与会专家就资金保障、跨部门协调、技术标准统一等议题展开了深入交流。</p>
|
|
</article>
|
|
</body></html>
|
|
"""
|
|
article = extract_article(
|
|
html,
|
|
source_id="eastmoney",
|
|
url="https://finance.eastmoney.com/a/123456789.html",
|
|
)
|
|
assert article.publish_time is not None
|
|
assert article.publish_time.year == 2026
|
|
assert article.publish_time.month == 6
|
|
assert article.publish_time.day == 16
|
|
assert "2026" in (article.publish_time_raw or "")
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 工具
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def test_url_hash_stable_and_short() -> None:
|
|
h1 = _url_hash("https://example.com/a")
|
|
h2 = _url_hash("https://example.com/a")
|
|
h3 = _url_hash("https://example.com/b")
|
|
assert h1 == h2 != h3
|
|
assert len(h1) == 16
|
|
|
|
|
|
def test_count_chinese() -> None:
|
|
assert _count_chinese("hello 你好 world") == 2
|
|
assert _count_chinese("ABC123") == 0
|
|
assert _count_chinese("中文测试") == 4
|
|
|
|
|
|
def test_article_short_summary() -> None:
|
|
a = Article(
|
|
source_id="cls",
|
|
url="https://x/1",
|
|
url_hash="abc123",
|
|
title="测试标题",
|
|
content="一段正文 " * 20,
|
|
publish_time=datetime(2026, 6, 15, 14, 30),
|
|
word_count=20,
|
|
)
|
|
s = a.short_summary()
|
|
assert "cls" in s
|
|
assert "2026-06-15" in s
|
|
assert "测试标题" in s
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Article 模型字段约束
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def test_article_requires_min_length_title_content() -> None:
|
|
with pytest.raises(Exception): # noqa: B017 - pydantic ValidationError
|
|
Article(
|
|
source_id="cls",
|
|
url="https://x/1",
|
|
url_hash="h",
|
|
title="",
|
|
content="",
|
|
)
|
|
|
|
|
|
def test_article_serializes_to_json(tmp_path: Path) -> None:
|
|
a = Article(
|
|
source_id="cls",
|
|
url="https://x/1",
|
|
url_hash="abc",
|
|
title="标题",
|
|
content="一二三四五六七八九十一二三四五六七八九十一二三四五六七八九十",
|
|
publish_time=datetime(2026, 6, 15, 14, 30),
|
|
word_count=30,
|
|
)
|
|
p = tmp_path / "a.json"
|
|
p.write_text(a.model_dump_json(), encoding="utf-8")
|
|
obj = json.loads(p.read_text(encoding="utf-8"))
|
|
assert obj["source_id"] == "cls"
|
|
assert obj["publish_time"].startswith("2026-06-15")
|