144 lines
4.8 KiB
Python
144 lines
4.8 KiB
Python
"""M2 正文提取模块测试"""
|
|
|
|
from pathlib import Path
|
|
|
|
from extractor.extractor import (
|
|
MIN_CONTENT_WORDS,
|
|
_clean_markdown,
|
|
_count_words,
|
|
_extract_author,
|
|
_extract_title,
|
|
extract_article,
|
|
)
|
|
|
|
# ════════════════════════════════════════════════
|
|
# 辅助函数
|
|
# ════════════════════════════════════════════════
|
|
|
|
def test_count_words():
|
|
assert _count_words("hello world") == 2
|
|
assert _count_words("") == 0
|
|
assert _count_words(None) == 0
|
|
|
|
|
|
def test_extract_title():
|
|
html = "<html><head><title>Breaking News: Markets Rally</title></head></html>"
|
|
assert "Breaking News" in _extract_title(html)
|
|
|
|
assert _extract_title("") == ""
|
|
|
|
|
|
def test_extract_author():
|
|
html = '<meta name="author" content="John Doe">'
|
|
assert _extract_author(html) == "John Doe"
|
|
|
|
assert _extract_author("") == ""
|
|
|
|
|
|
def test_clean_markdown():
|
|
md = """ADVERTISEMENT - Continue Reading Below
|
|
[Sign In](https://example.com/signin)
|
|
# Real Article Title
|
|
This is the actual content of the article.
|
|
It has multiple paragraphs."""
|
|
|
|
cleaned = _clean_markdown(md)
|
|
assert "ADVERTISEMENT" not in cleaned
|
|
assert "Sign In" not in cleaned
|
|
assert "Real Article Title" in cleaned
|
|
assert "actual content" in cleaned
|
|
|
|
|
|
def test_clean_markdown_preserves_content():
|
|
md = "# Market Update\n\nStocks rose today.\n\n[Read More](https://example.com)"
|
|
cleaned = _clean_markdown(md)
|
|
assert "Market Update" in cleaned
|
|
assert "Stocks rose" in cleaned
|
|
|
|
|
|
# ════════════════════════════════════════════════
|
|
# 提取引擎
|
|
# ════════════════════════════════════════════════
|
|
|
|
def test_extract_article_trafilatura(tmp_path: Path):
|
|
"""用 trafilatura 从 HTML 提取正文"""
|
|
html = tmp_path / "test.html"
|
|
text = (
|
|
"<!DOCTYPE html><html><head><title>Fed Raises Rates</title>"
|
|
'<meta name="author" content="Jane Smith">'
|
|
"</head><body><nav>Menu items</nav><article>"
|
|
"<p>The Federal Reserve raised interest rates by 25 basis points "
|
|
"today in a widely expected move. Chair Powell noted that inflation "
|
|
"remains above target but is trending downward.</p>"
|
|
"<p>Markets reacted positively, with the S&P 500 gaining 1.2%.</p>"
|
|
"</article><footer>Copyright 2026</footer></body></html>"
|
|
)
|
|
html.write_text(text)
|
|
|
|
result = extract_article(
|
|
source_id="test",
|
|
source_name="Test",
|
|
url="https://example.com/article",
|
|
url_hash="abc123",
|
|
html_path=str(html),
|
|
md_path="",
|
|
)
|
|
|
|
assert result.status == "success"
|
|
assert result.extractor == "trafilatura"
|
|
assert "Federal Reserve" in result.content
|
|
assert "Jane Smith" in result.author
|
|
assert result.word_count >= 30
|
|
|
|
|
|
def test_extract_article_no_content(tmp_path: Path):
|
|
"""无有效正文时降级"""
|
|
html = tmp_path / "empty.html"
|
|
html.write_text("<html><head></head><body>Short.</body></html>")
|
|
|
|
result = extract_article(
|
|
source_id="test", source_name="Test",
|
|
url="https://example.com/nocontent",
|
|
url_hash="def456",
|
|
html_path=str(html), md_path="",
|
|
)
|
|
|
|
assert result.status == "no_content"
|
|
|
|
|
|
def test_extract_article_md_fallback(tmp_path: Path):
|
|
"""HTML 不可用但 MD 可用时回退"""
|
|
md = tmp_path / "test.md"
|
|
md_text = (
|
|
"# Market Analysis\n\n"
|
|
"This is a detailed analysis of market conditions today. "
|
|
"The dow jones industrial average showed significant movement "
|
|
"as investors reacted to economic data. "
|
|
"Trading volume was above average across major exchanges. "
|
|
"Analysts noted that technical indicators suggested continued "
|
|
"upward momentum in the near term. "
|
|
"Several key sectors led the rally including technology "
|
|
"financials and healthcare stocks. "
|
|
"The bond market also saw increased activity as yields moved lower."
|
|
)
|
|
md.write_text(md_text)
|
|
|
|
result = extract_article(
|
|
source_id="test", source_name="Test",
|
|
url="https://example.com/mdonly",
|
|
url_hash="ghi789",
|
|
html_path="/nonexistent/file.html",
|
|
md_path=str(md),
|
|
)
|
|
|
|
assert result.status == "success"
|
|
assert result.extractor == "crawl4ai_md"
|
|
assert result.word_count >= 30
|
|
assert "Market Analysis" in result.content
|
|
|
|
|
|
def test_min_content_threshold():
|
|
"""MIN_CONTENT_WORDS 阈值合理"""
|
|
assert MIN_CONTENT_WORDS >= 30
|
|
assert MIN_CONTENT_WORDS <= 100
|