初始化

2026-07-18 16:13:52 +08:00
parent c0070f0a5c
commit fe8b417ab6
75 changed files with 12898 additions and 1 deletions
@@ -0,0 +1,143 @@
+"""M2 正文提取模块测试"""
+
+from pathlib import Path
+
+from extractor.extractor import (
+    MIN_CONTENT_WORDS,
+    _clean_markdown,
+    _count_words,
+    _extract_author,
+    _extract_title,
+    extract_article,
+)
+
+# ════════════════════════════════════════════════
+# 辅助函数
+# ════════════════════════════════════════════════
+
+def test_count_words():
+    assert _count_words("hello world") == 2
+    assert _count_words("") == 0
+    assert _count_words(None) == 0
+
+
+def test_extract_title():
+    html = "<html><head><title>Breaking News: Markets Rally</title></head></html>"
+    assert "Breaking News" in _extract_title(html)
+
+    assert _extract_title("") == ""
+
+
+def test_extract_author():
+    html = '<meta name="author" content="John Doe">'
+    assert _extract_author(html) == "John Doe"
+
+    assert _extract_author("") == ""
+
+
+def test_clean_markdown():
+    md = """ADVERTISEMENT - Continue Reading Below
+[Sign In](https://example.com/signin)
+# Real Article Title
+This is the actual content of the article.
+It has multiple paragraphs."""
+
+    cleaned = _clean_markdown(md)
+    assert "ADVERTISEMENT" not in cleaned
+    assert "Sign In" not in cleaned
+    assert "Real Article Title" in cleaned
+    assert "actual content" in cleaned
+
+
+def test_clean_markdown_preserves_content():
+    md = "# Market Update\n\nStocks rose today.\n\n[Read More](https://example.com)"
+    cleaned = _clean_markdown(md)
+    assert "Market Update" in cleaned
+    assert "Stocks rose" in cleaned
+
+
+# ════════════════════════════════════════════════
+# 提取引擎
+# ════════════════════════════════════════════════
+
+def test_extract_article_trafilatura(tmp_path: Path):
+    """用 trafilatura 从 HTML 提取正文"""
+    html = tmp_path / "test.html"
+    text = (
+        "<!DOCTYPE html><html><head><title>Fed Raises Rates</title>"
+        '<meta name="author" content="Jane Smith">'
+        "</head><body><nav>Menu items</nav><article>"
+        "<p>The Federal Reserve raised interest rates by 25 basis points "
+        "today in a widely expected move. Chair Powell noted that inflation "
+        "remains above target but is trending downward.</p>"
+        "<p>Markets reacted positively, with the S&P 500 gaining 1.2%.</p>"
+        "</article><footer>Copyright 2026</footer></body></html>"
+    )
+    html.write_text(text)
+
+    result = extract_article(
+        source_id="test",
+        source_name="Test",
+        url="https://example.com/article",
+        url_hash="abc123",
+        html_path=str(html),
+        md_path="",
+    )
+
+    assert result.status == "success"
+    assert result.extractor == "trafilatura"
+    assert "Federal Reserve" in result.content
+    assert "Jane Smith" in result.author
+    assert result.word_count >= 30
+
+
+def test_extract_article_no_content(tmp_path: Path):
+    """无有效正文时降级"""
+    html = tmp_path / "empty.html"
+    html.write_text("<html><head></head><body>Short.</body></html>")
+
+    result = extract_article(
+        source_id="test", source_name="Test",
+        url="https://example.com/nocontent",
+        url_hash="def456",
+        html_path=str(html), md_path="",
+    )
+
+    assert result.status == "no_content"
+
+
+def test_extract_article_md_fallback(tmp_path: Path):
+    """HTML 不可用但 MD 可用时回退"""
+    md = tmp_path / "test.md"
+    md_text = (
+        "# Market Analysis\n\n"
+        "This is a detailed analysis of market conditions today. "
+        "The dow jones industrial average showed significant movement "
+        "as investors reacted to economic data. "
+        "Trading volume was above average across major exchanges. "
+        "Analysts noted that technical indicators suggested continued "
+        "upward momentum in the near term. "
+        "Several key sectors led the rally including technology "
+        "financials and healthcare stocks. "
+        "The bond market also saw increased activity as yields moved lower."
+    )
+    md.write_text(md_text)
+
+    result = extract_article(
+        source_id="test", source_name="Test",
+        url="https://example.com/mdonly",
+        url_hash="ghi789",
+        html_path="/nonexistent/file.html",
+        md_path=str(md),
+    )
+
+    assert result.status == "success"
+    assert result.extractor == "crawl4ai_md"
+    assert result.word_count >= 30
+    assert "Market Analysis" in result.content
+
+
+def test_min_content_threshold():
+    """MIN_CONTENT_WORDS 阈值合理"""
+    assert MIN_CONTENT_WORDS >= 30
+    assert MIN_CONTENT_WORDS <= 100