"""M2 正文提取模块测试""" from pathlib import Path from extractor.extractor import ( MIN_CONTENT_WORDS, _clean_markdown, _count_words, _extract_author, _extract_title, extract_article, ) # ════════════════════════════════════════════════ # 辅助函数 # ════════════════════════════════════════════════ def test_count_words(): assert _count_words("hello world") == 2 assert _count_words("") == 0 assert _count_words(None) == 0 def test_extract_title(): html = "Breaking News: Markets Rally" assert "Breaking News" in _extract_title(html) assert _extract_title("") == "" def test_extract_author(): html = '' assert _extract_author(html) == "John Doe" assert _extract_author("") == "" def test_clean_markdown(): md = """ADVERTISEMENT - Continue Reading Below [Sign In](https://example.com/signin) # Real Article Title This is the actual content of the article. It has multiple paragraphs.""" cleaned = _clean_markdown(md) assert "ADVERTISEMENT" not in cleaned assert "Sign In" not in cleaned assert "Real Article Title" in cleaned assert "actual content" in cleaned def test_clean_markdown_preserves_content(): md = "# Market Update\n\nStocks rose today.\n\n[Read More](https://example.com)" cleaned = _clean_markdown(md) assert "Market Update" in cleaned assert "Stocks rose" in cleaned # ════════════════════════════════════════════════ # 提取引擎 # ════════════════════════════════════════════════ def test_extract_article_trafilatura(tmp_path: Path): """用 trafilatura 从 HTML 提取正文""" html = tmp_path / "test.html" text = ( "Fed Raises Rates" '' "
" "

The Federal Reserve raised interest rates by 25 basis points " "today in a widely expected move. Chair Powell noted that inflation " "remains above target but is trending downward.

" "

Markets reacted positively, with the S&P 500 gaining 1.2%.

" "
" ) html.write_text(text) result = extract_article( source_id="test", source_name="Test", url="https://example.com/article", url_hash="abc123", html_path=str(html), md_path="", ) assert result.status == "success" assert result.extractor == "trafilatura" assert "Federal Reserve" in result.content assert "Jane Smith" in result.author assert result.word_count >= 30 def test_extract_article_no_content(tmp_path: Path): """无有效正文时降级""" html = tmp_path / "empty.html" html.write_text("Short.") result = extract_article( source_id="test", source_name="Test", url="https://example.com/nocontent", url_hash="def456", html_path=str(html), md_path="", ) assert result.status == "no_content" def test_extract_article_md_fallback(tmp_path: Path): """HTML 不可用但 MD 可用时回退""" md = tmp_path / "test.md" md_text = ( "# Market Analysis\n\n" "This is a detailed analysis of market conditions today. " "The dow jones industrial average showed significant movement " "as investors reacted to economic data. " "Trading volume was above average across major exchanges. " "Analysts noted that technical indicators suggested continued " "upward momentum in the near term. " "Several key sectors led the rally including technology " "financials and healthcare stocks. " "The bond market also saw increased activity as yields moved lower." ) md.write_text(md_text) result = extract_article( source_id="test", source_name="Test", url="https://example.com/mdonly", url_hash="ghi789", html_path="/nonexistent/file.html", md_path=str(md), ) assert result.status == "success" assert result.extractor == "crawl4ai_md" assert result.word_count >= 30 assert "Market Analysis" in result.content def test_min_content_threshold(): """MIN_CONTENT_WORDS 阈值合理""" assert MIN_CONTENT_WORDS >= 30 assert MIN_CONTENT_WORDS <= 100