Initial commit

2026-07-18 15:51:01 +08:00
commit f2c80c5a9c
799 changed files with 133475 additions and 0 deletions
@@ -0,0 +1,382 @@
+"""M3 三层去重模块单元测试。"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+
+import pytest
+
+from dedup import (
+    DEFAULT_HAMMING_THRESHOLD,
+    Deduper,
+    DedupLayer,
+    Fingerprint,
+    FingerprintStore,
+    article_to_fingerprint,
+    content_hash,
+    hamming,
+    normalize_content,
+    simhash64,
+)
+from extractor import Article
+
+# --------------------------------------------------------------------------- #
+# fixtures
+# --------------------------------------------------------------------------- #
+
+def _article(
+    *,
+    url: str = "https://www.cls.cn/detail/1",
+    url_hash: str = "abc1234567890000",
+    source_id: str = "cls",
+    title: str = "宁德时代发布新一代麒麟电池",
+    content: str = (
+        "宁德时代今日正式发布了新一代麒麟电池产品，能量密度达到 255 Wh/kg，"
+        "显著优于上一代产品。该电池将于2026年第三季度量产。"
+    ),
+    publish_time: datetime | None = datetime(2026, 6, 16, 10, 0),
+) -> Article:
+    return Article(
+        source_id=source_id,
+        url=url,
+        url_hash=url_hash,
+        title=title,
+        content=content,
+        publish_time=publish_time,
+        word_count=len(content),
+    )
+
+
+@pytest.fixture
+def tmp_db(tmp_path: Path) -> Path:
+    return tmp_path / "fp.sqlite3"
+
+
+# --------------------------------------------------------------------------- #
+# hasher
+# --------------------------------------------------------------------------- #
+
+def test_normalize_content_strips_punct_and_whitespace() -> None:
+    norm = normalize_content("你好,  世界!\n这是 中文。")
+    assert norm == "你好世界这是中文"
+
+
+def test_normalize_content_handles_empty() -> None:
+    assert normalize_content("") == ""
+    assert normalize_content("   \n\t  ") == ""
+
+
+def test_content_hash_deterministic_and_punct_invariant() -> None:
+    a = "今天天气很好。"
+    b = "今天，天气，很好！！！"
+    assert content_hash(a) == content_hash(b)
+
+
+def test_content_hash_differs_for_different_text() -> None:
+    assert content_hash("今天天气很好") != content_hash("今天天气不好")
+
+
+def test_simhash_identical_text_same_value() -> None:
+    text = "宁德时代发布新一代麒麟电池产品 能量密度大幅提升"
+    assert simhash64(text) == simhash64(text)
+
+
+def test_simhash_minor_changes_close_distance() -> None:
+    """长文本(贴近真实新闻)的轻度改写,hamming 距离应在阈值内。"""
+    base = (
+        "宁德时代今日正式发布新一代麒麟电池产品，能量密度达到 255 瓦时每公斤，"
+        "显著优于上一代产品。该电池将于 2026 年第三季度量产，首批应用于多款新能源汽车。"
+        "公司股价应声上涨 5.2%，分析师认为这将进一步巩固宁德时代在全球动力电池领域的领先地位。"
+    ) * 2
+    rewritten = "财联社讯：" + base + "(完)"
+    d = hamming(simhash64(base), simhash64(rewritten))
+    assert d <= DEFAULT_HAMMING_THRESHOLD, f"长文本前后加标识汉明距离 {d} 不应超过阈值"
+
+
+def test_simhash_unrelated_text_far_distance() -> None:
+    """完全不相关的两段长文本汉明距离应远大于阈值。"""
+    a = "宁德时代发布新一代麒麟电池产品，能量密度达到255瓦时每公斤。" * 3
+    b = "美联储宣布维持利率不变，市场普遍预期下次会议将开启降息周期。" * 3
+    d = hamming(simhash64(a), simhash64(b))
+    assert d > DEFAULT_HAMMING_THRESHOLD * 2
+
+
+def test_simhash_empty_returns_zero() -> None:
+    assert simhash64("") == 0
+    assert simhash64("   ") == 0
+
+
+def test_hamming_basics() -> None:
+    assert hamming(0, 0) == 0
+    assert hamming(0xFF, 0x00) == 8
+    assert hamming(0xFF00FF00, 0x00FF00FF) == 32
+
+
+# --------------------------------------------------------------------------- #
+# FingerprintStore
+# --------------------------------------------------------------------------- #
+
+def test_store_upsert_and_get(tmp_db: Path) -> None:
+    fp = Fingerprint(
+        url_hash="hash1",
+        content_hash="ch1",
+        simhash=0xDEADBEEFCAFEBABE,
+        source_id="cls",
+        url="https://x/1",
+        title="A",
+        publish_date="2026-06-16",
+    )
+    with FingerprintStore(tmp_db) as store:
+        store.upsert(fp)
+        got = store.get_by_url_hash("hash1")
+        assert got is not None
+        assert got.content_hash == "ch1"
+        assert got.simhash == 0xDEADBEEFCAFEBABE
+        assert got.publish_date == "2026-06-16"
+
+
+def test_store_upsert_replaces_existing(tmp_db: Path) -> None:
+    base = Fingerprint(
+        url_hash="h",
+        content_hash="ch1",
+        simhash=1,
+        source_id="cls",
+        url="u",
+        title="t",
+    )
+    updated = base.model_copy(update={"content_hash": "ch2", "simhash": 999})
+    with FingerprintStore(tmp_db) as store:
+        store.upsert(base)
+        store.upsert(updated)
+        got = store.get_by_url_hash("h")
+        assert got is not None
+        assert got.content_hash == "ch2"
+        assert got.simhash == 999
+        assert store.count() == 1
+
+
+def test_store_find_by_content_hash(tmp_db: Path) -> None:
+    with FingerprintStore(tmp_db) as store:
+        store.upsert(Fingerprint(
+            url_hash="h1", content_hash="ch", simhash=0,
+            source_id="cls", url="u1", title="t1"
+        ))
+        assert store.find_by_content_hash("ch") is not None
+        assert store.find_by_content_hash("nope") is None
+
+
+def test_store_candidates_within_window(tmp_db: Path) -> None:
+    with FingerprintStore(tmp_db) as store:
+        for d, h in [("2026-05-01", "old"), ("2026-06-15", "near"), ("2026-07-30", "far")]:
+            store.upsert(Fingerprint(
+                url_hash=h, content_hash=h, simhash=0,
+                source_id="cls", url=f"u/{h}", title=h, publish_date=d,
+            ))
+        cands = store.candidates_for_simhash("2026-06-16", window_days=7)
+        url_hashes = sorted(c.url_hash for c in cands)
+        assert url_hashes == ["near"]
+
+
+def test_store_candidates_no_date_returns_all(tmp_db: Path) -> None:
+    with FingerprintStore(tmp_db) as store:
+        store.upsert(Fingerprint(
+            url_hash="h1", content_hash="c1", simhash=0,
+            source_id="cls", url="u", title="t", publish_date=None
+        ))
+        cands = store.candidates_for_simhash(None, 30)
+        assert len(cands) == 1
+
+
+def test_store_simhash_handles_high_bit(tmp_db: Path) -> None:
+    """64 位 SimHash 高位为 1 时,hex 存取应保持无符号。"""
+    high = (1 << 63) | 0x1234
+    with FingerprintStore(tmp_db) as store:
+        store.upsert(Fingerprint(
+            url_hash="h", content_hash="c", simhash=high,
+            source_id="cls", url="u", title="t",
+        ))
+        got = store.get_by_url_hash("h")
+        assert got is not None
+        assert got.simhash == high
+
+
+def test_store_count_by_source(tmp_db: Path) -> None:
+    with FingerprintStore(tmp_db) as store:
+        for i, src in enumerate(["cls", "cls", "sina"]):
+            store.upsert(Fingerprint(
+                url_hash=f"h{i}", content_hash=f"c{i}", simhash=i,
+                source_id=src, url=f"u{i}", title=f"t{i}",
+            ))
+        counts = store.count_by_source()
+        assert counts == {"cls": 2, "sina": 1}
+
+
+# --------------------------------------------------------------------------- #
+# Deduper - 三层判重
+# --------------------------------------------------------------------------- #
+
+def test_dedup_first_article_is_unique(tmp_db: Path) -> None:
+    art = _article()
+    with Deduper(db_path=tmp_db) as d:
+        result = d.ingest(art)
+        assert not result.is_duplicate
+        assert result.matched_layer is None
+        assert d.stats().total == 1
+
+
+def test_dedup_layer1_url_hash(tmp_db: Path) -> None:
+    """同一 url_hash 直接命中 L1。"""
+    a1 = _article()
+    a2 = _article()  # 同 url_hash 同 url
+    with Deduper(db_path=tmp_db) as d:
+        d.ingest(a1)
+        result = d.ingest(a2)
+        assert result.is_duplicate
+        assert result.matched_layer == DedupLayer.URL
+        assert d.stats().total == 1, "L1 命中应不写入新指纹"
+
+
+def test_dedup_layer2_content_hash(tmp_db: Path) -> None:
+    """url 不同但 content 完全一致 -> L2。"""
+    a1 = _article(url="https://a.com/1", url_hash="hash1aaaaaaaaaaa")
+    a2 = _article(url="https://b.com/2", url_hash="hash2bbbbbbbbbbb")
+    with Deduper(db_path=tmp_db) as d:
+        d.ingest(a1)
+        result = d.ingest(a2)
+        assert result.is_duplicate
+        assert result.matched_layer == DedupLayer.CONTENT
+        assert result.matched_url_hash == "hash1aaaaaaaaaaa"
+
+
+def test_dedup_layer2_punctuation_difference_still_caught(tmp_db: Path) -> None:
+    """标点/空白差异不应阻止 L2 命中(normalize_content 应剥离)。"""
+    base = "今天天气很好我们去公园散步"
+    a1 = _article(
+        url="https://a/1", url_hash="aaaa", content="今天天气很好。我们去公园散步！"
+    )
+    a2 = _article(
+        url="https://b/2", url_hash="bbbb", content="今天天气，很好；我们去公园 散步！！"
+    )
+    assert content_hash(a1.content) == content_hash(a2.content)
+    assert normalize_content(a1.content) == base
+    with Deduper(db_path=tmp_db) as d:
+        d.ingest(a1)
+        result = d.ingest(a2)
+        assert result.matched_layer == DedupLayer.CONTENT
+
+
+def test_dedup_layer3_simhash_minor_rewrite(tmp_db: Path) -> None:
+    """长文本 + 转载前后缀,落入 SimHash 层(贴近真实跨源转载场景)。"""
+    long_body = (
+        "宁德时代今日正式发布新一代麒麟电池产品，能量密度达到 255 瓦时每公斤，"
+        "显著优于上一代产品。该电池将于 2026 年第三季度量产，首批应用于多款新能源汽车。"
+        "公司股价应声上涨 5.2%，分析师认为这将进一步巩固宁德时代在全球动力电池领域的领先地位。"
+    ) * 2
+    rewritten = "财联社讯：" + long_body + "(完)"
+    a1 = _article(url="https://a/1", url_hash="aaaaa", content=long_body)
+    a2 = _article(url="https://b/2", url_hash="bbbbb", content=rewritten)
+    # 必要前提:content_hash 不同(否则会被 L2 截胡)
+    assert content_hash(a1.content) != content_hash(a2.content)
+
+    with Deduper(db_path=tmp_db) as d:
+        d.ingest(a1)
+        result = d.ingest(a2)
+        assert result.is_duplicate
+        assert result.matched_layer == DedupLayer.SIMHASH
+        assert result.hamming_distance is not None
+        assert result.hamming_distance <= DEFAULT_HAMMING_THRESHOLD
+
+
+def test_dedup_layer3_unrelated_articles_kept(tmp_db: Path) -> None:
+    a1 = _article(url="https://a/1", url_hash="aaaaa",
+                  content="宁德时代发布新一代麒麟电池产品，能量密度达到 255 瓦时每公斤。" * 5)
+    a2 = _article(url="https://b/2", url_hash="bbbbb",
+                  content="美联储宣布维持联邦基金利率不变，市场预期下次会议将开启降息。" * 5,
+                  title="美联储利率决议")
+    with Deduper(db_path=tmp_db) as d:
+        d.ingest(a1)
+        result = d.ingest(a2)
+        assert not result.is_duplicate
+        assert d.stats().total == 2
+
+
+def test_dedup_layer3_outside_time_window_kept(tmp_db: Path) -> None:
+    """SimHash 相近,但 publish_date 距离过远(> 30 天)不去重。"""
+    body = (
+        "宁德时代今日正式发布新一代麒麟电池产品，能量密度达到 255 瓦时每公斤，"
+        "显著优于上一代产品。该电池将于第三季度量产，首批应用于多款新能源汽车。" * 2
+    )
+    a1 = _article(
+        url="https://a/1", url_hash="aaaa1", content=body,
+        publish_time=datetime(2026, 1, 1, 9, 0),
+    )
+    a2 = _article(
+        url="https://b/2", url_hash="bbbb2", content=body[3:],  # 微改 -> 走 L3
+        publish_time=datetime(2026, 6, 16, 9, 0),
+    )
+    assert content_hash(a1.content) != content_hash(a2.content)
+    with Deduper(db_path=tmp_db, time_window_days=30) as d:
+        d.ingest(a1)
+        result = d.ingest(a2)
+        assert not result.is_duplicate, "时间窗口外不应命中 SimHash"
+
+
+def test_dedup_threshold_zero_only_exact_simhash(tmp_db: Path) -> None:
+    """阈值 0 -> 仅当 SimHash 完全相同才视为重复(且会先被 L2 拦截)。"""
+    a1 = _article(url="https://a/1", url_hash="aaaa1",
+                  content="宁德时代发布新一代麒麟电池产品 能量密度大幅提升")
+    a2 = _article(url="https://b/2", url_hash="bbbb2",
+                  content="财联社讯 宁德时代今天发布了新一代麒麟电池 能量密度提升明显")
+    with Deduper(db_path=tmp_db, simhash_threshold=0) as d:
+        d.ingest(a1)
+        result = d.ingest(a2)
+        # 两段相似但不同的文本,阈值 0 时不应判重
+        assert not result.is_duplicate
+
+
+# --------------------------------------------------------------------------- #
+# Deduper - check 不写入
+# --------------------------------------------------------------------------- #
+
+def test_check_does_not_write(tmp_db: Path) -> None:
+    art = _article()
+    with Deduper(db_path=tmp_db) as d:
+        result = d.check(art)
+        assert not result.is_duplicate
+        assert d.stats().total == 0  # check 不应入库
+
+
+def test_article_to_fingerprint_fields() -> None:
+    art = _article()
+    fp = article_to_fingerprint(art)
+    assert fp.url_hash == art.url_hash
+    assert fp.simhash == simhash64(art.content)
+    assert fp.content_hash == content_hash(art.content)
+    assert fp.publish_date == "2026-06-16"
+
+
+def test_article_to_fingerprint_handles_none_publish_time() -> None:
+    art = _article(publish_time=None)
+    fp = article_to_fingerprint(art)
+    assert fp.publish_date is None
+
+
+# --------------------------------------------------------------------------- #
+# stats
+# --------------------------------------------------------------------------- #
+
+def test_stats_aggregates_by_source(tmp_db: Path) -> None:
+    with Deduper(db_path=tmp_db) as d:
+        d.ingest(_article(source_id="cls", url="https://cls/1",
+                          url_hash="cls0000000000001"))
+        d.ingest(_article(source_id="cls", url="https://cls/2",
+                          url_hash="cls0000000000002",
+                          content="完全不同的另一篇文章" * 30))
+        d.ingest(_article(source_id="sina", url="https://sina/1",
+                          url_hash="sina000000000001",
+                          content="第三篇 完全不同 主题 美联储 利率" * 20))
+        stats = d.stats()
+        assert stats.total == 3
+        assert stats.by_source == {"cls": 2, "sina": 1}
+        assert stats.earliest is not None