Initial commit

2026-07-18 15:51:01 +08:00
commit f2c80c5a9c
799 changed files with 133475 additions and 0 deletions
@@ -0,0 +1,44 @@
+"""三层新闻去重模块 (M3)。
+
+公共 API:
+    - Deduper:         主类(check / ingest / stats)
+    - FingerprintStore: SQLite 指纹库(底层,通常无需直接用)
+    - DedupResult / DedupLayer / DedupStats / Fingerprint: 数据模型
+    - simhash64 / hamming / content_hash / normalize_content: 指纹算法
+"""
+
+from .deduper import (
+    DEFAULT_TIME_WINDOW_DAYS,
+    Deduper,
+    article_to_fingerprint,
+)
+from .hasher import (
+    DEFAULT_HAMMING_THRESHOLD,
+    NGRAM_SIZE,
+    SIMHASH_BITS,
+    content_hash,
+    hamming,
+    normalize_content,
+    simhash64,
+)
+from .models import DedupLayer, DedupResult, DedupStats, Fingerprint
+from .store import DEFAULT_DB_PATH, FingerprintStore
+
+__all__ = [
+    "DEFAULT_DB_PATH",
+    "DEFAULT_HAMMING_THRESHOLD",
+    "DEFAULT_TIME_WINDOW_DAYS",
+    "NGRAM_SIZE",
+    "SIMHASH_BITS",
+    "DedupLayer",
+    "DedupResult",
+    "DedupStats",
+    "Deduper",
+    "Fingerprint",
+    "FingerprintStore",
+    "article_to_fingerprint",
+    "content_hash",
+    "hamming",
+    "normalize_content",
+    "simhash64",
+]