初始化

2026-07-18 16:13:52 +08:00
parent c0070f0a5c
commit fe8b417ab6
75 changed files with 12898 additions and 1 deletions
@@ -0,0 +1,53 @@
+"""三层新闻去重模块 (M3)。
+
+公共 API:
+    - Deduper:         主类（check / ingest / stats）
+    - FingerprintStore: SQLite 指纹库（底层，通常无需直接用）
+    - DedupResult / DedupLayer / DedupStats / Fingerprint: 数据模型
+    - simhash64 / hamming / content_hash / normalize_content: 指纹算法
+    - dedup_all_sources / dedup_source: 批量去重管道
+"""
+
+from dedup.deduper import (
+    DEFAULT_TIME_WINDOW_DAYS,
+    Deduper,
+    article_to_fingerprint,
+)
+from dedup.hasher import (
+    DEFAULT_HAMMING_THRESHOLD,
+    NGRAM_SIZE,
+    SIMHASH_BITS,
+    content_hash,
+    hamming,
+    normalize_content,
+    simhash64,
+)
+from dedup.models import DedupLayer, DedupResult, DedupStats, Fingerprint
+from dedup.pipeline import dedup_all_sources, dedup_source
+from dedup.store import DEFAULT_DB_PATH, FingerprintStore
+
+__all__ = [
+    # 主类
+    "Deduper",
+    "FingerprintStore",
+    # 管道
+    "dedup_all_sources",
+    "dedup_source",
+    # 模型
+    "DedupLayer",
+    "DedupResult",
+    "DedupStats",
+    "Fingerprint",
+    # 指纹算法
+    "article_to_fingerprint",
+    "content_hash",
+    "hamming",
+    "normalize_content",
+    "simhash64",
+    # 常量
+    "DEFAULT_DB_PATH",
+    "DEFAULT_HAMMING_THRESHOLD",
+    "DEFAULT_TIME_WINDOW_DAYS",
+    "NGRAM_SIZE",
+    "SIMHASH_BITS",
+]