45 lines
1.0 KiB
Python
45 lines
1.0 KiB
Python
"""三层新闻去重模块 (M3)。
|
|
|
|
公共 API:
|
|
- Deduper: 主类(check / ingest / stats)
|
|
- FingerprintStore: SQLite 指纹库(底层,通常无需直接用)
|
|
- DedupResult / DedupLayer / DedupStats / Fingerprint: 数据模型
|
|
- simhash64 / hamming / content_hash / normalize_content: 指纹算法
|
|
"""
|
|
|
|
from .deduper import (
|
|
DEFAULT_TIME_WINDOW_DAYS,
|
|
Deduper,
|
|
article_to_fingerprint,
|
|
)
|
|
from .hasher import (
|
|
DEFAULT_HAMMING_THRESHOLD,
|
|
NGRAM_SIZE,
|
|
SIMHASH_BITS,
|
|
content_hash,
|
|
hamming,
|
|
normalize_content,
|
|
simhash64,
|
|
)
|
|
from .models import DedupLayer, DedupResult, DedupStats, Fingerprint
|
|
from .store import DEFAULT_DB_PATH, FingerprintStore
|
|
|
|
__all__ = [
|
|
"DEFAULT_DB_PATH",
|
|
"DEFAULT_HAMMING_THRESHOLD",
|
|
"DEFAULT_TIME_WINDOW_DAYS",
|
|
"NGRAM_SIZE",
|
|
"SIMHASH_BITS",
|
|
"DedupLayer",
|
|
"DedupResult",
|
|
"DedupStats",
|
|
"Deduper",
|
|
"Fingerprint",
|
|
"FingerprintStore",
|
|
"article_to_fingerprint",
|
|
"content_hash",
|
|
"hamming",
|
|
"normalize_content",
|
|
"simhash64",
|
|
]
|