54 lines
1.3 KiB
Python
54 lines
1.3 KiB
Python
"""三层新闻去重模块 (M3)。
|
||
|
||
公共 API:
|
||
- Deduper: 主类（check / ingest / stats）
|
||
- FingerprintStore: SQLite 指纹库（底层，通常无需直接用）
|
||
- DedupResult / DedupLayer / DedupStats / Fingerprint: 数据模型
|
||
- simhash64 / hamming / content_hash / normalize_content: 指纹算法
|
||
- dedup_all_sources / dedup_source: 批量去重管道
|
||
"""
|
||
|
||
from dedup.deduper import (
|
||
DEFAULT_TIME_WINDOW_DAYS,
|
||
Deduper,
|
||
article_to_fingerprint,
|
||
)
|
||
from dedup.hasher import (
|
||
DEFAULT_HAMMING_THRESHOLD,
|
||
NGRAM_SIZE,
|
||
SIMHASH_BITS,
|
||
content_hash,
|
||
hamming,
|
||
normalize_content,
|
||
simhash64,
|
||
)
|
||
from dedup.models import DedupLayer, DedupResult, DedupStats, Fingerprint
|
||
from dedup.pipeline import dedup_all_sources, dedup_source
|
||
from dedup.store import DEFAULT_DB_PATH, FingerprintStore
|
||
|
||
__all__ = [
|
||
# 主类
|
||
"Deduper",
|
||
"FingerprintStore",
|
||
# 管道
|
||
"dedup_all_sources",
|
||
"dedup_source",
|
||
# 模型
|
||
"DedupLayer",
|
||
"DedupResult",
|
||
"DedupStats",
|
||
"Fingerprint",
|
||
# 指纹算法
|
||
"article_to_fingerprint",
|
||
"content_hash",
|
||
"hamming",
|
||
"normalize_content",
|
||
"simhash64",
|
||
# 常量
|
||
"DEFAULT_DB_PATH",
|
||
"DEFAULT_HAMMING_THRESHOLD",
|
||
"DEFAULT_TIME_WINDOW_DAYS",
|
||
"NGRAM_SIZE",
|
||
"SIMHASH_BITS",
|
||
]
|