simon
65ead54b4f
- 删除 5 个过时/残留文档(project_plan/agent_prompt/optimization_plan/report_db_design/deploy/README) - 新建 docs/architecture.md(项目架构:11 包职责+数据模型+配置+产物) - 重写 docs/user-guide.md(CLI 全量+增量/断点续跑+MCP+FAQ) - 重写 README.md(精简入口+文档索引) - 更新 continuation.md(追加本次记录) - 更新 .gitignore(排除 data/* 运行产物)
45 lines
1.0 KiB
Python
45 lines
1.0 KiB
Python
"""三层新闻去重模块 (M3)。
|
|
|
|
公共 API:
|
|
- Deduper: 主类(check / ingest / stats)
|
|
- FingerprintStore: SQLite 指纹库(底层,通常无需直接用)
|
|
- DedupResult / DedupLayer / DedupStats / Fingerprint: 数据模型
|
|
- simhash64 / hamming / content_hash / normalize_content: 指纹算法
|
|
"""
|
|
|
|
from .deduper import (
|
|
DEFAULT_TIME_WINDOW_DAYS,
|
|
Deduper,
|
|
article_to_fingerprint,
|
|
)
|
|
from .hasher import (
|
|
DEFAULT_HAMMING_THRESHOLD,
|
|
NGRAM_SIZE,
|
|
SIMHASH_BITS,
|
|
content_hash,
|
|
hamming,
|
|
normalize_content,
|
|
simhash64,
|
|
)
|
|
from .models import DedupLayer, DedupResult, DedupStats, Fingerprint
|
|
from .store import DEFAULT_DB_PATH, FingerprintStore
|
|
|
|
__all__ = [
|
|
"DEFAULT_DB_PATH",
|
|
"DEFAULT_HAMMING_THRESHOLD",
|
|
"DEFAULT_TIME_WINDOW_DAYS",
|
|
"NGRAM_SIZE",
|
|
"SIMHASH_BITS",
|
|
"DedupLayer",
|
|
"DedupResult",
|
|
"DedupStats",
|
|
"Deduper",
|
|
"Fingerprint",
|
|
"FingerprintStore",
|
|
"article_to_fingerprint",
|
|
"content_hash",
|
|
"hamming",
|
|
"normalize_content",
|
|
"simhash64",
|
|
]
|