初始化

dedup/__init__.py

@@ -0,0 +1,53 @@
+"""三层新闻去重模块 (M3)。
+
+公共 API:
+    - Deduper:         主类（check / ingest / stats）
+    - FingerprintStore: SQLite 指纹库（底层，通常无需直接用）
+    - DedupResult / DedupLayer / DedupStats / Fingerprint: 数据模型
+    - simhash64 / hamming / content_hash / normalize_content: 指纹算法
+    - dedup_all_sources / dedup_source: 批量去重管道
+"""
+
+from dedup.deduper import (
+    DEFAULT_TIME_WINDOW_DAYS,
+    Deduper,
+    article_to_fingerprint,
+)
+from dedup.hasher import (
+    DEFAULT_HAMMING_THRESHOLD,
+    NGRAM_SIZE,
+    SIMHASH_BITS,
+    content_hash,
+    hamming,
+    normalize_content,
+    simhash64,
+)
+from dedup.models import DedupLayer, DedupResult, DedupStats, Fingerprint
+from dedup.pipeline import dedup_all_sources, dedup_source
+from dedup.store import DEFAULT_DB_PATH, FingerprintStore
+
+__all__ = [
+    # 主类
+    "Deduper",
+    "FingerprintStore",
+    # 管道
+    "dedup_all_sources",
+    "dedup_source",
+    # 模型
+    "DedupLayer",
+    "DedupResult",
+    "DedupStats",
+    "Fingerprint",
+    # 指纹算法
+    "article_to_fingerprint",
+    "content_hash",
+    "hamming",
+    "normalize_content",
+    "simhash64",
+    # 常量
+    "DEFAULT_DB_PATH",
+    "DEFAULT_HAMMING_THRESHOLD",
+    "DEFAULT_TIME_WINDOW_DAYS",
+    "NGRAM_SIZE",
+    "SIMHASH_BITS",
+]

dedup/deduper.py

@@ -0,0 +1,178 @@
+"""三层去重主流程。
+
+调用顺序: check / ingest 内部按 L1 → L2 → L3 顺序判定，任意层命中即返回。
+
+Deduper 不要求线程安全；批处理串行调用即可。
+"""
+
+import logging
+from datetime import datetime
+from pathlib import Path
+
+import yaml
+
+from dedup.hasher import DEFAULT_HAMMING_THRESHOLD, content_hash, hamming, simhash64
+from dedup.models import DedupLayer, DedupResult, DedupStats, Fingerprint
+from dedup.store import DEFAULT_DB_PATH, FingerprintStore
+from extractor.models import ProcessedArticle
+
+logger = logging.getLogger(__name__)
+
+# 默认时间窗口（±N 天）
+DEFAULT_TIME_WINDOW_DAYS = 30
+
+
+def _load_dedup_config() -> dict:
+    """从 system.yaml 加载去重配置。"""
+    config_path = Path("configs/system.yaml")
+    if config_path.exists():
+        try:
+            with open(config_path, encoding="utf-8") as f:
+                raw = yaml.safe_load(f)
+            return raw.get("dedup", {})
+        except Exception:
+            logger.warning("加载 dedup 配置失败，使用默认值")
+    return {}
+
+
+def _publish_date(article: ProcessedArticle) -> str | None:
+    """从 ProcessedArticle.publish_time 取 YYYY-MM-DD 字符串。"""
+    if not article.publish_time:
+        return None
+    # publish_time 格式为 ISO 8601，如 "2026-06-21T10:30:00"
+    try:
+        return article.publish_time[:10]
+    except (IndexError, TypeError):
+        return None
+
+
+def article_to_fingerprint(article: ProcessedArticle) -> Fingerprint:
+    """构造 Fingerprint（用于 ingest 写入或对外只读）。"""
+    return Fingerprint(
+        url_hash=article.url_hash,
+        content_hash=content_hash(article.content),
+        simhash=simhash64(article.content),
+        source_id=article.source_id,
+        url=article.url,
+        title=article.title,
+        publish_date=_publish_date(article),
+        ingested_at=datetime.now(),
+    )
+
+
+class Deduper:
+    """三层去重器。
+
+    构造完毕后:
+        - check(article)  仅判断，不写入
+        - ingest(article) 判断，不重复则写入指纹库，返回结果
+    """
+
+    def __init__(
+        self,
+        db_path: str | Path = DEFAULT_DB_PATH,
+        simhash_threshold: int | None = None,
+        time_window_days: int | None = None,
+    ) -> None:
+        config = _load_dedup_config()
+
+        self.store = FingerprintStore(db_path)
+        self.simhash_threshold = (
+            simhash_threshold
+            if simhash_threshold is not None
+            else config.get("hamming_distance_threshold", DEFAULT_HAMMING_THRESHOLD)
+        )
+        self.time_window_days = (
+            time_window_days
+            if time_window_days is not None
+            else config.get("simhash_window_days", DEFAULT_TIME_WINDOW_DAYS)
+        )
+
+    def close(self) -> None:
+        self.store.close()
+
+    def __enter__(self) -> "Deduper":
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self.close()
+
+    # ------------------------------------------------------------------ #
+    # 公共 API
+    # ------------------------------------------------------------------ #
+
+    def check(self, article: ProcessedArticle) -> DedupResult:
+        """三层判重（只读，不写入指纹库）。"""
+        fp = article_to_fingerprint(article)
+
+        # L1: URL hash
+        existing = self.store.get_by_url_hash(fp.url_hash)
+        if existing is not None:
+            return DedupResult(
+                url_hash=fp.url_hash,
+                is_duplicate=True,
+                matched_layer=DedupLayer.URL,
+                matched_url_hash=existing.url_hash,
+                matched_url=existing.url,
+                matched_title=existing.title,
+            )
+
+        # L2: 内容 hash
+        existing = self.store.find_by_content_hash(fp.content_hash)
+        if existing is not None:
+            return DedupResult(
+                url_hash=fp.url_hash,
+                is_duplicate=True,
+                matched_layer=DedupLayer.CONTENT,
+                matched_url_hash=existing.url_hash,
+                matched_url=existing.url,
+                matched_title=existing.title,
+            )
+
+        # L3: SimHash 模糊
+        candidates = self.store.candidates_for_simhash(
+            fp.publish_date, self.time_window_days
+        )
+        best_dist: int | None = None
+        best_match: Fingerprint | None = None
+        for c in candidates:
+            d = hamming(fp.simhash, c.simhash)
+            if d <= self.simhash_threshold and (best_dist is None or d < best_dist):
+                best_dist = d
+                best_match = c
+                if d == 0:  # 不可能更近，提前结束
+                    break
+
+        if best_match is not None:
+            return DedupResult(
+                url_hash=fp.url_hash,
+                is_duplicate=True,
+                matched_layer=DedupLayer.SIMHASH,
+                matched_url_hash=best_match.url_hash,
+                matched_url=best_match.url,
+                matched_title=best_match.title,
+                hamming_distance=best_dist,
+            )
+
+        return DedupResult(url_hash=fp.url_hash, is_duplicate=False)
+
+    def ingest(self, article: ProcessedArticle) -> DedupResult:
+        """判重 + 不重复则入库。"""
+        result = self.check(article)
+        if not result.is_duplicate:
+            fp = article_to_fingerprint(article)
+            self.store.upsert(fp)
+            logger.debug("指纹入库: %s %s", fp.url_hash, fp.title[:40])
+        else:
+            logger.debug("命中重复: %s", result.short_summary())
+        return result
+
+    def stats(self) -> DedupStats:
+        """指纹库统计信息。"""
+        lo, hi = self.store.date_range()
+        return DedupStats(
+            total=self.store.count(),
+            by_source=self.store.count_by_source(),
+            earliest=lo,
+            latest=hi,
+        )

dedup/hasher.py

@@ -0,0 +1,89 @@
+"""三层去重的指纹算法。
+
+核心:
+    - normalize_content: 把 content 折叠成纯净文本，用于跨源比对
+    - content_hash: normalize 后 SHA1[:16]
+    - simhash64: 字符 3-gram + md5 加权累加，产出 64 位无符号整数
+    - hamming: 两个 SimHash 的汉明距离
+
+设计取舍:
+    SimHash 的"分词"用字符 3-gram 而非英文分词库。理由:
+        1. 字符 3-gram 对英文和中文同样有效，无需外部 NLP 依赖
+        2. 英文字符级 3-gram 天然捕获词根、前缀、后缀信息
+        3. 跨语言场景（英文源可能引用中文/日文公司名）字符级更鲁棒
+"""
+
+import hashlib
+import unicodedata
+
+# 64 位 SimHash 位宽
+SIMHASH_BITS = 64
+SIMHASH_MASK = (1 << SIMHASH_BITS) - 1
+
+# 默认 SimHash 汉明距离阈值（≤ 此值视为重复）
+DEFAULT_HAMMING_THRESHOLD = 3
+
+# 字符 n-gram 长度
+NGRAM_SIZE = 3
+
+
+def normalize_content(text: str) -> str:
+    """把 content 折叠成"无空白无标点"形式，用于 L2 内容 hash 与 SimHash 输入。
+
+    使用 Unicode 类别判断:
+        - P*  Punctuation（所有中英文标点）
+        - Z*  Separator（空格 / 行 / 段分隔符）
+        - C*  Control（NUL / 换行控制等）
+    保留 L*（字母）、N*（数字）、S*（符号，如 +/-、% 等），以及 CJK 字符。
+    """
+    if not text:
+        return ""
+    return "".join(
+        ch for ch in text if unicodedata.category(ch)[0] not in ("P", "Z", "C")
+    )
+
+
+def content_hash(text: str) -> str:
+    """对 normalize_content(text) 做 SHA1，取前 16 hex 字符。"""
+    norm = normalize_content(text)
+    return hashlib.sha1(norm.encode("utf-8")).hexdigest()[:16]
+
+
+def _ngrams(text: str, n: int = NGRAM_SIZE) -> list[str]:
+    """字符级 n-gram。文本短于 n 时，直接整体作为单个 token。"""
+    if len(text) < n:
+        return [text] if text else []
+    return [text[i : i + n] for i in range(len(text) - n + 1)]
+
+
+def simhash64(text: str) -> int:
+    """64 位 SimHash。返回无符号整数，空文本返回 0。"""
+    norm = normalize_content(text)
+    if not norm:
+        return 0
+
+    grams = _ngrams(norm)
+    if not grams:
+        return 0
+
+    v = [0] * SIMHASH_BITS
+    for gram in grams:
+        h = int(hashlib.md5(gram.encode("utf-8"), usedforsecurity=False).hexdigest(), 16)
+        # 取低 64 位
+        h64 = h & SIMHASH_MASK
+        for i in range(SIMHASH_BITS):
+            if (h64 >> i) & 1:
+                v[i] += 1
+            else:
+                v[i] -= 1
+
+    fp = 0
+    for i in range(SIMHASH_BITS):
+        if v[i] > 0:
+            fp |= 1 << i
+    return fp
+
+
+def hamming(a: int, b: int) -> int:
+    """两个 SimHash 的汉明距离。"""
+    return bin((a ^ b) & SIMHASH_MASK).count("1")

dedup/models.py

@@ -0,0 +1,57 @@
+"""三层去重模块的数据模型。"""
+
+from datetime import datetime
+from enum import StrEnum
+
+from pydantic import BaseModel, Field
+
+
+class DedupLayer(StrEnum):
+    """命中去重的层。"""
+
+    URL = "url"          # L1: 完全相同 URL
+    CONTENT = "content"  # L2: 标准化后 content 完全一致
+    SIMHASH = "simhash"  # L3: SimHash 汉明距离 ≤ 阈值
+
+
+class Fingerprint(BaseModel):
+    """单篇文章的指纹记录，持久化到 SQLite。"""
+
+    url_hash: str = Field(..., description="主键，与 ProcessedArticle.url_hash 一致")
+    content_hash: str = Field(..., description="标准化 content 的 SHA1[:16]")
+    simhash: int = Field(..., description="64 位 SimHash 整数（无符号）")
+    source_id: str
+    url: str
+    title: str
+    publish_date: str | None = Field(default=None, description="YYYY-MM-DD，用于时间窗口")
+    ingested_at: datetime = Field(default_factory=datetime.now)
+
+
+class DedupResult(BaseModel):
+    """对单篇文章的判重结果。"""
+
+    url_hash: str
+    is_duplicate: bool
+    matched_layer: DedupLayer | None = None
+    matched_url_hash: str | None = None
+    matched_url: str | None = None
+    matched_title: str | None = None
+    hamming_distance: int | None = Field(
+        default=None, description="仅 SimHash 层有值"
+    )
+
+    def short_summary(self) -> str:
+        if not self.is_duplicate:
+            return f"[UNIQUE] {self.url_hash}"
+        layer = self.matched_layer.value if self.matched_layer else "?"
+        extra = f" hd={self.hamming_distance}" if self.hamming_distance is not None else ""
+        return f"[DUP/{layer}] {self.url_hash} ~ {self.matched_url_hash}{extra}"
+
+
+class DedupStats(BaseModel):
+    """指纹库统计。"""
+
+    total: int = 0
+    by_source: dict[str, int] = Field(default_factory=dict)
+    earliest: str | None = None
+    latest: str | None = None

dedup/pipeline.py

@@ -0,0 +1,226 @@
+"""批量去重管道：扫描 processed 目录 → 判重 → 唯一条目写入 deduped。
+
+输入:  data/processed/{source_id}/{YYYYMMDD}/{url_hash}.json
+输出:  data/deduped/{YYYYMMDD}/uniques/{url_hash}.json
+"""
+
+import json
+import logging
+from datetime import datetime
+from pathlib import Path
+
+from crawler.utils import get_news_day
+from dedup.deduper import Deduper
+from dedup.models import DedupResult
+from extractor.models import ProcessedArticle
+
+logger = logging.getLogger(__name__)
+
+
+def _get_processed_sources(base_dir: str = "data/processed") -> list[str]:
+    """扫描 data/processed/ 下所有源 ID。
+
+    Args:
+        base_dir: processed 数据根目录
+
+    Returns:
+        源 ID 列表
+    """
+    raw_path = Path(base_dir)
+    if not raw_path.exists():
+        return []
+    return sorted([
+        d.name for d in raw_path.iterdir()
+        if d.is_dir() and not d.name.startswith(".")
+    ])
+
+
+def _load_processed_articles(
+    source_id: str,
+    date_str: str,
+) -> list[ProcessedArticle]:
+    """加载指定源/日期的已处理文章。
+
+    Args:
+        source_id: 新闻源 ID
+        date_str: 日期 YYYYMMDD
+
+    Returns:
+        ProcessedArticle 列表
+    """
+    base_dir = Path(f"data/processed/{source_id}/{date_str}")
+    if not base_dir.exists():
+        return []
+
+    articles: list[ProcessedArticle] = []
+    for json_file in sorted(base_dir.glob("*.json")):
+        # 跳过 index.jsonl
+        if json_file.name == "index.jsonl":
+            continue
+        try:
+            data = json.loads(json_file.read_text(encoding="utf-8"))
+            articles.append(ProcessedArticle(**data))
+        except (json.JSONDecodeError, Exception) as e:
+            logger.warning("解析 processed JSON 失败 %s: %s", json_file, e)
+
+    return articles
+
+
+def dedup_source(
+    source_id: str,
+    deduper: Deduper,
+    date_str: str | None = None,
+) -> dict:
+    """对单个源的已处理文章执行去重。
+
+    Args:
+        source_id: 新闻源 ID
+        deduper: 去重器实例
+        date_str: 日期 YYYYMMDD，默认当前新闻日
+
+    Returns:
+        统计 dict
+    """
+    if date_str is None:
+        date_str = get_news_day()
+
+    logger.info("━━━ 去重 [%s] %s ━━━", source_id, date_str)
+
+    articles = _load_processed_articles(source_id, date_str)
+    if not articles:
+        logger.warning("[%s] %s 无待处理文章", source_id, date_str)
+        return {"source_id": source_id, "total": 0, "unique": 0, "duplicate": 0}
+
+    # 输出目录
+    out_dir = Path(f"data/deduped/{date_str}/uniques")
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    unique_count = 0
+    dup_count = 0
+
+    for article in articles:
+        result = deduper.ingest(article)
+
+        if result.is_duplicate:
+            dup_count += 1
+            logger.debug("[%s] 🔁 %s → L%d: %s",
+                         source_id,
+                         article.title[:40],
+                         _layer_num(result),
+                         result.short_summary())
+        else:
+            unique_count += 1
+            # 写入唯一条目
+            out_file = out_dir / f"{article.url_hash}.json"
+            out_file.write_text(
+                article.model_dump_json(indent=2, ensure_ascii=False),
+                encoding="utf-8",
+            )
+            logger.debug("[%s] ✅ %s (%d words)",
+                         source_id, article.title[:40], article.word_count)
+
+    logger.info("[%s] 去重完成: 唯一 %d / 重复 %d / 总计 %d",
+                source_id, unique_count, dup_count, len(articles))
+
+    return {
+        "source_id": source_id,
+        "total": len(articles),
+        "unique": unique_count,
+        "duplicate": dup_count,
+    }
+
+
+def _layer_num(result: DedupResult) -> int:
+    """DedupResult → 命中层编号。"""
+    if result.matched_layer is None:
+        return 0
+    mapping = {"url": 1, "content": 2, "simhash": 3}
+    return mapping.get(result.matched_layer.value, 0)
+
+
+def dedup_all_sources(
+    source_filter: str | None = None,
+    date_str: str | None = None,
+) -> dict:
+    """对所有源的已处理文章执行去重。
+
+    Args:
+        source_filter: 可选，只处理指定源
+        date_str: 日期，默认当前新闻日
+
+    Returns:
+        统计 dict
+    """
+    if date_str is None:
+        date_str = get_news_day()
+
+    start_time = datetime.now()
+
+    if source_filter:
+        sources = [source_filter] if source_filter in _get_processed_sources() else []
+    else:
+        sources = _get_processed_sources()
+
+    logger.info("══════ 开始去重 %d 个源，日期: %s ══════", len(sources), date_str)
+
+    total_unique = 0
+    total_dup = 0
+    total_articles = 0
+
+    with Deduper() as deduper:
+        for src in sources:
+            result = dedup_source(src, deduper, date_str)
+            total_articles += result["total"]
+            total_unique += result["unique"]
+            total_dup += result["duplicate"]
+
+        # 输出 dedup 索引
+        _write_dedup_index(deduper, date_str, total_unique)
+
+    elapsed = (datetime.now() - start_time).total_seconds()
+    logger.info("══════ 去重完成: 唯一 %d / 重复 %d / 总计 %d，耗时 %.1f 秒 ══════",
+                total_unique, total_dup, total_articles, elapsed)
+
+    return {
+        "sources_processed": len(sources),
+        "total_articles": total_articles,
+        "unique": total_unique,
+        "duplicate": total_dup,
+        "elapsed_sec": elapsed,
+        "date": date_str,
+    }
+
+
+def _write_dedup_index(
+    deduper: Deduper,
+    date_str: str,
+    unique_count: int,
+) -> None:
+    """写出去重索引文件。
+
+    Args:
+        deduper: 去重器实例
+        date_str: 日期
+        unique_count: 唯一文章数
+    """
+    out_dir = Path(f"data/deduped/{date_str}")
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    stats = deduper.stats()
+
+    index_data = {
+        "date": date_str,
+        "unique_articles": unique_count,
+        "fingerprint_db_total": stats.total,
+        "fingerprint_db_by_source": stats.by_source,
+        "fingerprint_db_earliest": stats.earliest,
+        "fingerprint_db_latest": stats.latest,
+        "generated_at": datetime.now().isoformat(),
+    }
+
+    index_path = out_dir / "index.json"
+    index_path.write_text(
+        json.dumps(index_data, indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+    logger.info("去重索引已写入: %s", index_path)

dedup/store.py

@@ -0,0 +1,177 @@
+"""SQLite 指纹存储。
+
+注意: SimHash 是 64 位无符号整数，SQLite INTEGER 是 64 位有符号
+(范围 [-2^63, 2^63-1])。直接存可能溢出/转负数，虽然 XOR 仍然
+正确但语义混乱。这里统一存为 16 位 hex TEXT，避免符号问题。
+"""
+
+import logging
+import sqlite3
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+from dedup.models import Fingerprint
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_DB_PATH = Path("data/dedup/fingerprints.sqlite3")
+
+_SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS fingerprints (
+    url_hash TEXT PRIMARY KEY,
+    content_hash TEXT NOT NULL,
+    simhash_hex TEXT NOT NULL,
+    source_id TEXT NOT NULL,
+    url TEXT NOT NULL,
+    title TEXT NOT NULL,
+    publish_date TEXT,
+    ingested_at TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_content_hash ON fingerprints(content_hash);
+CREATE INDEX IF NOT EXISTS idx_publish_date ON fingerprints(publish_date);
+CREATE INDEX IF NOT EXISTS idx_source_id ON fingerprints(source_id);
+"""
+
+
+def _to_hex(simhash: int) -> str:
+    """64 位无符号整数 → 16 位 hex 字符串。"""
+    return f"{simhash:016x}"
+
+
+def _from_hex(hex_str: str) -> int:
+    """16 位 hex 字符串 → 64 位无符号整数。"""
+    return int(hex_str, 16)
+
+
+def _row_to_fp(row: sqlite3.Row) -> Fingerprint:
+    """sqlite3.Row → Fingerprint 模型。"""
+    return Fingerprint(
+        url_hash=row["url_hash"],
+        content_hash=row["content_hash"],
+        simhash=_from_hex(row["simhash_hex"]),
+        source_id=row["source_id"],
+        url=row["url"],
+        title=row["title"],
+        publish_date=row["publish_date"],
+        ingested_at=datetime.fromisoformat(row["ingested_at"]),
+    )
+
+
+class FingerprintStore:
+    """SQLite 指纹库。线程不安全（每个线程请新建实例）。"""
+
+    def __init__(self, db_path: str | Path = DEFAULT_DB_PATH) -> None:
+        self.db_path = Path(db_path)
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._conn: sqlite3.Connection = sqlite3.connect(
+            self.db_path, isolation_level=None
+        )
+        self._conn.row_factory = sqlite3.Row
+        self._conn.executescript(_SCHEMA_SQL)
+        logger.debug("打开指纹库: %s", self.db_path)
+
+    def close(self) -> None:
+        self._conn.close()
+
+    def __enter__(self) -> "FingerprintStore":
+        return self
+
+    def __exit__(self, *_: Any) -> None:
+        self.close()
+
+    # ------------------------------------------------------------------ #
+    # 查询
+    # ------------------------------------------------------------------ #
+
+    def get_by_url_hash(self, url_hash: str) -> Fingerprint | None:
+        """按 url_hash 精确查询。"""
+        row = self._conn.execute(
+            "SELECT * FROM fingerprints WHERE url_hash = ?", (url_hash,)
+        ).fetchone()
+        return _row_to_fp(row) if row else None
+
+    def find_by_content_hash(self, content_hash: str) -> Fingerprint | None:
+        """返回任一 content_hash 匹配项。"""
+        row = self._conn.execute(
+            "SELECT * FROM fingerprints WHERE content_hash = ? LIMIT 1",
+            (content_hash,),
+        ).fetchone()
+        return _row_to_fp(row) if row else None
+
+    def candidates_for_simhash(
+        self,
+        publish_date: str | None,
+        window_days: int,
+    ) -> list[Fingerprint]:
+        """返回 publish_date ± window_days 内的指纹候选。
+
+        publish_date 为 None 时，不限定窗口（返回全部，慎用）。
+        """
+        if publish_date is None or window_days < 0:
+            rows = self._conn.execute("SELECT * FROM fingerprints").fetchall()
+            return [_row_to_fp(r) for r in rows]
+
+        try:
+            center = datetime.strptime(publish_date, "%Y-%m-%d")
+        except ValueError:
+            logger.debug("publish_date 不可解析: %r，退化为全表扫描", publish_date)
+            rows = self._conn.execute("SELECT * FROM fingerprints").fetchall()
+            return [_row_to_fp(r) for r in rows]
+
+        lo = (center - timedelta(days=window_days)).strftime("%Y-%m-%d")
+        hi = (center + timedelta(days=window_days)).strftime("%Y-%m-%d")
+        rows = self._conn.execute(
+            "SELECT * FROM fingerprints "
+            "WHERE publish_date IS NULL OR (publish_date >= ? AND publish_date <= ?)",
+            (lo, hi),
+        ).fetchall()
+        return [_row_to_fp(r) for r in rows]
+
+    def count(self) -> int:
+        """指纹总数。"""
+        return self._conn.execute("SELECT COUNT(*) FROM fingerprints").fetchone()[0]
+
+    def count_by_source(self) -> dict[str, int]:
+        """按 source_id 统计。"""
+        rows = self._conn.execute(
+            "SELECT source_id, COUNT(*) AS n FROM fingerprints GROUP BY source_id"
+        ).fetchall()
+        return {r["source_id"]: r["n"] for r in rows}
+
+    def date_range(self) -> tuple[str | None, str | None]:
+        """指纹库中最早和最晚的 publish_date。"""
+        row = self._conn.execute(
+            "SELECT MIN(publish_date) AS lo, MAX(publish_date) AS hi FROM fingerprints"
+        ).fetchone()
+        return (row["lo"], row["hi"]) if row else (None, None)
+
+    # ------------------------------------------------------------------ #
+    # 写入
+    # ------------------------------------------------------------------ #
+
+    def upsert(self, fp: Fingerprint) -> None:
+        """插入或替换指纹。"""
+        self._conn.execute(
+            "INSERT OR REPLACE INTO fingerprints "
+            "(url_hash, content_hash, simhash_hex, source_id, url, title, "
+            " publish_date, ingested_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+            (
+                fp.url_hash,
+                fp.content_hash,
+                _to_hex(fp.simhash),
+                fp.source_id,
+                fp.url,
+                fp.title,
+                fp.publish_date,
+                fp.ingested_at.isoformat(),
+            ),
+        )
+
+    def delete(self, url_hash: str) -> None:
+        """删除指定指纹。"""
+        self._conn.execute("DELETE FROM fingerprints WHERE url_hash = ?", (url_hash,))
+
+    def clear(self) -> None:
+        """清空指纹库，主要用于测试。"""
+        self._conn.execute("DELETE FROM fingerprints")