初始化

dedup/models.py

@@ -0,0 +1,57 @@
+"""三层去重模块的数据模型。"""
+
+from datetime import datetime
+from enum import StrEnum
+
+from pydantic import BaseModel, Field
+
+
+class DedupLayer(StrEnum):
+    """命中去重的层。"""
+
+    URL = "url"          # L1: 完全相同 URL
+    CONTENT = "content"  # L2: 标准化后 content 完全一致
+    SIMHASH = "simhash"  # L3: SimHash 汉明距离 ≤ 阈值
+
+
+class Fingerprint(BaseModel):
+    """单篇文章的指纹记录，持久化到 SQLite。"""
+
+    url_hash: str = Field(..., description="主键，与 ProcessedArticle.url_hash 一致")
+    content_hash: str = Field(..., description="标准化 content 的 SHA1[:16]")
+    simhash: int = Field(..., description="64 位 SimHash 整数（无符号）")
+    source_id: str
+    url: str
+    title: str
+    publish_date: str | None = Field(default=None, description="YYYY-MM-DD，用于时间窗口")
+    ingested_at: datetime = Field(default_factory=datetime.now)
+
+
+class DedupResult(BaseModel):
+    """对单篇文章的判重结果。"""
+
+    url_hash: str
+    is_duplicate: bool
+    matched_layer: DedupLayer | None = None
+    matched_url_hash: str | None = None
+    matched_url: str | None = None
+    matched_title: str | None = None
+    hamming_distance: int | None = Field(
+        default=None, description="仅 SimHash 层有值"
+    )
+
+    def short_summary(self) -> str:
+        if not self.is_duplicate:
+            return f"[UNIQUE] {self.url_hash}"
+        layer = self.matched_layer.value if self.matched_layer else "?"
+        extra = f" hd={self.hamming_distance}" if self.hamming_distance is not None else ""
+        return f"[DUP/{layer}] {self.url_hash} ~ {self.matched_url_hash}{extra}"
+
+
+class DedupStats(BaseModel):
+    """指纹库统计。"""
+
+    total: int = 0
+    by_source: dict[str, int] = Field(default_factory=dict)
+    earliest: str | None = None
+    latest: str | None = None