simon/news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
simon
2026-07-18 15:51:01 +08:00
commit f2c80c5a9c
799 changed files with 133475 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dedup/__init__.py
+44
View File
@@ -0,0 +1,44 @@
"""三层新闻去重模块 (M3)。
公共 API:
- Deduper: 主类(check / ingest / stats)
- FingerprintStore: SQLite 指纹库(底层,通常无需直接用)
- DedupResult / DedupLayer / DedupStats / Fingerprint: 数据模型
- simhash64 / hamming / content_hash / normalize_content: 指纹算法
"""
from .deduper import (
DEFAULT_TIME_WINDOW_DAYS,
Deduper,
article_to_fingerprint,
)
from .hasher import (
DEFAULT_HAMMING_THRESHOLD,
NGRAM_SIZE,
SIMHASH_BITS,
content_hash,
hamming,
normalize_content,
simhash64,
)
from .models import DedupLayer, DedupResult, DedupStats, Fingerprint
from .store import DEFAULT_DB_PATH, FingerprintStore
__all__ = [
"DEFAULT_DB_PATH",
"DEFAULT_HAMMING_THRESHOLD",
"DEFAULT_TIME_WINDOW_DAYS",
"NGRAM_SIZE",
"SIMHASH_BITS",
"DedupLayer",
"DedupResult",
"DedupStats",
"Deduper",
"Fingerprint",
"FingerprintStore",
"article_to_fingerprint",
"content_hash",
"hamming",
"normalize_content",
"simhash64",
]
dedup/deduper.py
+155
View File
@@ -0,0 +1,155 @@
"""三层去重主流程。
调用顺序:check / ingest 内部按 L1 -> L2 -> L3 顺序判定,任意层命中即返回。
Deduper 不要求线程安全;批处理串行调用即可。
"""
from __future__ import annotations
from datetime import datetime
from pathlib import Path
from loguru import logger
from extractor import Article
from .hasher import (
DEFAULT_HAMMING_THRESHOLD,
content_hash,
hamming,
simhash64,
)
from .models import DedupLayer, DedupResult, DedupStats, Fingerprint
from .store import DEFAULT_DB_PATH, FingerprintStore
# 默认时间窗口(±N 天)
DEFAULT_TIME_WINDOW_DAYS = 30
def _publish_date(article: Article) -> str | None:
"""从 Article.publish_time 取 YYYY-MM-DD 字符串。"""
if article.publish_time is None:
return None
return article.publish_time.strftime("%Y-%m-%d")
def article_to_fingerprint(article: Article) -> Fingerprint:
"""构造 Fingerprint(用于 ingest 写入或对外只读)。"""
return Fingerprint(
url_hash=article.url_hash,
content_hash=content_hash(article.content),
simhash=simhash64(article.content),
source_id=article.source_id,
url=article.url,
title=article.title,
publish_date=_publish_date(article),
ingested_at=datetime.now(),
)
class Deduper:
"""三层去重器。
构造完毕后:
- check(article) 仅判断,不写入;
- ingest(article) 判断,不重复则写入指纹库,返回结果。
"""
def __init__(
self,
db_path: str | Path = DEFAULT_DB_PATH,
simhash_threshold: int = DEFAULT_HAMMING_THRESHOLD,
time_window_days: int = DEFAULT_TIME_WINDOW_DAYS,
) -> None:
self.store = FingerprintStore(db_path)
self.simhash_threshold = simhash_threshold
self.time_window_days = time_window_days
def close(self) -> None:
self.store.close()
def __enter__(self) -> Deduper:
return self
def __exit__(self, *_: object) -> None:
self.close()
# ------------------------------------------------------------------ #
# 公共 API
# ------------------------------------------------------------------ #
def check(self, article: Article) -> DedupResult:
"""三层判重(只读)。"""
fp = article_to_fingerprint(article)
# L1: URL hash
existing = self.store.get_by_url_hash(fp.url_hash)
if existing is not None:
return DedupResult(
url_hash=fp.url_hash,
is_duplicate=True,
matched_layer=DedupLayer.URL,
matched_url_hash=existing.url_hash,
matched_url=existing.url,
matched_title=existing.title,
)
# L2: 内容 hash
existing = self.store.find_by_content_hash(fp.content_hash)
if existing is not None:
return DedupResult(
url_hash=fp.url_hash,
is_duplicate=True,
matched_layer=DedupLayer.CONTENT,
matched_url_hash=existing.url_hash,
matched_url=existing.url,
matched_title=existing.title,
)
# L3: SimHash 模糊
candidates = self.store.candidates_for_simhash(
fp.publish_date, self.time_window_days
)
best_dist: int | None = None
best_match: Fingerprint | None = None
for c in candidates:
d = hamming(fp.simhash, c.simhash)
if d <= self.simhash_threshold and (best_dist is None or d < best_dist):
best_dist = d
best_match = c
if d == 0: # 不可能更近,提前结束
break
if best_match is not None:
return DedupResult(
url_hash=fp.url_hash,
is_duplicate=True,
matched_layer=DedupLayer.SIMHASH,
matched_url_hash=best_match.url_hash,
matched_url=best_match.url,
matched_title=best_match.title,
hamming_distance=best_dist,
)
return DedupResult(url_hash=fp.url_hash, is_duplicate=False)
def ingest(self, article: Article) -> DedupResult:
"""判重 + 不重复则入库。"""
result = self.check(article)
if not result.is_duplicate:
fp = article_to_fingerprint(article)
self.store.upsert(fp)
logger.debug("入库: {} {}", fp.url_hash, fp.title[:30])
else:
logger.debug("命中重复: {}", result.short_summary())
return result
def stats(self) -> DedupStats:
lo, hi = self.store.date_range()
return DedupStats(
total=self.store.count(),
by_source=self.store.count_by_source(),
earliest=lo,
latest=hi,
)
dedup/hasher.py
+93
View File
@@ -0,0 +1,93 @@
"""三层去重的指纹算法。
核心:
- normalize_content:把 content 折叠成纯净文本,用于跨源比对;
- content_hash:normalize 后 SHA1[:16];
- simhash64:字符 3-gram + md5 加权累加,产出 64 位无符号整数;
- hamming:两个 SimHash 的汉明距离。
设计取舍:
SimHash 的"分词"用字符 3-gram 而非 jieba。理由:
1. 中文场景下字符 3-gram 与词级 SimHash 在重复识别上效果接近,
而前者无外部依赖、ARM/嵌入式友好;
2. M5 Embedding 后续不依赖 jieba,引入只为 M3 不划算;
3. 重复率验收门槛 ≤ 5%(project_plan.md 第七章),3-gram 经验上
足以分辨。
"""
from __future__ import annotations
import hashlib
import unicodedata
# 64 位 SimHash 位宽
SIMHASH_BITS = 64
SIMHASH_MASK = (1 << SIMHASH_BITS) - 1
# 默认 SimHash 汉明距离阈值(<= 此值视为重复)
DEFAULT_HAMMING_THRESHOLD = 3
# 字符 n-gram 长度
NGRAM_SIZE = 3
def normalize_content(text: str) -> str:
"""把 content 折叠成"无空白无标点"形式,用于 L2 内容 hash 与 SimHash 输入。
使用 Unicode 类别判断:
- P* Punctuation(所有中英文标点)
- Z* Separator(空格 / 行 / 段分隔符)
- C* Control(NUL / 换行控制等)
保留 L*(字母)、N*(数字)、S*(符号,如 +/-、% 等),以及 CJK 字符。
"""
if not text:
return ""
return "".join(
ch for ch in text if unicodedata.category(ch)[0] not in ("P", "Z", "C")
)
def content_hash(text: str) -> str:
"""对 normalize_content(text) 做 SHA1,取前 16 hex 字符。"""
norm = normalize_content(text)
return hashlib.sha1(norm.encode("utf-8")).hexdigest()[:16]
def _ngrams(text: str, n: int = NGRAM_SIZE) -> list[str]:
"""字符级 n-gram。文本短于 n 时,直接整体作为单个 token。"""
if len(text) < n:
return [text] if text else []
return [text[i : i + n] for i in range(len(text) - n + 1)]
def simhash64(text: str) -> int:
"""64 位 SimHash。返回无符号整数,空文本返回 0。"""
norm = normalize_content(text)
if not norm:
return 0
grams = _ngrams(norm)
if not grams:
return 0
v = [0] * SIMHASH_BITS
for gram in grams:
h = int(hashlib.md5(gram.encode("utf-8"), usedforsecurity=False).hexdigest(), 16)
# 取低 64 位
h64 = h & SIMHASH_MASK
for i in range(SIMHASH_BITS):
if (h64 >> i) & 1:
v[i] += 1
else:
v[i] -= 1
fp = 0
for i in range(SIMHASH_BITS):
if v[i] > 0:
fp |= 1 << i
return fp
def hamming(a: int, b: int) -> int:
"""两个 SimHash 的汉明距离。"""
return bin((a ^ b) & SIMHASH_MASK).count("1")
dedup/models.py
+64
View File
@@ -0,0 +1,64 @@
"""三层去重模块的数据模型。"""
from __future__ import annotations
from datetime import datetime
from enum import StrEnum
from typing import Literal
from pydantic import BaseModel, Field
class DedupLayer(StrEnum):
"""命中去重的层。"""
URL = "url" # L1: 完全相同 URL
CONTENT = "content" # L2: 标准化后 content 完全一致
SIMHASH = "simhash" # L3: SimHash 汉明距离 <= 阈值
class Fingerprint(BaseModel):
"""单篇文章的指纹记录,持久化到 SQLite。"""
url_hash: str = Field(..., description="主键,与 Article.url_hash 一致")
content_hash: str = Field(..., description="标准化 content 的 SHA1[:16]")
simhash: int = Field(..., description="64 位 SimHash 整数(无符号)")
source_id: str
url: str
title: str
publish_date: str | None = Field(default=None, description="YYYY-MM-DD,用于时间窗口")
ingested_at: datetime = Field(default_factory=datetime.now)
class DedupResult(BaseModel):
"""对单篇文章的判重结果。"""
url_hash: str
is_duplicate: bool
matched_layer: DedupLayer | None = None
matched_url_hash: str | None = None
matched_url: str | None = None
matched_title: str | None = None
hamming_distance: int | None = Field(
default=None, description="仅 SimHash 层有值"
)
def short_summary(self) -> str:
if not self.is_duplicate:
return f"[UNIQUE] {self.url_hash}"
layer = self.matched_layer.value if self.matched_layer else "?"
extra = f" hd={self.hamming_distance}" if self.hamming_distance is not None else ""
return f"[DUP/{layer}] {self.url_hash} ~ {self.matched_url_hash}{extra}"
class DedupStats(BaseModel):
"""指纹库统计。"""
total: int = 0
by_source: dict[str, int] = Field(default_factory=dict)
earliest: str | None = None
latest: str | None = None
# 类型别名,便于在批处理日志中归类
DedupVerdict = Literal["unique", "duplicate"]
dedup/store.py
+169
View File
@@ -0,0 +1,169 @@
"""SQLite 指纹存储。
注意:SimHash 是 64 位无符号整数,SQLite INTEGER 是 64 位有符号
(范围 [-2^63, 2^63-1])。直接存可能溢出/转负数,虽然 XOR 仍然
正确但语义混乱。这里统一存为 16 位 hex TEXT,避免符号问题。
"""
from __future__ import annotations
import sqlite3
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any
from loguru import logger
from .models import Fingerprint
DEFAULT_DB_PATH = Path("data/dedup/fingerprints.sqlite3")
_SCHEMA_SQL = """
CREATE TABLE IF NOT EXISTS fingerprints (
url_hash TEXT PRIMARY KEY,
content_hash TEXT NOT NULL,
simhash_hex TEXT NOT NULL,
source_id TEXT NOT NULL,
url TEXT NOT NULL,
title TEXT NOT NULL,
publish_date TEXT,
ingested_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_content_hash ON fingerprints(content_hash);
CREATE INDEX IF NOT EXISTS idx_publish_date ON fingerprints(publish_date);
CREATE INDEX IF NOT EXISTS idx_source_id ON fingerprints(source_id);
"""
def _to_hex(simhash: int) -> str:
return f"{simhash:016x}"
def _from_hex(hex_str: str) -> int:
return int(hex_str, 16)
def _row_to_fp(row: sqlite3.Row) -> Fingerprint:
return Fingerprint(
url_hash=row["url_hash"],
content_hash=row["content_hash"],
simhash=_from_hex(row["simhash_hex"]),
source_id=row["source_id"],
url=row["url"],
title=row["title"],
publish_date=row["publish_date"],
ingested_at=datetime.fromisoformat(row["ingested_at"]),
)
class FingerprintStore:
"""SQLite 包装。线程不安全(每个线程请新建实例)。"""
def __init__(self, db_path: str | Path = DEFAULT_DB_PATH) -> None:
self.db_path = Path(db_path)
self.db_path.parent.mkdir(parents=True, exist_ok=True)
self._conn: sqlite3.Connection = sqlite3.connect(
self.db_path, isolation_level=None
)
self._conn.row_factory = sqlite3.Row
self._conn.executescript(_SCHEMA_SQL)
logger.debug("打开指纹库: {}", self.db_path)
def close(self) -> None:
self._conn.close()
def __enter__(self) -> FingerprintStore:
return self
def __exit__(self, *_: Any) -> None:
self.close()
# ------------------------------------------------------------------ #
# 查询
# ------------------------------------------------------------------ #
def get_by_url_hash(self, url_hash: str) -> Fingerprint | None:
row = self._conn.execute(
"SELECT * FROM fingerprints WHERE url_hash = ?", (url_hash,)
).fetchone()
return _row_to_fp(row) if row else None
def find_by_content_hash(self, content_hash: str) -> Fingerprint | None:
"""返回任一匹配项。"""
row = self._conn.execute(
"SELECT * FROM fingerprints WHERE content_hash = ? LIMIT 1",
(content_hash,),
).fetchone()
return _row_to_fp(row) if row else None
def candidates_for_simhash(
self,
publish_date: str | None,
window_days: int,
) -> list[Fingerprint]:
"""返回 publish_date ± window_days 内的指纹候选。
publish_date 为 None 时,不限定窗口(返回全部,慎用)。
"""
if publish_date is None or window_days < 0:
rows = self._conn.execute("SELECT * FROM fingerprints").fetchall()
return [_row_to_fp(r) for r in rows]
try:
center = datetime.strptime(publish_date, "%Y-%m-%d")
except ValueError:
logger.debug("publish_date 不可解析: {!r},退化为全表扫描", publish_date)
rows = self._conn.execute("SELECT * FROM fingerprints").fetchall()
return [_row_to_fp(r) for r in rows]
lo = (center - timedelta(days=window_days)).strftime("%Y-%m-%d")
hi = (center + timedelta(days=window_days)).strftime("%Y-%m-%d")
rows = self._conn.execute(
"SELECT * FROM fingerprints "
"WHERE publish_date IS NULL OR (publish_date >= ? AND publish_date <= ?)",
(lo, hi),
).fetchall()
return [_row_to_fp(r) for r in rows]
def count(self) -> int:
return self._conn.execute("SELECT COUNT(*) FROM fingerprints").fetchone()[0]
def count_by_source(self) -> dict[str, int]:
rows = self._conn.execute(
"SELECT source_id, COUNT(*) AS n FROM fingerprints GROUP BY source_id"
).fetchall()
return {r["source_id"]: r["n"] for r in rows}
def date_range(self) -> tuple[str | None, str | None]:
row = self._conn.execute(
"SELECT MIN(publish_date) AS lo, MAX(publish_date) AS hi FROM fingerprints"
).fetchone()
return (row["lo"], row["hi"]) if row else (None, None)
# ------------------------------------------------------------------ #
# 写入
# ------------------------------------------------------------------ #
def upsert(self, fp: Fingerprint) -> None:
self._conn.execute(
"INSERT OR REPLACE INTO fingerprints "
"(url_hash, content_hash, simhash_hex, source_id, url, title, "
" publish_date, ingested_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
(
fp.url_hash,
fp.content_hash,
_to_hex(fp.simhash),
fp.source_id,
fp.url,
fp.title,
fp.publish_date,
fp.ingested_at.isoformat(),
),
)
def delete(self, url_hash: str) -> None:
self._conn.execute("DELETE FROM fingerprints WHERE url_hash = ?", (url_hash,))
def clear(self) -> None:
"""清空指纹库,主要用于测试。"""
self._conn.execute("DELETE FROM fingerprints")