simon
8fa27ad65b
- run_dedup: 生产模式重复率仅作 WARNING 告警,不影响退出码(执行成功即 0)
- run_dedup: 新增 --strict 验收模式(重复率 > 5% 返回 1),保留 M3 验收门槛
- run_dedup: 新增统计快照 data/deduped/{date}/stats.json(原子写)
- run_dedup: 顺带修复空日场景 sources.json 写入 FileNotFoundError
- pipeline: 移除 dedup rc=1 特判,恢复'非 0 即失败'统一语义
- 新增 tests/test_run_dedup.py 7 个测试;全量 266 passed
317 lines
12 KiB
Python
317 lines
12 KiB
Python
"""M3 批量去重入口脚本。
|
|
|
|
输入: data/processed/{source}/{YYYYMMDD}/*.json (M2 产物)
|
|
输出:
|
|
- 指纹库:data/dedup/fingerprints.sqlite3 (source_ids 列记录多源)
|
|
- 唯一文章:data/deduped/{YYYYMMDD}/uniques/{url_hash}.json (含 sources 多源字段)
|
|
- 多源记录:data/deduped/{YYYYMMDD}/sources.json
|
|
{url_hash: [source_id, ...]},一条唯一新闻的全部来源
|
|
- 重复记录:data/deduped/{YYYYMMDD}/duplicates.jsonl
|
|
(含 matched_source_id / matched_source_ids)
|
|
|
|
用法:
|
|
uv run python -m scripts.run_dedup # 处理今日全部源
|
|
uv run python -m scripts.run_dedup --date 20260616
|
|
uv run python -m scripts.run_dedup --source sina --date 20260616
|
|
uv run python -m scripts.run_dedup --reset # 清空指纹库重新建立
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from collections import Counter
|
|
from datetime import date, datetime
|
|
from pathlib import Path
|
|
|
|
from loguru import logger
|
|
from pydantic import ValidationError
|
|
|
|
from dedup import Deduper
|
|
from extractor import Article
|
|
|
|
# 重复率告警阈值(超过则输出 WARNING 日志;--strict 验收模式下作为返回码门槛)
|
|
_DUP_RATE_THRESHOLD = 0.05
|
|
|
|
|
|
def _setup_logger(level: str) -> None:
|
|
logger.remove()
|
|
logger.add(
|
|
sys.stderr,
|
|
level=level,
|
|
format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}",
|
|
)
|
|
log_path = Path("logs") / "dedup.log"
|
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
logger.add(log_path, level="DEBUG", rotation="10 MB", retention=5, encoding="utf-8")
|
|
|
|
|
|
def _load_article(json_path: Path) -> Article | None:
|
|
try:
|
|
data = json.loads(json_path.read_text(encoding="utf-8"))
|
|
return Article.model_validate(data)
|
|
except (json.JSONDecodeError, ValidationError) as e:
|
|
logger.warning("跳过无法解析的 article 文件 {}: {}", json_path, e)
|
|
return None
|
|
|
|
|
|
def _list_source_dirs(processed_root: Path) -> list[str]:
|
|
if not processed_root.is_dir():
|
|
return []
|
|
return sorted(p.name for p in processed_root.iterdir() if p.is_dir())
|
|
|
|
|
|
def _write_unique(
|
|
url_hash: str,
|
|
article: Article,
|
|
uniques_dir: Path,
|
|
sources_map: dict[str, list[str]],
|
|
) -> None:
|
|
"""写 uniques JSON,附加 sources 多源字段(向后兼容:下游 Pydantic 忽略多余字段)。"""
|
|
data = json.loads(article.model_dump_json())
|
|
data["sources"] = list(dict.fromkeys(sources_map.get(url_hash, [article.source_id])))
|
|
out_path = uniques_dir / f"{url_hash}.json"
|
|
out_path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
|
|
def _update_unique_sources(
|
|
url_hash: str,
|
|
uniques_dir: Path,
|
|
sources_map: dict[str, list[str]],
|
|
) -> None:
|
|
"""仅更新已存在 uniques 文件的 sources 字段(不覆盖原文内容)。
|
|
|
|
跨日命中时对应 uniques 文件在历史日期目录,不在本次处理范围,以指纹库为准。
|
|
"""
|
|
uniq_path = uniques_dir / f"{url_hash}.json"
|
|
if not uniq_path.is_file():
|
|
return
|
|
try:
|
|
data = json.loads(uniq_path.read_text(encoding="utf-8"))
|
|
data["sources"] = list(dict.fromkeys(sources_map.get(url_hash, [])))
|
|
uniq_path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
except (json.JSONDecodeError, OSError) as e:
|
|
logger.warning("更新 uniques 多源失败 {}: {}", uniq_path, e)
|
|
|
|
|
|
def _merge_sources(url_hash: str, new_source: str, sources_map: dict[str, list[str]]) -> None:
|
|
"""把新源并入 url_hash 的源列表(去重保序,主源居首)。"""
|
|
cur = sources_map.setdefault(url_hash, [])
|
|
if new_source not in cur:
|
|
cur.append(new_source)
|
|
|
|
|
|
def _process_source_day(
|
|
source_id: str,
|
|
day: str,
|
|
processed_root: Path,
|
|
out_root: Path,
|
|
deduper: Deduper,
|
|
sources_map: dict[str, list[str]],
|
|
) -> tuple[int, int, Counter]:
|
|
"""处理单源单日。返回 (uniques, duplicates, layer_counter)。
|
|
|
|
sources_map: 本次去重涉及内容组的 url_hash -> 全部来源列表(跨源累积,
|
|
最终写入 data/deduped/{day}/sources.json,供「显示新闻源」使用;
|
|
跨日命中的历史内容组也会记录,权威多源以指纹库 source_ids 列为准)。
|
|
"""
|
|
src_dir = processed_root / source_id / day
|
|
if not src_dir.is_dir():
|
|
logger.info("源 {} 日期 {} 无 processed 目录,跳过", source_id, day)
|
|
return 0, 0, Counter()
|
|
|
|
files = sorted(src_dir.glob("*.json"))
|
|
if not files:
|
|
logger.info("源 {} 日期 {} 无文章,跳过", source_id, day)
|
|
return 0, 0, Counter()
|
|
|
|
uniques_dir = out_root / day / "uniques"
|
|
uniques_dir.mkdir(parents=True, exist_ok=True)
|
|
dup_log = out_root / day / "duplicates.jsonl"
|
|
|
|
uniq_cnt = 0
|
|
dup_cnt = 0
|
|
layer_cnt: Counter = Counter()
|
|
|
|
with dup_log.open("a", encoding="utf-8") as dup_f:
|
|
for fp in files:
|
|
article = _load_article(fp)
|
|
if article is None:
|
|
continue
|
|
result = deduper.ingest(article)
|
|
if result.is_duplicate:
|
|
dup_cnt += 1
|
|
if result.matched_layer is not None:
|
|
layer_cnt[result.matched_layer.value] += 1
|
|
# 记录多源:把被去重文章的源并入对应唯一新闻
|
|
if result.matched_url_hash:
|
|
_merge_sources(result.matched_url_hash, article.source_id, sources_map)
|
|
# 若该唯一新闻文件在当天目录,同步更新其 sources 字段
|
|
_update_unique_sources(result.matched_url_hash, uniques_dir, sources_map)
|
|
dup_f.write(
|
|
json.dumps(
|
|
{
|
|
"source_id": article.source_id,
|
|
"url": article.url,
|
|
"url_hash": article.url_hash,
|
|
"title": article.title,
|
|
"matched_layer": (
|
|
result.matched_layer.value
|
|
if result.matched_layer
|
|
else None
|
|
),
|
|
"matched_url": result.matched_url,
|
|
"matched_url_hash": result.matched_url_hash,
|
|
"matched_title": result.matched_title,
|
|
"matched_source_id": result.matched_source_id,
|
|
"matched_source_ids": result.all_source_ids,
|
|
"hamming_distance": result.hamming_distance,
|
|
},
|
|
ensure_ascii=False,
|
|
)
|
|
+ "\n"
|
|
)
|
|
else:
|
|
uniq_cnt += 1
|
|
sources_map[article.url_hash] = [article.source_id]
|
|
_write_unique(article.url_hash, article, uniques_dir, sources_map)
|
|
|
|
total = uniq_cnt + dup_cnt
|
|
rate = dup_cnt / max(total, 1)
|
|
logger.info(
|
|
"源 {} 日期 {}: 唯一 {} / 重复 {} (重复率 {:.1%}) layers={}",
|
|
source_id,
|
|
day,
|
|
uniq_cnt,
|
|
dup_cnt,
|
|
rate,
|
|
dict(layer_cnt),
|
|
)
|
|
return uniq_cnt, dup_cnt, layer_cnt
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="A 股新闻三层去重 (M3)")
|
|
parser.add_argument("--processed-root", default="data/processed")
|
|
parser.add_argument("--out-root", default="data/deduped")
|
|
parser.add_argument("--db", default="data/dedup/fingerprints.sqlite3")
|
|
parser.add_argument("--source", default=None, help="只处理单源")
|
|
parser.add_argument(
|
|
"--date", default=date.today().strftime("%Y%m%d"), help="日期 YYYYMMDD"
|
|
)
|
|
parser.add_argument("--simhash-threshold", type=int, default=3)
|
|
parser.add_argument("--window-days", type=int, default=30)
|
|
parser.add_argument("--reset", action="store_true", help="处理前清空指纹库")
|
|
parser.add_argument(
|
|
"--strict", action="store_true",
|
|
help="验收模式:重复率超过阈值(5%)时返回 1。"
|
|
"默认为生产模式,重复率仅作统计告警,不影响退出码",
|
|
)
|
|
parser.add_argument("--log-level", default="INFO")
|
|
args = parser.parse_args()
|
|
|
|
_setup_logger(args.log_level)
|
|
processed_root = Path(args.processed_root)
|
|
out_root = Path(args.out_root)
|
|
|
|
sources = [args.source] if args.source else _list_source_dirs(processed_root)
|
|
if not sources:
|
|
logger.error("{} 下无源目录", processed_root)
|
|
return 2
|
|
|
|
with Deduper(
|
|
db_path=args.db,
|
|
simhash_threshold=args.simhash_threshold,
|
|
time_window_days=args.window_days,
|
|
) as deduper:
|
|
if args.reset:
|
|
logger.warning("--reset:清空指纹库 {}", args.db)
|
|
deduper.store.clear()
|
|
|
|
# 清掉同日 duplicates.jsonl 避免重复追加(uniques 用 url_hash 文件名,会自然覆盖)
|
|
dup_log = out_root / args.date / "duplicates.jsonl"
|
|
if dup_log.exists():
|
|
dup_log.unlink()
|
|
|
|
total_uniq = 0
|
|
total_dup = 0
|
|
total_layers: Counter = Counter()
|
|
# 当天唯一新闻 url_hash -> 全部来源列表(跨源累积,多源记录)
|
|
sources_map: dict[str, list[str]] = {}
|
|
for src in sources:
|
|
u, d, lc = _process_source_day(
|
|
src, args.date, processed_root, out_root, deduper, sources_map
|
|
)
|
|
total_uniq += u
|
|
total_dup += d
|
|
total_layers.update(lc)
|
|
|
|
# 多源记录汇总:data/deduped/{day}/sources.json
|
|
# {url_hash: [source_id, ...]},配合 uniques/{url_hash}.json 的 sources 字段
|
|
# 与指纹库 source_ids 列,提供「一条唯一新闻多个来源」的完整记录。
|
|
sources_path = out_root / args.date / "sources.json"
|
|
sources_path.parent.mkdir(parents=True, exist_ok=True)
|
|
sources_path.write_text(
|
|
json.dumps(
|
|
{k: v for k, v in sources_map.items() if v},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
logger.info(
|
|
"多源记录已写入 {} ({} 条唯一新闻,{} 条含多源)",
|
|
sources_path,
|
|
len(sources_map),
|
|
sum(1 for v in sources_map.values() if len(v) > 1),
|
|
)
|
|
|
|
total = total_uniq + total_dup
|
|
rate = total_dup / max(total, 1)
|
|
log_fn = logger.warning if rate > _DUP_RATE_THRESHOLD else logger.info
|
|
log_fn(
|
|
"全部完成: 唯一 {} / 重复 {} (重复率 {:.1%}) layers={}",
|
|
total_uniq,
|
|
total_dup,
|
|
rate,
|
|
dict(total_layers),
|
|
)
|
|
|
|
# 统计快照: data/deduped/{date}/stats.json(原子写),供运维排查与监控
|
|
stats_payload = {
|
|
"date": args.date,
|
|
"generated_at": datetime.now().isoformat(),
|
|
"unique": total_uniq,
|
|
"duplicates": total_dup,
|
|
"total": total,
|
|
"dup_rate": round(rate, 4),
|
|
"dup_rate_threshold": _DUP_RATE_THRESHOLD,
|
|
"layers": dict(total_layers),
|
|
"fingerprint_total": deduper.stats().total,
|
|
}
|
|
stats_path = out_root / args.date / "stats.json"
|
|
stats_path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = stats_path.with_suffix(".json.tmp")
|
|
tmp.write_text(json.dumps(stats_payload, ensure_ascii=False, indent=2),
|
|
encoding="utf-8")
|
|
tmp.replace(stats_path)
|
|
|
|
if rate > _DUP_RATE_THRESHOLD and total > 0:
|
|
logger.warning(
|
|
"重复率 {:.1%} 超过阈值 {:.0%}(唯一 {},重复 {})。"
|
|
"同日多次调度/跨源转载属常见现象;若为当日首次处理仍异常偏高,"
|
|
"请检查指纹库与抓取源(详见 {})",
|
|
rate, _DUP_RATE_THRESHOLD, total_uniq, total_dup, stats_path,
|
|
)
|
|
|
|
# 生产模式:重复率是统计指标,不影响退出码(执行成功即 0);
|
|
# 验收模式(--strict):保留 M3 验收门槛(重复率 ≤ 5%),超标返回 1。
|
|
if args.strict and total > 0 and rate > _DUP_RATE_THRESHOLD:
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|