Initial commit

scripts/run_dedup.py

@@ -0,0 +1,198 @@
+"""M3 批量去重入口脚本。
+
+输入: data/processed/{source}/{YYYYMMDD}/*.json (M2 产物)
+输出:
+    - 指纹库:data/dedup/fingerprints.sqlite3
+    - 唯一文章:data/deduped/{YYYYMMDD}/uniques/{url_hash}.json
+    - 重复记录:data/deduped/{YYYYMMDD}/duplicates.jsonl
+
+用法:
+    uv run python -m scripts.run_dedup                       # 处理今日全部源
+    uv run python -m scripts.run_dedup --date 20260616
+    uv run python -m scripts.run_dedup --source sina --date 20260616
+    uv run python -m scripts.run_dedup --reset               # 清空指纹库重新建立
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections import Counter
+from datetime import date
+from pathlib import Path
+
+from loguru import logger
+from pydantic import ValidationError
+
+from dedup import Deduper
+from extractor import Article
+
+
+def _setup_logger(level: str) -> None:
+    logger.remove()
+    logger.add(
+        sys.stderr,
+        level=level,
+        format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}",
+    )
+    log_path = Path("logs") / "dedup.log"
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    logger.add(log_path, level="DEBUG", rotation="10 MB", retention=5, encoding="utf-8")
+
+
+def _load_article(json_path: Path) -> Article | None:
+    try:
+        data = json.loads(json_path.read_text(encoding="utf-8"))
+        return Article.model_validate(data)
+    except (json.JSONDecodeError, ValidationError) as e:
+        logger.warning("跳过无法解析的 article 文件 {}: {}", json_path, e)
+        return None
+
+
+def _list_source_dirs(processed_root: Path) -> list[str]:
+    if not processed_root.is_dir():
+        return []
+    return sorted(p.name for p in processed_root.iterdir() if p.is_dir())
+
+
+def _process_source_day(
+    source_id: str,
+    day: str,
+    processed_root: Path,
+    out_root: Path,
+    deduper: Deduper,
+) -> tuple[int, int, Counter]:
+    """处理单源单日。返回 (uniques, duplicates, layer_counter)。"""
+    src_dir = processed_root / source_id / day
+    if not src_dir.is_dir():
+        logger.info("源 {} 日期 {} 无 processed 目录,跳过", source_id, day)
+        return 0, 0, Counter()
+
+    files = sorted(src_dir.glob("*.json"))
+    if not files:
+        logger.info("源 {} 日期 {} 无文章,跳过", source_id, day)
+        return 0, 0, Counter()
+
+    uniques_dir = out_root / day / "uniques"
+    uniques_dir.mkdir(parents=True, exist_ok=True)
+    dup_log = out_root / day / "duplicates.jsonl"
+
+    uniq_cnt = 0
+    dup_cnt = 0
+    layer_cnt: Counter = Counter()
+
+    with dup_log.open("a", encoding="utf-8") as dup_f:
+        for fp in files:
+            article = _load_article(fp)
+            if article is None:
+                continue
+            result = deduper.ingest(article)
+            if result.is_duplicate:
+                dup_cnt += 1
+                if result.matched_layer is not None:
+                    layer_cnt[result.matched_layer.value] += 1
+                dup_f.write(
+                    json.dumps(
+                        {
+                            "source_id": article.source_id,
+                            "url": article.url,
+                            "url_hash": article.url_hash,
+                            "title": article.title,
+                            "matched_layer": (
+                                result.matched_layer.value
+                                if result.matched_layer
+                                else None
+                            ),
+                            "matched_url": result.matched_url,
+                            "matched_url_hash": result.matched_url_hash,
+                            "matched_title": result.matched_title,
+                            "hamming_distance": result.hamming_distance,
+                        },
+                        ensure_ascii=False,
+                    )
+                    + "\n"
+                )
+            else:
+                uniq_cnt += 1
+                out_path = uniques_dir / f"{article.url_hash}.json"
+                out_path.write_text(article.model_dump_json(indent=2), encoding="utf-8")
+
+    total = uniq_cnt + dup_cnt
+    rate = dup_cnt / max(total, 1)
+    logger.info(
+        "源 {} 日期 {}: 唯一 {} / 重复 {} (重复率 {:.1%}) layers={}",
+        source_id,
+        day,
+        uniq_cnt,
+        dup_cnt,
+        rate,
+        dict(layer_cnt),
+    )
+    return uniq_cnt, dup_cnt, layer_cnt
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="A 股新闻三层去重 (M3)")
+    parser.add_argument("--processed-root", default="data/processed")
+    parser.add_argument("--out-root", default="data/deduped")
+    parser.add_argument("--db", default="data/dedup/fingerprints.sqlite3")
+    parser.add_argument("--source", default=None, help="只处理单源")
+    parser.add_argument(
+        "--date", default=date.today().strftime("%Y%m%d"), help="日期 YYYYMMDD"
+    )
+    parser.add_argument("--simhash-threshold", type=int, default=3)
+    parser.add_argument("--window-days", type=int, default=30)
+    parser.add_argument("--reset", action="store_true", help="处理前清空指纹库")
+    parser.add_argument("--log-level", default="INFO")
+    args = parser.parse_args()
+
+    _setup_logger(args.log_level)
+    processed_root = Path(args.processed_root)
+    out_root = Path(args.out_root)
+
+    sources = [args.source] if args.source else _list_source_dirs(processed_root)
+    if not sources:
+        logger.error("{} 下无源目录", processed_root)
+        return 2
+
+    with Deduper(
+        db_path=args.db,
+        simhash_threshold=args.simhash_threshold,
+        time_window_days=args.window_days,
+    ) as deduper:
+        if args.reset:
+            logger.warning("--reset:清空指纹库 {}", args.db)
+            deduper.store.clear()
+
+        # 清掉同日 duplicates.jsonl 避免重复追加(uniques 用 url_hash 文件名,会自然覆盖)
+        dup_log = out_root / args.date / "duplicates.jsonl"
+        if dup_log.exists():
+            dup_log.unlink()
+
+        total_uniq = 0
+        total_dup = 0
+        total_layers: Counter = Counter()
+        for src in sources:
+            u, d, lc = _process_source_day(
+                src, args.date, processed_root, out_root, deduper
+            )
+            total_uniq += u
+            total_dup += d
+            total_layers.update(lc)
+
+        total = total_uniq + total_dup
+        rate = total_dup / max(total, 1)
+        logger.info(
+            "全部完成: 唯一 {} / 重复 {} (重复率 {:.1%}) layers={}",
+            total_uniq,
+            total_dup,
+            rate,
+            dict(total_layers),
+        )
+        # 验收门槛: ≤ 5%
+        return 0 if rate <= 0.05 or total == 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())