"""M3 批量去重入口脚本。

输入: data/processed/{source}/{YYYYMMDD}/*.json (M2 产物)
输出:
    - 指纹库:data/dedup/fingerprints.sqlite3
    - 唯一文章:data/deduped/{YYYYMMDD}/uniques/{url_hash}.json
    - 重复记录:data/deduped/{YYYYMMDD}/duplicates.jsonl

用法:
    uv run python -m scripts.run_dedup                       # 处理今日全部源
    uv run python -m scripts.run_dedup --date 20260616
    uv run python -m scripts.run_dedup --source sina --date 20260616
    uv run python -m scripts.run_dedup --reset               # 清空指纹库重新建立
"""

from __future__ import annotations

import argparse
import json
import sys
from collections import Counter
from datetime import date
from pathlib import Path

from loguru import logger
from pydantic import ValidationError

from dedup import Deduper
from extractor import Article


def _setup_logger(level: str) -> None:
    logger.remove()
    logger.add(
        sys.stderr,
        level=level,
        format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}",
    )
    log_path = Path("logs") / "dedup.log"
    log_path.parent.mkdir(parents=True, exist_ok=True)
    logger.add(log_path, level="DEBUG", rotation="10 MB", retention=5, encoding="utf-8")


def _load_article(json_path: Path) -> Article | None:
    try:
        data = json.loads(json_path.read_text(encoding="utf-8"))
        return Article.model_validate(data)
    except (json.JSONDecodeError, ValidationError) as e:
        logger.warning("跳过无法解析的 article 文件 {}: {}", json_path, e)
        return None


def _list_source_dirs(processed_root: Path) -> list[str]:
    if not processed_root.is_dir():
        return []
    return sorted(p.name for p in processed_root.iterdir() if p.is_dir())


def _process_source_day(
    source_id: str,
    day: str,
    processed_root: Path,
    out_root: Path,
    deduper: Deduper,
) -> tuple[int, int, Counter]:
    """处理单源单日。返回 (uniques, duplicates, layer_counter)。"""
    src_dir = processed_root / source_id / day
    if not src_dir.is_dir():
        logger.info("源 {} 日期 {} 无 processed 目录,跳过", source_id, day)
        return 0, 0, Counter()

    files = sorted(src_dir.glob("*.json"))
    if not files:
        logger.info("源 {} 日期 {} 无文章,跳过", source_id, day)
        return 0, 0, Counter()

    uniques_dir = out_root / day / "uniques"
    uniques_dir.mkdir(parents=True, exist_ok=True)
    dup_log = out_root / day / "duplicates.jsonl"

    uniq_cnt = 0
    dup_cnt = 0
    layer_cnt: Counter = Counter()

    with dup_log.open("a", encoding="utf-8") as dup_f:
        for fp in files:
            article = _load_article(fp)
            if article is None:
                continue
            result = deduper.ingest(article)
            if result.is_duplicate:
                dup_cnt += 1
                if result.matched_layer is not None:
                    layer_cnt[result.matched_layer.value] += 1
                dup_f.write(
                    json.dumps(
                        {
                            "source_id": article.source_id,
                            "url": article.url,
                            "url_hash": article.url_hash,
                            "title": article.title,
                            "matched_layer": (
                                result.matched_layer.value
                                if result.matched_layer
                                else None
                            ),
                            "matched_url": result.matched_url,
                            "matched_url_hash": result.matched_url_hash,
                            "matched_title": result.matched_title,
                            "hamming_distance": result.hamming_distance,
                        },
                        ensure_ascii=False,
                    )
                    + "\n"
                )
            else:
                uniq_cnt += 1
                out_path = uniques_dir / f"{article.url_hash}.json"
                out_path.write_text(article.model_dump_json(indent=2), encoding="utf-8")

    total = uniq_cnt + dup_cnt
    rate = dup_cnt / max(total, 1)
    logger.info(
        "源 {} 日期 {}: 唯一 {} / 重复 {} (重复率 {:.1%}) layers={}",
        source_id,
        day,
        uniq_cnt,
        dup_cnt,
        rate,
        dict(layer_cnt),
    )
    return uniq_cnt, dup_cnt, layer_cnt


def main() -> int:
    parser = argparse.ArgumentParser(description="A 股新闻三层去重 (M3)")
    parser.add_argument("--processed-root", default="data/processed")
    parser.add_argument("--out-root", default="data/deduped")
    parser.add_argument("--db", default="data/dedup/fingerprints.sqlite3")
    parser.add_argument("--source", default=None, help="只处理单源")
    parser.add_argument(
        "--date", default=date.today().strftime("%Y%m%d"), help="日期 YYYYMMDD"
    )
    parser.add_argument("--simhash-threshold", type=int, default=3)
    parser.add_argument("--window-days", type=int, default=30)
    parser.add_argument("--reset", action="store_true", help="处理前清空指纹库")
    parser.add_argument("--log-level", default="INFO")
    args = parser.parse_args()

    _setup_logger(args.log_level)
    processed_root = Path(args.processed_root)
    out_root = Path(args.out_root)

    sources = [args.source] if args.source else _list_source_dirs(processed_root)
    if not sources:
        logger.error("{} 下无源目录", processed_root)
        return 2

    with Deduper(
        db_path=args.db,
        simhash_threshold=args.simhash_threshold,
        time_window_days=args.window_days,
    ) as deduper:
        if args.reset:
            logger.warning("--reset:清空指纹库 {}", args.db)
            deduper.store.clear()

        # 清掉同日 duplicates.jsonl 避免重复追加(uniques 用 url_hash 文件名,会自然覆盖)
        dup_log = out_root / args.date / "duplicates.jsonl"
        if dup_log.exists():
            dup_log.unlink()

        total_uniq = 0
        total_dup = 0
        total_layers: Counter = Counter()
        for src in sources:
            u, d, lc = _process_source_day(
                src, args.date, processed_root, out_root, deduper
            )
            total_uniq += u
            total_dup += d
            total_layers.update(lc)

        total = total_uniq + total_dup
        rate = total_dup / max(total, 1)
        logger.info(
            "全部完成: 唯一 {} / 重复 {} (重复率 {:.1%}) layers={}",
            total_uniq,
            total_dup,
            rate,
            dict(total_layers),
        )
        # 验收门槛: ≤ 5%
        return 0 if rate <= 0.05 or total == 0 else 1


if __name__ == "__main__":
    raise SystemExit(main())