"""正文提取管道：批量处理 raw 目录下的文章"""

import json
import logging
from datetime import datetime
from pathlib import Path

from crawler.storage import load_index
from crawler.utils import get_news_day
from extractor.extractor import extract_article
from extractor.models import ProcessedArticle

logger = logging.getLogger(__name__)


def get_raw_data_sources(base_dir: str = "data/raw") -> list[str]:
    """扫描 data/raw/ 下所有源 ID

    Args:
        base_dir: raw 数据根目录

    Returns:
        源 ID 列表
    """
    raw_path = Path(base_dir)
    if not raw_path.exists():
        return []
    return sorted([
        d.name for d in raw_path.iterdir()
        if d.is_dir() and not d.name.startswith(".")
    ])


def process_source(
    source_id: str,
    date_str: str | None = None,
) -> list[ProcessedArticle]:
    """处理单个源的文章正文提取

    Args:
        source_id: 新闻源 ID
        date_str: 日期 YYYYMMDD，默认当前新闻日

    Returns:
        ProcessedArticle 列表
    """
    if date_str is None:
        date_str = get_news_day()

    logger.info("━━━ 正文提取 [%s] %s ━━━", source_id, date_str)

    # 读取 raw index
    articles = load_index(source_id, date_str)
    if not articles:
        logger.warning("[%s] %s 无待处理文章", source_id, date_str)
        return []

    # 输出目录
    out_dir = Path(f"data/processed/{source_id}/{date_str}")
    out_dir.mkdir(parents=True, exist_ok=True)

    results: list[ProcessedArticle] = []
    success = 0
    no_content = 0
    failed = 0

    # 增量：跳过已处理的文章
    skipped = 0
    for article in articles[:]:
        out_file = out_dir / f"{article.url_hash}.json"
        if out_file.exists():
            articles.remove(article)
            skipped += 1
    if skipped > 0:
        logger.info("[%s] 增量跳过 %d 篇已处理，剩余 %d 篇",
                    source_id, skipped, len(articles))

    for article in articles:
        # 确保 html_path / md_path 是绝对路径（相对于 data/raw/）
        html_path = str(Path("data/raw") / source_id / date_str / f"{article.url_hash}.html")
        md_path = str(Path("data/raw") / source_id / date_str / f"{article.url_hash}.md")

        # 如果 index 中已有路径，优先使用
        if article.html_path:
            html_path = article.html_path
        if article.md_path:
            md_path = article.md_path

        processed = extract_article(
            source_id=source_id,
            source_name=article.source_name,
            url=article.url,
            url_hash=article.url_hash,
            html_path=html_path,
            md_path=md_path,
            crawl_publish_time=article.publish_time,
        )

        # 写入 JSON 输出
        out_file = out_dir / f"{article.url_hash}.json"
        out_file.write_text(
            processed.model_dump_json(indent=2, ensure_ascii=False),
            encoding="utf-8",
        )

        results.append(processed)

        if processed.status == "success":
            success += 1
            logger.debug("[%s] ✅ %s (%d words, %s)",
                         source_id, processed.title[:40], processed.word_count, processed.extractor)
        elif processed.status == "no_content":
            no_content += 1
            logger.warning("[%s] ⚠️ 无正文: %s", source_id, processed.url[:80])
        else:
            failed += 1

    logger.info("[%s] 完成: 成功 %d / 无内容 %d / 失败 %d",
                source_id, success, no_content, failed)

    # 写入处理索引
    index_path = out_dir / "index.jsonl"
    with open(index_path, "a", encoding="utf-8") as f:
        for r in results:
            if r.status == "success":
                f.write(json.dumps({
                    "url_hash": r.url_hash,
                    "title": r.title,
                    "word_count": r.word_count,
                    "extractor": r.extractor,
                    "publish_time": r.publish_time,
                }, ensure_ascii=False) + "\n")

    return results


def process_all_sources(
    source_filter: str | None = None,
    date_str: str | None = None,
) -> dict:
    """处理所有源的文章正文提取

    Args:
        source_filter: 可选，只处理指定源
        date_str: 日期，默认当前新闻日

    Returns:
        统计 dict
    """
    if date_str is None:
        date_str = get_news_day()

    start_time = datetime.now()

    if source_filter:
        sources = [source_filter] if source_filter in get_raw_data_sources() else []
    else:
        sources = get_raw_data_sources()

    logger.info("══════ 开始正文提取 %d 个源，日期: %s ══════", len(sources), date_str)

    total = 0
    for src in sources:
        results = process_source(src, date_str)
        total += len([r for r in results if r.status == "success"])

    elapsed = (datetime.now() - start_time).total_seconds()
    logger.info("══════ 提取完成: %d 篇文章，耗时 %.1f 秒 ══════", total, elapsed)

    return {
        "sources_processed": len(sources),
        "total_articles": total,
        "elapsed_sec": elapsed,
        "date": date_str,
    }