"""Crawl4AI 异步抓取引擎。

设计:
    1. 一个 AsyncWebCrawler 实例服务所有源(共用浏览器,降低开销);
    2. 每个 URL 独立的 CrawlerRunConfig(wait_for/timeout 来自源配置);
    3. 全局 Semaphore 控制并发(默认 3);
    4. 每个 URL 用 tenacity 重试(默认 3 次,指数退避);
    5. 两阶段: 先抓首页 -> 解析文章链接 -> 并发抓文章详情。
"""

from __future__ import annotations

import asyncio
import re
from typing import Any
from urllib.parse import urljoin, urlparse

import httpx
from bs4 import BeautifulSoup
from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig
from loguru import logger
from markdownify import markdownify as _md_convert

from .models import (
    ArticleLink,
    CrawlerConfig,
    CrawlerSettings,
    CrawlResult,
    CrawlStage,
    SourceConfig,
)
from .storage import mark_url_seen, save_result

# --------------------------------------------------------------------------- #
# 工具函数
# --------------------------------------------------------------------------- #

def _markdown_text(crawl4ai_markdown: Any) -> str:
    """从 Crawl4AI 的 markdown 字段中安全取出字符串。

    新版 Crawl4AI(0.4+) 返回 MarkdownGenerationResult 对象;
    旧版可能直接是 str。这里做兼容。
    """
    if crawl4ai_markdown is None:
        return ""
    if isinstance(crawl4ai_markdown, str):
        return crawl4ai_markdown
    raw = getattr(crawl4ai_markdown, "raw_markdown", None)
    if isinstance(raw, str):
        return raw
    fit = getattr(crawl4ai_markdown, "fit_markdown", None)
    if isinstance(fit, str):
        return fit
    return str(crawl4ai_markdown)


def extract_article_links(
    html: str,
    base_url: str,
    source: SourceConfig,
) -> list[ArticleLink]:
    """从列表页 HTML 中抽取文章链接,使用源配置的正则筛选。

    步骤:
        1. 若设置了 article_link_selector,先 narrow 到选择器范围;
        2. 找出所有 <a href>;
        3. urljoin 转绝对 URL;
        4. 用 article_url_pattern 正则筛选;
        5. 去重保持顺序;
        6. 截断到 max_articles_per_run。
    """
    soup = BeautifulSoup(html, "html.parser")
    scope = (
        soup.select_one(source.article_link_selector)
        if source.article_link_selector
        else soup
    )
    if scope is None:
        logger.warning(
            "源 {} 设置了 article_link_selector={!r},但页面中未匹配到", source.id, source.article_link_selector
        )
        return []

    pattern = re.compile(source.article_url_pattern)
    seen: set[str] = set()
    links: list[ArticleLink] = []

    for a in scope.find_all("a", href=True):
        href = a["href"].strip()
        if not href or href.startswith(("javascript:", "#", "mailto:")):
            continue
        absolute = urljoin(base_url, href)
        # 去掉 fragment
        absolute = absolute.split("#", 1)[0]
        if not pattern.match(absolute):
            continue
        if absolute in seen:
            continue
        seen.add(absolute)
        anchor = (a.get_text() or "").strip() or None
        links.append(ArticleLink(source_id=source.id, url=absolute, anchor_text=anchor))
        if len(links) >= source.max_articles_per_run:
            break

    logger.info(
        "源 {} 列表页抽取链接 {} 条 (上限 {})",
        source.id,
        len(links),
        source.max_articles_per_run,
    )
    return links


# --------------------------------------------------------------------------- #
# 单 URL 抓取
# --------------------------------------------------------------------------- #

def _build_run_config(source: SourceConfig) -> CrawlerRunConfig:
    """根据源配置构造 CrawlerRunConfig。"""
    return CrawlerRunConfig(
        cache_mode=CacheMode.BYPASS,
        wait_for=source.wait_for,
        page_timeout=source.page_timeout_ms,
        excluded_tags=["script", "style", "noscript"],
    )


class _TimeoutGuard:
    """单源连续超时计数器，达到阈值后标记跳过，防止整个进程被拖死。"""

    def __init__(self, source_id: str, max_consecutive: int = 2) -> None:
        self.source_id = source_id
        self.max_consecutive = max_consecutive
        self._count = 0
        self.skip = False

    def record(self, error: str | None, success: bool) -> None:
        """根据抓取结果更新计数器。"""
        if self.skip:
            return
        if success:
            self._count = 0
            return
        if error and "timeout" in error.lower():
            self._count += 1
            if self._count >= self.max_consecutive:
                self.skip = True
                logger.warning(
                    "源 {} 连续超时 {} 次，跳过剩余请求",
                    self.source_id,
                    self._count,
                )

    def skipped_result(self, url: str, stage: CrawlStage) -> CrawlResult:
        """生成"已跳过"结果。"""
        return CrawlResult(
            source_id=self.source_id,
            stage=stage,
            url=url,
            success=False,
            error="Skipped: 源连续超时已跳过",
            attempts=0,
        )


async def _fetch_static(
    url: str,
    source: SourceConfig,
    stage: CrawlStage,
    attempt: int,
) -> CrawlResult:
    """使用 httpx 直连抓取静态页面（js_render=False），绕过 Playwright 反爬检测。"""
    headers = {
        "User-Agent": (
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
        ),
    }
    timeout_sec = source.page_timeout_ms / 1000.0
    try:
        async with httpx.AsyncClient(
            timeout=timeout_sec,
            follow_redirects=True,
            headers=headers,
        ) as client:
            r = await client.get(url)
        html = r.text
        # HTML → Markdown：先清洗再转换
        markdown = ""
        if html:
            try:
                soup = BeautifulSoup(html, "html.parser")
                for tag in soup(["script", "style", "noscript", "meta", "link"]):
                    tag.decompose()
                markdown = _md_convert(str(soup)) or ""
            except Exception:
                markdown = ""
        return CrawlResult(
            source_id=source.id,
            stage=stage,
            url=url,
            success=True,
            status_code=r.status_code,
            html=html,
            markdown=markdown,
            attempts=attempt,
        )
    except Exception as e:
        logger.warning("静态抓取异常 {} {}: {}", source.id, url, e)
        return CrawlResult(
            source_id=source.id,
            stage=stage,
            url=url,
            success=False,
            error=f"{type(e).__name__}: {e}",
            attempts=attempt,
        )


async def _crawl_once(
    crawler: AsyncWebCrawler,
    url: str,
    source: SourceConfig,
    stage: CrawlStage,
    attempt: int,
) -> CrawlResult:
    """单次抓取(无重试),用于被 retry 循环包裹。

    - js_render=False 且无 wait_for → httpx 直连,绕过浏览器反爬;
    - 其他情况 → Playwright。
    """
    # 静态页面走 httpx 直连
    if not source.js_render and not source.wait_for:
        return await _fetch_static(url, source, stage, attempt)

    # 动态页面走 Playwright
    run_config = _build_run_config(source)
    try:
        c4_result = await crawler.arun(url=url, config=run_config)
    except Exception as e:
        logger.warning("抓取异常 {} {}: {}", source.id, url, e)
        return CrawlResult(
            source_id=source.id,
            stage=stage,
            url=url,
            success=False,
            error=f"{type(e).__name__}: {e}",
            attempts=attempt,
        )

    success = bool(getattr(c4_result, "success", False))
    return CrawlResult(
        source_id=source.id,
        stage=stage,
        url=url,
        success=success,
        status_code=getattr(c4_result, "status_code", None),
        html=getattr(c4_result, "html", "") or "",
        markdown=_markdown_text(getattr(c4_result, "markdown", None)),
        error=None if success else (getattr(c4_result, "error_message", None) or "unknown"),
        attempts=attempt,
    )


async def crawl_url_with_retry(
    crawler: AsyncWebCrawler,
    url: str,
    source: SourceConfig,
    stage: CrawlStage,
    settings: CrawlerSettings,
    semaphore: asyncio.Semaphore,
    guard: _TimeoutGuard | None = None,
) -> CrawlResult:
    """带并发限制与指数退避重试的单 URL 抓取。

    重试策略:
        最多 settings.retry_max_attempts 次;
        失败 (success=False) 触发重试,等待时间 min(min_wait * 2^(n-1), max_wait)。

    若传入 guard,每次重试前检查是否已被跳过。
    """
    async with semaphore:
        # 检查是否已被跳过
        if guard and guard.skip:
            return guard.skipped_result(url, stage)

        max_attempts = settings.retry_max_attempts
        result: CrawlResult | None = None

        for attempt_no in range(1, max_attempts + 1):
            result = await _crawl_once(crawler, url, source, stage, attempt_no)
            if result.success:
                if guard:
                    guard.record(None, True)
                return result
            if attempt_no >= max_attempts:
                break
            # 重试前检查是否已被其他并发请求触发跳过
            if guard and guard.skip:
                return guard.skipped_result(url, stage)
            wait = min(
                settings.retry_min_wait_sec * (2 ** (attempt_no - 1)),
                settings.retry_max_wait_sec,
            )
            logger.info(
                "源 {} {} 第 {}/{} 次失败,{} 秒后重试: {}",
                source.id,
                url,
                attempt_no,
                max_attempts,
                wait,
                result.error,
            )
            await asyncio.sleep(wait)

        # 此处 result 必非 None(循环至少跑一次)
        assert result is not None
        if guard:
            guard.record(result.error, False)
        return result


# --------------------------------------------------------------------------- #
# 抓取流程
# --------------------------------------------------------------------------- #

async def crawl_source(
    crawler: AsyncWebCrawler,
    source: SourceConfig,
    settings: CrawlerSettings,
    semaphore: asyncio.Semaphore,
    save: bool = True,
) -> list[CrawlResult]:
    """抓取单个新闻源:遍历所有入口 → 抽取链接(去重) → 并发抓文章。"""
    homepages = source.all_homepages()
    logger.info(
        "===== 开始抓取源: {} ({}) {} 个入口 =====",
        source.id, source.name, len(homepages),
    )

    all_list_results: list[CrawlResult] = []
    all_links: list[ArticleLink] = []

    # 增量:加载历史 URL hash
    from .storage import load_seen_urls
    from .storage import url_hash as storage_url_hash
    seen_hashes = load_seen_urls(source.id)
    if seen_hashes:
        logger.info("源 {} 增量模式: 已记录 {} 条历史 URL", source.id, len(seen_hashes))

    # 遍历每个入口
    for hp_url in homepages:
        list_result = await crawl_url_with_retry(
            crawler=crawler, url=hp_url, source=source,
            stage=CrawlStage.LIST, settings=settings, semaphore=semaphore,
        )
        if save:
            save_result(list_result, settings.output_root)
        all_list_results.append(list_result)

        if not list_result.success:
            logger.warning("源 {} 入口 {} 抓取失败,跳过: {}", source.id, hp_url, list_result.error)
            continue

        parsed_base = urlparse(hp_url)
        base = f"{parsed_base.scheme}://{parsed_base.netloc}"
        links = extract_article_links(list_result.html, base, source)

        # 去重 + 增量:按 hash 去重(当前运行跨入口 + 历史记录)
        new_links = [ln for ln in links if storage_url_hash(ln.url) not in seen_hashes]
        for ln in new_links:
            seen_hashes.add(storage_url_hash(ln.url))
        all_links.extend(new_links)
        # 旧链接数(跨入口去重前)
        old_count = len(links) - len(new_links)
        logger.info("源 {} 入口 {} 抽取链接 {} 条(去重后新增 {}, 跳过旧 {} 条)",
                     source.id, hp_url, len(links), len(new_links), old_count)

    if not all_links:
        logger.warning("源 {} 所有入口未发现任何文章链接", source.id)
        return all_list_results

    # 连续超时保护：同源连续超时 2 次后跳过剩余请求
    guard = _TimeoutGuard(source.id)

    article_tasks = [
        crawl_url_with_retry(
            crawler=crawler, url=link.url, source=source,
            stage=CrawlStage.ARTICLE, settings=settings, semaphore=semaphore,
            guard=guard,
        )
        for link in all_links
    ]
    article_results = await asyncio.gather(*article_tasks, return_exceptions=False)

    if save:
        for r in article_results:
            save_result(r, settings.output_root)

    success_cnt = sum(1 for r in article_results if r.success)
    # 记录成功抓取的 URL(增量去重)
    new_saved = 0
    for r in article_results:
        if r.success:
            mark_url_seen(source.id, r.url)
            new_saved += 1

    logger.info(
        "源 {} 完成: 文章 {}/{} 成功, 新增 {} 条已记录",
        source.id, success_cnt, len(article_results), new_saved,
    )
    return [*all_list_results, *article_results]


async def crawl_all(
    config: CrawlerConfig,
    save: bool = True,
) -> list[CrawlResult]:
    """抓取所有启用的源,返回扁平的 CrawlResult 列表。"""
    settings = config.settings
    enabled = config.enabled_sources()
    if not enabled:
        logger.warning("没有启用的新闻源,直接返回")
        return []

    browser_config = BrowserConfig(
        headless=settings.headless,
        user_agent=settings.user_agent,
        verbose=False,
    )
    semaphore = asyncio.Semaphore(settings.concurrency)
    all_results: list[CrawlResult] = []

    async with AsyncWebCrawler(config=browser_config) as crawler:
        for source in enabled:
            try:
                results = await crawl_source(crawler, source, settings, semaphore, save=save)
                all_results.extend(results)
            except Exception as e:  # 单源异常不应中断整体抓取
                logger.exception("源 {} 抓取异常: {}", source.id, e)

    total = len(all_results)
    succ = sum(1 for r in all_results if r.success)
    logger.info("全部完成: {}/{} 成功 (成功率 {:.0%})", succ, total, succ / max(total, 1))
    return all_results