"""Crawl4AI 异步新闻抓取引擎

海外服务器运行，约束：内存 ≤ 2GB，串行抓取，单源 ≤ 2 小时。
所有业务参数从 configs/system.yaml 的 crawler 节读取。
"""

import asyncio
import gc
import hashlib
import json
import logging
import os
import re
from datetime import datetime
from pathlib import Path
from urllib.parse import urljoin

import psutil
import yaml
from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig

from crawler.models import ArticleItem, CrawlResult, SourceConfig
from crawler.utils import get_news_day

logger = logging.getLogger(__name__)

# ── 从配置文件加载常量 ──────────────────────────────────


def _load_crawler_config() -> dict:
    """从 system.yaml 读取 crawler 配置节（含 Profile 覆盖）"""
    try:
        from crawler.config import load_system_config
        cfg = load_system_config()
        return cfg.get("crawler", {})
    except Exception:
        logger.warning("读取 system.yaml 失败，使用默认值")
    return {}

_cfg = _load_crawler_config()

MAX_MEMORY_MB = int(_cfg.get("max_memory_mb", 1800))
ARTICLE_DELAY_SEC = float(_cfg.get("article_delay_sec", 3.0))
PAGE_TIMEOUT_MS = int(_cfg.get("page_timeout_sec", 45)) * 1000
SOURCE_TIMEOUT_SEC = int(_cfg.get("source_timeout_sec", 7200))
_VIEWPORT_WIDTH = int(_cfg.get("viewport_width", 1024))
_VIEWPORT_HEIGHT = int(_cfg.get("viewport_height", 768))
_USER_AGENT = _cfg.get(
    "user_agent",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
    "AppleWebKit/537.36 (KHTML, like Gecko) "
    "Chrome/131.0.0.0 Safari/537.36",
)
_HEADFUL = bool(_cfg.get("headful", False))
_XVFB_DISPLAY = _cfg.get("xvfb_display", ":99")

# ── 代理配置 ──────────────────────────────────────────


def _load_proxy_config() -> dict:
    """从 system.yaml 读取 proxy 配置节（含 Profile 覆盖）"""
    try:
        from crawler.config import load_system_config
        cfg = load_system_config()
        return cfg.get("proxy", {})
    except Exception:
        pass
    return {}

_proxy_cfg = _load_proxy_config()
PROXY_ENABLED = bool(_proxy_cfg.get("enabled", False))
PROXY_URL = _proxy_cfg.get("url", "socks5://127.0.0.1:1080")

# ── 工具函数 ──────────────────────────────────────────


def compute_url_hash(url: str) -> str:
    """计算 URL 的 SHA256 前 16 个字符作为短文件名"""
    return hashlib.sha256(url.encode("utf-8")).hexdigest()[:16]


def _get_memory_mb() -> float:
    """获取当前进程内存使用量（MB）"""
    proc = psutil.Process(os.getpid())
    return proc.memory_info().rss / (1024 * 1024)


def _check_memory(source_id: str) -> None:
    """检查内存，超限时警告"""
    mem = _get_memory_mb()
    if mem > MAX_MEMORY_MB:
        logger.warning("[%s] ⚠️ 内存使用 %.0f MB 超过上限 %.0f MB，触发 GC",
                       source_id, mem, MAX_MEMORY_MB)
        gc.collect()
        mem_after = _get_memory_mb()
        logger.info("[%s] GC 后内存: %.0f MB (回收 %.0f MB)",
                    source_id, mem_after, mem - mem_after)

# ── 浏览器配置 ──────────────────────────────────────────


def _make_browser_config(source: SourceConfig) -> BrowserConfig:
    """浏览器配置：根据 anti_bot_mode 和系统配置选择策略

    - None:      标准轻量 headless（默认）
    - "stealth": 反检测 headless（隐藏自动化特征）
    - "headful": 非 headless 模式（有头浏览器，最像真人）

    代理：如果 system.yaml proxy.enabled=true，浏览器流量走 SOCKS5 代理。
    headful 模式自动设置 DISPLAY 环境变量（支持 xvfb）。
    """
    mode = source.anti_bot_mode
    # headless 判定：
    #   - 系统级 _HEADFUL=true  → 强制有头（覆盖所有 source 模式）
    #   - source mode="headful"  → 有头
    #   - 其他                   → headless
    if _HEADFUL:
        headless = False        # 系统级强制 headful
    elif mode == "headful":
        headless = False
    else:
        headless = True

    # 基础反检测参数
    extra_args = [
        "--disable-dev-shm-usage",
        "--disable-gpu",
        "--no-sandbox",
    ]

    # 反检测参数：stealth 模式或系统级 headful 都启用
    if mode == "stealth" or _HEADFUL:
        extra_args += [
            "--disable-blink-features=AutomationControlled",
            "--disable-features=IsolateOrigins,site-per-process",
            "--disable-webrtc",  # 防止 WebRTC 泄露真实 IP
        ]

    # ── SOCKS5 代理 ──────────────────────────────────
    if PROXY_ENABLED and PROXY_URL:
        extra_args.append(f"--proxy-server={PROXY_URL}")
        logger.info("🌐 浏览器代理已启用: %s", PROXY_URL)

    # ── headful 模式：设置虚拟显示器 ──────────────────
    if not headless:
        display = os.environ.get("DISPLAY", "")
        if not display:
            os.environ["DISPLAY"] = _XVFB_DISPLAY
            logger.info("🖥️  headful 模式: DISPLAY=%s", _XVFB_DISPLAY)

    # stealth 反检测
    enable_stealth = (mode == "stealth") or _HEADFUL

    return BrowserConfig(
        browser_type="chromium",
        headless=headless,
        viewport_width=_VIEWPORT_WIDTH,
        viewport_height=_VIEWPORT_HEIGHT,
        verbose=False,
        text_mode=headless,             # headful 模式下不禁用图片（更像真人）
        light_mode=headless,
        user_agent=_USER_AGENT,
        extra_args=extra_args,
        enable_stealth=enable_stealth,  # Playwright stealth 反检测
    )


def _make_run_config(source: SourceConfig) -> CrawlerRunConfig:
    """抓取运行时配置"""
    mode = source.anti_bot_mode or ""
    is_stealth = (mode in ("stealth", "headful")) or _HEADFUL

    return CrawlerRunConfig(
        cache_mode=CacheMode.BYPASS,
        page_timeout=PAGE_TIMEOUT_MS,
        wait_until="domcontentloaded",                    # networkidle 会被 Cloudflare 挑战阻塞
        scan_full_page=False,
        simulate_user=True,
        override_navigator=True,
        magic=is_stealth,                                   # Crawl4AI 全自动反检测
        delay_before_return_html=3.0 if is_stealth else 0.1,  # 等待 Cloudflare JS 挑战完成
        remove_consent_popups=True,                 # 自动关闭 Cookie/隐私弹窗
    )

def _extract_domain(url: str) -> str:
    """从 URL 提取域名（去掉 www 前缀）。"""
    from urllib.parse import urlparse
    host = urlparse(url).hostname or ""
    return host.removeprefix("www.").lower()


# ── 链接提取 ──────────────────────────────────────────


async def _extract_article_urls(
    crawler: AsyncWebCrawler,
    source: SourceConfig,
) -> list[str]:
    """从首页抓取符合 article_url_pattern 的文章链接（增量：跳过已抓取过的 URL）"""
    config = _make_run_config(source)

    try:
        result = await crawler.arun(url=source.homepage, config=config)
    except Exception as e:
        logger.error("抓取首页失败 [%s] %s: %s", source.id, source.homepage, e)
        return []

    if not result.success:
        logger.error("首页返回失败 [%s]: %s", source.id, result.error_message)
        return []

    html = result.html or ""
    pattern = re.compile(r'href=["\']([^"\']*?)["\']', re.IGNORECASE)
    raw_urls: list[str] = pattern.findall(html)

    article_url_pattern = re.compile(source.article_url_pattern, re.IGNORECASE)
    base_without_fragment = source.homepage.split("#")[0]
    base_domain = _extract_domain(source.homepage)

    # 非内容类扩展名
    _SKIP_EXT = re.compile(
        r"\.(png|ico|gif|jpg|jpeg|svg|webp|css|js|xml|json|rss|pdf|zip|woff2?|ttf|eot)"
        r"([?#]|$)",
        re.IGNORECASE,
    )
    # 非文章技术路径
    _SKIP_PATH = re.compile(
        r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon"
        r"|most-popular-news|trending|popular-news|latest-news)"
        r"[/.]",
        re.IGNORECASE,
    )

    article_urls: list[str] = []
    for raw in raw_urls:
        raw_stripped = raw.strip()
        if raw_stripped.startswith("#") or not raw_stripped:
            continue
        if raw_stripped.lower().startswith("javascript:"):
            continue

        full = urljoin(source.homepage, raw_stripped)
        full_clean = full.split("#")[0]

        # 过滤非内容类 URL
        if _SKIP_EXT.search(full_clean):
            continue
        if _SKIP_PATH.search(full_clean):
            continue
        # 过滤非同域链接（广告、CDN 等）
        if _extract_domain(full_clean) != base_domain:
            continue

        if full_clean.rstrip("/") == base_without_fragment.rstrip("/"):
            continue
        if article_url_pattern.search(full_clean):
            article_urls.append(full_clean)

    # 本批去重
    seen: set[str] = set()
    unique: list[str] = []
    for u in article_urls:
        if u not in seen:
            seen.add(u)
            unique.append(u)

    # 增量过滤：读取已抓取的 url_hash，跳过已存在 URL
    today = get_news_day()
    index_path = Path(f"data/raw/{source.id}/{today}/index.jsonl")
    existing_hashes: set[str] = set()
    if index_path.exists():
        with open(index_path, encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                try:
                    item = json.loads(line)
                    existing_hashes.add(item.get("url_hash", ""))
                except json.JSONDecodeError:
                    continue

    filtered: list[str] = []
    skipped = 0
    for u in unique:
        uh = compute_url_hash(u)
        if uh in existing_hashes:
            skipped += 1
            logger.debug("[%s] 跳过已抓取: %s", source.id, u[:80])
            continue
        filtered.append(u)
        if len(filtered) >= source.max_articles_per_run:
            break

    if skipped > 0:
        logger.info("[%s] 增量过滤: 跳过 %d 篇已抓取，剩余 %d 篇待抓取",
                    source.id, skipped, len(filtered))

    logger.info("[%s] 从首页提取到 %d 个文章链接（去重后 %d，增量后 %d）",
                source.id, len(article_urls), len(unique), len(filtered))
    return filtered

# ── 单篇抓取 ──────────────────────────────────────────


async def _crawl_single_article(
    crawler: AsyncWebCrawler,
    url: str,
    source: SourceConfig,
    index: int,
    total: int,
) -> ArticleItem:
    """抓取单篇文章（串行模式）"""
    url_hash = compute_url_hash(url)
    now = datetime.now()
    today = get_news_day()

    article = ArticleItem(
        source_id=source.id,
        source_name=source.name,
        url=url,
        url_hash=url_hash,
        title="",
        crawl_time=now.isoformat(),
        html_path="",
        status="failed",
    )

    try:
        config = _make_run_config(source)
        result = await crawler.arun(url=url, config=config)

        if not result.success:
            article.error = result.error_message or "Unknown error"
            logger.warning("[%s] [%d/%d] 抓取失败: %s",
                           source.id, index, total, article.error[:100])
            return article

        # 提取标题
        article.title = (result.metadata.get("title") if result.metadata else "") or ""

        # 保存输出
        out_dir = Path(f"data/raw/{source.id}/{today}")
        out_dir.mkdir(parents=True, exist_ok=True)

        # HTML
        html_path = out_dir / f"{url_hash}.html"
        if result.html:
            html_path.write_text(result.html, encoding="utf-8")
            article.html_path = str(html_path)

        # Markdown
        md_path = out_dir / f"{url_hash}.md"
        md_content = ""
        if hasattr(result, "markdown") and result.markdown:
            md_content = str(result.markdown)
            md_path.write_text(md_content, encoding="utf-8")
            article.md_path = str(md_path)

        article.word_count = len(md_content.split()) if md_content else 0
        article.status = "success"

        # 每篇都打日志方便追踪进度
        mem_mb = _get_memory_mb()
        logger.info("[%s] [%d/%d] ✅ %s (%d words, %.0f MB)",
                    source.id, index, total, article.title[:50], article.word_count, mem_mb)

    except Exception as e:
        article.error = str(e)
        logger.error("[%s] [%d/%d] 抓取异常: %s", source.id, index, total, e)

    return article

# ── 单源抓取 ──────────────────────────────────────────


async def crawl_source(source: SourceConfig) -> CrawlResult:
    """抓取单个新闻源：首页 → 串行抓取每篇文章

    约束：内存 ≤ 2GB，超时 ≤ 2 小时
    """
    start_time = datetime.now()
    result = CrawlResult(
        source_id=source.id,
        source_name=source.name,
        start_time=start_time.isoformat(),
    )

    mem_start = _get_memory_mb()
    logger.info("━━━ 开始 [%s] %s (内存: %.0f MB) ━━━", source.id, source.name, mem_start)

    browser_config = _make_browser_config(source)

    try:
        async with asyncio.timeout(SOURCE_TIMEOUT_SEC):
            async with AsyncWebCrawler(config=browser_config) as crawler:
                # 1. 首页提取链接
                article_urls = await _extract_article_urls(crawler, source)
                result.total_found = len(article_urls)

                if not article_urls:
                    logger.warning("[%s] 未提取到任何文章链接", source.id)
                    result.end_time = datetime.now().isoformat()
                    return result

                # 2. 串行抓取每篇文章
                total = len(article_urls)
                for i, url in enumerate(article_urls, 1):
                    _check_memory(source.id)

                    article = await _crawl_single_article(
                        crawler, url, source, index=i, total=total,
                    )

                    if article.status == "success":
                        result.total_success += 1
                    else:
                        result.total_failed += 1
                    result.articles.append(article)

                    # 冷却间隔
                    if i < total:
                        await asyncio.sleep(ARTICLE_DELAY_SEC)

    except TimeoutError:
        logger.error("[%s] ⏰ 超时 %.0f 秒（上限 %d 秒），中止",
                     source.id, (datetime.now() - start_time).total_seconds(),
                     SOURCE_TIMEOUT_SEC)
        result.error = "Source timeout"
    except Exception as e:
        logger.exception("[%s] 抓取过程异常: %s", source.id, e)
        result.error = str(e)

    # 主动回收
    gc.collect()

    result.end_time = datetime.now().isoformat()
    elapsed = (datetime.now() - start_time).total_seconds()
    mem_end = _get_memory_mb()
    logger.info(
        "[%s] 完成: 发现 %d / 成功 %d / 失败 %d，耗时 %.1f 秒，内存 %.0f→%.0f MB",
        source.id, result.total_found, result.total_success, result.total_failed,
        elapsed, mem_start, mem_end,
    )

    return result