feat: 增强反爬策略 + forexlive 迁移 investinglive

- crawler.py: 启用 magic/stealth 全自动反检测, --disable-webrtc,
  delay_before_return_html=3s, remove_consent_popups, 过滤列表页URL
- rss_crawler.py: 新增 _make_rss_client() 支持 httpx 代理,
  Google News RSS 跳过 article_url_pattern 域名过滤
- sources.yaml: investing.com 改用 Google News RSS 代理绕过Cloudflare;
  forexlive -> investinglive (域名已301重定向)
- domestic_crawl_8g.sh: HTTP_PROXY/HTTPS_PROXY 在 .env 加载前设置

Co-Authored-By: Claude <noreply@anthropic.com>

crawler/crawler.py

@@ -133,6 +133,7 @@ def _make_browser_config(source: SourceConfig) -> BrowserConfig:
         extra_args += [
             "--disable-blink-features=AutomationControlled",
             "--disable-features=IsolateOrigins,site-per-process",
+            "--disable-webrtc",  # 防止 WebRTC 泄露真实 IP
         ]
 
     # ── SOCKS5 代理 ──────────────────────────────────
@@ -147,6 +148,9 @@ def _make_browser_config(source: SourceConfig) -> BrowserConfig:
             os.environ["DISPLAY"] = _XVFB_DISPLAY
             logger.info("🖥️  headful 模式: DISPLAY=%s", _XVFB_DISPLAY)
 
+    # stealth 反检测
+    enable_stealth = (mode == "stealth") or _HEADFUL
+
     return BrowserConfig(
         browser_type="chromium",
         headless=headless,
@@ -157,18 +161,25 @@ def _make_browser_config(source: SourceConfig) -> BrowserConfig:
         light_mode=headless,
         user_agent=_USER_AGENT,
         extra_args=extra_args,
+        enable_stealth=enable_stealth,  # Playwright stealth 反检测
     )
 
 
 def _make_run_config(source: SourceConfig) -> CrawlerRunConfig:
     """抓取运行时配置"""
+    mode = source.anti_bot_mode or ""
+    is_stealth = (mode in ("stealth", "headful")) or _HEADFUL
+
     return CrawlerRunConfig(
         cache_mode=CacheMode.BYPASS,
         page_timeout=PAGE_TIMEOUT_MS,
-        wait_until="domcontentloaded",
+        wait_until="domcontentloaded",                    # networkidle 会被 Cloudflare 挑战阻塞
         scan_full_page=False,
         simulate_user=True,
         override_navigator=True,
+        magic=is_stealth,                                   # Crawl4AI 全自动反检测
+        delay_before_return_html=3.0 if is_stealth else 0.1,  # 等待 Cloudflare JS 挑战完成
+        remove_consent_popups=True,                 # 自动关闭 Cookie/隐私弹窗
     )
 
 def _extract_domain(url: str) -> str:
@@ -214,7 +225,8 @@ async def _extract_article_urls(
     )
     # 非文章技术路径
     _SKIP_PATH = re.compile(
-        r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon)"
+        r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon"
+        r"|most-popular-news|trending|popular-news|latest-news)"
         r"[/.]",
         re.IGNORECASE,
     )

crawler/rss_crawler.py

@@ -78,14 +78,18 @@ def _is_valid_article_url(url: str, source: SourceConfig) -> bool:
         return False
 
     # 源级 article_url_pattern 校验
-    pattern = getattr(source, "article_url_pattern", "")
-    if pattern:
-        try:
-            if not re.search(pattern, url, re.IGNORECASE):
-                logger.debug("RSS 跳过不匹配 article_url_pattern: %s", url[:80])
-                return False
-        except re.error:
-            pass  # 正则异常不阻塞
+    # Google News RSS 跳过（链接为 news.google.com 格式，不匹配源 pattern）
+    rss_url = getattr(source, "rss_url", "")
+    is_google_news = "news.google.com" in rss_url
+    if not is_google_news:
+        pattern = getattr(source, "article_url_pattern", "")
+        if pattern:
+            try:
+                if not re.search(pattern, url, re.IGNORECASE):
+                    logger.debug("RSS 跳过不匹配 article_url_pattern: %s", url[:80])
+                    return False
+            except re.error:
+                pass  # 正则异常不阻塞
 
     # 域名一致性校验（RSS 跨站污染防护）
     # 例外: Google News RSS 的链接是 news.google.com 跳转 URL，跳过域名校验
@@ -237,6 +241,20 @@ def _attr(element: ElementTree.Element, tag: str, attr: str) -> str:
 # ── 主抓取接口 ───────────────────────────────────────
 
 
+def _make_rss_client() -> httpx.Client:
+    """创建 httpx 客户端，自动配置代理"""
+    try:
+        from crawler.config import load_system_config
+        cfg = load_system_config()
+        p = cfg.get("proxy", {})
+        if p.get("enabled") and p.get("url"):
+            return httpx.Client(proxy=p["url"], follow_redirects=True, timeout=30.0)
+    except Exception:
+        pass
+    return httpx.Client(follow_redirects=True, timeout=30.0)
+
+
+
 def crawl_rss_source(source: SourceConfig) -> CrawlResult:
     """通过 RSS/Atom Feed 抓取单个新闻源。
 
@@ -268,9 +286,9 @@ def crawl_rss_source(source: SourceConfig) -> CrawlResult:
     crawl_time = start_time.isoformat()
 
     try:
-        resp = httpx.get(
+        client = _make_rss_client()
+        resp = client.get(
             rss_url,
-            follow_redirects=True,
             timeout=30.0,
             headers={
                 "User-Agent": (