feat: 增强反爬策略 + forexlive 迁移 investinglive

- crawler.py: 启用 magic/stealth 全自动反检测, --disable-webrtc,
  delay_before_return_html=3s, remove_consent_popups, 过滤列表页URL
- rss_crawler.py: 新增 _make_rss_client() 支持 httpx 代理,
  Google News RSS 跳过 article_url_pattern 域名过滤
- sources.yaml: investing.com 改用 Google News RSS 代理绕过Cloudflare;
  forexlive -> investinglive (域名已301重定向)
- domestic_crawl_8g.sh: HTTP_PROXY/HTTPS_PROXY 在 .env 加载前设置

Co-Authored-By: Claude <noreply@anthropic.com>

configs/sources.yaml

@@ -75,7 +75,9 @@ sources:
     article_url_pattern: "/news/"
     js_render: true
     max_articles_per_run: 25
-    rss_url: "https://www.investing.com/rss/news_1063.rss"  # RSS 绕过反爬
+    rss_url: "https://news.google.com/rss/search?q=site:investing.com+news&hl=en-US&gl=US&ceid=US:en"  # Google News RSS 代理（绕过 Cloudflare，仅摘要）
+    anti_bot_mode: "stealth"       # Cloudflare 反爬（RSS Google News 代理绕过，Web headful 兜底）
+    # ⚠️ Google News RSS 仅获取标题+摘要，全文需 Web 回退 headful Playwright 抓取
 
   - id: "seekingalpha"
     name: "Seeking Alpha"
@@ -122,15 +124,16 @@ sources:
     anti_bot_mode: "stealth"
     rss_url: "https://www.economist.com/finance-and-economics/rss.xml"  # Finance & economics RSS（含标题+摘要+链接）
 
-  - id: "forexlive"
+  - id: "investinglive"
-    name: "ForexLive"
+    name: "InvestingLive"
     enabled: true
-    homepage: "https://www.forexlive.com/"
+    homepage: "https://investinglive.com/"
-    article_url_pattern: "/(news|technical-analysis|Education)/"
+    article_url_pattern: "/(news|central-banks|commodities|stocks|technical-analysis)/"
-    js_render: false
+    js_render: true
-    max_articles_per_run: 20
+    max_articles_per_run: 25
-    rss_url: "https://www.forexlive.com/feed"  # Breaking News RSS（含正文，~91KB/次）
+    anti_bot_mode: "stealth"
-
+    rss_url: "https://investinglive.com/feed"  # RSS 含完整正文（~120KB/次，25篇）
+    # 2026-07-23 更新: forexlive.com 已 301 重定向到 investinglive.com
   - id: "zerohedge"
     name: "ZeroHedge"
     enabled: true

crawler/crawler.py

@@ -133,6 +133,7 @@ def _make_browser_config(source: SourceConfig) -> BrowserConfig:
         extra_args += [
             "--disable-blink-features=AutomationControlled",
             "--disable-features=IsolateOrigins,site-per-process",
+            "--disable-webrtc",  # 防止 WebRTC 泄露真实 IP
         ]
 
     # ── SOCKS5 代理 ──────────────────────────────────
@@ -147,6 +148,9 @@ def _make_browser_config(source: SourceConfig) -> BrowserConfig:
             os.environ["DISPLAY"] = _XVFB_DISPLAY
             logger.info("🖥️  headful 模式: DISPLAY=%s", _XVFB_DISPLAY)
 
+    # stealth 反检测
+    enable_stealth = (mode == "stealth") or _HEADFUL
+
     return BrowserConfig(
         browser_type="chromium",
         headless=headless,
@@ -157,18 +161,25 @@ def _make_browser_config(source: SourceConfig) -> BrowserConfig:
         light_mode=headless,
         user_agent=_USER_AGENT,
         extra_args=extra_args,
+        enable_stealth=enable_stealth,  # Playwright stealth 反检测
     )
 
 
 def _make_run_config(source: SourceConfig) -> CrawlerRunConfig:
     """抓取运行时配置"""
+    mode = source.anti_bot_mode or ""
+    is_stealth = (mode in ("stealth", "headful")) or _HEADFUL
+
     return CrawlerRunConfig(
         cache_mode=CacheMode.BYPASS,
         page_timeout=PAGE_TIMEOUT_MS,
-        wait_until="domcontentloaded",
+        wait_until="domcontentloaded",                    # networkidle 会被 Cloudflare 挑战阻塞
         scan_full_page=False,
         simulate_user=True,
         override_navigator=True,
+        magic=is_stealth,                                   # Crawl4AI 全自动反检测
+        delay_before_return_html=3.0 if is_stealth else 0.1,  # 等待 Cloudflare JS 挑战完成
+        remove_consent_popups=True,                 # 自动关闭 Cookie/隐私弹窗
     )
 
 def _extract_domain(url: str) -> str:
@@ -214,7 +225,8 @@ async def _extract_article_urls(
     )
     # 非文章技术路径
     _SKIP_PATH = re.compile(
-        r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon)"
+        r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon"
+        r"|most-popular-news|trending|popular-news|latest-news)"
         r"[/.]",
         re.IGNORECASE,
     )

crawler/rss_crawler.py

@@ -78,14 +78,18 @@ def _is_valid_article_url(url: str, source: SourceConfig) -> bool:
         return False
 
     # 源级 article_url_pattern 校验
-    pattern = getattr(source, "article_url_pattern", "")
+    # Google News RSS 跳过（链接为 news.google.com 格式，不匹配源 pattern）
-    if pattern:
+    rss_url = getattr(source, "rss_url", "")
-        try:
+    is_google_news = "news.google.com" in rss_url
-            if not re.search(pattern, url, re.IGNORECASE):
+    if not is_google_news:
-                logger.debug("RSS 跳过不匹配 article_url_pattern: %s", url[:80])
+        pattern = getattr(source, "article_url_pattern", "")
-                return False
+        if pattern:
-        except re.error:
+            try:
-            pass  # 正则异常不阻塞
+                if not re.search(pattern, url, re.IGNORECASE):
+                    logger.debug("RSS 跳过不匹配 article_url_pattern: %s", url[:80])
+                    return False
+            except re.error:
+                pass  # 正则异常不阻塞
 
     # 域名一致性校验（RSS 跨站污染防护）
     # 例外: Google News RSS 的链接是 news.google.com 跳转 URL，跳过域名校验
@@ -237,6 +241,20 @@ def _attr(element: ElementTree.Element, tag: str, attr: str) -> str:
 # ── 主抓取接口 ───────────────────────────────────────
 
 
+def _make_rss_client() -> httpx.Client:
+    """创建 httpx 客户端，自动配置代理"""
+    try:
+        from crawler.config import load_system_config
+        cfg = load_system_config()
+        p = cfg.get("proxy", {})
+        if p.get("enabled") and p.get("url"):
+            return httpx.Client(proxy=p["url"], follow_redirects=True, timeout=30.0)
+    except Exception:
+        pass
+    return httpx.Client(follow_redirects=True, timeout=30.0)
+
+
+
 def crawl_rss_source(source: SourceConfig) -> CrawlResult:
     """通过 RSS/Atom Feed 抓取单个新闻源。
 
@@ -268,9 +286,9 @@ def crawl_rss_source(source: SourceConfig) -> CrawlResult:
     crawl_time = start_time.isoformat()
 
     try:
-        resp = httpx.get(
+        client = _make_rss_client()
+        resp = client.get(
             rss_url,
-            follow_redirects=True,
             timeout=30.0,
             headers={
                 "User-Agent": (

scripts/domestic_crawl_8g.sh

@@ -31,6 +31,8 @@ LOG "浏览器模式: headful (Xvfb :99)"
 LOG "HTTP 代理: 127.0.0.1:3128 (Privoxy → SOCKS5)"
 
 # ── 加载 .env ──
+export HTTP_PROXY=http://127.0.0.1:3128
+export HTTPS_PROXY=http://127.0.0.1:3128
 export $(grep -v '^#' .env | grep -v '^$' | xargs 2>/dev/null || true)
 
 # ── 启动 Xvfb（如果尚未运行）──