From 5c106120bf898e85d82cb5041b43da24d6e5878b Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 23 Jul 2026 09:25:23 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E5=BC=BA=E5=8F=8D=E7=88=AC?= =?UTF-8?q?=E7=AD=96=E7=95=A5=20+=20forexlive=20=E8=BF=81=E7=A7=BB=20inves?= =?UTF-8?q?tinglive?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - crawler.py: 启用 magic/stealth 全自动反检测, --disable-webrtc, delay_before_return_html=3s, remove_consent_popups, 过滤列表页URL - rss_crawler.py: 新增 _make_rss_client() 支持 httpx 代理, Google News RSS 跳过 article_url_pattern 域名过滤 - sources.yaml: investing.com 改用 Google News RSS 代理绕过Cloudflare; forexlive -> investinglive (域名已301重定向) - domestic_crawl_8g.sh: HTTP_PROXY/HTTPS_PROXY 在 .env 加载前设置 Co-Authored-By: Claude --- configs/sources.yaml | 21 +++++++++++--------- crawler/crawler.py | 16 +++++++++++++-- crawler/rss_crawler.py | 38 ++++++++++++++++++++++++++---------- scripts/domestic_crawl_8g.sh | 2 ++ 4 files changed, 56 insertions(+), 21 deletions(-) diff --git a/configs/sources.yaml b/configs/sources.yaml index 4df871a..d9cb8d8 100644 --- a/configs/sources.yaml +++ b/configs/sources.yaml @@ -75,7 +75,9 @@ sources: article_url_pattern: "/news/" js_render: true max_articles_per_run: 25 - rss_url: "https://www.investing.com/rss/news_1063.rss" # RSS 绕过反爬 + rss_url: "https://news.google.com/rss/search?q=site:investing.com+news&hl=en-US&gl=US&ceid=US:en" # Google News RSS 代理（绕过 Cloudflare，仅摘要） + anti_bot_mode: "stealth" # Cloudflare 反爬（RSS Google News 代理绕过，Web headful 兜底） + # ⚠️ Google News RSS 仅获取标题+摘要，全文需 Web 回退 headful Playwright 抓取 - id: "seekingalpha" name: "Seeking Alpha" @@ -122,15 +124,16 @@ sources: anti_bot_mode: "stealth" rss_url: "https://www.economist.com/finance-and-economics/rss.xml" # Finance & economics RSS（含标题+摘要+链接） - - id: "forexlive" - name: "ForexLive" + - id: "investinglive" + name: "InvestingLive" enabled: true - homepage: "https://www.forexlive.com/" - article_url_pattern: "/(news|technical-analysis|Education)/" - js_render: false - max_articles_per_run: 20 - rss_url: "https://www.forexlive.com/feed" # Breaking News RSS（含正文，~91KB/次） - + homepage: "https://investinglive.com/" + article_url_pattern: "/(news|central-banks|commodities|stocks|technical-analysis)/" + js_render: true + max_articles_per_run: 25 + anti_bot_mode: "stealth" + rss_url: "https://investinglive.com/feed" # RSS 含完整正文（~120KB/次，25篇） + # 2026-07-23 更新: forexlive.com 已 301 重定向到 investinglive.com - id: "zerohedge" name: "ZeroHedge" enabled: true diff --git a/crawler/crawler.py b/crawler/crawler.py index 68f40a3..302ab45 100644 --- a/crawler/crawler.py +++ b/crawler/crawler.py @@ -133,6 +133,7 @@ def _make_browser_config(source: SourceConfig) -> BrowserConfig: extra_args += [ "--disable-blink-features=AutomationControlled", "--disable-features=IsolateOrigins,site-per-process", + "--disable-webrtc", # 防止 WebRTC 泄露真实 IP ] # ── SOCKS5 代理 ────────────────────────────────── @@ -147,6 +148,9 @@ def _make_browser_config(source: SourceConfig) -> BrowserConfig: os.environ["DISPLAY"] = _XVFB_DISPLAY logger.info("🖥️ headful 模式: DISPLAY=%s", _XVFB_DISPLAY) + # stealth 反检测 + enable_stealth = (mode == "stealth") or _HEADFUL + return BrowserConfig( browser_type="chromium", headless=headless, @@ -157,18 +161,25 @@ def _make_browser_config(source: SourceConfig) -> BrowserConfig: light_mode=headless, user_agent=_USER_AGENT, extra_args=extra_args, + enable_stealth=enable_stealth, # Playwright stealth 反检测 ) def _make_run_config(source: SourceConfig) -> CrawlerRunConfig: """抓取运行时配置""" + mode = source.anti_bot_mode or "" + is_stealth = (mode in ("stealth", "headful")) or _HEADFUL + return CrawlerRunConfig( cache_mode=CacheMode.BYPASS, page_timeout=PAGE_TIMEOUT_MS, - wait_until="domcontentloaded", + wait_until="domcontentloaded", # networkidle 会被 Cloudflare 挑战阻塞 scan_full_page=False, simulate_user=True, override_navigator=True, + magic=is_stealth, # Crawl4AI 全自动反检测 + delay_before_return_html=3.0 if is_stealth else 0.1, # 等待 Cloudflare JS 挑战完成 + remove_consent_popups=True, # 自动关闭 Cookie/隐私弹窗 ) def _extract_domain(url: str) -> str: @@ -214,7 +225,8 @@ async def _extract_article_urls( ) # 非文章技术路径 _SKIP_PATH = re.compile( - r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon)" + r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon" + r"|most-popular-news|trending|popular-news|latest-news)" r"[/.]", re.IGNORECASE, ) diff --git a/crawler/rss_crawler.py b/crawler/rss_crawler.py index 5c19ffb..c237701 100644 --- a/crawler/rss_crawler.py +++ b/crawler/rss_crawler.py @@ -78,14 +78,18 @@ def _is_valid_article_url(url: str, source: SourceConfig) -> bool: return False # 源级 article_url_pattern 校验 - pattern = getattr(source, "article_url_pattern", "") - if pattern: - try: - if not re.search(pattern, url, re.IGNORECASE): - logger.debug("RSS 跳过不匹配 article_url_pattern: %s", url[:80]) - return False - except re.error: - pass # 正则异常不阻塞 + # Google News RSS 跳过（链接为 news.google.com 格式，不匹配源 pattern） + rss_url = getattr(source, "rss_url", "") + is_google_news = "news.google.com" in rss_url + if not is_google_news: + pattern = getattr(source, "article_url_pattern", "") + if pattern: + try: + if not re.search(pattern, url, re.IGNORECASE): + logger.debug("RSS 跳过不匹配 article_url_pattern: %s", url[:80]) + return False + except re.error: + pass # 正则异常不阻塞 # 域名一致性校验（RSS 跨站污染防护） # 例外: Google News RSS 的链接是 news.google.com 跳转 URL，跳过域名校验 @@ -237,6 +241,20 @@ def _attr(element: ElementTree.Element, tag: str, attr: str) -> str: # ── 主抓取接口 ─────────────────────────────────────── +def _make_rss_client() -> httpx.Client: + """创建 httpx 客户端，自动配置代理""" + try: + from crawler.config import load_system_config + cfg = load_system_config() + p = cfg.get("proxy", {}) + if p.get("enabled") and p.get("url"): + return httpx.Client(proxy=p["url"], follow_redirects=True, timeout=30.0) + except Exception: + pass + return httpx.Client(follow_redirects=True, timeout=30.0) + + + def crawl_rss_source(source: SourceConfig) -> CrawlResult: """通过 RSS/Atom Feed 抓取单个新闻源。 @@ -268,9 +286,9 @@ def crawl_rss_source(source: SourceConfig) -> CrawlResult: crawl_time = start_time.isoformat() try: - resp = httpx.get( + client = _make_rss_client() + resp = client.get( rss_url, - follow_redirects=True, timeout=30.0, headers={ "User-Agent": ( diff --git a/scripts/domestic_crawl_8g.sh b/scripts/domestic_crawl_8g.sh index 0f3aa8c..4b6587e 100755 --- a/scripts/domestic_crawl_8g.sh +++ b/scripts/domestic_crawl_8g.sh @@ -31,6 +31,8 @@ LOG "浏览器模式: headful (Xvfb :99)" LOG "HTTP 代理: 127.0.0.1:3128 (Privoxy → SOCKS5)" # ── 加载 .env ── +export HTTP_PROXY=http://127.0.0.1:3128 +export HTTPS_PROXY=http://127.0.0.1:3128 export $(grep -v '^#' .env | grep -v '^$' | xargs 2>/dev/null || true) # ── 启动 Xvfb（如果尚未运行）──