feat: 增强反爬策略 + forexlive 迁移 investinglive
- crawler.py: 启用 magic/stealth 全自动反检测, --disable-webrtc, delay_before_return_html=3s, remove_consent_popups, 过滤列表页URL - rss_crawler.py: 新增 _make_rss_client() 支持 httpx 代理, Google News RSS 跳过 article_url_pattern 域名过滤 - sources.yaml: investing.com 改用 Google News RSS 代理绕过Cloudflare; forexlive -> investinglive (域名已301重定向) - domestic_crawl_8g.sh: HTTP_PROXY/HTTPS_PROXY 在 .env 加载前设置 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
+12
-9
@@ -75,7 +75,9 @@ sources:
|
||||
article_url_pattern: "/news/"
|
||||
js_render: true
|
||||
max_articles_per_run: 25
|
||||
rss_url: "https://www.investing.com/rss/news_1063.rss" # RSS 绕过反爬
|
||||
rss_url: "https://news.google.com/rss/search?q=site:investing.com+news&hl=en-US&gl=US&ceid=US:en" # Google News RSS 代理（绕过 Cloudflare，仅摘要）
|
||||
anti_bot_mode: "stealth" # Cloudflare 反爬（RSS Google News 代理绕过，Web headful 兜底）
|
||||
# ⚠️ Google News RSS 仅获取标题+摘要，全文需 Web 回退 headful Playwright 抓取
|
||||
|
||||
- id: "seekingalpha"
|
||||
name: "Seeking Alpha"
|
||||
@@ -122,15 +124,16 @@ sources:
|
||||
anti_bot_mode: "stealth"
|
||||
rss_url: "https://www.economist.com/finance-and-economics/rss.xml" # Finance & economics RSS（含标题+摘要+链接）
|
||||
|
||||
- id: "forexlive"
|
||||
name: "ForexLive"
|
||||
- id: "investinglive"
|
||||
name: "InvestingLive"
|
||||
enabled: true
|
||||
homepage: "https://www.forexlive.com/"
|
||||
article_url_pattern: "/(news|technical-analysis|Education)/"
|
||||
js_render: false
|
||||
max_articles_per_run: 20
|
||||
rss_url: "https://www.forexlive.com/feed" # Breaking News RSS（含正文，~91KB/次）
|
||||
|
||||
homepage: "https://investinglive.com/"
|
||||
article_url_pattern: "/(news|central-banks|commodities|stocks|technical-analysis)/"
|
||||
js_render: true
|
||||
max_articles_per_run: 25
|
||||
anti_bot_mode: "stealth"
|
||||
rss_url: "https://investinglive.com/feed" # RSS 含完整正文（~120KB/次，25篇）
|
||||
# 2026-07-23 更新: forexlive.com 已 301 重定向到 investinglive.com
|
||||
- id: "zerohedge"
|
||||
name: "ZeroHedge"
|
||||
enabled: true
|
||||
|
||||
+14
-2
@@ -133,6 +133,7 @@ def _make_browser_config(source: SourceConfig) -> BrowserConfig:
|
||||
extra_args += [
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--disable-features=IsolateOrigins,site-per-process",
|
||||
"--disable-webrtc", # 防止 WebRTC 泄露真实 IP
|
||||
]
|
||||
|
||||
# ── SOCKS5 代理 ──────────────────────────────────
|
||||
@@ -147,6 +148,9 @@ def _make_browser_config(source: SourceConfig) -> BrowserConfig:
|
||||
os.environ["DISPLAY"] = _XVFB_DISPLAY
|
||||
logger.info("🖥️ headful 模式: DISPLAY=%s", _XVFB_DISPLAY)
|
||||
|
||||
# stealth 反检测
|
||||
enable_stealth = (mode == "stealth") or _HEADFUL
|
||||
|
||||
return BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=headless,
|
||||
@@ -157,18 +161,25 @@ def _make_browser_config(source: SourceConfig) -> BrowserConfig:
|
||||
light_mode=headless,
|
||||
user_agent=_USER_AGENT,
|
||||
extra_args=extra_args,
|
||||
enable_stealth=enable_stealth, # Playwright stealth 反检测
|
||||
)
|
||||
|
||||
|
||||
def _make_run_config(source: SourceConfig) -> CrawlerRunConfig:
|
||||
"""抓取运行时配置"""
|
||||
mode = source.anti_bot_mode or ""
|
||||
is_stealth = (mode in ("stealth", "headful")) or _HEADFUL
|
||||
|
||||
return CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
page_timeout=PAGE_TIMEOUT_MS,
|
||||
wait_until="domcontentloaded",
|
||||
wait_until="domcontentloaded", # networkidle 会被 Cloudflare 挑战阻塞
|
||||
scan_full_page=False,
|
||||
simulate_user=True,
|
||||
override_navigator=True,
|
||||
magic=is_stealth, # Crawl4AI 全自动反检测
|
||||
delay_before_return_html=3.0 if is_stealth else 0.1, # 等待 Cloudflare JS 挑战完成
|
||||
remove_consent_popups=True, # 自动关闭 Cookie/隐私弹窗
|
||||
)
|
||||
|
||||
def _extract_domain(url: str) -> str:
|
||||
@@ -214,7 +225,8 @@ async def _extract_article_urls(
|
||||
)
|
||||
# 非文章技术路径
|
||||
_SKIP_PATH = re.compile(
|
||||
r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon)"
|
||||
r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon"
|
||||
r"|most-popular-news|trending|popular-news|latest-news)"
|
||||
r"[/.]",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
+20
-2
@@ -78,6 +78,10 @@ def _is_valid_article_url(url: str, source: SourceConfig) -> bool:
|
||||
return False
|
||||
|
||||
# 源级 article_url_pattern 校验
|
||||
# Google News RSS 跳过（链接为 news.google.com 格式，不匹配源 pattern）
|
||||
rss_url = getattr(source, "rss_url", "")
|
||||
is_google_news = "news.google.com" in rss_url
|
||||
if not is_google_news:
|
||||
pattern = getattr(source, "article_url_pattern", "")
|
||||
if pattern:
|
||||
try:
|
||||
@@ -237,6 +241,20 @@ def _attr(element: ElementTree.Element, tag: str, attr: str) -> str:
|
||||
# ── 主抓取接口 ───────────────────────────────────────
|
||||
|
||||
|
||||
def _make_rss_client() -> httpx.Client:
|
||||
"""创建 httpx 客户端，自动配置代理"""
|
||||
try:
|
||||
from crawler.config import load_system_config
|
||||
cfg = load_system_config()
|
||||
p = cfg.get("proxy", {})
|
||||
if p.get("enabled") and p.get("url"):
|
||||
return httpx.Client(proxy=p["url"], follow_redirects=True, timeout=30.0)
|
||||
except Exception:
|
||||
pass
|
||||
return httpx.Client(follow_redirects=True, timeout=30.0)
|
||||
|
||||
|
||||
|
||||
def crawl_rss_source(source: SourceConfig) -> CrawlResult:
|
||||
"""通过 RSS/Atom Feed 抓取单个新闻源。
|
||||
|
||||
@@ -268,9 +286,9 @@ def crawl_rss_source(source: SourceConfig) -> CrawlResult:
|
||||
crawl_time = start_time.isoformat()
|
||||
|
||||
try:
|
||||
resp = httpx.get(
|
||||
client = _make_rss_client()
|
||||
resp = client.get(
|
||||
rss_url,
|
||||
follow_redirects=True,
|
||||
timeout=30.0,
|
||||
headers={
|
||||
"User-Agent": (
|
||||
|
||||
@@ -31,6 +31,8 @@ LOG "浏览器模式: headful (Xvfb :99)"
|
||||
LOG "HTTP 代理: 127.0.0.1:3128 (Privoxy → SOCKS5)"
|
||||
|
||||
# ── 加载 .env ──
|
||||
export HTTP_PROXY=http://127.0.0.1:3128
|
||||
export HTTPS_PROXY=http://127.0.0.1:3128
|
||||
export $(grep -v '^#' .env | grep -v '^$' | xargs 2>/dev/null || true)
|
||||
|
||||
# ── 启动 Xvfb（如果尚未运行）──
|
||||
|
||||
Reference in New Issue
Block a user