feat: 增强反爬策略 + forexlive 迁移 investinglive
- crawler.py: 启用 magic/stealth 全自动反检测, --disable-webrtc, delay_before_return_html=3s, remove_consent_popups, 过滤列表页URL - rss_crawler.py: 新增 _make_rss_client() 支持 httpx 代理, Google News RSS 跳过 article_url_pattern 域名过滤 - sources.yaml: investing.com 改用 Google News RSS 代理绕过Cloudflare; forexlive -> investinglive (域名已301重定向) - domestic_crawl_8g.sh: HTTP_PROXY/HTTPS_PROXY 在 .env 加载前设置 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
+28
-10
@@ -78,14 +78,18 @@ def _is_valid_article_url(url: str, source: SourceConfig) -> bool:
|
||||
return False
|
||||
|
||||
# 源级 article_url_pattern 校验
|
||||
pattern = getattr(source, "article_url_pattern", "")
|
||||
if pattern:
|
||||
try:
|
||||
if not re.search(pattern, url, re.IGNORECASE):
|
||||
logger.debug("RSS 跳过不匹配 article_url_pattern: %s", url[:80])
|
||||
return False
|
||||
except re.error:
|
||||
pass # 正则异常不阻塞
|
||||
# Google News RSS 跳过(链接为 news.google.com 格式,不匹配源 pattern)
|
||||
rss_url = getattr(source, "rss_url", "")
|
||||
is_google_news = "news.google.com" in rss_url
|
||||
if not is_google_news:
|
||||
pattern = getattr(source, "article_url_pattern", "")
|
||||
if pattern:
|
||||
try:
|
||||
if not re.search(pattern, url, re.IGNORECASE):
|
||||
logger.debug("RSS 跳过不匹配 article_url_pattern: %s", url[:80])
|
||||
return False
|
||||
except re.error:
|
||||
pass # 正则异常不阻塞
|
||||
|
||||
# 域名一致性校验(RSS 跨站污染防护)
|
||||
# 例外: Google News RSS 的链接是 news.google.com 跳转 URL,跳过域名校验
|
||||
@@ -237,6 +241,20 @@ def _attr(element: ElementTree.Element, tag: str, attr: str) -> str:
|
||||
# ── 主抓取接口 ───────────────────────────────────────
|
||||
|
||||
|
||||
def _make_rss_client() -> httpx.Client:
|
||||
"""创建 httpx 客户端,自动配置代理"""
|
||||
try:
|
||||
from crawler.config import load_system_config
|
||||
cfg = load_system_config()
|
||||
p = cfg.get("proxy", {})
|
||||
if p.get("enabled") and p.get("url"):
|
||||
return httpx.Client(proxy=p["url"], follow_redirects=True, timeout=30.0)
|
||||
except Exception:
|
||||
pass
|
||||
return httpx.Client(follow_redirects=True, timeout=30.0)
|
||||
|
||||
|
||||
|
||||
def crawl_rss_source(source: SourceConfig) -> CrawlResult:
|
||||
"""通过 RSS/Atom Feed 抓取单个新闻源。
|
||||
|
||||
@@ -268,9 +286,9 @@ def crawl_rss_source(source: SourceConfig) -> CrawlResult:
|
||||
crawl_time = start_time.isoformat()
|
||||
|
||||
try:
|
||||
resp = httpx.get(
|
||||
client = _make_rss_client()
|
||||
resp = client.get(
|
||||
rss_url,
|
||||
follow_redirects=True,
|
||||
timeout=30.0,
|
||||
headers={
|
||||
"User-Agent": (
|
||||
|
||||
Reference in New Issue
Block a user